2023-10-09T09:27:56.833 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T09:27:56.834 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635 --name teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --verbose --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635/orig.config.yaml 2023-10-09T09:27:56.895 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T09:27:56.980 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7417635' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 3687 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi012.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi110.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T09:27:56.981 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T09:27:56.982 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T09:27:56.982 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T09:27:56.983 INFO:teuthology.task.internal:Checking packages... 2023-10-09T09:27:57.006 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T09:27:57.006 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T09:27:57.006 INFO:teuthology.packaging:ref: None 2023-10-09T09:27:57.006 INFO:teuthology.packaging:tag: None 2023-10-09T09:27:57.006 INFO:teuthology.packaging:branch: reef 2023-10-09T09:27:57.007 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:27:57.007 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-10-09T09:27:57.247 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610.g397de1b0 2023-10-09T09:27:57.247 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T09:27:57.253 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T09:27:57.253 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T09:27:57.296 INFO:teuthology.task.internal:Saving configuration 2023-10-09T09:27:57.317 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T09:27:57.329 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T09:27:57.356 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi012.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 09:22:55.238270', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T09:27:57.378 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi110.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 09:22:55.240068', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T09:27:57.378 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T09:27:57.385 INFO:teuthology.task.internal:roles: ubuntu@smithi012.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T09:27:57.385 INFO:teuthology.task.internal:roles: ubuntu@smithi110.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T09:27:57.385 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T09:27:57.476 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fdb267aaaf0>, signals=[15]) 2023-10-09T09:27:57.477 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T09:27:57.486 INFO:teuthology.task.internal:Opening connections... 2023-10-09T09:27:57.486 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi012.front.sepia.ceph.com 2023-10-09T09:27:57.488 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T09:27:57.556 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi110.front.sepia.ceph.com 2023-10-09T09:27:57.557 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T09:27:57.624 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T09:27:57.631 DEBUG:teuthology.orchestra.run.smithi012:> uname -m 2023-10-09T09:27:57.651 INFO:teuthology.orchestra.run.smithi012.stdout:x86_64 2023-10-09T09:27:57.652 DEBUG:teuthology.orchestra.run.smithi012:> cat /etc/os-release 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:ID="rhel" 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:ID_LIKE="fedora" 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION_ID="8.6" 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:PLATFORM_ID="platform:el8" 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:ANSI_COLOR="0;31" 2023-10-09T09:27:57.713 INFO:teuthology.orchestra.run.smithi012.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T09:27:57.714 INFO:teuthology.orchestra.run.smithi012.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T09:27:57.714 INFO:teuthology.orchestra.run.smithi012.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T09:27:57.714 INFO:teuthology.orchestra.run.smithi012.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T09:27:57.714 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:27:57.714 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T09:27:57.714 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T09:27:57.714 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T09:27:57.714 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T09:27:57.715 INFO:teuthology.lock.ops:Updating smithi012.front.sepia.ceph.com on lock server 2023-10-09T09:27:57.740 DEBUG:teuthology.orchestra.run.smithi110:> uname -m 2023-10-09T09:27:57.760 INFO:teuthology.orchestra.run.smithi110.stdout:x86_64 2023-10-09T09:27:57.760 DEBUG:teuthology.orchestra.run.smithi110:> cat /etc/os-release 2023-10-09T09:27:57.819 INFO:teuthology.orchestra.run.smithi110.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T09:27:57.819 INFO:teuthology.orchestra.run.smithi110.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T09:27:57.819 INFO:teuthology.orchestra.run.smithi110.stdout:ID="rhel" 2023-10-09T09:27:57.820 INFO:teuthology.orchestra.run.smithi110.stdout:ID_LIKE="fedora" 2023-10-09T09:27:57.820 INFO:teuthology.orchestra.run.smithi110.stdout:VERSION_ID="8.6" 2023-10-09T09:27:57.820 INFO:teuthology.orchestra.run.smithi110.stdout:PLATFORM_ID="platform:el8" 2023-10-09T09:27:57.820 INFO:teuthology.orchestra.run.smithi110.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T09:27:57.820 INFO:teuthology.orchestra.run.smithi110.stdout:ANSI_COLOR="0;31" 2023-10-09T09:27:57.820 INFO:teuthology.orchestra.run.smithi110.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T09:27:57.820 INFO:teuthology.orchestra.run.smithi110.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T09:27:57.821 INFO:teuthology.orchestra.run.smithi110.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T09:27:57.821 INFO:teuthology.orchestra.run.smithi110.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T09:27:57.821 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:27:57.821 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T09:27:57.821 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T09:27:57.821 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T09:27:57.821 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T09:27:57.822 INFO:teuthology.lock.ops:Updating smithi110.front.sepia.ceph.com on lock server 2023-10-09T09:27:57.840 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T09:27:57.850 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T09:27:57.871 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T09:27:57.871 DEBUG:teuthology.orchestra.run.smithi012:> test '!' -e /home/ubuntu/cephtest 2023-10-09T09:27:57.873 DEBUG:teuthology.orchestra.run.smithi110:> test '!' -e /home/ubuntu/cephtest 2023-10-09T09:27:57.891 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T09:27:57.900 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T09:27:57.900 DEBUG:teuthology.orchestra.run.smithi012:> test -z $(ls -A /var/lib/ceph) 2023-10-09T09:27:57.933 DEBUG:teuthology.orchestra.run.smithi110:> test -z $(ls -A /var/lib/ceph) 2023-10-09T09:27:57.958 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T09:27:58.014 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T09:27:58.028 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T09:27:58.028 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-09T09:27:58.029 DEBUG:teuthology.orchestra.run.smithi012:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T09:27:58.029 DEBUG:teuthology.orchestra.run.smithi110:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T09:27:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.047 DEBUG:teuthology.orchestra.run.smithi110:> uname -r 2023-10-09T09:27:58.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:27:58.048 DEBUG:teuthology.orchestra.run.smithi012:> uname -r 2023-10-09T09:27:58.106 INFO:teuthology.orchestra.run.smithi110.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T09:27:58.106 INFO:teuthology.task.kernel:Running kernel on smithi110: 4.18.0-372.9.1.el8.x86_64 2023-10-09T09:27:58.106 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum install -y kernel 2023-10-09T09:27:58.109 INFO:teuthology.orchestra.run.smithi012.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T09:27:58.109 INFO:teuthology.task.kernel:Running kernel on smithi012: 4.18.0-372.9.1.el8.x86_64 2023-10-09T09:27:58.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum install -y kernel 2023-10-09T09:27:59.363 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:27:59.364 INFO:teuthology.orchestra.run.smithi110.stdout:Unable to read consumer identity 2023-10-09T09:27:59.368 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:27:59.368 INFO:teuthology.orchestra.run.smithi012.stdout:Unable to read consumer identity 2023-10-09T09:28:00.830 INFO:teuthology.orchestra.run.smithi012.stdout:Copr repo for python3-asyncssh owned by ceph 14 kB/s | 3.5 kB 00:00 2023-10-09T09:28:00.908 INFO:teuthology.orchestra.run.smithi110.stdout:Copr repo for python3-asyncssh owned by ceph 15 kB/s | 3.5 kB 00:00 2023-10-09T09:28:02.825 INFO:teuthology.orchestra.run.smithi110.stdout:Extra Packages for Enterprise Linux 8.4 MB/s | 16 MB 00:01 2023-10-09T09:28:03.056 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 7.2 MB/s | 16 MB 00:02 2023-10-09T09:28:06.291 INFO:teuthology.orchestra.run.smithi110.stdout:lab-extras 368 kB/s | 24 kB 00:00 2023-10-09T09:28:06.746 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 337 kB/s | 24 kB 00:00 2023-10-09T09:28:06.969 INFO:teuthology.orchestra.run.smithi110.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T09:28:07.018 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:28:07.019 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:28:07.019 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:28:07.144 DEBUG:teuthology.orchestra.run.smithi110:> echo no | sudo yum reinstall kernel || true 2023-10-09T09:28:07.458 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T09:28:07.476 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:28:07.476 INFO:teuthology.orchestra.run.smithi110.stdout:Unable to read consumer identity 2023-10-09T09:28:07.516 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:28:07.517 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:28:07.517 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:28:07.658 DEBUG:teuthology.orchestra.run.smithi012:> echo no | sudo yum reinstall kernel || true 2023-10-09T09:28:07.736 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:01 ago on Mon 09 Oct 2023 09:28:06 AM UTC. 2023-10-09T09:28:07.782 INFO:teuthology.orchestra.run.smithi110.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T09:28:07.812 INFO:teuthology.orchestra.run.smithi110.stderr:Error: No packages marked for reinstall. 2023-10-09T09:28:07.858 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum reinstall -y kernel || true 2023-10-09T09:28:08.009 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:28:08.009 INFO:teuthology.orchestra.run.smithi012.stdout:Unable to read consumer identity 2023-10-09T09:28:08.185 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:28:08.185 INFO:teuthology.orchestra.run.smithi110.stdout:Unable to read consumer identity 2023-10-09T09:28:08.288 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 09:28:06 AM UTC. 2023-10-09T09:28:08.333 INFO:teuthology.orchestra.run.smithi012.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T09:28:08.384 INFO:teuthology.orchestra.run.smithi012.stderr:Error: No packages marked for reinstall. 2023-10-09T09:28:08.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum reinstall -y kernel || true 2023-10-09T09:28:08.437 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 09:28:06 AM UTC. 2023-10-09T09:28:08.478 INFO:teuthology.orchestra.run.smithi110.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T09:28:08.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: No packages marked for reinstall. 2023-10-09T09:28:08.551 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T09:28:08.631 INFO:teuthology.orchestra.run.smithi110.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T09:28:08.631 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T09:28:08.631 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T09:28:08.632 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T09:28:08.632 DEBUG:teuthology.orchestra.run.smithi110:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T09:28:08.660 INFO:teuthology.orchestra.run.smithi110.stdout:ttyS1 2023-10-09T09:28:08.680 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:08.785 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:28:08.785 INFO:teuthology.orchestra.run.smithi012.stdout:Unable to read consumer identity 2023-10-09T09:28:09.060 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:03 ago on Mon 09 Oct 2023 09:28:06 AM UTC. 2023-10-09T09:28:09.108 INFO:teuthology.orchestra.run.smithi012.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T09:28:09.146 INFO:teuthology.orchestra.run.smithi012.stderr:Error: No packages marked for reinstall. 2023-10-09T09:28:09.195 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T09:28:09.278 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T09:28:09.278 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T09:28:09.278 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T09:28:09.278 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T09:28:09.279 DEBUG:teuthology.orchestra.run.smithi012:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T09:28:09.305 INFO:teuthology.orchestra.run.smithi012.stdout:ttyS1 2023-10-09T09:28:09.329 DEBUG:teuthology.parallel:result is None 2023-10-09T09:28:09.329 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T09:28:09.336 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T09:28:09.336 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T09:28:09.371 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T09:28:09.389 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T09:28:09.396 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T09:28:09.420 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T09:28:09.420 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T09:28:09.428 DEBUG:teuthology.orchestra.run.smithi110:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T09:28:09.458 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T09:28:09.465 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T09:28:09.465 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T09:28:09.495 DEBUG:teuthology.orchestra.run.smithi110:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T09:28:09.530 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T09:28:09.540 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T09:28:09.543 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T09:28:09.554 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T09:28:09.555 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T09:28:09.562 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T09:28:09.562 DEBUG:teuthology.orchestra.run.smithi012:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T09:28:09.584 DEBUG:teuthology.orchestra.run.smithi110:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T09:28:09.611 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T09:28:09.620 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T09:28:09.620 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T09:28:09.652 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T09:28:09.668 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T09:28:09.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T09:28:09.811 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T09:28:09.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T09:28:09.895 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:28:09.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T09:28:09.964 DEBUG:teuthology.orchestra.run.smithi110:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T09:28:10.007 DEBUG:teuthology.orchestra.run.smithi110:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T09:28:10.062 DEBUG:teuthology.orchestra.run.smithi110:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T09:28:10.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T09:28:10.188 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:28:10.188 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T09:28:10.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo service rsyslog restart 2023-10-09T09:28:10.258 DEBUG:teuthology.orchestra.run.smithi110:> sudo service rsyslog restart 2023-10-09T09:28:10.316 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T09:28:10.354 INFO:teuthology.orchestra.run.smithi110.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T09:28:10.703 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T09:28:10.711 INFO:teuthology.task.internal:Starting timer... 2023-10-09T09:28:10.711 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T09:28:10.757 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T09:28:10.784 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-09T09:28:10.784 DEBUG:teuthology.orchestra.run.smithi012:> sudo service auditd rotate 2023-10-09T09:28:10.839 INFO:teuthology.orchestra.run.smithi012.stdout:Rotating logs: 2023-10-09T09:28:10.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo service auditd rotate 2023-10-09T09:28:10.907 INFO:teuthology.orchestra.run.smithi110.stdout:Rotating logs: 2023-10-09T09:28:10.909 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T09:28:10.909 DEBUG:teuthology.orchestra.run.smithi012:> /usr/sbin/getenforce 2023-10-09T09:28:10.938 INFO:teuthology.orchestra.run.smithi012.stdout:Permissive 2023-10-09T09:28:10.938 DEBUG:teuthology.orchestra.run.smithi110:> /usr/sbin/getenforce 2023-10-09T09:28:10.957 INFO:teuthology.orchestra.run.smithi110.stdout:Permissive 2023-10-09T09:28:10.957 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi012.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi110.front.sepia.ceph.com': 'permissive'} 2023-10-09T09:28:10.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T09:28:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.006 DEBUG:teuthology.orchestra.run.smithi110:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T09:28:11.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:28:11.034 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T09:28:11.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo /usr/sbin/setenforce permissive 2023-10-09T09:28:11.073 DEBUG:teuthology.orchestra.run.smithi110:> sudo /usr/sbin/setenforce permissive 2023-10-09T09:28:11.103 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T09:28:11.113 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-09T09:28:11.119 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-09T09:28:11.226 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T09:28:11.240 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T09:28:11.240 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi012.front.sepia.ceph.com,smithi110.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T09:37:09.003 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi012.front.sepia.ceph.com'), Remote(name='ubuntu@smithi110.front.sepia.ceph.com')] 2023-10-09T09:37:09.004 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2023-10-09T09:37:09.004 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T09:37:09.075 DEBUG:teuthology.orchestra.run.smithi012:> true 2023-10-09T09:37:09.144 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi012.front.sepia.ceph.com' 2023-10-09T09:37:09.144 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2023-10-09T09:37:09.145 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T09:37:09.212 DEBUG:teuthology.orchestra.run.smithi110:> true 2023-10-09T09:37:09.280 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi110.front.sepia.ceph.com' 2023-10-09T09:37:09.280 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T09:37:09.291 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T09:37:09.291 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T09:37:09.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T09:37:09.294 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T09:37:09.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T09:37:09.326 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T09:37:09.342 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T09:37:09.351 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T09:37:09.366 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T09:37:09.415 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: ntpd: command not found 2023-10-09T09:37:09.428 INFO:teuthology.orchestra.run.smithi012.stdout:506 Cannot talk to daemon 2023-10-09T09:37:09.440 INFO:teuthology.orchestra.run.smithi110.stderr:sudo: ntpd: command not found 2023-10-09T09:37:09.443 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T09:37:09.453 INFO:teuthology.orchestra.run.smithi110.stdout:506 Cannot talk to daemon 2023-10-09T09:37:09.458 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T09:37:09.467 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T09:37:09.481 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T09:37:09.543 INFO:teuthology.orchestra.run.smithi012.stderr:bash: ntpq: command not found 2023-10-09T09:37:09.546 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T09:37:09.546 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2023-10-09T09:37:09.546 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T09:37:09.546 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T09:37:09.546 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T09:37:09.546 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T09:37:09.548 INFO:teuthology.orchestra.run.smithi110.stderr:bash: ntpq: command not found 2023-10-09T09:37:09.552 INFO:teuthology.orchestra.run.smithi110.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T09:37:09.552 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================== 2023-10-09T09:37:09.552 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T09:37:09.552 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T09:37:09.552 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T09:37:09.552 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T09:37:09.553 INFO:teuthology.run_tasks:Running task pexec... 2023-10-09T09:37:09.563 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-09T09:37:09.564 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi012.front.sepia.ceph.com 2023-10-09T09:37:09.564 DEBUG:teuthology.orchestra.run.smithi012:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T09:37:09.564 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi110.front.sepia.ceph.com 2023-10-09T09:37:09.564 DEBUG:teuthology.orchestra.run.smithi110:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T09:37:09.967 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:37:09.981 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:37:10.779 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:39 ago on Mon 09 Oct 2023 09:35:31 AM UTC. 2023-10-09T09:37:10.830 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:24 ago on Mon 09 Oct 2023 09:35:46 AM UTC. 2023-10-09T09:37:12.654 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:37:12.655 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:37:12.655 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2023-10-09T09:37:12.655 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:37:12.655 INFO:teuthology.orchestra.run.smithi110.stdout:Resetting modules: 2023-10-09T09:37:12.655 INFO:teuthology.orchestra.run.smithi110.stdout: container-tools 2023-10-09T09:37:12.656 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:37:12.656 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:37:12.656 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:37:12.656 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:37:12.804 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:37:12.805 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:37:12.805 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2023-10-09T09:37:12.805 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:37:12.805 INFO:teuthology.orchestra.run.smithi012.stdout:Resetting modules: 2023-10-09T09:37:12.805 INFO:teuthology.orchestra.run.smithi012.stdout: container-tools 2023-10-09T09:37:12.805 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:12.806 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:37:12.806 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:37:12.806 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:13.626 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:42 ago on Mon 09 Oct 2023 09:35:31 AM UTC. 2023-10-09T09:37:13.807 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:27 ago on Mon 09 Oct 2023 09:35:46 AM UTC. 2023-10-09T09:37:16.248 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:37:16.450 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:37:16.713 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:37:16.945 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:37:17.605 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:46 ago on Mon 09 Oct 2023 09:35:31 AM UTC. 2023-10-09T09:37:17.779 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:31 ago on Mon 09 Oct 2023 09:35:46 AM UTC. 2023-10-09T09:37:19.330 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:37:19.424 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================================ 2023-10-09T09:37:19.424 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:37:19.424 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================================ 2023-10-09T09:37:19.424 INFO:teuthology.orchestra.run.smithi110.stdout:Installing group/module packages: 2023-10-09T09:37:19.424 INFO:teuthology.orchestra.run.smithi110.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-10-09T09:37:19.424 INFO:teuthology.orchestra.run.smithi110.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-10-09T09:37:19.424 INFO:teuthology.orchestra.run.smithi110.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout:Installing module profiles: 2023-10-09T09:37:19.425 INFO:teuthology.orchestra.run.smithi110.stdout: container-tools/common 2023-10-09T09:37:19.426 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:37:19.426 INFO:teuthology.orchestra.run.smithi110.stdout:Enabling module streams: 2023-10-09T09:37:19.426 INFO:teuthology.orchestra.run.smithi110.stdout: container-tools rhel8 2023-10-09T09:37:19.426 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:37:19.426 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:37:19.426 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================================ 2023-10-09T09:37:19.426 INFO:teuthology.orchestra.run.smithi110.stdout:Install 10 Packages 2023-10-09T09:37:19.426 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:37:19.427 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 18 M 2023-10-09T09:37:19.427 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 65 M 2023-10-09T09:37:19.427 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:37:19.492 INFO:teuthology.orchestra.run.smithi110.stdout:(1/10): yajl-2.1.0-10.el8.x86_64.rpm 619 kB/s | 41 kB 00:00 2023-10-09T09:37:19.518 INFO:teuthology.orchestra.run.smithi110.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 272 kB/s | 25 kB 00:00 2023-10-09T09:37:19.543 INFO:teuthology.orchestra.run.smithi110.stdout:(3/10): python3-pyxdg-0.25-16.el8.noarch.rpm 805 kB/s | 94 kB 00:00 2023-10-09T09:37:19.568 INFO:teuthology.orchestra.run.smithi110.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 653 kB/s | 49 kB 00:00 2023-10-09T09:37:19.602 INFO:teuthology.orchestra.run.smithi110.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 6.1 MB/s | 524 kB 00:00 2023-10-09T09:37:19.627 INFO:teuthology.orchestra.run.smithi110.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.5 MB/s | 149 kB 00:00 2023-10-09T09:37:19.661 INFO:teuthology.orchestra.run.smithi110.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 3.5 MB/s | 213 kB 00:00 2023-10-09T09:37:19.761 INFO:teuthology.orchestra.run.smithi110.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 22 MB/s | 2.2 MB 00:00 2023-10-09T09:37:19.784 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:37:19.861 INFO:teuthology.orchestra.run.smithi110.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 30 MB/s | 7.0 MB 00:00 2023-10-09T09:37:19.884 INFO:teuthology.orchestra.run.smithi012.stdout:============================================================================================================ 2023-10-09T09:37:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:37:20.037 INFO:teuthology.orchestra.run.smithi012.stdout:============================================================================================================ 2023-10-09T09:37:20.038 INFO:teuthology.orchestra.run.smithi012.stdout:Installing group/module packages: 2023-10-09T09:37:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-10-09T09:37:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout:Installing module profiles: 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: container-tools/common 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout:Enabling module streams: 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: container-tools rhel8 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout:============================================================================================================ 2023-10-09T09:37:20.040 INFO:teuthology.orchestra.run.smithi012.stdout:Install 10 Packages 2023-10-09T09:37:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:20.042 INFO:teuthology.orchestra.run.smithi110.stdout:(10/10): buildah-1.26.2-1.module+el8.6.0+16771+ 19 MB/s | 8.0 MB 00:00 2023-10-09T09:37:20.042 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:37:20.042 INFO:teuthology.orchestra.run.smithi110.stdout:Total 35 MB/s | 18 MB 00:00 2023-10-09T09:37:20.042 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 18 M 2023-10-09T09:37:20.043 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 65 M 2023-10-09T09:37:20.043 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:37:20.043 INFO:teuthology.orchestra.run.smithi012.stdout:(1/10): python3-pyxdg-0.25-16.el8.noarch.rpm 1.4 MB/s | 94 kB 00:00 2023-10-09T09:37:20.043 INFO:teuthology.orchestra.run.smithi012.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 274 kB/s | 25 kB 00:00 2023-10-09T09:37:20.043 INFO:teuthology.orchestra.run.smithi012.stdout:(3/10): yajl-2.1.0-10.el8.x86_64.rpm 352 kB/s | 41 kB 00:00 2023-10-09T09:37:20.043 INFO:teuthology.orchestra.run.smithi012.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 654 kB/s | 49 kB 00:00 2023-10-09T09:37:20.061 INFO:teuthology.orchestra.run.smithi012.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 6.1 MB/s | 524 kB 00:00 2023-10-09T09:37:20.087 INFO:teuthology.orchestra.run.smithi012.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.5 MB/s | 149 kB 00:00 2023-10-09T09:37:20.112 INFO:teuthology.orchestra.run.smithi012.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 4.1 MB/s | 213 kB 00:00 2023-10-09T09:37:20.154 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:37:20.173 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:37:20.173 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:37:20.204 INFO:teuthology.orchestra.run.smithi012.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 24 MB/s | 2.2 MB 00:00 2023-10-09T09:37:20.321 INFO:teuthology.orchestra.run.smithi012.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 30 MB/s | 7.0 MB 00:00 2023-10-09T09:37:20.354 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:37:20.358 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:37:20.412 INFO:teuthology.orchestra.run.smithi012.stdout:(10/10): buildah-1.26.2-1.module+el8.6.0+16771+ 19 MB/s | 8.0 MB 00:00 2023-10-09T09:37:20.413 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:37:20.413 INFO:teuthology.orchestra.run.smithi012.stdout:Total 35 MB/s | 18 MB 00:00 2023-10-09T09:37:20.620 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:37:20.723 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:37:20.724 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:37:20.834 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:37:20.838 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:37:20.868 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:37:21.066 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-10-09T09:37:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-10-09T09:37:21.384 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:37:21.467 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-10-09T09:37:21.539 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-10-09T09:37:21.685 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-10-09T09:37:21.704 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-10-09T09:37:21.891 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-10-09T09:37:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-10-09T09:37:22.102 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-10-09T09:37:22.434 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-10-09T09:37:22.872 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-10-09T09:37:23.219 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-10-09T09:37:23.747 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-10-09T09:37:23.955 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-10-09T09:37:24.144 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-10-09T09:37:24.164 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-10-09T09:37:24.341 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-10-09T09:37:24.531 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-10-09T09:37:24.607 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-10-09T09:37:24.607 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-10-09T09:37:24.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-10-09T09:37:24.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-10-09T09:37:24.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-10-09T09:37:24.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-10-09T09:37:24.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-10-09T09:37:24.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-10-09T09:37:24.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-10-09T09:37:24.608 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-10-09T09:37:24.725 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-10-09T09:37:24.893 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-10-09T09:37:25.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-10-09T09:37:25.791 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-10-09T09:37:25.791 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:54 ago on Mon 09 Oct 2023 09:35:31 AM UTC. 2023-10-09T09:37:26.365 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-10-09T09:37:26.365 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:39 ago on Mon 09 Oct 2023 09:35:46 AM UTC. 2023-10-09T09:37:28.334 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:37:28.527 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-10-09T09:37:28.528 INFO:teuthology.orchestra.run.smithi110.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T09:37:28.529 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:37:28.529 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:37:28.664 DEBUG:teuthology.parallel:result is None 2023-10-09T09:37:29.143 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:37:29.357 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:29.357 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:37:29.357 INFO:teuthology.orchestra.run.smithi012.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-10-09T09:37:29.357 INFO:teuthology.orchestra.run.smithi012.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-10-09T09:37:29.357 INFO:teuthology.orchestra.run.smithi012.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-10-09T09:37:29.357 INFO:teuthology.orchestra.run.smithi012.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-10-09T09:37:29.358 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-10-09T09:37:29.358 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-10-09T09:37:29.358 INFO:teuthology.orchestra.run.smithi012.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-10-09T09:37:29.358 INFO:teuthology.orchestra.run.smithi012.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-10-09T09:37:29.358 INFO:teuthology.orchestra.run.smithi012.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-10-09T09:37:29.358 INFO:teuthology.orchestra.run.smithi012.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T09:37:29.358 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:29.358 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:37:29.509 DEBUG:teuthology.parallel:result is None 2023-10-09T09:37:29.509 INFO:teuthology.run_tasks:Running task install... 2023-10-09T09:37:29.519 DEBUG:teuthology.task.install:project ceph 2023-10-09T09:37:29.519 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T09:37:29.519 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T09:37:29.519 INFO:teuthology.task.install:Using flavor: default 2023-10-09T09:37:29.526 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T09:37:29.526 INFO:teuthology.task.install:extra packages: [] 2023-10-09T09:37:29.526 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T09:37:29.526 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T09:37:29.527 INFO:teuthology.packaging:ref: None 2023-10-09T09:37:29.527 INFO:teuthology.packaging:tag: None 2023-10-09T09:37:29.527 INFO:teuthology.packaging:branch: pacific 2023-10-09T09:37:29.527 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:37:29.527 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-09T09:37:29.529 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T09:37:29.529 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T09:37:29.529 INFO:teuthology.packaging:ref: None 2023-10-09T09:37:29.529 INFO:teuthology.packaging:tag: None 2023-10-09T09:37:29.529 INFO:teuthology.packaging:branch: pacific 2023-10-09T09:37:29.529 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:37:29.529 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-09T09:37:29.744 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/ 2023-10-09T09:37:29.745 INFO:teuthology.task.install.rpm:Package version is 16.2.14-48.gaeb2e425 2023-10-09T09:37:29.759 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/ 2023-10-09T09:37:29.760 INFO:teuthology.task.install.rpm:Package version is 16.2.14-48.gaeb2e425 2023-10-09T09:37:29.872 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T09:37:29.872 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:37:29.873 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T09:37:29.906 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T09:37:29.906 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T09:37:29.906 INFO:teuthology.packaging:ref: None 2023-10-09T09:37:29.906 INFO:teuthology.packaging:tag: None 2023-10-09T09:37:29.906 INFO:teuthology.packaging:branch: pacific 2023-10-09T09:37:29.907 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:37:29.907 DEBUG:teuthology.orchestra.run.smithi110:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T09:37:29.919 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T09:37:29.920 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:37:29.920 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T09:37:29.949 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T09:37:29.949 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T09:37:29.950 INFO:teuthology.packaging:ref: None 2023-10-09T09:37:29.950 INFO:teuthology.packaging:tag: None 2023-10-09T09:37:29.950 INFO:teuthology.packaging:branch: pacific 2023-10-09T09:37:29.950 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:37:29.950 DEBUG:teuthology.orchestra.run.smithi012:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T09:37:29.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T09:37:30.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T09:37:30.058 DEBUG:teuthology.orchestra.run.smithi110:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T09:37:30.097 DEBUG:teuthology.orchestra.run.smithi012:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T09:37:30.126 INFO:teuthology.orchestra.run.smithi110.stdout:check_obsoletes = 1 2023-10-09T09:37:30.127 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum clean all 2023-10-09T09:37:30.164 INFO:teuthology.orchestra.run.smithi012.stdout:check_obsoletes = 1 2023-10-09T09:37:30.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum clean all 2023-10-09T09:37:30.500 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:37:30.556 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:37:30.740 INFO:teuthology.orchestra.run.smithi110.stdout:44 files removed 2023-10-09T09:37:30.786 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-radosgw 2023-10-09T09:37:30.793 INFO:teuthology.orchestra.run.smithi012.stdout:44 files removed 2023-10-09T09:37:30.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-radosgw 2023-10-09T09:37:31.118 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:37:31.192 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:37:31.798 INFO:teuthology.orchestra.run.smithi012.stdout:ceph packages for x86_64 315 kB/s | 76 kB 00:00 2023-10-09T09:37:31.816 INFO:teuthology.orchestra.run.smithi110.stdout:ceph packages for x86_64 224 kB/s | 76 kB 00:00 2023-10-09T09:37:32.114 INFO:teuthology.orchestra.run.smithi012.stdout:ceph noarch packages 54 kB/s | 15 kB 00:00 2023-10-09T09:37:32.158 INFO:teuthology.orchestra.run.smithi110.stdout:ceph noarch packages 49 kB/s | 15 kB 00:00 2023-10-09T09:37:32.467 INFO:teuthology.orchestra.run.smithi110.stdout:ceph source packages 5.0 kB/s | 1.4 kB 00:00 2023-10-09T09:37:32.490 INFO:teuthology.orchestra.run.smithi012.stdout:ceph source packages 4.1 kB/s | 1.4 kB 00:00 2023-10-09T09:37:32.618 INFO:teuthology.orchestra.run.smithi110.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2023-10-09T09:37:32.632 INFO:teuthology.orchestra.run.smithi012.stdout:Copr repo for python3-asyncssh owned by ceph 29 kB/s | 3.5 kB 00:00 2023-10-09T09:37:33.919 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-10-09T09:37:36.419 INFO:teuthology.orchestra.run.smithi110.stdout:Extra Packages for Enterprise Linux 4.2 MB/s | 16 MB 00:03 2023-10-09T09:37:37.750 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 461 kB/s | 24 kB 00:00 2023-10-09T09:37:38.684 INFO:teuthology.orchestra.run.smithi012.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 62 MB/s | 53 MB 00:00 2023-10-09T09:37:39.877 INFO:teuthology.orchestra.run.smithi110.stdout:lab-extras 423 kB/s | 24 kB 00:00 2023-10-09T09:37:40.746 INFO:teuthology.orchestra.run.smithi110.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 59 MB/s | 47 MB 00:00 2023-10-09T09:37:47.327 INFO:teuthology.orchestra.run.smithi012.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-10-09T09:37:48.939 INFO:teuthology.orchestra.run.smithi110.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 63 MB/s | 53 MB 00:00 2023-10-09T09:37:59.297 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:37:59.300 INFO:teuthology.orchestra.run.smithi012.stdout:=================================================================================================== 2023-10-09T09:37:59.300 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:37:59.300 INFO:teuthology.orchestra.run.smithi012.stdout:=================================================================================================== 2023-10-09T09:37:59.300 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:37:59.300 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-radosgw x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 12 M 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout:Upgrading: 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: librados2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.7 M 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: librbd1 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 4.2 M 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-base x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 6.1 M 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-common x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 24 M 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-selinux x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 26 k 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 824 k 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T09:37:59.301 INFO:teuthology.orchestra.run.smithi012.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: libradosstriper1 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 493 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: librgw2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.8 M 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-argparse x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 45 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-common x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 99 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cephfs x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 215 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rados x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 390 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rbd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 369 k 2023-10-09T09:37:59.302 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rgw x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 114 k 2023-10-09T09:37:59.303 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:59.303 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:37:59.303 INFO:teuthology.orchestra.run.smithi012.stdout:=================================================================================================== 2023-10-09T09:37:59.303 INFO:teuthology.orchestra.run.smithi012.stdout:Install 21 Packages 2023-10-09T09:37:59.303 INFO:teuthology.orchestra.run.smithi012.stdout:Upgrade 2 Packages 2023-10-09T09:37:59.303 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:37:59.303 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 57 M 2023-10-09T09:37:59.303 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:38:00.114 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:38:00.117 INFO:teuthology.orchestra.run.smithi110.stdout:=================================================================================================== 2023-10-09T09:38:00.117 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:38:00.117 INFO:teuthology.orchestra.run.smithi110.stdout:=================================================================================================== 2023-10-09T09:38:00.118 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:38:00.118 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-radosgw x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 12 M 2023-10-09T09:38:00.118 INFO:teuthology.orchestra.run.smithi110.stdout:Upgrading: 2023-10-09T09:38:00.118 INFO:teuthology.orchestra.run.smithi110.stdout: librados2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.7 M 2023-10-09T09:38:00.118 INFO:teuthology.orchestra.run.smithi110.stdout: librbd1 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 4.2 M 2023-10-09T09:38:00.118 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:38:00.118 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-base x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 6.1 M 2023-10-09T09:38:00.118 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-common x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 24 M 2023-10-09T09:38:00.119 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-selinux x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 26 k 2023-10-09T09:38:00.119 INFO:teuthology.orchestra.run.smithi110.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T09:38:00.119 INFO:teuthology.orchestra.run.smithi110.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-10-09T09:38:00.119 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 824 k 2023-10-09T09:38:00.119 INFO:teuthology.orchestra.run.smithi110.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T09:38:00.119 INFO:teuthology.orchestra.run.smithi110.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T09:38:00.119 INFO:teuthology.orchestra.run.smithi110.stdout: libradosstriper1 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 493 k 2023-10-09T09:38:00.119 INFO:teuthology.orchestra.run.smithi110.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T09:38:00.120 INFO:teuthology.orchestra.run.smithi110.stdout: librgw2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.8 M 2023-10-09T09:38:00.120 INFO:teuthology.orchestra.run.smithi110.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T09:38:00.120 INFO:teuthology.orchestra.run.smithi110.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T09:38:00.120 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-argparse x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 45 k 2023-10-09T09:38:00.120 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-common x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 99 k 2023-10-09T09:38:00.120 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cephfs x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 215 k 2023-10-09T09:38:00.120 INFO:teuthology.orchestra.run.smithi110.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T09:38:00.120 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rados x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 390 k 2023-10-09T09:38:00.121 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rbd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 369 k 2023-10-09T09:38:00.121 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rgw x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 114 k 2023-10-09T09:38:00.121 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:38:00.121 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:38:00.121 INFO:teuthology.orchestra.run.smithi110.stdout:=================================================================================================== 2023-10-09T09:38:00.121 INFO:teuthology.orchestra.run.smithi110.stdout:Install 21 Packages 2023-10-09T09:38:00.121 INFO:teuthology.orchestra.run.smithi110.stdout:Upgrade 2 Packages 2023-10-09T09:38:00.121 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:38:00.122 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 57 M 2023-10-09T09:38:00.122 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:38:00.418 INFO:teuthology.orchestra.run.smithi012.stdout:(1/23): ceph-base-16.2.14-48.gaeb2e425.el8.x86_ 6.7 MB/s | 6.1 MB 00:00 2023-10-09T09:38:00.484 INFO:teuthology.orchestra.run.smithi012.stdout:(2/23): ceph-selinux-16.2.14-48.gaeb2e425.el8.x 384 kB/s | 26 kB 00:00 2023-10-09T09:38:00.701 INFO:teuthology.orchestra.run.smithi012.stdout:(3/23): libcephfs2-16.2.14-48.gaeb2e425.el8.x86 3.7 MB/s | 824 kB 00:00 2023-10-09T09:38:00.860 INFO:teuthology.orchestra.run.smithi012.stdout:(4/23): libradosstriper1-16.2.14-48.gaeb2e425.e 3.0 MB/s | 493 kB 00:00 2023-10-09T09:38:00.994 INFO:teuthology.orchestra.run.smithi012.stdout:(5/23): ceph-radosgw-16.2.14-48.gaeb2e425.el8.x 7.8 MB/s | 12 MB 00:01 2023-10-09T09:38:01.086 INFO:teuthology.orchestra.run.smithi012.stdout:(6/23): python3-ceph-argparse-16.2.14-48.gaeb2e 491 kB/s | 45 kB 00:00 2023-10-09T09:38:01.202 INFO:teuthology.orchestra.run.smithi012.stdout:(7/23): python3-ceph-common-16.2.14-48.gaeb2e42 846 kB/s | 99 kB 00:00 2023-10-09T09:38:01.311 INFO:teuthology.orchestra.run.smithi012.stdout:(8/23): python3-cephfs-16.2.14-48.gaeb2e425.el8 1.9 MB/s | 215 kB 00:00 2023-10-09T09:38:01.478 INFO:teuthology.orchestra.run.smithi012.stdout:(9/23): librgw2-16.2.14-48.gaeb2e425.el8.x86_64 6.1 MB/s | 3.8 MB 00:00 2023-10-09T09:38:01.712 INFO:teuthology.orchestra.run.smithi012.stdout:(10/23): python3-rbd-16.2.14-48.gaeb2e425.el8.x 1.5 MB/s | 369 kB 00:00 2023-10-09T09:38:01.795 INFO:teuthology.orchestra.run.smithi012.stdout:(11/23): python3-rgw-16.2.14-48.gaeb2e425.el8.x 1.3 MB/s | 114 kB 00:00 2023-10-09T09:38:01.862 INFO:teuthology.orchestra.run.smithi012.stdout:(12/23): python3-rados-16.2.14-48.gaeb2e425.el8 709 kB/s | 390 kB 00:00 2023-10-09T09:38:01.923 INFO:teuthology.orchestra.run.smithi110.stdout:(1/23): ceph-base-16.2.14-48.gaeb2e425.el8.x86_ 3.4 MB/s | 6.1 MB 00:01 2023-10-09T09:38:01.964 INFO:teuthology.orchestra.run.smithi110.stdout:(2/23): ceph-selinux-16.2.14-48.gaeb2e425.el8.x 617 kB/s | 26 kB 00:00 2023-10-09T09:38:02.098 INFO:teuthology.orchestra.run.smithi110.stdout:(3/23): libcephfs2-16.2.14-48.gaeb2e425.el8.x86 6.0 MB/s | 824 kB 00:00 2023-10-09T09:38:02.177 INFO:teuthology.orchestra.run.smithi012.stdout:(13/23): ceph-common-16.2.14-48.gaeb2e425.el8.x 9.0 MB/s | 24 MB 00:02 2023-10-09T09:38:02.190 INFO:teuthology.orchestra.run.smithi110.stdout:(4/23): libradosstriper1-16.2.14-48.gaeb2e425.e 5.2 MB/s | 493 kB 00:00 2023-10-09T09:38:02.582 INFO:teuthology.orchestra.run.smithi110.stdout:(5/23): ceph-radosgw-16.2.14-48.gaeb2e425.el8.x 4.8 MB/s | 12 MB 00:02 2023-10-09T09:38:02.632 INFO:teuthology.orchestra.run.smithi110.stdout:(6/23): python3-ceph-argparse-16.2.14-48.gaeb2e 905 kB/s | 45 kB 00:00 2023-10-09T09:38:02.638 INFO:teuthology.orchestra.run.smithi012.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 128 kB/s | 59 kB 00:00 2023-10-09T09:38:02.725 INFO:teuthology.orchestra.run.smithi110.stdout:(7/23): librgw2-16.2.14-48.gaeb2e425.el8.x86_64 7.1 MB/s | 3.8 MB 00:00 2023-10-09T09:38:02.746 INFO:teuthology.orchestra.run.smithi012.stdout:(15/23): leveldb-1.22-1.el8.x86_64.rpm 205 kB/s | 181 kB 00:00 2023-10-09T09:38:02.749 INFO:teuthology.orchestra.run.smithi110.stdout:(8/23): python3-ceph-common-16.2.14-48.gaeb2e42 843 kB/s | 99 kB 00:00 2023-10-09T09:38:02.775 INFO:teuthology.orchestra.run.smithi110.stdout:(9/23): python3-cephfs-16.2.14-48.gaeb2e425.el8 4.3 MB/s | 215 kB 00:00 2023-10-09T09:38:02.805 INFO:teuthology.orchestra.run.smithi012.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 805 kB/s | 47 kB 00:00 2023-10-09T09:38:02.822 INFO:teuthology.orchestra.run.smithi012.stdout:(17/23): libunwind-1.3.1-3.el8.x86_64.rpm 407 kB/s | 75 kB 00:00 2023-10-09T09:38:02.825 INFO:teuthology.orchestra.run.smithi110.stdout:(10/23): python3-rados-16.2.14-48.gaeb2e425.el8 5.1 MB/s | 390 kB 00:00 2023-10-09T09:38:02.847 INFO:teuthology.orchestra.run.smithi012.stdout:(18/23): gperftools-libs-2.7-9.el8.x86_64.rpm 291 kB/s | 306 kB 00:01 2023-10-09T09:38:02.850 INFO:teuthology.orchestra.run.smithi110.stdout:(11/23): python3-rbd-16.2.14-48.gaeb2e425.el8.x 4.8 MB/s | 369 kB 00:00 2023-10-09T09:38:02.876 INFO:teuthology.orchestra.run.smithi110.stdout:(12/23): python3-rgw-16.2.14-48.gaeb2e425.el8.x 2.2 MB/s | 114 kB 00:00 2023-10-09T09:38:02.881 INFO:teuthology.orchestra.run.smithi012.stdout:(19/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 3.3 MB/s | 259 kB 00:00 2023-10-09T09:38:02.915 INFO:teuthology.orchestra.run.smithi012.stdout:(20/23): python3-prettytable-0.7.2-14.el8.noarc 478 kB/s | 44 kB 00:00 2023-10-09T09:38:02.948 INFO:teuthology.orchestra.run.smithi012.stdout:(21/23): librdkafka-0.11.4-3.el8.x86_64.rpm 3.8 MB/s | 354 kB 00:00 2023-10-09T09:38:03.257 INFO:teuthology.orchestra.run.smithi012.stdout:(22/23): librbd1-16.2.14-48.gaeb2e425.el8.x86_6 12 MB/s | 4.2 MB 00:00 2023-10-09T09:38:03.468 INFO:teuthology.orchestra.run.smithi110.stdout:(13/23): leveldb-1.22-1.el8.x86_64.rpm 306 kB/s | 181 kB 00:00 2023-10-09T09:38:03.560 INFO:teuthology.orchestra.run.smithi110.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 432 kB/s | 306 kB 00:00 2023-10-09T09:38:03.577 INFO:teuthology.orchestra.run.smithi110.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 542 kB/s | 59 kB 00:00 2023-10-09T09:38:03.636 INFO:teuthology.orchestra.run.smithi110.stdout:(16/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 4.3 MB/s | 259 kB 00:00 2023-10-09T09:38:03.657 INFO:teuthology.orchestra.run.smithi012.stdout:(23/23): librados2-16.2.14-48.gaeb2e425.el8.x86 4.9 MB/s | 3.7 MB 00:00 2023-10-09T09:38:03.662 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:38:03.662 INFO:teuthology.orchestra.run.smithi012.stdout:Total 13 MB/s | 57 MB 00:04 2023-10-09T09:38:03.669 INFO:teuthology.orchestra.run.smithi110.stdout:(17/23): python3-prettytable-0.7.2-14.el8.noarc 1.3 MB/s | 44 kB 00:00 2023-10-09T09:38:03.694 INFO:teuthology.orchestra.run.smithi110.stdout:(18/23): libunwind-1.3.1-3.el8.x86_64.rpm 559 kB/s | 75 kB 00:00 2023-10-09T09:38:03.711 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:38:03.728 INFO:teuthology.orchestra.run.smithi110.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.9 MB/s | 354 kB 00:00 2023-10-09T09:38:03.762 INFO:teuthology.orchestra.run.smithi110.stdout:(20/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 702 kB/s | 47 kB 00:00 2023-10-09T09:38:03.763 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:38:03.763 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:38:04.104 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:38:04.108 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:38:04.237 INFO:teuthology.orchestra.run.smithi110.stdout:(21/23): librados2-16.2.14-48.gaeb2e425.el8.x86 7.5 MB/s | 3.7 MB 00:00 2023-10-09T09:38:04.312 INFO:teuthology.orchestra.run.smithi110.stdout:(22/23): librbd1-16.2.14-48.gaeb2e425.el8.x86_6 7.6 MB/s | 4.2 MB 00:00 2023-10-09T09:38:05.137 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:38:05.328 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T09:38:05.734 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T09:38:05.919 INFO:teuthology.orchestra.run.smithi012.stdout: Upgrading : librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T09:38:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T09:38:06.233 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 3/25 2023-10-09T09:38:06.589 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T09:38:06.744 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T09:38:07.186 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T09:38:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: Upgrading : librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T09:38:07.603 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T09:38:07.771 INFO:teuthology.orchestra.run.smithi110.stdout:(23/23): ceph-common-16.2.14-48.gaeb2e425.el8.x 3.2 MB/s | 24 MB 00:07 2023-10-09T09:38:07.775 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:38:07.775 INFO:teuthology.orchestra.run.smithi110.stdout:Total 7.5 MB/s | 57 MB 00:07 2023-10-09T09:38:07.785 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T09:38:07.824 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:38:07.857 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T09:38:07.870 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:38:07.870 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:38:08.198 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:38:08.201 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:38:08.377 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-10-09T09:38:08.535 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 9/25 2023-10-09T09:38:08.621 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 9/25 2023-10-09T09:38:08.791 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-10-09T09:38:08.942 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x 11/25 2023-10-09T09:38:09.089 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 12/25 2023-10-09T09:38:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:38:09.297 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 13/25 2023-10-09T09:38:09.446 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T09:38:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 14/25 2023-10-09T09:38:09.610 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 15/25 2023-10-09T09:38:09.666 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 15/25 2023-10-09T09:38:09.819 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T09:38:09.843 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T09:38:10.021 INFO:teuthology.orchestra.run.smithi110.stdout: Upgrading : librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T09:38:10.030 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T09:38:10.120 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T09:38:10.261 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T09:38:10.377 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 3/25 2023-10-09T09:38:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86 19/25 2023-10-09T09:38:10.619 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T09:38:10.813 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T09:38:11.243 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T09:38:11.463 INFO:teuthology.orchestra.run.smithi110.stdout: Upgrading : librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T09:38:11.527 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T09:38:11.761 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-10-09T09:38:12.271 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T09:38:12.637 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T09:38:12.813 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 9/25 2023-10-09T09:38:12.892 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 9/25 2023-10-09T09:38:13.093 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-10-09T09:38:13.259 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x 11/25 2023-10-09T09:38:13.407 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 12/25 2023-10-09T09:38:13.605 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 13/25 2023-10-09T09:38:13.804 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 14/25 2023-10-09T09:38:13.971 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 15/25 2023-10-09T09:38:14.062 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 15/25 2023-10-09T09:38:14.213 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T09:38:14.235 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T09:38:14.421 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T09:38:14.434 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T09:38:14.673 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T09:38:14.804 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86 19/25 2023-10-09T09:38:15.172 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T09:38:15.431 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 21/25 2023-10-09T09:38:15.515 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 21/25 2023-10-09T09:38:15.659 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T09:38:18.497 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T09:38:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T09:38:19.408 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T09:38:19.715 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 21/25 2023-10-09T09:38:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 21/25 2023-10-09T09:38:19.945 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T09:38:34.779 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T09:38:34.779 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /sys 2023-10-09T09:38:34.779 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /proc 2023-10-09T09:38:34.779 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /dev 2023-10-09T09:38:34.780 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /run 2023-10-09T09:38:34.780 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /mnt 2023-10-09T09:38:34.780 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /var/tmp 2023-10-09T09:38:34.780 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /home 2023-10-09T09:38:34.780 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /tmp 2023-10-09T09:38:34.780 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /dev 2023-10-09T09:38:34.780 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:38:35.976 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 23/25 2023-10-09T09:38:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 23/25 2023-10-09T09:38:36.038 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T09:38:36.223 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T09:38:36.223 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T09:38:36.845 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 1/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 3/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 4/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 7/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x 8/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86 9/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 10/25 2023-10-09T09:38:36.846 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 11/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 12/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 13/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T09:38:36.847 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T09:38:36.848 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 24/25 2023-10-09T09:38:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T09:38:38.103 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /sys 2023-10-09T09:38:38.103 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /proc 2023-10-09T09:38:38.103 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /dev 2023-10-09T09:38:38.103 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /run 2023-10-09T09:38:38.103 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /mnt 2023-10-09T09:38:38.104 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /var/tmp 2023-10-09T09:38:38.104 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /home 2023-10-09T09:38:38.104 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /tmp 2023-10-09T09:38:38.104 INFO:teuthology.orchestra.run.smithi110.stdout:skipping the directory /dev 2023-10-09T09:38:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:38:38.212 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T09:38:38.212 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:52 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:38:39.274 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 23/25 2023-10-09T09:38:39.338 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 23/25 2023-10-09T09:38:39.339 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T09:38:39.528 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T09:38:39.528 INFO:teuthology.orchestra.run.smithi110.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T09:38:40.111 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T09:38:40.111 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 1/25 2023-10-09T09:38:40.111 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T09:38:40.111 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 3/25 2023-10-09T09:38:40.111 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 4/25 2023-10-09T09:38:40.111 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 7/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x 8/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86 9/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 10/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 11/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 12/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 13/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-10-09T09:38:40.112 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-10-09T09:38:40.113 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T09:38:40.113 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-10-09T09:38:40.113 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-10-09T09:38:40.113 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-10-09T09:38:40.113 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-10-09T09:38:40.113 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T09:38:40.113 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T09:38:40.113 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 24/25 2023-10-09T09:38:40.728 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:38:40.944 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:38:40.944 INFO:teuthology.orchestra.run.smithi012.stdout:Upgraded: 2023-10-09T09:38:40.944 INFO:teuthology.orchestra.run.smithi012.stdout: librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.944 INFO:teuthology.orchestra.run.smithi012.stdout: librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.944 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:38:40.944 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: leveldb-1.22-1.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.945 INFO:teuthology.orchestra.run.smithi012.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:38:40.946 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:38:41.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-test 2023-10-09T09:38:41.494 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T09:38:41.494 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:53 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:38:41.510 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:38:42.342 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:56 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:38:43.896 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:38:43.900 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:38:43.901 INFO:teuthology.orchestra.run.smithi012.stdout:=========================================================================================== 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout:=========================================================================================== 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-test x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 52 M 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout: libcephsqlite x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 174 k 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T09:38:43.902 INFO:teuthology.orchestra.run.smithi012.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T09:38:43.903 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:38:43.903 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:38:43.903 INFO:teuthology.orchestra.run.smithi012.stdout:=========================================================================================== 2023-10-09T09:38:43.903 INFO:teuthology.orchestra.run.smithi012.stdout:Install 6 Packages 2023-10-09T09:38:43.903 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:38:43.903 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 53 M 2023-10-09T09:38:43.904 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 195 M 2023-10-09T09:38:43.904 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:38:43.948 INFO:teuthology.orchestra.run.smithi012.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.5 MB/s | 70 kB 00:00 2023-10-09T09:38:44.032 INFO:teuthology.orchestra.run.smithi012.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.3 MB/s | 323 kB 00:00 2023-10-09T09:38:44.073 INFO:teuthology.orchestra.run.smithi012.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-10-09T09:38:44.085 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:38:44.085 INFO:teuthology.orchestra.run.smithi110.stdout:Upgraded: 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout: librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout: librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T09:38:44.086 INFO:teuthology.orchestra.run.smithi110.stdout: leveldb-1.22-1.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T09:38:44.087 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.088 INFO:teuthology.orchestra.run.smithi110.stdout: python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.088 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.088 INFO:teuthology.orchestra.run.smithi110.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T09:38:44.088 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.088 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.089 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:44.089 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:38:44.089 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:38:44.116 INFO:teuthology.orchestra.run.smithi012.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-10-09T09:38:44.157 INFO:teuthology.orchestra.run.smithi012.stdout:(5/6): libcephsqlite-16.2.14-48.gaeb2e425.el8.x 686 kB/s | 174 kB 00:00 2023-10-09T09:38:44.258 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-test 2023-10-09T09:38:44.605 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:38:45.449 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:00:57 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:38:46.812 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:38:46.813 INFO:teuthology.orchestra.run.smithi110.stdout:=========================================================================================== 2023-10-09T09:38:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:38:46.813 INFO:teuthology.orchestra.run.smithi110.stdout:=========================================================================================== 2023-10-09T09:38:46.813 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:38:46.813 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-test x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 52 M 2023-10-09T09:38:46.813 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: libcephsqlite x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 174 k 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout:=========================================================================================== 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout:Install 6 Packages 2023-10-09T09:38:46.814 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:38:46.815 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 53 M 2023-10-09T09:38:46.815 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 195 M 2023-10-09T09:38:46.815 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:38:46.848 INFO:teuthology.orchestra.run.smithi110.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.1 MB/s | 70 kB 00:00 2023-10-09T09:38:46.915 INFO:teuthology.orchestra.run.smithi110.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.7 MB/s | 323 kB 00:00 2023-10-09T09:38:46.965 INFO:teuthology.orchestra.run.smithi110.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-10-09T09:38:46.999 INFO:teuthology.orchestra.run.smithi110.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 6.0 MB/s | 202 kB 00:00 2023-10-09T09:38:47.208 INFO:teuthology.orchestra.run.smithi110.stdout:(5/6): libcephsqlite-16.2.14-48.gaeb2e425.el8.x 443 kB/s | 174 kB 00:00 2023-10-09T09:38:48.199 INFO:teuthology.orchestra.run.smithi012.stdout:(6/6): ceph-test-16.2.14-48.gaeb2e425.el8.x86_6 12 MB/s | 52 MB 00:04 2023-10-09T09:38:48.201 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:38:48.201 INFO:teuthology.orchestra.run.smithi012.stdout:Total 12 MB/s | 53 MB 00:04 2023-10-09T09:38:48.241 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:38:48.263 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:38:48.264 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:38:48.554 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:38:48.558 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:38:49.029 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:38:49.142 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T09:38:49.226 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T09:38:49.440 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T09:38:49.577 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T09:38:49.716 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T09:38:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 5/6 2023-10-09T09:38:54.524 INFO:teuthology.orchestra.run.smithi110.stdout:(6/6): ceph-test-16.2.14-48.gaeb2e425.el8.x86_6 6.7 MB/s | 52 MB 00:07 2023-10-09T09:38:54.526 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:38:54.526 INFO:teuthology.orchestra.run.smithi110.stdout:Total 6.8 MB/s | 53 MB 00:07 2023-10-09T09:38:54.563 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:38:54.584 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:38:54.584 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:38:54.700 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 5/6 2023-10-09T09:38:54.857 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:38:54.861 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:38:54.880 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 6/6 2023-10-09T09:38:55.121 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 6/6 2023-10-09T09:38:55.121 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 1/6 2023-10-09T09:38:55.121 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 2/6 2023-10-09T09:38:55.121 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T09:38:55.121 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T09:38:55.121 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T09:38:55.345 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:38:55.482 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T09:38:55.596 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T09:38:55.786 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T09:38:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T09:38:56.071 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T09:38:56.257 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 5/6 2023-10-09T09:38:56.303 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T09:38:56.303 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:10 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:38:58.804 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:38:59.010 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:38:59.010 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:38:59.010 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:59.010 INFO:teuthology.orchestra.run.smithi012.stdout: jq-1.6-3.el8.x86_64 2023-10-09T09:38:59.010 INFO:teuthology.orchestra.run.smithi012.stdout: libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:38:59.011 INFO:teuthology.orchestra.run.smithi012.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T09:38:59.011 INFO:teuthology.orchestra.run.smithi012.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T09:38:59.011 INFO:teuthology.orchestra.run.smithi012.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T09:38:59.011 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:38:59.011 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:38:59.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph 2023-10-09T09:38:59.521 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:00.366 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:14 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:00.868 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 5/6 2023-10-09T09:39:01.070 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 6/6 2023-10-09T09:39:01.292 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 6/6 2023-10-09T09:39:01.292 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 1/6 2023-10-09T09:39:01.292 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 2/6 2023-10-09T09:39:01.292 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T09:39:01.292 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T09:39:01.292 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T09:39:01.895 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:39:01.898 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================================== 2023-10-09T09:39:01.898 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:39:01.898 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================================== 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout: ceph x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 6.4 k 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mds x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 2.4 M 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 1.6 M 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-modules-core noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 244 k 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mon x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 5.0 M 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-osd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 18 M 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T09:39:01.899 INFO:teuthology.orchestra.run.smithi012.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T09:39:01.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T09:39:01.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T09:39:01.902 INFO:teuthology.orchestra.run.smithi012.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T09:39:01.902 INFO:teuthology.orchestra.run.smithi012.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T09:39:01.902 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:01.902 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:39:01.902 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================================== 2023-10-09T09:39:01.902 INFO:teuthology.orchestra.run.smithi012.stdout:Install 32 Packages 2023-10-09T09:39:01.902 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:01.903 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 31 M 2023-10-09T09:39:01.903 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 101 M 2023-10-09T09:39:01.903 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:39:02.056 INFO:teuthology.orchestra.run.smithi012.stdout:(1/32): ceph-16.2.14-48.gaeb2e425.el8.x86_64.rp 56 kB/s | 6.4 kB 00:00 2023-10-09T09:39:02.454 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T09:39:02.455 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:14 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:02.548 INFO:teuthology.orchestra.run.smithi012.stdout:(2/32): ceph-mon-16.2.14-48.gaeb2e425.el8.x86_6 10 MB/s | 5.0 MB 00:00 2023-10-09T09:39:02.590 INFO:teuthology.orchestra.run.smithi012.stdout:(3/32): ceph-mgr-16.2.14-48.gaeb2e425.el8.x86_6 2.5 MB/s | 1.6 MB 00:00 2023-10-09T09:39:02.651 INFO:teuthology.orchestra.run.smithi012.stdout:(4/32): ceph-mds-16.2.14-48.gaeb2e425.el8.x86_6 3.5 MB/s | 2.4 MB 00:00 2023-10-09T09:39:02.699 INFO:teuthology.orchestra.run.smithi012.stdout:(5/32): ceph-mgr-modules-core-16.2.14-48.gaeb2e 2.2 MB/s | 244 kB 00:00 2023-10-09T09:39:02.749 INFO:teuthology.orchestra.run.smithi012.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 435 kB/s | 44 kB 00:00 2023-10-09T09:39:02.774 INFO:teuthology.orchestra.run.smithi012.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 2.4 MB/s | 185 kB 00:00 2023-10-09T09:39:02.808 INFO:teuthology.orchestra.run.smithi012.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.9 MB/s | 173 kB 00:00 2023-10-09T09:39:02.833 INFO:teuthology.orchestra.run.smithi012.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 6.4 MB/s | 384 kB 00:00 2023-10-09T09:39:02.859 INFO:teuthology.orchestra.run.smithi012.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 220 kB/s | 11 kB 00:00 2023-10-09T09:39:02.884 INFO:teuthology.orchestra.run.smithi012.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 351 kB/s | 18 kB 00:00 2023-10-09T09:39:02.909 INFO:teuthology.orchestra.run.smithi012.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 972 kB/s | 49 kB 00:00 2023-10-09T09:39:02.934 INFO:teuthology.orchestra.run.smithi012.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 1.1 MB/s | 59 kB 00:00 2023-10-09T09:39:02.960 INFO:teuthology.orchestra.run.smithi012.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 5.5 MB/s | 283 kB 00:00 2023-10-09T09:39:02.985 INFO:teuthology.orchestra.run.smithi012.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 311 kB/s | 16 kB 00:00 2023-10-09T09:39:03.010 INFO:teuthology.orchestra.run.smithi012.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 372 kB/s | 19 kB 00:00 2023-10-09T09:39:03.037 INFO:teuthology.orchestra.run.smithi012.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 476 kB/s | 24 kB 00:00 2023-10-09T09:39:03.061 INFO:teuthology.orchestra.run.smithi012.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 584 kB/s | 29 kB 00:00 2023-10-09T09:39:03.086 INFO:teuthology.orchestra.run.smithi012.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 549 kB/s | 27 kB 00:00 2023-10-09T09:39:03.111 INFO:teuthology.orchestra.run.smithi012.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 5.0 MB/s | 255 kB 00:00 2023-10-09T09:39:03.137 INFO:teuthology.orchestra.run.smithi012.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 4.9 MB/s | 251 kB 00:00 2023-10-09T09:39:03.154 INFO:teuthology.orchestra.run.smithi012.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 2.0 MB/s | 85 kB 00:00 2023-10-09T09:39:03.179 INFO:teuthology.orchestra.run.smithi012.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 550 kB/s | 23 kB 00:00 2023-10-09T09:39:03.254 INFO:teuthology.orchestra.run.smithi012.stdout:(24/32): python3-cryptography-3.2.1-5.el8.x86_6 7.3 MB/s | 559 kB 00:00 2023-10-09T09:39:03.280 INFO:teuthology.orchestra.run.smithi012.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 871 kB/s | 109 kB 00:00 2023-10-09T09:39:03.322 INFO:teuthology.orchestra.run.smithi012.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 3.5 MB/s | 238 kB 00:00 2023-10-09T09:39:03.347 INFO:teuthology.orchestra.run.smithi012.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 806 kB/s | 54 kB 00:00 2023-10-09T09:39:03.372 INFO:teuthology.orchestra.run.smithi012.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 774 kB/s | 39 kB 00:00 2023-10-09T09:39:03.398 INFO:teuthology.orchestra.run.smithi012.stdout:(29/32): python3-mako-1.0.6-13.el8.noarch.rpm 3.1 MB/s | 157 kB 00:00 2023-10-09T09:39:03.431 INFO:teuthology.orchestra.run.smithi012.stdout:(30/32): python3-werkzeug-0.12.2-4.el8.noarch.r 7.6 MB/s | 457 kB 00:00 2023-10-09T09:39:03.448 INFO:teuthology.orchestra.run.smithi012.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-10-09T09:39:03.773 INFO:teuthology.orchestra.run.smithi012.stdout:(32/32): ceph-osd-16.2.14-48.gaeb2e425.el8.x86_ 15 MB/s | 18 MB 00:01 2023-10-09T09:39:03.776 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:03.776 INFO:teuthology.orchestra.run.smithi012.stdout:Total 17 MB/s | 31 MB 00:01 2023-10-09T09:39:03.871 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:39:03.912 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:39:03.912 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:39:04.143 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:39:04.148 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:39:04.901 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:39:05.090 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout: jq-1.6-3.el8.x86_64 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout: libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:05.091 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:39:05.211 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph 2023-10-09T09:39:05.323 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:39:05.507 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T09:39:05.545 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:05.779 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T09:39:05.977 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-10-09T09:39:06.195 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-10-09T09:39:06.353 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:18 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:06.559 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-10-09T09:39:06.737 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-10-09T09:39:06.963 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-10-09T09:39:07.293 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-10-09T09:39:07.551 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-10-09T09:39:07.688 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-10-09T09:39:07.724 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:39:07.728 INFO:teuthology.orchestra.run.smithi110.stdout:====================================================================================================== 2023-10-09T09:39:07.728 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:39:07.728 INFO:teuthology.orchestra.run.smithi110.stdout:====================================================================================================== 2023-10-09T09:39:07.728 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:39:07.728 INFO:teuthology.orchestra.run.smithi110.stdout: ceph x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 6.4 k 2023-10-09T09:39:07.728 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:39:07.728 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mds x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 2.4 M 2023-10-09T09:39:07.728 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 1.6 M 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-modules-core noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 244 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mon x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 5.0 M 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-osd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 18 M 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T09:39:07.729 INFO:teuthology.orchestra.run.smithi110.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T09:39:07.730 INFO:teuthology.orchestra.run.smithi110.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout:====================================================================================================== 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout:Install 32 Packages 2023-10-09T09:39:07.731 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:07.732 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 31 M 2023-10-09T09:39:07.732 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 101 M 2023-10-09T09:39:07.732 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:39:07.875 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-10-09T09:39:07.882 INFO:teuthology.orchestra.run.smithi110.stdout:(1/32): ceph-16.2.14-48.gaeb2e425.el8.x86_64.rp 54 kB/s | 6.4 kB 00:00 2023-10-09T09:39:08.055 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-10-09T09:39:08.108 INFO:teuthology.orchestra.run.smithi110.stdout:(2/32): ceph-mgr-16.2.14-48.gaeb2e425.el8.x86_6 4.7 MB/s | 1.6 MB 00:00 2023-10-09T09:39:08.270 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T09:39:08.350 INFO:teuthology.orchestra.run.smithi110.stdout:(3/32): ceph-mds-16.2.14-48.gaeb2e425.el8.x86_6 4.2 MB/s | 2.4 MB 00:00 2023-10-09T09:39:08.417 INFO:teuthology.orchestra.run.smithi110.stdout:(4/32): ceph-mgr-modules-core-16.2.14-48.gaeb2e 3.6 MB/s | 244 kB 00:00 2023-10-09T09:39:08.462 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T09:39:08.576 INFO:teuthology.orchestra.run.smithi110.stdout:(5/32): ceph-mon-16.2.14-48.gaeb2e425.el8.x86_6 7.3 MB/s | 5.0 MB 00:00 2023-10-09T09:39:08.584 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T09:39:08.750 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T09:39:08.900 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T09:39:08.984 INFO:teuthology.orchestra.run.smithi110.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 77 kB/s | 44 kB 00:00 2023-10-09T09:39:09.028 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T09:39:09.204 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T09:39:09.226 INFO:teuthology.orchestra.run.smithi110.stdout:(7/32): ceph-osd-16.2.14-48.gaeb2e425.el8.x86_6 16 MB/s | 18 MB 00:01 2023-10-09T09:39:09.356 INFO:teuthology.orchestra.run.smithi110.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 239 kB/s | 185 kB 00:00 2023-10-09T09:39:09.385 INFO:teuthology.orchestra.run.smithi110.stdout:(9/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 431 kB/s | 173 kB 00:00 2023-10-09T09:39:09.502 INFO:teuthology.orchestra.run.smithi110.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 78 kB/s | 11 kB 00:00 2023-10-09T09:39:09.527 INFO:teuthology.orchestra.run.smithi110.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 124 kB/s | 18 kB 00:00 2023-10-09T09:39:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T09:39:09.652 INFO:teuthology.orchestra.run.smithi110.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 325 kB/s | 49 kB 00:00 2023-10-09T09:39:09.686 INFO:teuthology.orchestra.run.smithi110.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 370 kB/s | 59 kB 00:00 2023-10-09T09:39:09.828 INFO:teuthology.orchestra.run.smithi110.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 110 kB/s | 16 kB 00:00 2023-10-09T09:39:09.953 INFO:teuthology.orchestra.run.smithi110.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 940 kB/s | 283 kB 00:00 2023-10-09T09:39:09.987 INFO:teuthology.orchestra.run.smithi110.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 118 kB/s | 19 kB 00:00 2023-10-09T09:39:10.087 INFO:teuthology.orchestra.run.smithi110.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 179 kB/s | 24 kB 00:00 2023-10-09T09:39:10.119 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T09:39:10.129 INFO:teuthology.orchestra.run.smithi110.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 206 kB/s | 29 kB 00:00 2023-10-09T09:39:10.155 INFO:teuthology.orchestra.run.smithi110.stdout:(19/32): python3-cherrypy-18.4.0-1.el8.noarch.r 414 kB/s | 384 kB 00:00 2023-10-09T09:39:10.222 INFO:teuthology.orchestra.run.smithi110.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 205 kB/s | 27 kB 00:00 2023-10-09T09:39:10.339 INFO:teuthology.orchestra.run.smithi110.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.3 MB/s | 251 kB 00:00 2023-10-09T09:39:10.356 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T09:39:10.364 INFO:teuthology.orchestra.run.smithi110.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 602 kB/s | 85 kB 00:00 2023-10-09T09:39:10.389 INFO:teuthology.orchestra.run.smithi110.stdout:(23/32): python3-waitress-1.4.3-1.el8.noarch.rp 984 kB/s | 255 kB 00:00 2023-10-09T09:39:10.440 INFO:teuthology.orchestra.run.smithi110.stdout:(24/32): python3-pytz-2017.2-9.el8.noarch.rpm 720 kB/s | 54 kB 00:00 2023-10-09T09:39:10.465 INFO:teuthology.orchestra.run.smithi110.stdout:(25/32): python3-zc-lockfile-2.0-2.el8.noarch.r 182 kB/s | 23 kB 00:00 2023-10-09T09:39:10.512 INFO:teuthology.orchestra.run.smithi110.stdout:(26/32): python3-markupsafe-0.23-19.el8.x86_64. 318 kB/s | 39 kB 00:00 2023-10-09T09:39:10.549 INFO:teuthology.orchestra.run.smithi110.stdout:(27/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.4 MB/s | 157 kB 00:00 2023-10-09T09:39:10.574 INFO:teuthology.orchestra.run.smithi110.stdout:(28/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.8 MB/s | 103 kB 00:00 2023-10-09T09:39:10.591 INFO:teuthology.orchestra.run.smithi110.stdout:(29/32): python3-pycparser-2.14-14.el8.noarch.r 2.5 MB/s | 109 kB 00:00 2023-10-09T09:39:10.596 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T09:39:10.633 INFO:teuthology.orchestra.run.smithi110.stdout:(30/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.7 MB/s | 457 kB 00:00 2023-10-09T09:39:10.658 INFO:teuthology.orchestra.run.smithi110.stdout:(31/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 3.5 MB/s | 238 kB 00:00 2023-10-09T09:39:10.683 INFO:teuthology.orchestra.run.smithi110.stdout:(32/32): python3-cryptography-3.2.1-5.el8.x86_6 5.0 MB/s | 559 kB 00:00 2023-10-09T09:39:10.688 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:10.688 INFO:teuthology.orchestra.run.smithi110.stdout:Total 11 MB/s | 31 MB 00:02 2023-10-09T09:39:10.780 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:39:10.783 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T09:39:10.818 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:39:10.819 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:39:11.034 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:39:11.038 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:39:11.064 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T09:39:11.355 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T09:39:11.661 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.n 27/32 2023-10-09T09:39:11.801 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 28/32 2023-10-09T09:39:12.143 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:39:12.417 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T09:39:12.658 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T09:39:12.898 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-10-09T09:39:13.252 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-10-09T09:39:13.383 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 28/32 2023-10-09T09:39:13.554 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 29/32 2023-10-09T09:39:13.758 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-10-09T09:39:13.930 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-10-09T09:39:14.005 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 29/32 2023-10-09T09:39:14.129 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 30/32 2023-10-09T09:39:14.162 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-10-09T09:39:14.358 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 30/32 2023-10-09T09:39:14.452 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-10-09T09:39:14.512 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 31/32 2023-10-09T09:39:14.531 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 31/32 2023-10-09T09:39:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 32/32 2023-10-09T09:39:14.743 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-10-09T09:39:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-10-09T09:39:15.022 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 32/32 2023-10-09T09:39:15.022 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 1/32 2023-10-09T09:39:15.022 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 2/32 2023-10-09T09:39:15.022 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 3/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 4/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 5/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.n 6/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T09:39:15.023 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-10-09T09:39:15.024 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-10-09T09:39:15.025 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-10-09T09:39:15.025 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-10-09T09:39:15.025 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-10-09T09:39:15.025 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-10-09T09:39:15.214 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-10-09T09:39:15.422 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-10-09T09:39:15.660 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T09:39:15.878 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T09:39:16.034 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T09:39:16.216 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T09:39:16.432 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T09:39:16.526 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-10-09T09:39:16.526 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:30 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:16.560 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T09:39:16.766 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T09:39:17.008 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T09:39:17.235 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T09:39:17.507 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T09:39:17.776 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T09:39:17.964 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T09:39:18.311 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T09:39:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T09:39:18.950 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.n 27/32 2023-10-09T09:39:19.033 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:39:19.099 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 28/32 2023-10-09T09:39:19.242 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:19.242 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:39:19.242 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T09:39:19.243 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T09:39:19.244 INFO:teuthology.orchestra.run.smithi012.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:19.245 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:39:19.404 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-base 2023-10-09T09:39:19.776 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:20.584 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 28/32 2023-10-09T09:39:20.602 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:34 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:20.821 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 29/32 2023-10-09T09:39:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 29/32 2023-10-09T09:39:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 30/32 2023-10-09T09:39:21.587 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 30/32 2023-10-09T09:39:21.737 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 31/32 2023-10-09T09:39:21.755 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 31/32 2023-10-09T09:39:21.868 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 32/32 2023-10-09T09:39:22.026 INFO:teuthology.orchestra.run.smithi012.stdout:Package ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:39:22.151 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:39:22.151 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:39:22.151 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:39:22.215 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 32/32 2023-10-09T09:39:22.215 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 1/32 2023-10-09T09:39:22.215 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 2/32 2023-10-09T09:39:22.215 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 3/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 4/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 5/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.n 6/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T09:39:22.216 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T09:39:22.217 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-10-09T09:39:22.218 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-10-09T09:39:22.218 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-10-09T09:39:22.218 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-10-09T09:39:22.218 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-10-09T09:39:22.218 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-10-09T09:39:22.218 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-10-09T09:39:22.278 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install cephadm 2023-10-09T09:39:22.646 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:23.490 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:37 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:23.790 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-10-09T09:39:23.790 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:35 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:25.087 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:39:25.087 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:39:25.087 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:39:25.087 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:39:25.088 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:39:25.088 INFO:teuthology.orchestra.run.smithi012.stdout: cephadm noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 84 k 2023-10-09T09:39:25.088 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:25.088 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:39:25.088 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:39:25.088 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2023-10-09T09:39:25.088 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:25.089 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 84 k 2023-10-09T09:39:25.089 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 348 k 2023-10-09T09:39:25.089 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:39:25.361 INFO:teuthology.orchestra.run.smithi012.stdout:cephadm-16.2.14-48.gaeb2e425.el8.noarch.rpm 309 kB/s | 84 kB 00:00 2023-10-09T09:39:25.361 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:25.361 INFO:teuthology.orchestra.run.smithi012.stdout:Total 308 kB/s | 84 kB 00:00 2023-10-09T09:39:25.362 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:39:25.370 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:39:25.370 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:39:25.372 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:39:25.376 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:39:25.422 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:39:26.114 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:26.305 INFO:teuthology.orchestra.run.smithi110.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T09:39:26.306 INFO:teuthology.orchestra.run.smithi110.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T09:39:26.307 INFO:teuthology.orchestra.run.smithi110.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T09:39:26.308 INFO:teuthology.orchestra.run.smithi110.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T09:39:26.308 INFO:teuthology.orchestra.run.smithi110.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T09:39:26.308 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:26.308 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:39:26.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-base 2023-10-09T09:39:26.800 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:27.608 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:39 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:28.343 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T09:39:28.485 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T09:39:28.660 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T09:39:28.878 INFO:teuthology.orchestra.run.smithi110.stdout:Package ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:39:28.987 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:39:28.988 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:39:28.988 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:39:29.089 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install cephadm 2023-10-09T09:39:29.433 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:29.778 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T09:39:29.778 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:43 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:30.358 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:42 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:31.814 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout: cephadm noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 84 k 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-10-09T09:39:31.815 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:31.816 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 84 k 2023-10-09T09:39:31.816 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 348 k 2023-10-09T09:39:31.816 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:39:31.981 INFO:teuthology.orchestra.run.smithi110.stdout:cephadm-16.2.14-48.gaeb2e425.el8.noarch.rpm 510 kB/s | 84 kB 00:00 2023-10-09T09:39:31.982 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:31.982 INFO:teuthology.orchestra.run.smithi110.stdout:Total 506 kB/s | 84 kB 00:00 2023-10-09T09:39:31.982 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:39:31.989 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:39:31.990 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:39:31.991 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:39:31.995 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:39:32.045 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:39:32.349 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:39:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:32.555 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:39:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:39:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:32.555 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:39:32.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T09:39:33.097 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:33.694 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T09:39:33.842 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T09:39:34.007 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T09:39:34.017 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:48 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:35.097 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T09:39:35.097 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:47 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:35.556 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 161 k 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2023-10-09T09:39:35.557 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:35.558 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 161 k 2023-10-09T09:39:35.558 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 470 k 2023-10-09T09:39:35.558 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:39:35.849 INFO:teuthology.orchestra.run.smithi012.stdout:ceph-immutable-object-cache-16.2.14-48.gaeb2e42 554 kB/s | 161 kB 00:00 2023-10-09T09:39:35.850 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:35.850 INFO:teuthology.orchestra.run.smithi012.stdout:Total 552 kB/s | 161 kB 00:00 2023-10-09T09:39:35.850 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:39:35.862 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:39:35.862 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:39:35.921 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:39:35.925 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:39:36.055 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:39:36.182 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T09:39:36.552 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T09:39:37.556 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:39:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T09:39:37.613 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:51 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:37.740 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:37.740 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:39:37.740 INFO:teuthology.orchestra.run.smithi110.stdout: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:39:37.740 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:37.741 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:39:37.885 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T09:39:38.243 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:39.110 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:51 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:40.419 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:39:40.472 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repo Size 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 161 k 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-10-09T09:39:40.473 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:40.474 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 161 k 2023-10-09T09:39:40.474 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 470 k 2023-10-09T09:39:40.474 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:39:40.624 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:40.624 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:39:40.624 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:40.624 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:40.624 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:39:40.770 INFO:teuthology.orchestra.run.smithi110.stdout:ceph-immutable-object-cache-16.2.14-48.gaeb2e42 544 kB/s | 161 kB 00:00 2023-10-09T09:39:40.771 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:40.771 INFO:teuthology.orchestra.run.smithi110.stdout:Total 542 kB/s | 161 kB 00:00 2023-10-09T09:39:40.771 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:39:40.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr 2023-10-09T09:39:40.782 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:39:40.782 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:39:40.834 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:39:40.838 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:39:40.997 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:39:41.145 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:41.160 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T09:39:41.527 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T09:39:42.022 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:56 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T09:39:42.573 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:01:54 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:43.461 INFO:teuthology.orchestra.run.smithi012.stdout:Package ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:39:43.582 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:39:43.583 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:39:43.583 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:39:43.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T09:39:44.073 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:45.670 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:59 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:47.232 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:39:47.233 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================================== 2023-10-09T09:39:47.233 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:39:47.233 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================================== 2023-10-09T09:39:47.233 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-dashboard noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 3.1 M 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-grafana-dashboards noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 24 k 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-prometheus-alerts noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 15 k 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================================== 2023-10-09T09:39:47.234 INFO:teuthology.orchestra.run.smithi012.stdout:Install 6 Packages 2023-10-09T09:39:47.235 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:47.235 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 3.4 M 2023-10-09T09:39:47.235 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 22 M 2023-10-09T09:39:47.235 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:39:47.554 INFO:teuthology.orchestra.run.smithi012.stdout:(1/6): ceph-grafana-dashboards-16.2.14-48.gaeb2 171 kB/s | 24 kB 00:00 2023-10-09T09:39:47.671 INFO:teuthology.orchestra.run.smithi012.stdout:(2/6): ceph-prometheus-alerts-16.2.14-48.gaeb2e 57 kB/s | 15 kB 00:00 2023-10-09T09:39:47.955 INFO:teuthology.orchestra.run.smithi012.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 84 kB/s | 34 kB 00:00 2023-10-09T09:39:48.130 INFO:teuthology.orchestra.run.smithi012.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 244 kB/s | 43 kB 00:00 2023-10-09T09:39:48.200 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:39:48.230 INFO:teuthology.orchestra.run.smithi012.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 351 kB/s | 196 kB 00:00 2023-10-09T09:39:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:48.388 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:39:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:39:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:48.388 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:39:48.463 INFO:teuthology.orchestra.run.smithi012.stdout:(6/6): ceph-mgr-dashboard-16.2.14-48.gaeb2e425. 3.0 MB/s | 3.1 MB 00:01 2023-10-09T09:39:48.467 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:48.467 INFO:teuthology.orchestra.run.smithi012.stdout:Total 2.8 MB/s | 3.4 MB 00:01 2023-10-09T09:39:48.481 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:39:48.493 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:39:48.493 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:39:48.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr 2023-10-09T09:39:48.603 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:39:48.607 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:39:48.851 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:49.045 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:39:49.207 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T09:39:49.383 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T09:39:49.557 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T09:39:49.662 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.no 4/6 2023-10-09T09:39:50.205 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:02 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:50.775 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.n 5/6 2023-10-09T09:39:51.016 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T09:39:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T09:39:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.n 1/6 2023-10-09T09:39:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 2/6 2023-10-09T09:39:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.no 3/6 2023-10-09T09:39:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T09:39:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T09:39:51.489 INFO:teuthology.orchestra.run.smithi110.stdout:Package ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:39:51.601 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:39:51.602 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:39:51.603 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:39:51.975 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T09:39:52.319 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T09:39:52.359 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:06 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:53.189 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:05 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:54.565 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:39:54.566 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================================== 2023-10-09T09:39:54.566 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:39:54.567 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================================== 2023-10-09T09:39:54.567 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:39:54.567 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-dashboard noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 3.1 M 2023-10-09T09:39:54.567 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:39:54.567 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-grafana-dashboards noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 24 k 2023-10-09T09:39:54.568 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-prometheus-alerts noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 15 k 2023-10-09T09:39:54.568 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T09:39:54.568 INFO:teuthology.orchestra.run.smithi110.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T09:39:54.568 INFO:teuthology.orchestra.run.smithi110.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T09:39:54.568 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:54.568 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:39:54.568 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================================== 2023-10-09T09:39:54.569 INFO:teuthology.orchestra.run.smithi110.stdout:Install 6 Packages 2023-10-09T09:39:54.569 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:39:54.569 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 3.4 M 2023-10-09T09:39:54.570 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 22 M 2023-10-09T09:39:54.570 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:39:54.851 INFO:teuthology.orchestra.run.smithi110.stdout:(1/6): ceph-grafana-dashboards-16.2.14-48.gaeb2 109 kB/s | 24 kB 00:00 2023-10-09T09:39:54.876 INFO:teuthology.orchestra.run.smithi110.stdout:(2/6): ceph-prometheus-alerts-16.2.14-48.gaeb2e 60 kB/s | 15 kB 00:00 2023-10-09T09:39:54.927 INFO:teuthology.orchestra.run.smithi110.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 447 kB/s | 34 kB 00:00 2023-10-09T09:39:54.952 INFO:teuthology.orchestra.run.smithi110.stdout:(4/6): python3-routes-2.4.1-12.el8.noarch.rpm 2.6 MB/s | 196 kB 00:00 2023-10-09T09:39:55.002 INFO:teuthology.orchestra.run.smithi110.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 567 kB/s | 43 kB 00:00 2023-10-09T09:39:55.133 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:39:55.336 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:55.337 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:39:55.369 INFO:teuthology.orchestra.run.smithi110.stdout:(6/6): ceph-mgr-dashboard-16.2.14-48.gaeb2e425. 4.2 MB/s | 3.1 MB 00:00 2023-10-09T09:39:55.371 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:55.371 INFO:teuthology.orchestra.run.smithi110.stdout:Total 4.3 MB/s | 3.4 MB 00:00 2023-10-09T09:39:55.382 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:39:55.392 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:39:55.392 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:39:55.460 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:39:55.464 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:39:55.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T09:39:55.720 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:39:55.855 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:39:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T09:39:56.114 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T09:39:56.318 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T09:39:56.464 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.no 4/6 2023-10-09T09:39:56.760 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:10 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:39:57.498 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.n 5/6 2023-10-09T09:39:57.772 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T09:39:58.016 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T09:39:58.016 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.n 1/6 2023-10-09T09:39:58.016 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 2/6 2023-10-09T09:39:58.016 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.no 3/6 2023-10-09T09:39:58.017 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T09:39:58.017 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T09:39:58.293 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:39:58.294 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================================================= 2023-10-09T09:39:58.294 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================================================= 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 8.6 M 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T09:39:58.295 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:58.296 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:39:58.296 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================================================= 2023-10-09T09:39:58.296 INFO:teuthology.orchestra.run.smithi012.stdout:Install 8 Packages 2023-10-09T09:39:58.296 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:39:58.296 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 37 M 2023-10-09T09:39:58.296 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 210 M 2023-10-09T09:39:58.297 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:39:58.375 INFO:teuthology.orchestra.run.smithi012.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 2.1 MB/s | 171 kB 00:00 2023-10-09T09:39:58.440 INFO:teuthology.orchestra.run.smithi012.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 4.4 MB/s | 644 kB 00:00 2023-10-09T09:39:58.576 INFO:teuthology.orchestra.run.smithi012.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 27 MB/s | 3.7 MB 00:00 2023-10-09T09:39:58.626 INFO:teuthology.orchestra.run.smithi012.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 4.4 MB/s | 225 kB 00:00 2023-10-09T09:39:58.852 INFO:teuthology.orchestra.run.smithi012.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 29 MB/s | 14 MB 00:00 2023-10-09T09:39:58.927 INFO:teuthology.orchestra.run.smithi012.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 16 MB/s | 4.9 MB 00:00 2023-10-09T09:39:59.027 INFO:teuthology.orchestra.run.smithi012.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 27 MB/s | 4.7 MB 00:00 2023-10-09T09:39:59.056 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T09:39:59.056 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:11 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:39:59.243 INFO:teuthology.orchestra.run.smithi012.stdout:(8/8): ceph-mgr-diskprediction-local-16.2.14-48 9.1 MB/s | 8.6 MB 00:00 2023-10-09T09:39:59.244 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:39:59.245 INFO:teuthology.orchestra.run.smithi012.stdout:Total 39 MB/s | 37 MB 00:00 2023-10-09T09:39:59.574 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:39:59.593 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:39:59.593 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:39:59.909 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:39:59.913 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:40:00.465 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:40:00.585 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T09:40:00.729 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T09:40:00.876 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T09:40:01.537 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:40:01.638 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T09:40:01.721 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:01.721 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:40:01.722 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:01.722 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:01.722 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:01.722 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T09:40:01.722 INFO:teuthology.orchestra.run.smithi110.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T09:40:01.722 INFO:teuthology.orchestra.run.smithi110.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T09:40:01.722 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:01.722 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:40:01.793 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T09:40:01.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T09:40:02.190 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:02.428 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T09:40:02.593 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T09:40:03.046 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:15 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:03.317 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T09:40:03.609 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T09:40:03.792 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T09:40:04.429 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:40:04.430 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================================= 2023-10-09T09:40:04.430 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:40:04.430 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================================= 2023-10-09T09:40:04.430 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 8.6 M 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:40:04.431 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================================= 2023-10-09T09:40:04.432 INFO:teuthology.orchestra.run.smithi110.stdout:Install 8 Packages 2023-10-09T09:40:04.432 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:04.432 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 37 M 2023-10-09T09:40:04.432 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 210 M 2023-10-09T09:40:04.432 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:40:04.574 INFO:teuthology.orchestra.run.smithi110.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 27 MB/s | 3.7 MB 00:00 2023-10-09T09:40:04.624 INFO:teuthology.orchestra.run.smithi110.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 4.4 MB/s | 225 kB 00:00 2023-10-09T09:40:04.850 INFO:teuthology.orchestra.run.smithi110.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 33 MB/s | 14 MB 00:00 2023-10-09T09:40:04.916 INFO:teuthology.orchestra.run.smithi110.stdout:(4/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 17 MB/s | 4.9 MB 00:00 2023-10-09T09:40:04.975 INFO:teuthology.orchestra.run.smithi110.stdout:(5/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 11 MB/s | 644 kB 00:00 2023-10-09T09:40:05.047 INFO:teuthology.orchestra.run.smithi110.stdout:(6/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 2.3 MB/s | 171 kB 00:00 2023-10-09T09:40:05.109 INFO:teuthology.orchestra.run.smithi110.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.7 MB 00:00 2023-10-09T09:40:05.334 INFO:teuthology.orchestra.run.smithi110.stdout:(8/8): ceph-mgr-diskprediction-local-16.2.14-48 9.5 MB/s | 8.6 MB 00:00 2023-10-09T09:40:05.335 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:05.335 INFO:teuthology.orchestra.run.smithi110.stdout:Total 41 MB/s | 37 MB 00:00 2023-10-09T09:40:05.574 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:40:05.592 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:40:05.592 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:40:05.649 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T09:40:05.890 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:40:05.893 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:40:06.443 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:40:06.622 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T09:40:06.761 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T09:40:06.906 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T09:40:07.000 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T09:40:07.183 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 8/8 2023-10-09T09:40:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 8/8 2023-10-09T09:40:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 1/8 2023-10-09T09:40:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T09:40:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-10-09T09:40:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-10-09T09:40:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T09:40:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T09:40:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-10-09T09:40:07.524 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T09:40:07.873 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T09:40:08.500 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T09:40:08.559 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-10-09T09:40:08.559 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:22 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:08.664 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T09:40:09.336 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T09:40:09.667 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T09:40:09.872 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T09:40:11.601 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T09:40:11.614 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:40:11.821 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:11.821 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:40:11.821 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:11.821 INFO:teuthology.orchestra.run.smithi012.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T09:40:11.821 INFO:teuthology.orchestra.run.smithi012.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T09:40:11.821 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T09:40:11.822 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T09:40:11.822 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T09:40:11.822 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T09:40:11.822 INFO:teuthology.orchestra.run.smithi012.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T09:40:11.822 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:11.822 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:40:12.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-rook 2023-10-09T09:40:12.373 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:12.869 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T09:40:13.047 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 8/8 2023-10-09T09:40:13.241 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:27 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 8/8 2023-10-09T09:40:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 1/8 2023-10-09T09:40:13.526 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-10-09T09:40:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-10-09T09:40:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-10-09T09:40:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-10-09T09:40:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-10-09T09:40:13.527 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-10-09T09:40:14.619 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-10-09T09:40:14.619 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:26 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:14.782 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================================= 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================================= 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-rook noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 27 k 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T09:40:14.784 INFO:teuthology.orchestra.run.smithi012.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:40:14.785 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================================= 2023-10-09T09:40:14.786 INFO:teuthology.orchestra.run.smithi012.stdout:Install 13 Packages 2023-10-09T09:40:14.786 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:14.786 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 1.9 M 2023-10-09T09:40:14.786 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 24 M 2023-10-09T09:40:14.787 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:40:14.940 INFO:teuthology.orchestra.run.smithi012.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 191 kB/s | 16 kB 00:00 2023-10-09T09:40:14.957 INFO:teuthology.orchestra.run.smithi012.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 323 kB/s | 33 kB 00:00 2023-10-09T09:40:14.999 INFO:teuthology.orchestra.run.smithi012.stdout:(3/13): ceph-mgr-rook-16.2.14-48.gaeb2e425.el8. 190 kB/s | 27 kB 00:00 2023-10-09T09:40:15.024 INFO:teuthology.orchestra.run.smithi012.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 970 kB/s | 81 kB 00:00 2023-10-09T09:40:15.066 INFO:teuthology.orchestra.run.smithi012.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 1.4 MB/s | 61 kB 00:00 2023-10-09T09:40:15.108 INFO:teuthology.orchestra.run.smithi012.stdout:(6/13): python3-rsa-4.9-2.el8.noarch.rpm 588 kB/s | 64 kB 00:00 2023-10-09T09:40:15.150 INFO:teuthology.orchestra.run.smithi012.stdout:(7/13): python3-kubernetes-11.0.0-6.el8.noarch. 6.0 MB/s | 1.2 MB 00:00 2023-10-09T09:40:15.227 INFO:teuthology.orchestra.run.smithi012.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 266 kB/s | 43 kB 00:00 2023-10-09T09:40:15.259 INFO:teuthology.orchestra.run.smithi012.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.1 MB/s | 155 kB 00:00 2023-10-09T09:40:15.276 INFO:teuthology.orchestra.run.smithi012.stdout:(10/13): python3-jsonpatch-1.21-2.el8.noarch.rp 646 kB/s | 27 kB 00:00 2023-10-09T09:40:15.293 INFO:teuthology.orchestra.run.smithi012.stdout:(11/13): python3-pyasn1-modules-0.3.7-6.el8.noa 817 kB/s | 110 kB 00:00 2023-10-09T09:40:15.310 INFO:teuthology.orchestra.run.smithi012.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 601 kB/s | 20 kB 00:00 2023-10-09T09:40:15.327 INFO:teuthology.orchestra.run.smithi012.stdout:(13/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.8 MB/s | 126 kB 00:00 2023-10-09T09:40:15.328 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:15.328 INFO:teuthology.orchestra.run.smithi012.stdout:Total 3.5 MB/s | 1.9 MB 00:00 2023-10-09T09:40:15.387 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:40:15.402 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:40:15.403 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:40:15.479 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:40:15.483 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:40:16.041 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:40:16.233 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T09:40:16.436 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T09:40:16.605 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T09:40:16.780 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-10-09T09:40:17.101 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-10-09T09:40:17.313 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-09T09:40:17.469 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-09T09:40:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T09:40:17.813 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T09:40:18.041 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T09:40:19.155 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T09:40:19.353 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:40:19.380 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T09:40:19.497 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 13/13 2023-10-09T09:40:19.543 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:19.543 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:40:19.543 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:19.544 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:40:19.675 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr-rook 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 13/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 1/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-10-09T09:40:19.739 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-10-09T09:40:19.740 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-10-09T09:40:19.740 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-10-09T09:40:20.024 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:20.898 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-10-09T09:40:20.899 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:34 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:20.911 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:32 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:22.375 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:40:22.377 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================= 2023-10-09T09:40:22.377 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:40:22.377 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================= 2023-10-09T09:40:22.377 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-rook noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 27 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T09:40:22.378 INFO:teuthology.orchestra.run.smithi110.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T09:40:22.379 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T09:40:22.379 INFO:teuthology.orchestra.run.smithi110.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T09:40:22.379 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:22.379 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:40:22.379 INFO:teuthology.orchestra.run.smithi110.stdout:======================================================================================================= 2023-10-09T09:40:22.379 INFO:teuthology.orchestra.run.smithi110.stdout:Install 13 Packages 2023-10-09T09:40:22.379 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:22.380 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 1.9 M 2023-10-09T09:40:22.380 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 24 M 2023-10-09T09:40:22.380 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:40:22.790 INFO:teuthology.orchestra.run.smithi110.stdout:(1/13): ceph-mgr-rook-16.2.14-48.gaeb2e425.el8. 188 kB/s | 27 kB 00:00 2023-10-09T09:40:22.895 INFO:teuthology.orchestra.run.smithi110.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 65 kB/s | 16 kB 00:00 2023-10-09T09:40:22.924 INFO:teuthology.orchestra.run.smithi110.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 118 kB/s | 33 kB 00:00 2023-10-09T09:40:23.058 INFO:teuthology.orchestra.run.smithi110.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 481 kB/s | 64 kB 00:00 2023-10-09T09:40:23.083 INFO:teuthology.orchestra.run.smithi110.stdout:(5/13): python3-google-auth-1.1.1-10.el8.noarch 278 kB/s | 81 kB 00:00 2023-10-09T09:40:23.133 INFO:teuthology.orchestra.run.smithi110.stdout:(6/13): python3-pyasn1-modules-0.3.7-6.el8.noar 2.2 MB/s | 110 kB 00:00 2023-10-09T09:40:23.150 INFO:teuthology.orchestra.run.smithi110.stdout:(7/13): python3-websocket-client-0.56.0-5.el8.n 662 kB/s | 61 kB 00:00 2023-10-09T09:40:23.176 INFO:teuthology.orchestra.run.smithi110.stdout:(8/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 633 kB/s | 27 kB 00:00 2023-10-09T09:40:23.218 INFO:teuthology.orchestra.run.smithi110.stdout:(9/13): python3-jsonpointer-1.10-11.el8.noarch. 553 kB/s | 20 kB 00:00 2023-10-09T09:40:23.251 INFO:teuthology.orchestra.run.smithi110.stdout:(10/13): python3-kubernetes-11.0.0-6.el8.noarch 3.3 MB/s | 1.2 MB 00:00 2023-10-09T09:40:23.276 INFO:teuthology.orchestra.run.smithi110.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.0 MB/s | 126 kB 00:00 2023-10-09T09:40:23.301 INFO:teuthology.orchestra.run.smithi110.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 510 kB/s | 43 kB 00:00 2023-10-09T09:40:23.327 INFO:teuthology.orchestra.run.smithi110.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 2.0 MB/s | 155 kB 00:00 2023-10-09T09:40:23.329 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:23.330 INFO:teuthology.orchestra.run.smithi110.stdout:Total 2.0 MB/s | 1.9 MB 00:00 2023-10-09T09:40:23.395 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:40:23.410 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:40:23.411 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:40:23.481 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:40:23.485 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:40:23.518 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:40:23.722 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:23.722 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T09:40:23.723 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T09:40:23.724 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T09:40:23.724 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T09:40:23.724 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T09:40:23.724 INFO:teuthology.orchestra.run.smithi012.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T09:40:23.724 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:23.724 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:40:23.872 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T09:40:23.985 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:40:24.220 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T09:40:24.250 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:24.439 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T09:40:24.723 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T09:40:24.949 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-10-09T09:40:25.127 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:39 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:25.143 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-10-09T09:40:25.344 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-10-09T09:40:25.557 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-10-09T09:40:25.761 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T09:40:25.994 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T09:40:26.251 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T09:40:26.672 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:40:26.674 INFO:teuthology.orchestra.run.smithi012.stdout:============================================================================================== 2023-10-09T09:40:26.674 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:40:26.674 INFO:teuthology.orchestra.run.smithi012.stdout:============================================================================================== 2023-10-09T09:40:26.674 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:40:26.674 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-cephadm noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 116 k 2023-10-09T09:40:26.674 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:40:26.675 INFO:teuthology.orchestra.run.smithi012.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-10-09T09:40:26.675 INFO:teuthology.orchestra.run.smithi012.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T09:40:26.675 INFO:teuthology.orchestra.run.smithi012.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-10-09T09:40:26.675 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T09:40:26.675 INFO:teuthology.orchestra.run.smithi012.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-10-09T09:40:26.675 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:26.675 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:40:26.676 INFO:teuthology.orchestra.run.smithi012.stdout:============================================================================================== 2023-10-09T09:40:26.676 INFO:teuthology.orchestra.run.smithi012.stdout:Install 6 Packages 2023-10-09T09:40:26.676 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:26.677 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 5.8 M 2023-10-09T09:40:26.677 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 24 M 2023-10-09T09:40:26.677 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:40:27.061 INFO:teuthology.orchestra.run.smithi012.stdout:(1/6): ceph-mgr-cephadm-16.2.14-48.gaeb2e425.el 624 kB/s | 116 kB 00:00 2023-10-09T09:40:27.253 INFO:teuthology.orchestra.run.smithi012.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 56 kB/s | 21 kB 00:00 2023-10-09T09:40:27.328 INFO:teuthology.orchestra.run.smithi012.stdout:(3/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 7.2 MB/s | 538 kB 00:00 2023-10-09T09:40:27.340 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T09:40:27.461 INFO:teuthology.orchestra.run.smithi012.stdout:(4/6): python3-babel-2.5.1-7.el8.noarch.rpm 36 MB/s | 4.8 MB 00:00 2023-10-09T09:40:27.487 INFO:teuthology.orchestra.run.smithi012.stdout:(5/6): python3-remoto-1.2.1-1.el8.noarch.rpm 115 kB/s | 49 kB 00:00 2023-10-09T09:40:27.521 INFO:teuthology.orchestra.run.smithi012.stdout:(6/6): python3-execnet-1.7.1-1.el8.noarch.rpm 551 kB/s | 355 kB 00:00 2023-10-09T09:40:27.523 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:27.523 INFO:teuthology.orchestra.run.smithi012.stdout:Total 6.9 MB/s | 5.8 MB 00:00 2023-10-09T09:40:27.580 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:40:27.592 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:40:27.592 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:40:27.619 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T09:40:27.676 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:40:27.680 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:40:27.803 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 13/13 2023-10-09T09:40:28.047 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 13/13 2023-10-09T09:40:28.047 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 1/13 2023-10-09T09:40:28.047 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T09:40:28.047 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T09:40:28.047 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T09:40:28.047 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T09:40:28.047 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T09:40:28.047 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T09:40:28.048 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-10-09T09:40:28.048 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-10-09T09:40:28.048 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-10-09T09:40:28.048 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-10-09T09:40:28.048 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-10-09T09:40:28.707 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:40:29.058 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-10-09T09:40:29.203 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-10-09T09:40:29.203 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:41 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:29.293 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-10-09T09:40:29.577 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-10-09T09:40:29.842 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-10-09T09:40:30.126 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-10-09T09:40:30.295 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T09:40:30.546 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T09:40:30.546 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/6 2023-10-09T09:40:30.546 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-10-09T09:40:30.546 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-10-09T09:40:30.546 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-10-09T09:40:30.546 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-10-09T09:40:31.818 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:40:31.862 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-10-09T09:40:31.862 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:45 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:32.004 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:32.004 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:40:32.004 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:32.004 INFO:teuthology.orchestra.run.smithi110.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T09:40:32.004 INFO:teuthology.orchestra.run.smithi110.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:32.005 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:40:32.127 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T09:40:32.479 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:33.376 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:45 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:34.591 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:40:34.783 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:40:34.784 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================== 2023-10-09T09:40:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:40:34.784 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================== 2023-10-09T09:40:34.784 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:40:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-cephadm noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 116 k 2023-10-09T09:40:34.784 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout:============================================================================================== 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout:Install 6 Packages 2023-10-09T09:40:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:34.786 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 5.8 M 2023-10-09T09:40:34.786 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 24 M 2023-10-09T09:40:34.786 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout: python3-apipkg-1.5-6.el8.noarch 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-10-09T09:40:34.793 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:34.794 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:40:34.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-fuse 2023-10-09T09:40:35.079 INFO:teuthology.orchestra.run.smithi110.stdout:(1/6): python3-apipkg-1.5-6.el8.noarch.rpm 81 kB/s | 21 kB 00:00 2023-10-09T09:40:35.104 INFO:teuthology.orchestra.run.smithi110.stdout:(2/6): ceph-mgr-cephadm-16.2.14-48.gaeb2e425.el 403 kB/s | 116 kB 00:00 2023-10-09T09:40:35.238 INFO:teuthology.orchestra.run.smithi110.stdout:(3/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.9 MB/s | 538 kB 00:00 2023-10-09T09:40:35.263 INFO:teuthology.orchestra.run.smithi110.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 266 kB/s | 49 kB 00:00 2023-10-09T09:40:35.288 INFO:teuthology.orchestra.run.smithi110.stdout:(5/6): python3-execnet-1.7.1-1.el8.noarch.rpm 753 kB/s | 355 kB 00:00 2023-10-09T09:40:35.323 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:35.455 INFO:teuthology.orchestra.run.smithi110.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 22 MB/s | 4.8 MB 00:00 2023-10-09T09:40:35.456 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:35.456 INFO:teuthology.orchestra.run.smithi110.stdout:Total 8.7 MB/s | 5.8 MB 00:00 2023-10-09T09:40:35.509 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:40:35.519 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:40:35.519 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:40:35.598 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:40:35.602 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:40:36.599 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:40:36.964 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-10-09T09:40:36.978 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:50 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:37.191 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-10-09T09:40:37.486 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-10-09T09:40:37.804 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-10-09T09:40:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-10-09T09:40:38.327 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T09:40:38.525 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:40:38.526 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:40:38.526 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2023-10-09T09:40:38.526 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:40:38.526 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-fuse 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout: x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 910 k 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout:Install 2 Packages 2023-10-09T09:40:38.527 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:38.528 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 993 k 2023-10-09T09:40:38.528 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 2.8 M 2023-10-09T09:40:38.528 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:40:38.572 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T09:40:38.572 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/6 2023-10-09T09:40:38.572 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-10-09T09:40:38.572 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-10-09T09:40:38.573 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-10-09T09:40:38.573 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-10-09T09:40:38.682 INFO:teuthology.orchestra.run.smithi012.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 538 kB/s | 83 kB 00:00 2023-10-09T09:40:38.807 INFO:teuthology.orchestra.run.smithi012.stdout:(2/2): ceph-fuse-16.2.14-48.gaeb2e425.el8.x86_6 3.2 MB/s | 910 kB 00:00 2023-10-09T09:40:38.808 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:38.808 INFO:teuthology.orchestra.run.smithi012.stdout:Total 3.5 MB/s | 993 kB 00:00 2023-10-09T09:40:38.819 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:40:38.841 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:40:38.842 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:40:38.922 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:40:38.926 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:40:39.099 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:40:39.380 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T09:40:39.533 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2/2 2023-10-09T09:40:39.627 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-10-09T09:40:39.627 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:51 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:39.877 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2/2 2023-10-09T09:40:39.877 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/2 2023-10-09T09:40:40.984 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T09:40:40.984 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:02:54 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:43.501 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: python3-apipkg-1.5-6.el8.noarch 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:43.924 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:40:43.925 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install ceph-fuse 2023-10-09T09:40:44.224 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:40:44.280 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:44.428 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:44.428 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:40:44.428 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T09:40:44.428 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:44.428 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:40:44.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librados-devel 2023-10-09T09:40:44.975 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:45.869 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:02:57 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:46.301 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:00 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:47.288 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:40:47.288 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:40:47.289 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repository Size 2023-10-09T09:40:47.289 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:40:47.289 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:40:47.289 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-fuse 2023-10-09T09:40:47.289 INFO:teuthology.orchestra.run.smithi110.stdout: x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 910 k 2023-10-09T09:40:47.289 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2023-10-09T09:40:47.289 INFO:teuthology.orchestra.run.smithi110.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T09:40:47.290 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:47.290 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:40:47.290 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:40:47.290 INFO:teuthology.orchestra.run.smithi110.stdout:Install 2 Packages 2023-10-09T09:40:47.290 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:47.291 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 993 k 2023-10-09T09:40:47.291 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 2.8 M 2023-10-09T09:40:47.292 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:40:47.495 INFO:teuthology.orchestra.run.smithi110.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 404 kB/s | 83 kB 00:00 2023-10-09T09:40:47.528 INFO:teuthology.orchestra.run.smithi110.stdout:(2/2): ceph-fuse-16.2.14-48.gaeb2e425.el8.x86_6 3.7 MB/s | 910 kB 00:00 2023-10-09T09:40:47.529 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:47.529 INFO:teuthology.orchestra.run.smithi110.stdout:Total 4.0 MB/s | 993 kB 00:00 2023-10-09T09:40:47.541 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:40:47.561 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:40:47.562 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:40:47.639 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:40:47.643 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:40:47.809 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:40:47.846 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:40:47.847 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:40:47.847 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2023-10-09T09:40:47.847 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:40:47.847 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:40:47.847 INFO:teuthology.orchestra.run.smithi012.stdout: librados-devel x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 131 k 2023-10-09T09:40:47.847 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:47.847 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:40:47.848 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:40:47.848 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2023-10-09T09:40:47.848 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:47.848 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 131 k 2023-10-09T09:40:47.848 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 432 k 2023-10-09T09:40:47.848 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:40:48.028 INFO:teuthology.orchestra.run.smithi012.stdout:librados-devel-16.2.14-48.gaeb2e425.el8.x86_64. 732 kB/s | 131 kB 00:00 2023-10-09T09:40:48.028 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:48.028 INFO:teuthology.orchestra.run.smithi012.stdout:Total 727 kB/s | 131 kB 00:00 2023-10-09T09:40:48.029 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:40:48.035 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:40:48.035 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:40:48.091 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T09:40:48.093 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:40:48.097 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:40:48.212 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:40:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2/2 2023-10-09T09:40:48.388 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:40:48.618 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:40:48.669 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2/2 2023-10-09T09:40:48.669 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/2 2023-10-09T09:40:49.723 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:40:49.723 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:03 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:49.737 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T09:40:49.737 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:01 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:55.112 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:40:55.298 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:55.298 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:40:55.298 INFO:teuthology.orchestra.run.smithi110.stdout: ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T09:40:55.298 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:55.298 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:40:55.427 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install librados-devel 2023-10-09T09:40:55.784 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:56.193 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:40:56.399 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:56.399 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:40:56.399 INFO:teuthology.orchestra.run.smithi012.stdout: librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:40:56.399 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:40:56.399 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:40:56.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install libcephfs2 2023-10-09T09:40:56.925 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:40:56.950 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:08 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:40:58.363 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:40:58.364 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:40:58.364 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repo Size 2023-10-09T09:40:58.364 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:40:58.364 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:40:58.364 INFO:teuthology.orchestra.run.smithi110.stdout: librados-devel x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 131 k 2023-10-09T09:40:58.364 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:58.364 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:40:58.364 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:40:58.365 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-10-09T09:40:58.365 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:40:58.365 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 131 k 2023-10-09T09:40:58.365 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 432 k 2023-10-09T09:40:58.365 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:40:58.388 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:12 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:40:58.659 INFO:teuthology.orchestra.run.smithi110.stdout:librados-devel-16.2.14-48.gaeb2e425.el8.x86_64. 448 kB/s | 131 kB 00:00 2023-10-09T09:40:58.659 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:40:58.659 INFO:teuthology.orchestra.run.smithi110.stdout:Total 446 kB/s | 131 kB 00:00 2023-10-09T09:40:58.659 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:40:58.664 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:40:58.664 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:40:58.719 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:40:58.722 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:40:58.838 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:40:58.997 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:40:59.227 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:40:59.862 INFO:teuthology.orchestra.run.smithi012.stdout:Package libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:40:59.984 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:40:59.985 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:40:59.985 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:00.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install libcephfs-devel 2023-10-09T09:41:00.301 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:00.301 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:12 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:00.491 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:01.604 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:15 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:02.862 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:41:03.048 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:03.048 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:41:03.048 INFO:teuthology.orchestra.run.smithi110.stdout: librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:03.048 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:03.048 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:03.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install libcephfs2 2023-10-09T09:41:03.167 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:03.167 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:03.167 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2023-10-09T09:41:03.167 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:03.167 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:41:03.167 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs-devel x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 25 k 2023-10-09T09:41:03.168 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:03.168 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:41:03.168 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:03.168 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2023-10-09T09:41:03.168 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:03.168 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 25 k 2023-10-09T09:41:03.168 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 107 k 2023-10-09T09:41:03.168 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:41:03.365 INFO:teuthology.orchestra.run.smithi012.stdout:libcephfs-devel-16.2.14-48.gaeb2e425.el8.x86_64 129 kB/s | 25 kB 00:00 2023-10-09T09:41:03.366 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:41:03.366 INFO:teuthology.orchestra.run.smithi012.stdout:Total 128 kB/s | 25 kB 00:00 2023-10-09T09:41:03.366 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:41:03.370 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:41:03.370 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:41:03.373 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:41:03.377 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:41:03.522 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:03.686 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:41:03.910 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:04.134 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:05.025 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:17 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:05.194 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:05.194 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:19 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:06.294 INFO:teuthology.orchestra.run.smithi110.stdout:Package libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:06.401 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:06.402 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:41:06.402 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:06.503 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install libcephfs-devel 2023-10-09T09:41:06.857 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:08.278 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:20 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:08.576 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:41:08.779 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:08.780 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:41:08.780 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:08.780 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:08.780 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:08.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librados2 2023-10-09T09:41:09.305 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:09.727 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:09.727 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repo Size 2023-10-09T09:41:09.728 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:09.728 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:41:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs-devel x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 25 k 2023-10-09T09:41:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:09.728 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:41:09.729 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:09.729 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-10-09T09:41:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:09.730 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 25 k 2023-10-09T09:41:09.730 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 107 k 2023-10-09T09:41:09.730 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:41:09.988 INFO:teuthology.orchestra.run.smithi110.stdout:libcephfs-devel-16.2.14-48.gaeb2e425.el8.x86_64 98 kB/s | 25 kB 00:00 2023-10-09T09:41:09.989 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:41:09.989 INFO:teuthology.orchestra.run.smithi110.stdout:Total 98 kB/s | 25 kB 00:00 2023-10-09T09:41:09.989 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:41:09.993 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:41:09.993 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:41:09.995 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:41:09.999 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:41:10.052 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:41:10.287 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:10.498 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:10.809 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:24 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:11.490 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:11.490 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:23 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:12.227 INFO:teuthology.orchestra.run.smithi012.stdout:Package librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:12.348 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:12.350 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:41:12.350 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:12.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librbd1 2023-10-09T09:41:12.853 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:14.510 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:28 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:15.809 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:41:15.934 INFO:teuthology.orchestra.run.smithi012.stdout:Package librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:15.993 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:15.993 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:41:15.994 INFO:teuthology.orchestra.run.smithi110.stdout: libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:15.994 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:15.994 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:16.055 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:16.056 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:41:16.056 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:16.127 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install librados2 2023-10-09T09:41:16.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rados 2023-10-09T09:41:16.481 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:16.556 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:17.441 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:29 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:17.536 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:31 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:18.789 INFO:teuthology.orchestra.run.smithi110.stdout:Package librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:18.900 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:18.901 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:41:18.901 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:18.970 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:19.015 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install librbd1 2023-10-09T09:41:19.091 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:19.093 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:41:19.093 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:19.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rgw 2023-10-09T09:41:19.368 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:19.606 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:20.358 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:32 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:20.536 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:34 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:21.618 INFO:teuthology.orchestra.run.smithi110.stdout:Package librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:21.729 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:21.730 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:41:21.730 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:21.868 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install python3-rados 2023-10-09T09:41:21.997 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:22.116 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:22.118 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:41:22.118 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:22.222 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:22.250 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-cephfs 2023-10-09T09:41:22.627 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:23.136 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:35 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:24.160 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:38 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:24.415 INFO:teuthology.orchestra.run.smithi110.stdout:Package python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:24.525 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:24.526 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:41:24.526 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:24.632 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install python3-rgw 2023-10-09T09:41:24.986 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:25.610 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:25.730 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:25.732 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:41:25.732 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:25.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rbd 2023-10-09T09:41:26.242 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:26.551 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:38 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:27.719 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:41 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:27.814 INFO:teuthology.orchestra.run.smithi110.stdout:Package python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:27.924 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:27.925 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:41:27.925 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:28.029 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install python3-cephfs 2023-10-09T09:41:28.381 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:29.098 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:29.219 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:29.220 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2023-10-09T09:41:29.220 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:29.225 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:41 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:29.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-fuse 2023-10-09T09:41:29.719 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:30.485 INFO:teuthology.orchestra.run.smithi110.stdout:Package python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:30.592 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:30.594 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:41:30.594 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:30.609 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:44 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:30.707 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install python3-rbd 2023-10-09T09:41:31.061 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:31.918 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:43 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:32.147 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:32.147 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:32.147 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2023-10-09T09:41:32.147 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:32.147 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:41:32.148 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-fuse x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 88 k 2023-10-09T09:41:32.148 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:32.148 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:41:32.148 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:32.148 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2023-10-09T09:41:32.148 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:32.149 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 88 k 2023-10-09T09:41:32.149 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 240 k 2023-10-09T09:41:32.149 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:41:32.308 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-fuse-16.2.14-48.gaeb2e425.el8.x86_64.rpm 555 kB/s | 88 kB 00:00 2023-10-09T09:41:32.309 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:41:32.309 INFO:teuthology.orchestra.run.smithi012.stdout:Total 551 kB/s | 88 kB 00:00 2023-10-09T09:41:32.309 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:41:32.319 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:41:32.319 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:41:32.377 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:41:32.381 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:41:32.516 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:41:32.770 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:33.000 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:33.199 INFO:teuthology.orchestra.run.smithi110.stdout:Package python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T09:41:33.308 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:33.310 INFO:teuthology.orchestra.run.smithi110.stdout:Nothing to do. 2023-10-09T09:41:33.310 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:33.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install rbd-fuse 2023-10-09T09:41:33.767 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:34.116 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:48 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:34.669 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:46 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:36.036 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:36.036 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:36.036 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2023-10-09T09:41:36.037 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:36.037 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:41:36.037 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-fuse x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 88 k 2023-10-09T09:41:36.037 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:36.037 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:41:36.037 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:36.037 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-10-09T09:41:36.037 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:36.038 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 88 k 2023-10-09T09:41:36.038 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 240 k 2023-10-09T09:41:36.038 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:41:36.332 INFO:teuthology.orchestra.run.smithi110.stdout:rbd-fuse-16.2.14-48.gaeb2e425.el8.x86_64.rpm 300 kB/s | 88 kB 00:00 2023-10-09T09:41:36.333 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:41:36.333 INFO:teuthology.orchestra.run.smithi110.stdout:Total 299 kB/s | 88 kB 00:00 2023-10-09T09:41:36.333 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:41:36.343 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:41:36.343 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:41:36.397 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:41:36.401 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:41:36.515 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:41:36.622 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:41:36.667 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:36.824 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:36.824 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:41:36.825 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:36.825 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:36.825 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:36.929 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:36.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-mirror 2023-10-09T09:41:37.353 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:38.023 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:38.023 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:50 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:38.209 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:52 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:39.767 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:39.768 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:39.768 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2023-10-09T09:41:39.768 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:39.768 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:41:39.768 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-mirror x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.7 M 2023-10-09T09:41:39.769 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:39.769 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:41:39.769 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:39.769 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2023-10-09T09:41:39.769 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:39.769 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 3.7 M 2023-10-09T09:41:39.770 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 15 M 2023-10-09T09:41:39.770 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:41:40.454 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-mirror-16.2.14-48.gaeb2e425.el8.x86_64.rpm 5.4 MB/s | 3.7 MB 00:00 2023-10-09T09:41:40.455 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:41:40.455 INFO:teuthology.orchestra.run.smithi012.stdout:Total 5.3 MB/s | 3.7 MB 00:00 2023-10-09T09:41:40.456 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:41:40.469 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:41:40.469 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:41:40.542 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:41:40.546 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:41:40.813 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:41:41.000 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:41.000 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:41:41.000 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:41.001 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:41.001 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:41.008 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:41:41.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install rbd-mirror 2023-10-09T09:41:41.154 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:41.478 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:41.527 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:42.427 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:54 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:42.596 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:42.596 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:03:56 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:43.873 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:43.874 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:43.874 INFO:teuthology.orchestra.run.smithi110.stdout: Package Arch Version Repo Size 2023-10-09T09:41:43.874 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:43.874 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:41:43.874 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-mirror x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.7 M 2023-10-09T09:41:43.874 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:43.874 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:41:43.874 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:43.875 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-10-09T09:41:43.875 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:43.875 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 3.7 M 2023-10-09T09:41:43.875 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 15 M 2023-10-09T09:41:43.875 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:41:44.519 INFO:teuthology.orchestra.run.smithi110.stdout:rbd-mirror-16.2.14-48.gaeb2e425.el8.x86_64.rpm 5.7 MB/s | 3.7 MB 00:00 2023-10-09T09:41:44.519 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:41:44.520 INFO:teuthology.orchestra.run.smithi110.stdout:Total 5.7 MB/s | 3.7 MB 00:00 2023-10-09T09:41:44.520 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:41:44.531 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:41:44.531 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:41:44.599 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:41:44.603 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:41:45.041 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:41:45.227 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:45.580 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:46.654 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:46.654 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:03:58 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:47.257 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:41:47.459 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:47.459 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:41:47.459 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:47.459 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:47.459 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:47.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-nbd 2023-10-09T09:41:47.996 INFO:teuthology.orchestra.run.smithi012.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:48.912 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:04:02 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:49.301 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:41:49.484 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:49.485 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:41:49.485 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:49.485 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:49.485 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:49.605 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum -y install rbd-nbd 2023-10-09T09:41:49.961 INFO:teuthology.orchestra.run.smithi110.stdout:Updating Subscription Management repositories. 2023-10-09T09:41:50.477 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2023-10-09T09:41:50.477 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2023-10-09T09:41:50.478 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:50.478 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2023-10-09T09:41:50.478 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-nbd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 173 k 2023-10-09T09:41:50.478 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:50.478 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2023-10-09T09:41:50.478 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2023-10-09T09:41:50.479 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2023-10-09T09:41:50.479 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:50.479 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 173 k 2023-10-09T09:41:50.479 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 515 k 2023-10-09T09:41:50.480 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2023-10-09T09:41:50.657 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-nbd-16.2.14-48.gaeb2e425.el8.x86_64.rpm 972 kB/s | 173 kB 00:00 2023-10-09T09:41:50.658 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:41:50.658 INFO:teuthology.orchestra.run.smithi012.stdout:Total 965 kB/s | 173 kB 00:00 2023-10-09T09:41:50.658 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2023-10-09T09:41:50.670 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2023-10-09T09:41:50.670 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2023-10-09T09:41:50.729 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2023-10-09T09:41:50.733 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2023-10-09T09:41:50.861 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2023-10-09T09:41:50.941 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:04:02 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:51.016 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:52.318 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2023-10-09T09:41:52.318 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2023-10-09T09:41:52.318 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:52.318 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2023-10-09T09:41:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-nbd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 173 k 2023-10-09T09:41:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:52.319 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2023-10-09T09:41:52.319 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2023-10-09T09:41:52.319 INFO:teuthology.orchestra.run.smithi110.stdout:Install 1 Package 2023-10-09T09:41:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:52.319 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 173 k 2023-10-09T09:41:52.320 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 515 k 2023-10-09T09:41:52.320 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2023-10-09T09:41:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:52.329 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:04:06 ago on Mon 09 Oct 2023 09:37:46 AM UTC. 2023-10-09T09:41:52.655 INFO:teuthology.orchestra.run.smithi110.stdout:rbd-nbd-16.2.14-48.gaeb2e425.el8.x86_64.rpm 517 kB/s | 173 kB 00:00 2023-10-09T09:41:52.656 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2023-10-09T09:41:52.656 INFO:teuthology.orchestra.run.smithi110.stdout:Total 515 kB/s | 173 kB 00:00 2023-10-09T09:41:52.656 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2023-10-09T09:41:52.667 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2023-10-09T09:41:52.667 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2023-10-09T09:41:52.721 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2023-10-09T09:41:52.725 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2023-10-09T09:41:52.837 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2023-10-09T09:41:53.073 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:53.295 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:54.359 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T09:41:54.359 INFO:teuthology.orchestra.run.smithi110.stdout:Last metadata expiration check: 0:04:06 ago on Mon 09 Oct 2023 09:37:48 AM UTC. 2023-10-09T09:41:56.700 INFO:teuthology.orchestra.run.smithi012.stdout:Installed products updated. 2023-10-09T09:41:56.902 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:56.902 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2023-10-09T09:41:56.902 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:56.903 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:41:56.903 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2023-10-09T09:41:57.049 DEBUG:teuthology.parallel:result is None 2023-10-09T09:41:59.220 INFO:teuthology.orchestra.run.smithi110.stdout:Installed products updated. 2023-10-09T09:41:59.406 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:59.406 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2023-10-09T09:41:59.406 INFO:teuthology.orchestra.run.smithi110.stdout: rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T09:41:59.406 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:41:59.406 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2023-10-09T09:41:59.546 DEBUG:teuthology.parallel:result is None 2023-10-09T09:41:59.546 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T09:41:59.546 INFO:teuthology.packaging:ref: None 2023-10-09T09:41:59.546 INFO:teuthology.packaging:tag: None 2023-10-09T09:41:59.546 INFO:teuthology.packaging:branch: pacific 2023-10-09T09:41:59.546 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:41:59.547 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-09T09:41:59.778 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T09:41:59.805 INFO:teuthology.orchestra.run.smithi012.stdout:16.2.14-48.gaeb2e425.el8 2023-10-09T09:41:59.806 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48.gaeb2e425.el8 2023-10-09T09:41:59.806 INFO:teuthology.task.install:The correct ceph version 16.2.14-48.gaeb2e425 is installed. 2023-10-09T09:41:59.806 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T09:41:59.806 INFO:teuthology.packaging:ref: None 2023-10-09T09:41:59.806 INFO:teuthology.packaging:tag: None 2023-10-09T09:41:59.806 INFO:teuthology.packaging:branch: pacific 2023-10-09T09:41:59.806 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:41:59.807 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-09T09:42:00.009 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T09:42:00.032 INFO:teuthology.orchestra.run.smithi110.stdout:16.2.14-48.gaeb2e425.el8 2023-10-09T09:42:00.032 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48.gaeb2e425.el8 2023-10-09T09:42:00.032 INFO:teuthology.task.install:The correct ceph version 16.2.14-48.gaeb2e425 is installed. 2023-10-09T09:42:00.033 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T09:42:00.033 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:42:00.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T09:42:00.067 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:42:00.067 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T09:42:00.100 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T09:42:00.101 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:42:00.101 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T09:42:00.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T09:42:00.209 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:42:00.209 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T09:42:00.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T09:42:00.302 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T09:42:00.302 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:42:00.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T09:42:00.333 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T09:42:00.404 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:42:00.404 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T09:42:00.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T09:42:00.496 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T09:42:00.496 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:42:00.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T09:42:00.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T09:42:00.598 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:42:00.598 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T09:42:00.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T09:42:00.694 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T09:42:00.780 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T09:42:00.780 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T09:42:00.781 INFO:tasks.cephadm:Cluster fsid is 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:00.781 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T09:42:00.781 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.12', 'mon.c': '[v2:172.21.15.12:3301,v1:172.21.15.12:6790]', 'mon.b': '172.21.15.110'} 2023-10-09T09:42:00.781 INFO:tasks.cephadm:First mon is mon.a on smithi012 2023-10-09T09:42:00.782 INFO:tasks.cephadm:First mgr is y 2023-10-09T09:42:00.782 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T09:42:00.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo hostname $(hostname -s) 2023-10-09T09:42:00.814 DEBUG:teuthology.orchestra.run.smithi110:> sudo hostname $(hostname -s) 2023-10-09T09:42:00.841 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-10-09T09:42:00.842 DEBUG:teuthology.orchestra.run.smithi012:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T09:42:01.064 INFO:teuthology.orchestra.run.smithi012.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Oct 9 09:42 /home/ubuntu/cephtest/cephadm 2023-10-09T09:42:01.064 DEBUG:teuthology.orchestra.run.smithi110:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T09:42:01.201 INFO:teuthology.orchestra.run.smithi110.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Oct 9 09:42 /home/ubuntu/cephtest/cephadm 2023-10-09T09:42:01.201 DEBUG:teuthology.orchestra.run.smithi012:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T09:42:01.223 DEBUG:teuthology.orchestra.run.smithi110:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T09:42:01.251 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-10-09T09:42:01.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T09:42:01.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T09:42:02.885 INFO:teuthology.orchestra.run.smithi110.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T09:42:02.933 INFO:teuthology.orchestra.run.smithi012.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T09:42:42.817 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T09:42:42.818 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T09:42:42.818 INFO:teuthology.orchestra.run.smithi110.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T09:42:42.818 INFO:teuthology.orchestra.run.smithi110.stdout: "repo_digests": [ 2023-10-09T09:42:42.818 INFO:teuthology.orchestra.run.smithi110.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T09:42:42.818 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:42:42.818 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T09:42:45.401 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:42:45.401 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T09:42:45.401 INFO:teuthology.orchestra.run.smithi012.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T09:42:45.402 INFO:teuthology.orchestra.run.smithi012.stdout: "repo_digests": [ 2023-10-09T09:42:45.402 INFO:teuthology.orchestra.run.smithi012.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T09:42:45.402 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:42:45.402 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:42:45.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /etc/ceph 2023-10-09T09:42:45.451 DEBUG:teuthology.orchestra.run.smithi110:> sudo mkdir -p /etc/ceph 2023-10-09T09:42:45.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 777 /etc/ceph 2023-10-09T09:42:45.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 777 /etc/ceph 2023-10-09T09:42:45.557 INFO:tasks.cephadm:Writing seed config... 2023-10-09T09:42:45.558 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T09:42:45.559 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T09:42:45.560 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:42:45.560 DEBUG:teuthology.orchestra.run.smithi012:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T09:42:45.577 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 1881d6a6-6688-11ee-8db5-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T09:42:45.577 DEBUG:teuthology.orchestra.run.smithi012:mon.a> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T09:42:45.619 DEBUG:teuthology.orchestra.run.smithi012:mgr.y> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T09:42:45.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:42:45.646 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T09:42:45.647 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.12 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T09:42:45.672 INFO:journalctl@ceph.mgr.y.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:42:45.856 INFO:teuthology.orchestra.run.smithi012.stderr:-------------------------------------------------------------------------------- 2023-10-09T09:42:45.856 INFO:teuthology.orchestra.run.smithi012.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '1881d6a6-6688-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.12', '--skip-admin-label'] 2023-10-09T09:42:45.894 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: 4.1.1 2023-10-09T09:42:46.006 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying podman|docker is present... 2023-10-09T09:42:46.041 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: 4.1.1 2023-10-09T09:42:46.149 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying lvm2 is present... 2023-10-09T09:42:46.149 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying time synchronization is in place... 2023-10-09T09:42:46.155 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T09:42:46.161 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2023-10-09T09:42:46.167 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: enabled 2023-10-09T09:42:46.173 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: active 2023-10-09T09:42:46.174 INFO:teuthology.orchestra.run.smithi012.stderr:Unit chronyd.service is enabled and running 2023-10-09T09:42:46.174 INFO:teuthology.orchestra.run.smithi012.stderr:Repeating the final host check... 2023-10-09T09:42:46.210 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: 4.1.1 2023-10-09T09:42:46.321 INFO:teuthology.orchestra.run.smithi012.stderr:podman (/bin/podman) version 4.1.1 is present 2023-10-09T09:42:46.321 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl is present 2023-10-09T09:42:46.321 INFO:teuthology.orchestra.run.smithi012.stderr:lvcreate is present 2023-10-09T09:42:46.326 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T09:42:46.332 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2023-10-09T09:42:46.338 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: enabled 2023-10-09T09:42:46.344 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: active 2023-10-09T09:42:46.345 INFO:teuthology.orchestra.run.smithi012.stderr:Unit chronyd.service is enabled and running 2023-10-09T09:42:46.345 INFO:teuthology.orchestra.run.smithi012.stderr:Host looks OK 2023-10-09T09:42:46.345 INFO:teuthology.orchestra.run.smithi012.stderr:Cluster fsid: 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:46.345 INFO:teuthology.orchestra.run.smithi012.stderr:Acquiring lock 140572345825768 on /run/cephadm/1881d6a6-6688-11ee-8db5-212e2dc638e7.lock 2023-10-09T09:42:46.345 INFO:teuthology.orchestra.run.smithi012.stderr:Lock 140572345825768 acquired on /run/cephadm/1881d6a6-6688-11ee-8db5-212e2dc638e7.lock 2023-10-09T09:42:46.345 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying IP 172.21.15.12 port 3300 ... 2023-10-09T09:42:46.346 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying IP 172.21.15.12 port 6789 ... 2023-10-09T09:42:46.346 INFO:teuthology.orchestra.run.smithi012.stderr:Base mon IP(s) is [172.21.15.12:3300, 172.21.15.12:6789], mon addrv is [v2:172.21.15.12:3300,v1:172.21.15.12:6789] 2023-10-09T09:42:46.349 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-10-09T09:42:46.349 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.12 metric 100 2023-10-09T09:42:46.352 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T09:42:46.352 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-10-09T09:42:46.352 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1798sec hoplimit 64 pref medium 2023-10-09T09:42:46.354 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T09:42:46.354 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: inet6 ::1/128 scope host 2023-10-09T09:42:46.355 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T09:42:46.355 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2023-10-09T09:42:46.355 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fef8:25ea/64 scope link 2023-10-09T09:42:46.355 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T09:42:46.355 INFO:teuthology.orchestra.run.smithi012.stderr:Mon IP `172.21.15.12` is in CIDR network `172.21.0.0/20` 2023-10-09T09:42:46.356 INFO:teuthology.orchestra.run.smithi012.stderr:Mon IP `172.21.15.12` is in CIDR network `172.21.0.0/20` 2023-10-09T09:42:46.356 INFO:teuthology.orchestra.run.smithi012.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T09:42:46.356 INFO:teuthology.orchestra.run.smithi012.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T09:42:46.357 INFO:teuthology.orchestra.run.smithi012.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T09:42:46.445 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Trying to pull quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T09:42:46.537 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Getting image source signatures 2023-10-09T09:42:46.537 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying blob sha256:24d54b00114a652388987a20e62df7ca52e6b59afd272176be9f22a3eeefd892 2023-10-09T09:42:46.537 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying blob sha256:be9ed931e0f930233626f1c4d225977fbcc33e056e1a8fc67c7f63553d9455a2 2023-10-09T09:42:46.537 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying config sha256:83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352 2023-10-09T09:42:46.586 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Writing manifest to image destination 2023-10-09T09:42:46.586 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Storing signatures 2023-10-09T09:42:46.731 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: 83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352 2023-10-09T09:42:47.360 INFO:teuthology.orchestra.run.smithi012.stderr:ceph: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T09:42:47.662 INFO:teuthology.orchestra.run.smithi012.stderr:Ceph version: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T09:42:47.662 INFO:teuthology.orchestra.run.smithi012.stderr:Extracting ceph user uid/gid from container image... 2023-10-09T09:42:48.045 INFO:teuthology.orchestra.run.smithi012.stderr:stat: 167 167 2023-10-09T09:42:48.314 INFO:teuthology.orchestra.run.smithi012.stderr:Creating initial keys... 2023-10-09T09:42:48.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-authtool: AQAYyyNlIQmrJhAAAGwhIBBa/a+QChKMKjAI1g== 2023-10-09T09:42:49.297 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-authtool: AQAZyyNlIof2EBAANcmcQbhpYn8PKEFy6TEdag== 2023-10-09T09:42:49.980 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-authtool: AQAZyyNlSj6DORAAq8f4WxC1VE/m3Nd2gSM/Tg== 2023-10-09T09:42:50.359 INFO:teuthology.orchestra.run.smithi012.stderr:Creating initial monmap... 2023-10-09T09:42:50.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T09:42:50.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:50.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T09:42:51.118 INFO:teuthology.orchestra.run.smithi012.stderr:monmaptool for a [v2:172.21.15.12:3300,v1:172.21.15.12:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T09:42:51.118 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: set fsid to 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:51.118 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T09:42:51.118 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:42:51.118 INFO:teuthology.orchestra.run.smithi012.stderr:Creating mon... 2023-10-09T09:42:51.654 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.601+0000 7f1e46650880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:42:51.654 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 1 imported monmap: 2023-10-09T09:42:51.654 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: epoch 0 2023-10-09T09:42:51.654 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:51.654 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: last_changed 2023-10-09T09:42:50.732807+0000 2023-10-09T09:42:51.654 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: created 2023-10-09T09:42:50.732807+0000 2023-10-09T09:42:51.654 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 0 /usr/bin/ceph-mon: set fsid to 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: DB SUMMARY 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-09T09:42:51.655 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.create_if_missing: 1 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.env: 0x558f2689d080 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.fs: Posix File System 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.info_log: 0x558f28317aa0 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.statistics: (nil) 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.use_fsync: 0 2023-10-09T09:42:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.db_log_dir: 2023-10-09T09:42:51.657 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:42:51.658 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.write_buffer_manager: 0x558f2831f890 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.unordered_write: 0 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:42:51.659 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.row_cache: None 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.wal_filter: None 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T09:42:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:42:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.max_open_files: -1 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Compression algorithms supported: 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: kZSTD supported: 0 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T09:42:51.662 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.602+0000 7f1e46650880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.663 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.merge_operator: 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_filter: None 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558f2822fd20) 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: index_type: 0 2023-10-09T09:42:51.664 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: checksum: 1 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: block_cache: 0x558f28264d10 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-10-09T09:42:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-10-09T09:42:51.666 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: format_version: 2 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: block_align: 0 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compression: NoCompression 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.num_levels: 7 2023-10-09T09:42:51.667 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:42:51.668 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:42:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T09:42:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T09:42:51.671 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.ttl: 2592000 2023-10-09T09:42:51.672 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.603+0000 7f1e46650880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.604+0000 7f1e46650880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.604+0000 7f1e46650880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.604+0000 7f1e46650880 4 rocksdb: DB pointer 0x558f2832d800 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.605+0000 7f1e2f4d8700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.605+0000 7f1e2f4d8700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:42:51.673 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T09:42:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.675 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:42:51.676 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.606+0000 7f1e46650880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-10-09T09:42:51.677 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.606+0000 7f1e46650880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-10-09T09:42:51.678 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-mon: debug 2023-10-09T09:42:51.606+0000 7f1e46650880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-09T09:42:51.974 INFO:teuthology.orchestra.run.smithi012.stderr:create mon.a on 2023-10-09T09:42:51.986 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T09:42:52.080 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-09T09:42:52.177 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7.target. 2023-10-09T09:42:52.177 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7.target. 2023-10-09T09:42:52.400 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Failed to reset failed state of unit ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T09:42:52.407 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7.target.wants/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@.service. 2023-10-09T09:42:52.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:52 smithi012 systemd[1]: Starting Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:42:52.998 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: disabled 2023-10-09T09:42:53.005 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2023-10-09T09:42:53.005 INFO:teuthology.orchestra.run.smithi012.stderr:firewalld.service is not enabled 2023-10-09T09:42:53.005 INFO:teuthology.orchestra.run.smithi012.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T09:42:53.006 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mon to start... 2023-10-09T09:42:53.006 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mon... 2023-10-09T09:42:53.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:52 smithi012 podman[96139]: 2023-10-09T09:42:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:52 smithi012 bash[96139]: e9301bab98cbe0587e16c9829faf842bdf9c50dff3ef47c609f9257a652c79a0 2023-10-09T09:42:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:52 smithi012 systemd[1]: Started Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:42:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: cluster 2023-10-09T09:42:53.020445+0000 mon.a (mon.0) 0 : [INF] mkfs 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: cluster 2023-10-09T09:42:52.991949+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T09:42:53.613 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: cluster: 2023-10-09T09:42:53.614 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: id: 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:53.614 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: services: 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.605487s) 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mgr: no daemons active 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: data: 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: pgs: 2023-10-09T09:42:53.615 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:42:53.955 INFO:teuthology.orchestra.run.smithi012.stderr:mon is available 2023-10-09T09:42:53.955 INFO:teuthology.orchestra.run.smithi012.stderr:Assimilating anything we can from ceph.conf... 2023-10-09T09:42:54.213 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: cluster 2023-10-09T09:42:53.007746+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T09:42:54.214 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: cluster 2023-10-09T09:42:53.007988+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0]} removed_ranks: {} 2023-10-09T09:42:54.214 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: cluster 2023-10-09T09:42:53.022528+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-09T09:42:54.214 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: cluster 2023-10-09T09:42:53.025303+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T09:42:54.214 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: cluster 2023-10-09T09:42:53.025769+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T09:42:54.214 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: audit 2023-10-09T09:42:53.613176+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.12:0/1880447184' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T09:42:54.571 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:42:54.571 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [global] 2023-10-09T09:42:54.571 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: fsid = 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:54.571 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.12:3300,v1:172.21.15.12:6789] 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [mgr] 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [osd] 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T09:42:54.572 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T09:42:54.898 INFO:teuthology.orchestra.run.smithi012.stderr:Generating new minimal ceph.conf... 2023-10-09T09:42:55.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: audit 2023-10-09T09:42:54.566469+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.12:0/861697505' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T09:42:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: audit 2023-10-09T09:42:54.569145+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.12:0/861697505' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T09:42:55.838 INFO:teuthology.orchestra.run.smithi012.stderr:Restarting the monitor... 2023-10-09T09:42:56.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:55 smithi012 systemd[1]: Stopping Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:42:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:55 smithi012 bash[96484]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon.a" found: no such container 2023-10-09T09:42:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: audit 2023-10-09T09:42:55.506976+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.12:0/1918369564' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:42:56.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: debug 2023-10-09T09:42:56.222+0000 7fb258a7e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:42:56.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96155]: debug 2023-10-09T09:42:56.222+0000 7fb258a7e700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T09:42:57.072 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:56 smithi012 bash[96525]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T09:42:57.401 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T09:42:57.401 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 systemd[1]: Stopped Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:42:57.401 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 systemd[1]: Starting Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:42:57.658 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 podman[96654]: 2023-10-09T09:42:57.658 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.582+0000 7f7862cdb880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.582+0000 7f7862cdb880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.582+0000 7f7862cdb880 0 pidfile_write: ignore empty --pid-file 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.586+0000 7f7862cdb880 0 load: jerasure load: lrc load: isa 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 bash[96654]: 3c03dd9903d913a9b84cc2c916730bf3755f3934f271da3639267bcab1a67524 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: DB SUMMARY 2023-10-09T09:42:57.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86671 ; 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.env: 0x556e03b20080 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.fs: Posix File System 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.info_log: 0x556e04ad3a80 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.statistics: (nil) 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.use_fsync: 0 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:42:57.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:42:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.db_log_dir: 2023-10-09T09:42:57.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T09:42:57.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:42:57.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T09:42:57.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T09:42:57.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:42:57.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.write_buffer_manager: 0x556e057ce330 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:42:57.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.unordered_write: 0 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.row_cache: None 2023-10-09T09:42:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.wal_filter: None 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T09:42:57.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T09:42:57.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T09:42:57.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T09:42:57.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:42:57.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:42:57.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:42:57.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:42:57.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:42:57.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:42:57.667 INFO:teuthology.orchestra.run.smithi012.stderr:Setting mon public_network to 172.21.0.0/20 2023-10-09T09:42:57.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:42:57.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:42:57.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_open_files: -1 2023-10-09T09:42:57.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:42:57.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:42:57.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:42:57.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Compression algorithms supported: 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: kZSTD supported: 0 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-10-09T09:42:57.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.merge_operator: 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_filter: None 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556e04a83d18) 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cache_index_and_filter_blocks: 1 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:42:57.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: pin_top_level_index_and_filter: 1 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: index_type: 0 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: data_block_index_type: 0 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: index_shortening: 1 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: hash_index_allow_collision: 1 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: checksum: 1 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: no_block_cache: 0 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: block_cache: 0x556e04ab8f10 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: block_cache_name: BinnedLRUCache 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: block_cache_options: 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: capacity : 536870912 2023-10-09T09:42:57.672 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: num_shard_bits : 4 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: strict_capacity_limit : 0 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: high_pri_pool_ratio: 0.000 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: block_cache_compressed: (nil) 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: persistent_cache: (nil) 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: block_size: 4096 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: block_size_deviation: 10 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: block_restart_interval: 16 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: index_block_restart_interval: 1 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: metadata_block_size: 4096 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: partition_filters: 0 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: use_delta_encoding: 1 2023-10-09T09:42:57.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: whole_key_filtering: 1 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: verify_compression: 0 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: read_amp_bytes_per_bit: 0 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: format_version: 2 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: enable_index_compression: 1 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: block_align: 0 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compression: NoCompression 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:42:57.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.num_levels: 7 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:42:57.675 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:42:57.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:42:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:42:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:42:57.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.ttl: 2592000 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.587+0000 7f7862cdb880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.588+0000 7f7862cdb880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.588+0000 7f7862cdb880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.588+0000 7f7862cdb880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844577588620, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.588+0000 7f7862cdb880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.588+0000 7f7862cdb880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.589+0000 7f7862cdb880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844577589508, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83271, "table_properties": {"data_size": 81544, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75865, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696844577, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.589+0000 7f7862cdb880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-10-09T09:42:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.589+0000 7f7862cdb880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844577589883, "job": 1, "event": "recovery_finished"} 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.590+0000 7f7862cdb880 4 rocksdb: DB pointer 0x556e04b81800 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.590+0000 7f7849ae2700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.590+0000 7f7849ae2700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ** DB Stats ** 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:42:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ** Compaction Stats [default] ** 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: L0 2/0 82.97 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Sum 2/0 82.97 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ** Compaction Stats [default] ** 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2023-10-09T09:42:57.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Cumulative compaction: 0.00 GB write, 29.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Interval compaction: 0.00 GB write, 29.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ** Compaction Stats [default] ** 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: L0 2/0 82.97 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Sum 2/0 82.97 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ** Compaction Stats [default] ** 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:42:57.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Cumulative compaction: 0.00 GB write, 29.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.590+0000 7f7862cdb880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] at bind addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.591+0000 7f7862cdb880 1 mon.a@-1(???) e1 preinit fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:42:57.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.592+0000 7f7862cdb880 0 mon.a@-1(???).mds e1 new map 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.592+0000 7f7862cdb880 0 mon.a@-1(???).mds e1 print_map 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: e1 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: legacy client fscid: -1 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: No filesystems configured 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.592+0000 7f7862cdb880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.592+0000 7f7862cdb880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.592+0000 7f7862cdb880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:42:57.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.592+0000 7f7862cdb880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:42:57.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:42:55.506976+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.12:0/1918369564' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:42:57.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:42:57.592+0000 7f7862cdb880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T09:42:57.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 systemd[1]: Started Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:42:57.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:42:57.607945+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T09:42:57.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:42:57.607987+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0]} removed_ranks: {} 2023-10-09T09:42:57.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:42:57.608368+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-09T09:42:57.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:42:57.608400+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T09:42:57.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:42:57.608769+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T09:42:58.608 INFO:teuthology.orchestra.run.smithi012.stderr:Wrote config to /etc/ceph/ceph.conf 2023-10-09T09:42:58.608 INFO:teuthology.orchestra.run.smithi012.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T09:42:58.608 INFO:teuthology.orchestra.run.smithi012.stderr:Creating mgr... 2023-10-09T09:42:58.609 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying port 9283 ... 2023-10-09T09:42:58.743 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Failed to reset failed state of unit ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T09:42:58.749 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7.target.wants/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@.service. 2023-10-09T09:42:59.116 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:42:58 smithi012 systemd[1]: Starting Ceph mgr.y for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:42:59.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:42:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:42:58.253465+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.12:0/2111086581' entity='client.admin' 2023-10-09T09:42:59.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:42:59 smithi012 podman[96929]: 2023-10-09T09:42:59.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:42:59 smithi012 bash[96929]: bcfbb2bc52d44753f2da549ae3f01ff6881adaef66cfa65093c0b3640f3c2389 2023-10-09T09:42:59.444 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: disabled 2023-10-09T09:42:59.451 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2023-10-09T09:42:59.452 INFO:teuthology.orchestra.run.smithi012.stderr:firewalld.service is not enabled 2023-10-09T09:42:59.452 INFO:teuthology.orchestra.run.smithi012.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T09:42:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: disabled 2023-10-09T09:42:59.467 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2023-10-09T09:42:59.468 INFO:teuthology.orchestra.run.smithi012.stderr:firewalld.service is not enabled 2023-10-09T09:42:59.468 INFO:teuthology.orchestra.run.smithi012.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T09:42:59.468 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mgr to start... 2023-10-09T09:42:59.468 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mgr... 2023-10-09T09:42:59.686 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:42:59 smithi012 systemd[1]: Started Ceph mgr.y for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:42:59.686 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:42:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:42:59.509+0000 7f27f7acd680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:42:59.686 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:42:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:42:59.607+0000 7f27f7acd680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:43:00.095 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:43:00.098 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsid": "1881d6a6-6688-11ee-8db5-212e2dc638e7", 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "health": { 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T09:43:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 0 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "a" 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T09:43:00.100 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T09:43:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T09:43:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T09:43:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T09:43:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T09:43:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T09:43:00.101 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T09:43:00.102 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:00.102 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T09:43:00.102 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T09:43:00.102 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T09:43:00.102 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T09:43:00.102 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T09:43:00.102 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T09:43:00.105 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": false, 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "iostat", 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "nfs", 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "restful" 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2023-10-09T09:43:00.106 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:00.107 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T09:43:00.107 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:00.107 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modified": "2023-10-09T09:42:53.020983+0000", 2023-10-09T09:43:00.107 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2023-10-09T09:43:00.107 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:00.107 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T09:43:00.107 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2023-10-09T09:43:00.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:00.095028+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.12:0/2744233553' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T09:43:00.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:00.070+0000 7f27f7acd680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:43:00.515 INFO:teuthology.orchestra.run.smithi012.stderr:mgr not available, waiting (1/15)... 2023-10-09T09:43:01.357 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:01.016+0000 7f27f7acd680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:43:01.357 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:01.111+0000 7f27f7acd680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:43:01.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:01.357+0000 7f27f7acd680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:43:02.751 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:02.491+0000 7f27f7acd680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:43:02.751 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:02.660+0000 7f27f7acd680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:43:03.093 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:02.759+0000 7f27f7acd680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:43:03.093 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:02.871+0000 7f27f7acd680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:43:03.093 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:03.092+0000 7f27f7acd680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:43:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:43:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2023-10-09T09:43:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsid": "1881d6a6-6688-11ee-8db5-212e2dc638e7", 2023-10-09T09:43:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "health": { 2023-10-09T09:43:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T09:43:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T09:43:03.137 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T09:43:03.138 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:03.139 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T09:43:03.139 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 0 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "a" 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_age": 5, 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:03.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T09:43:03.141 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:03.141 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T09:43:03.141 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T09:43:03.141 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T09:43:03.144 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T09:43:03.144 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T09:43:03.144 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T09:43:03.144 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:03.144 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T09:43:03.144 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T09:43:03.144 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:03.145 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": false, 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "iostat", 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "nfs", 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "restful" 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:03.146 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modified": "2023-10-09T09:42:53.020983+0000", 2023-10-09T09:43:03.147 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2023-10-09T09:43:03.147 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:03.147 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T09:43:03.147 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2023-10-09T09:43:03.562 INFO:teuthology.orchestra.run.smithi012.stderr:mgr not available, waiting (2/15)... 2023-10-09T09:43:03.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:03.136092+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.12:0/123071459' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T09:43:03.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:03.471+0000 7f27f7acd680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:43:03.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:03.589+0000 7f27f7acd680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:43:04.886 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:04.471+0000 7f27f7acd680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:43:04.886 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:04.568+0000 7f27f7acd680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:43:04.887 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:04.678+0000 7f27f7acd680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:43:05.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:04.886+0000 7f27f7acd680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:43:05.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:05.171+0000 7f27f7acd680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:43:05.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:05.330+0000 7f27f7acd680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:43:05.767 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:05.497+0000 7f27f7acd680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:43:05.768 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:05.715+0000 7f27f7acd680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:43:06.178 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:43:06.178 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2023-10-09T09:43:06.178 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsid": "1881d6a6-6688-11ee-8db5-212e2dc638e7", 2023-10-09T09:43:06.178 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "health": { 2023-10-09T09:43:06.178 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 0 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T09:43:06.179 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "a" 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_age": 8, 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:06.180 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T09:43:06.181 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": false, 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T09:43:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "iostat", 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "nfs", 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "restful" 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modified": "2023-10-09T09:42:53.020983+0000", 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:06.183 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T09:43:06.184 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2023-10-09T09:43:06.184 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:05.818+0000 7f27f7acd680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:43:06.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:05.832490+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-10-09T09:43:06.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:05.834975+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00256992s) 2023-10-09T09:43:06.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.835996+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:43:06.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.836236+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:43:06.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.836429+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:43:06.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.836580+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:43:06.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.836960+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:43:06.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:05.851063+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-10-09T09:43:06.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.862152+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' 2023-10-09T09:43:06.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.866059+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:43:06.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.868898+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:43:06.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.874419+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' 2023-10-09T09:43:06.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:05.880553+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.12:0/4115350569' entity='mgr.y' 2023-10-09T09:43:06.708 INFO:teuthology.orchestra.run.smithi012.stderr:mgr not available, waiting (3/15)... 2023-10-09T09:43:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:06.171880+0000 mon.a (mon.0) 22 : audit [DBG] from='client.? 172.21.15.12:0/1138859409' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T09:43:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:06.838102+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e3: y(active, since 1.00569s) 2023-10-09T09:43:09.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:08.599661+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-10-09T09:43:10.965 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:43:10.965 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2023-10-09T09:43:10.965 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsid": "1881d6a6-6688-11ee-8db5-212e2dc638e7", 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "health": { 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 0 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T09:43:10.966 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "a" 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_age": 13, 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T09:43:10.967 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T09:43:10.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T09:43:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": true, 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "iostat", 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "nfs", 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "restful" 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modified": "2023-10-09T09:42:53.020983+0000", 2023-10-09T09:43:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2023-10-09T09:43:10.974 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2023-10-09T09:43:10.974 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T09:43:10.974 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2023-10-09T09:43:11.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:10.963478+0000 mon.a (mon.0) 25 : audit [DBG] from='client.? 172.21.15.12:0/3306766583' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T09:43:13.971 INFO:teuthology.orchestra.run.smithi012.stderr:mgr is available 2023-10-09T09:43:16.283 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:43:16.283 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [global] 2023-10-09T09:43:16.284 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: fsid = 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:43:16.284 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T09:43:16.284 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T09:43:16.284 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T09:43:16.284 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T09:43:16.285 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T09:43:16.285 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:43:16.285 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [mgr] 2023-10-09T09:43:16.285 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T09:43:16.285 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2023-10-09T09:43:16.285 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [osd] 2023-10-09T09:43:16.285 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T09:43:16.285 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T09:43:16.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:16.281815+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.12:0/3380810365' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T09:43:17.327 INFO:teuthology.orchestra.run.smithi012.stderr:Enabling cephadm module... 2023-10-09T09:43:18.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:18.277010+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.12:0/935565083' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T09:43:18.893 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ignoring --setuser ceph since I am not root 2023-10-09T09:43:18.893 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ignoring --setgroup ceph since I am not root 2023-10-09T09:43:18.893 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:18.795+0000 7f8a7037d680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:43:19.164 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:18.892+0000 7f8a7037d680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:43:19.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:19.354+0000 7f8a7037d680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:43:19.896 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2023-10-09T09:43:19.897 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 5, 2023-10-09T09:43:19.897 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": true, 2023-10-09T09:43:19.897 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T09:43:19.897 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T09:43:19.897 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2023-10-09T09:43:19.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:18.602728+0000 mon.a (mon.0) 28 : audit [INF] from='client.? 172.21.15.12:0/935565083' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T09:43:19.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:18.602803+0000 mon.a (mon.0) 29 : cluster [DBG] mgrmap e5: y(active, since 12s) 2023-10-09T09:43:20.219 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for the mgr to restart... 2023-10-09T09:43:20.219 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mgr epoch 5... 2023-10-09T09:43:20.601 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:20.310+0000 7f8a7037d680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:43:20.601 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:20.405+0000 7f8a7037d680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:43:20.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:19.895853+0000 mon.a (mon.0) 30 : audit [DBG] from='client.? 172.21.15.12:0/1452482869' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T09:43:20.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:20.660+0000 7f8a7037d680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:43:22.043 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:21.789+0000 7f8a7037d680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:43:22.043 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:21.949+0000 7f8a7037d680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:43:22.369 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:22.043+0000 7f8a7037d680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:43:22.369 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:22.152+0000 7f8a7037d680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:43:22.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:22.369+0000 7f8a7037d680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:43:23.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:22.732+0000 7f8a7037d680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:43:23.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:22.848+0000 7f8a7037d680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:43:24.130 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:23.715+0000 7f8a7037d680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:43:24.130 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:23.812+0000 7f8a7037d680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:43:24.130 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:23.922+0000 7f8a7037d680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:43:24.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:24.130+0000 7f8a7037d680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:43:24.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:24.415+0000 7f8a7037d680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:43:24.677 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:24.575+0000 7f8a7037d680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:43:25.054 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:24.742+0000 7f8a7037d680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:43:25.054 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:24.955+0000 7f8a7037d680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:43:25.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:25.054+0000 7f8a7037d680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:25.068139+0000 mon.a (mon.0) 31 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:25.068376+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:25.071130+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:25.071246+0000 mon.a (mon.0) 34 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00296166s) 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.072455+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.073069+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.073764+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.073925+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:43:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.074094+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:43:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:25.088453+0000 mon.a (mon.0) 40 : cluster [INF] Manager daemon y is now available 2023-10-09T09:43:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.102856+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:26.085 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2023-10-09T09:43:26.085 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-10-09T09:43:26.085 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T09:43:26.085 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2023-10-09T09:43:26.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:43:25.100260+0000 mgr.y (mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-09T09:43:26.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.222592+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:26.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.223769+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:26.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.225244+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:26.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.227351+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:26.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.241658+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:43:26.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:25.244648+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:43:26.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:26.074375+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e7: y(active, since 1.00608s) 2023-10-09T09:43:26.505 INFO:teuthology.orchestra.run.smithi012.stderr:mgr epoch 5 is available 2023-10-09T09:43:26.506 INFO:teuthology.orchestra.run.smithi012.stderr:Setting orchestrator backend to cephadm... 2023-10-09T09:43:27.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:26.073486+0000 mgr.y (mgr.14118) 2 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T09:43:27.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:26.084897+0000 mgr.y (mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T09:43:28.540 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: value unchanged 2023-10-09T09:43:28.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:27.224722+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-10-09T09:43:28.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:27.432153+0000 mgr.y (mgr.14118) 4 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:28.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:27.436387+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:28.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:27.450272+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:28.895 INFO:teuthology.orchestra.run.smithi012.stderr:Generating ssh key... 2023-10-09T09:43:29.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:28.539943+0000 mgr.y (mgr.14118) 5 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:30.332 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: Generating public/private rsa key pair. 2023-10-09T09:43:30.332 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: Your identification has been saved in /tmp/tmp6o5_aods/key. 2023-10-09T09:43:30.332 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: Your public key has been saved in /tmp/tmp6o5_aods/key.pub. 2023-10-09T09:43:30.333 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: The key fingerprint is: 2023-10-09T09:43:30.333 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: SHA256:lZhoP0iuxtgdzvpyZVT47GpwPm7ObeotDFtRF4JR1Hk ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:43:30.333 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: The key's randomart image is: 2023-10-09T09:43:30.333 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: +---[RSA 3072]----+ 2023-10-09T09:43:30.334 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | .*+..o | 2023-10-09T09:43:30.334 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | .oooo+ E | 2023-10-09T09:43:30.334 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | + o=o. . | 2023-10-09T09:43:30.334 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | + oo.o | 2023-10-09T09:43:30.334 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | +.So | 2023-10-09T09:43:30.334 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | + =o.=.. | 2023-10-09T09:43:30.334 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | . = +@ . | 2023-10-09T09:43:30.335 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | ...o.Oo. | 2023-10-09T09:43:30.335 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: | .+.**=o | 2023-10-09T09:43:30.335 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: +----[SHA256]-----+ 2023-10-09T09:43:31.140 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgEBNve2LIr2/QSSNYtSlbQAFA5iQJt3eH6F8OD9zH/3r5s69xisP87Mup+0dwYZoGES9lM97dAtmZ69KfvsyWaDBn4apPdsAQ4uvQYBvIMoNhf6TwwrdM1kHv2xWTcKd4YqtAzs6Gmnr4w1rebtJihyvDUn61DaP85ye1TuvD4hMvqoW0l3+1U9cr1EuNo8P3KuP7ygR/nS00bFA6NWSquYIdjiiTWUEGzO2UVwnnCZA4gzrMOJGp1rKb71b6CxpZrPuBxDQ4jZ2FWw6UvkKD2SWzwae0d6EfpB+6+zCJSRV7tOviIoyFXRDMmwDSudu9+D9zrubmvDViOLACmSoRLb21h2rKY9okjADkXm119I0zrO7oSiFS53b1EKfMan3EC9MbAnJ8OkOi4HmJdGyWXtpKlG4OtTkYOUqvxq0KjzQt2Gdbeeq7XpEwUhvVujAsqH9ol6xHBKtjRY/SJbhBaEoOiNtxwjJNlircXaMf9Cy5R1I8gsIGIjzUNj+3MAs= ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:43:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:29.694561+0000 mgr.y (mgr.14118) 6 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:43:29.694800+0000 mgr.y (mgr.14118) 7 : cephadm [INF] Generating ssh key... 2023-10-09T09:43:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:29.953192+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:29.955688+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:31.479 INFO:teuthology.orchestra.run.smithi012.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T09:43:31.479 INFO:teuthology.orchestra.run.smithi012.stderr:Adding key to root@localhost authorized_keys... 2023-10-09T09:43:31.480 INFO:teuthology.orchestra.run.smithi012.stderr:Adding host smithi012... 2023-10-09T09:43:32.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:31.140270+0000 mgr.y (mgr.14118) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:32.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [79B blob data] 2023-10-09T09:43:33.061 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Added host 'smithi012' with addr '172.21.15.12' 2023-10-09T09:43:33.423 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying unmanaged mon service... 2023-10-09T09:43:33.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:32.287593+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi012", "addr": "172.21.15.12", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:33.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:33.060223+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:33.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:33.076466+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:34.290 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled mon update... 2023-10-09T09:43:34.865 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying unmanaged mgr service... 2023-10-09T09:43:34.904 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:43:33.060545+0000 mgr.y (mgr.14118) 10 : cephadm [INF] Added host smithi012 2023-10-09T09:43:34.904 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:34.289907+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:35.920 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-10-09T09:43:35.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:34.286037+0000 mgr.y (mgr.14118) 11 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:35.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:43:34.287491+0000 mgr.y (mgr.14118) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-09T09:43:35.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:35.140995+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:35.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:35.512832+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:37.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:35.916892+0000 mgr.y (mgr.14118) 13 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:43:35.917961+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-09T09:43:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:35.920179+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:38.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:37.081864+0000 mon.a (mon.0) 60 : audit [INF] from='client.? 172.21.15.12:0/3220951515' entity='client.admin' 2023-10-09T09:43:38.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:37.155218+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14118 172.21.15.12:0/3391058534' entity='mgr.y' 2023-10-09T09:43:38.980 INFO:teuthology.orchestra.run.smithi012.stderr:Enabling the dashboard module... 2023-10-09T09:43:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:38.261961+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.12:0/3623658862' entity='client.admin' 2023-10-09T09:43:40.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:39.859490+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.12:0/366332051' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T09:43:40.554 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ignoring --setuser ceph since I am not root 2023-10-09T09:43:40.554 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ignoring --setgroup ceph since I am not root 2023-10-09T09:43:40.554 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:40.454+0000 7f66495c7680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:43:40.554 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:40.552+0000 7f66495c7680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:43:41.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:41.016+0000 7f66495c7680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:43:41.556 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2023-10-09T09:43:41.556 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 9, 2023-10-09T09:43:41.556 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": true, 2023-10-09T09:43:41.556 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T09:43:41.556 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T09:43:41.556 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2023-10-09T09:43:41.579 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:40.268917+0000 mon.a (mon.0) 64 : audit [INF] from='client.? 172.21.15.12:0/366332051' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T09:43:41.579 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:40.269000+0000 mon.a (mon.0) 65 : cluster [DBG] mgrmap e9: y(active, since 15s) 2023-10-09T09:43:41.870 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for the mgr to restart... 2023-10-09T09:43:41.871 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mgr epoch 9... 2023-10-09T09:43:42.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:41.960+0000 7f66495c7680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:43:42.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:42.056+0000 7f66495c7680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:43:42.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:41.555549+0000 mon.a (mon.0) 66 : audit [DBG] from='client.? 172.21.15.12:0/3814145971' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T09:43:42.677 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:42.315+0000 7f66495c7680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:43:43.706 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:43.448+0000 7f66495c7680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:43:43.706 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:43.610+0000 7f66495c7680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:43:43.706 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:43.705+0000 7f66495c7680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:43:44.033 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:43.814+0000 7f66495c7680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:43:44.396 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:44.032+0000 7f66495c7680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:43:44.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:44.395+0000 7f66495c7680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:43:44.677 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:44.512+0000 7f66495c7680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:43:45.590 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:45.380+0000 7f66495c7680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:43:45.590 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:45.479+0000 7f66495c7680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:43:45.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:45.589+0000 7f66495c7680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:43:45.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:45.799+0000 7f66495c7680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:43:46.412 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:46.083+0000 7f66495c7680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:43:46.412 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:46.243+0000 7f66495c7680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:43:46.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:46.411+0000 7f66495c7680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:43:46.677 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:46.625+0000 7f66495c7680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:43:46.992 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:43:46.726+0000 7f66495c7680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:43:46.992 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [79B blob data] 2023-10-09T09:43:46.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:46.740207+0000 mon.a (mon.0) 67 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:43:46.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:46.740445+0000 mon.a (mon.0) 68 : cluster [INF] Activating manager daemon y 2023-10-09T09:43:46.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:46.745973+0000 mon.a (mon.0) 69 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-09T09:43:46.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:46.746077+0000 mon.a (mon.0) 70 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00572608s) 2023-10-09T09:43:46.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.747257+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:43:46.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.747886+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:43:46.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.748824+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:43:46.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.749045+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:43:46.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.749209+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:43:46.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:46.751560+0000 mon.a (mon.0) 76 : cluster [INF] Manager daemon y is now available 2023-10-09T09:43:47.759 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2023-10-09T09:43:47.759 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-10-09T09:43:47.759 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T09:43:47.759 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2023-10-09T09:43:48.041 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.880549+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:48.041 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.882045+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:43:48.041 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.908977+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:43:48.041 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:46.912195+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:43:48.041 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:47.747997+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e11: y(active, since 1.00764s) 2023-10-09T09:43:48.077 INFO:teuthology.orchestra.run.smithi012.stderr:mgr epoch 9 is available 2023-10-09T09:43:48.078 INFO:teuthology.orchestra.run.smithi012.stderr:Generating a dashboard self-signed certificate... 2023-10-09T09:43:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:47.747347+0000 mgr.y (mgr.14148) 1 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T09:43:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:47.757740+0000 mgr.y (mgr.14148) 2 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T09:43:49.947 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Self-signed certificate created 2023-10-09T09:43:50.084 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:48.801389+0000 mon.a (mon.0) 82 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-10-09T09:43:50.084 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:49.452214+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:43:50.351 INFO:teuthology.orchestra.run.smithi012.stderr:Creating initial admin user... 2023-10-09T09:43:51.006 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:49.879835+0000 mgr.y (mgr.14148) 3 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:51.006 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:49.941732+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:43:51.006 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:49.946858+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:43:51.829 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$bzuRvbSQx.hvvng24/9yNOef.5i4y8m5TgUo8uHM37XuvTj73WsgG", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696844631, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T09:43:52.197 INFO:teuthology.orchestra.run.smithi012.stderr:Fetching dashboard port number... 2023-10-09T09:43:53.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 8443 2023-10-09T09:43:53.158 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:51.583619+0000 mgr.y (mgr.14148) 4 : audit [DBG] from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:43:53.158 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:51.828761+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:43:53.447 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: disabled 2023-10-09T09:43:53.454 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2023-10-09T09:43:53.454 INFO:teuthology.orchestra.run.smithi012.stderr:firewalld.service is not enabled 2023-10-09T09:43:53.454 INFO:teuthology.orchestra.run.smithi012.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T09:43:53.457 INFO:teuthology.orchestra.run.smithi012.stderr:Ceph Dashboard is now available at: 2023-10-09T09:43:53.457 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:53.457 INFO:teuthology.orchestra.run.smithi012.stderr: URL: https://smithi012:8443/ 2023-10-09T09:43:53.457 INFO:teuthology.orchestra.run.smithi012.stderr: User: admin 2023-10-09T09:43:53.457 INFO:teuthology.orchestra.run.smithi012.stderr: Password: 18omcp7jr7 2023-10-09T09:43:53.457 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:53.457 INFO:teuthology.orchestra.run.smithi012.stderr:Enabling autotune for osd_memory_target 2023-10-09T09:43:53.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:53.018485+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.12:0/3658834923' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T09:43:53.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:43:53.605751+0000 mon.a (mon.0) 88 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-09T09:43:55.578 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-10-09T09:43:55.939 INFO:teuthology.orchestra.run.smithi012.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T09:43:55.939 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr:Or, if you are only running a single cluster on this host: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr:Please consider enabling telemetry to help improve Ceph: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: ceph telemetry on 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr:For more information see: 2023-10-09T09:43:55.940 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:55.941 INFO:teuthology.orchestra.run.smithi012.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-10-09T09:43:55.941 INFO:teuthology.orchestra.run.smithi012.stderr: 2023-10-09T09:43:55.941 INFO:teuthology.orchestra.run.smithi012.stderr:Bootstrap complete. 2023-10-09T09:43:55.975 INFO:tasks.cephadm:Fetching config... 2023-10-09T09:43:55.975 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:43:55.975 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T09:43:55.992 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T09:43:55.992 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:43:55.992 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T09:43:56.050 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T09:43:56.050 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:43:56.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T09:43:56.119 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T09:43:56.120 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:43:56.120 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T09:43:56.177 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T09:43:56.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgEBNve2LIr2/QSSNYtSlbQAFA5iQJt3eH6F8OD9zH/3r5s69xisP87Mup+0dwYZoGES9lM97dAtmZ69KfvsyWaDBn4apPdsAQ4uvQYBvIMoNhf6TwwrdM1kHv2xWTcKd4YqtAzs6Gmnr4w1rebtJihyvDUn61DaP85ye1TuvD4hMvqoW0l3+1U9cr1EuNo8P3KuP7ygR/nS00bFA6NWSquYIdjiiTWUEGzO2UVwnnCZA4gzrMOJGp1rKb71b6CxpZrPuBxDQ4jZ2FWw6UvkKD2SWzwae0d6EfpB+6+zCJSRV7tOviIoyFXRDMmwDSudu9+D9zrubmvDViOLACmSoRLb21h2rKY9okjADkXm119I0zrO7oSiFS53b1EKfMan3EC9MbAnJ8OkOi4HmJdGyWXtpKlG4OtTkYOUqvxq0KjzQt2Gdbeeq7XpEwUhvVujAsqH9ol6xHBKtjRY/SJbhBaEoOiNtxwjJNlircXaMf9Cy5R1I8gsIGIjzUNj+3MAs= ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T09:43:56.258 INFO:teuthology.orchestra.run.smithi012.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgEBNve2LIr2/QSSNYtSlbQAFA5iQJt3eH6F8OD9zH/3r5s69xisP87Mup+0dwYZoGES9lM97dAtmZ69KfvsyWaDBn4apPdsAQ4uvQYBvIMoNhf6TwwrdM1kHv2xWTcKd4YqtAzs6Gmnr4w1rebtJihyvDUn61DaP85ye1TuvD4hMvqoW0l3+1U9cr1EuNo8P3KuP7ygR/nS00bFA6NWSquYIdjiiTWUEGzO2UVwnnCZA4gzrMOJGp1rKb71b6CxpZrPuBxDQ4jZ2FWw6UvkKD2SWzwae0d6EfpB+6+zCJSRV7tOviIoyFXRDMmwDSudu9+D9zrubmvDViOLACmSoRLb21h2rKY9okjADkXm119I0zrO7oSiFS53b1EKfMan3EC9MbAnJ8OkOi4HmJdGyWXtpKlG4OtTkYOUqvxq0KjzQt2Gdbeeq7XpEwUhvVujAsqH9ol6xHBKtjRY/SJbhBaEoOiNtxwjJNlircXaMf9Cy5R1I8gsIGIjzUNj+3MAs= ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:43:56.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgEBNve2LIr2/QSSNYtSlbQAFA5iQJt3eH6F8OD9zH/3r5s69xisP87Mup+0dwYZoGES9lM97dAtmZ69KfvsyWaDBn4apPdsAQ4uvQYBvIMoNhf6TwwrdM1kHv2xWTcKd4YqtAzs6Gmnr4w1rebtJihyvDUn61DaP85ye1TuvD4hMvqoW0l3+1U9cr1EuNo8P3KuP7ygR/nS00bFA6NWSquYIdjiiTWUEGzO2UVwnnCZA4gzrMOJGp1rKb71b6CxpZrPuBxDQ4jZ2FWw6UvkKD2SWzwae0d6EfpB+6+zCJSRV7tOviIoyFXRDMmwDSudu9+D9zrubmvDViOLACmSoRLb21h2rKY9okjADkXm119I0zrO7oSiFS53b1EKfMan3EC9MbAnJ8OkOi4HmJdGyWXtpKlG4OtTkYOUqvxq0KjzQt2Gdbeeq7XpEwUhvVujAsqH9ol6xHBKtjRY/SJbhBaEoOiNtxwjJNlircXaMf9Cy5R1I8gsIGIjzUNj+3MAs= ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T09:43:56.329 INFO:teuthology.orchestra.run.smithi110.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgEBNve2LIr2/QSSNYtSlbQAFA5iQJt3eH6F8OD9zH/3r5s69xisP87Mup+0dwYZoGES9lM97dAtmZ69KfvsyWaDBn4apPdsAQ4uvQYBvIMoNhf6TwwrdM1kHv2xWTcKd4YqtAzs6Gmnr4w1rebtJihyvDUn61DaP85ye1TuvD4hMvqoW0l3+1U9cr1EuNo8P3KuP7ygR/nS00bFA6NWSquYIdjiiTWUEGzO2UVwnnCZA4gzrMOJGp1rKb71b6CxpZrPuBxDQ4jZ2FWw6UvkKD2SWzwae0d6EfpB+6+zCJSRV7tOviIoyFXRDMmwDSudu9+D9zrubmvDViOLACmSoRLb21h2rKY9okjADkXm119I0zrO7oSiFS53b1EKfMan3EC9MbAnJ8OkOi4HmJdGyWXtpKlG4OtTkYOUqvxq0KjzQt2Gdbeeq7XpEwUhvVujAsqH9ol6xHBKtjRY/SJbhBaEoOiNtxwjJNlircXaMf9Cy5R1I8gsIGIjzUNj+3MAs= ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:43:56.341 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T09:43:56.908 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:55.575007+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.12:0/158747874' entity='client.admin' 2023-10-09T09:43:58.703 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T09:43:58.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T09:43:59.323 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:43:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:58.260483+0000 mon.a (mon.0) 90 : audit [INF] from='client.? 172.21.15.12:0/2755844006' entity='client.admin' 2023-10-09T09:44:00.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:59.476737+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:00.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:59.866772+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi012", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:44:00.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:59.870990+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:00.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:59.872918+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:00.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:43:59.877767+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:00.933 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi110 2023-10-09T09:44:00.933 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:44:00.933 DEBUG:teuthology.orchestra.run.smithi110:> dd of=/etc/ceph/ceph.conf 2023-10-09T09:44:00.954 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:44:00.954 DEBUG:teuthology.orchestra.run.smithi110:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:44:01.013 INFO:tasks.cephadm:Adding host smithi110 to orchestrator... 2023-10-09T09:44:01.013 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi110 2023-10-09T09:44:01.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:00.492969+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:01.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:00.543555+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:01.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:00.545091+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:01.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:00.545815+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:01.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:00.547061+0000 mgr.y (mgr.14148) 6 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:44:01.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:00.565233+0000 mgr.y (mgr.14148) 7 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:44:01.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:00.609545+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:01.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:00.611851+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:03.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:44:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [90B blob data] 2023-10-09T09:44:03.476 INFO:teuthology.orchestra.run.smithi012.stdout:Added host 'smithi110' with addr '172.21.15.110' 2023-10-09T09:44:03.865 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:44:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [80B blob data] 2023-10-09T09:44:03.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:02.669139+0000 mgr.y (mgr.14148) 8 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi110", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:44:03.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:03.475038+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:03.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:03.487827+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:03.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T09:44:05.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:03.475415+0000 mgr.y (mgr.14148) 9 : cephadm [INF] Added host smithi110 2023-10-09T09:44:05.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:04.041246+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:05.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:04.649770+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:05.504 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:44:05.504 INFO:teuthology.orchestra.run.smithi012.stdout:[{"addr": "172.21.15.12", "hostname": "smithi012", "labels": [], "status": ""}, {"addr": "172.21.15.110", "hostname": "smithi110", "labels": [], "status": ""}] 2023-10-09T09:44:05.914 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T09:44:05.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T09:44:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:05.504250+0000 mgr.y (mgr.14148) 10 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:44:08.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:06.750158+0000 mgr.y (mgr.14148) 11 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:08.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:07.534700+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:08.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:07.551238+0000 mon.a (mon.0) 107 : audit [INF] from='client.? 172.21.15.12:0/1802443796' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T09:44:08.539 INFO:teuthology.orchestra.run.smithi012.stderr:adjusted tunables profile to default 2023-10-09T09:44:08.979 INFO:tasks.cephadm:Adding mon.a on smithi012 2023-10-09T09:44:08.979 INFO:tasks.cephadm:Adding mon.c on smithi012 2023-10-09T09:44:08.980 INFO:tasks.cephadm:Adding mon.b on smithi110 2023-10-09T09:44:08.980 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi110:172.21.15.110=b' 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:07.983107+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:07.983875+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi110", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:07.984887+0000 mon.a (mon.0) 110 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:07.985723+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:07.987071+0000 mgr.y (mgr.14148) 12 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:08.027130+0000 mgr.y (mgr.14148) 13 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:08.054485+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:08.056985+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:08.535681+0000 mon.a (mon.0) 114 : audit [INF] from='client.? 172.21.15.12:0/1802443796' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T09:44:09.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:08.535762+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T09:44:10.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:08.750407+0000 mgr.y (mgr.14148) 14 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:10.570 INFO:teuthology.orchestra.run.smithi110.stdout:Scheduled mon update... 2023-10-09T09:44:10.961 DEBUG:teuthology.orchestra.run.smithi012:mon.c> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T09:44:10.966 DEBUG:teuthology.orchestra.run.smithi110:mon.b> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T09:44:10.968 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T09:44:10.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T09:44:10.997 INFO:journalctl@ceph.mon.b.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:44:11.004 INFO:journalctl@ceph.mon.c.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:44:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:10.564627+0000 mgr.y (mgr.14148) 15 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi110:172.21.15.110=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:44:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:10.567411+0000 mgr.y (mgr.14148) 16 : cephadm [INF] Saving service mon spec with placement smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi110:172.21.15.110=b;count:3 2023-10-09T09:44:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:10.570183+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:10.578200+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:10.580122+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:10.581149+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:10.587545+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:10.589778+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:10.590829+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:10.591719+0000 mgr.y (mgr.14148) 17 : cephadm [INF] Deploying daemon mon.b on smithi110 2023-10-09T09:44:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:10.750708+0000 mgr.y (mgr.14148) 18 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:12.847 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:44:12.848 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","modified":"2023-10-09T09:42:50.732807Z","created":"2023-10-09T09:42:50.732807Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T09:44:12.849 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2023-10-09T09:44:13.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:12.751067+0000 mgr.y (mgr.14148) 19 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:12.847577+0000 mon.a (mon.0) 123 : audit [DBG] from='client.? 172.21.15.110:0/1133432248' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:44:14.021 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:13 smithi110 systemd[1]: Starting Ceph mon.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:44:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 podman[97061]: 2023-10-09T09:44:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 bash[97061]: 99ff116de7cfd554ea2e7be41767b652c4e29ce51147da119cfbd0ab98d3043a 2023-10-09T09:44:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.197+0000 7fd1aaeef880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:44:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.197+0000 7fd1aaeef880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T09:44:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.197+0000 7fd1aaeef880 0 pidfile_write: ignore empty --pid-file 2023-10-09T09:44:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.201+0000 7fd1aaeef880 0 load: jerasure load: lrc load: isa 2023-10-09T09:44:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: DB SUMMARY 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 511 ; 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.env: 0x55e152799080 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.fs: Posix File System 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.info_log: 0x55e15404ba80 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.statistics: (nil) 2023-10-09T09:44:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.use_fsync: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:44:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.db_log_dir: 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:44:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.write_buffer_manager: 0x55e154d46330 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.unordered_write: 0 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:44:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.row_cache: None 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.wal_filter: None 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:44:14.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:44:14.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_open_files: -1 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Compression algorithms supported: 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: kZSTD supported: 0 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:44:14.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:44:14.453 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.merge_operator: 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_filter: None 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e153ffbd18) 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cache_index_and_filter_blocks: 1 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: pin_top_level_index_and_filter: 1 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: index_type: 0 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: data_block_index_type: 0 2023-10-09T09:44:14.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: index_shortening: 1 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: hash_index_allow_collision: 1 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: checksum: 1 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: no_block_cache: 0 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: block_cache: 0x55e154030f10 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: block_cache_name: BinnedLRUCache 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: block_cache_options: 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: capacity : 536870912 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: num_shard_bits : 4 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: strict_capacity_limit : 0 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: high_pri_pool_ratio: 0.000 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: block_cache_compressed: (nil) 2023-10-09T09:44:14.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: persistent_cache: (nil) 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: block_size: 4096 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: block_size_deviation: 10 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: block_restart_interval: 16 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: index_block_restart_interval: 1 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: metadata_block_size: 4096 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: partition_filters: 0 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: use_delta_encoding: 1 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: whole_key_filtering: 1 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: verify_compression: 0 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: read_amp_bytes_per_bit: 0 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: format_version: 2 2023-10-09T09:44:14.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: enable_index_compression: 1 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: block_align: 0 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compression: NoCompression 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.num_levels: 7 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:44:14.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:44:14.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:44:14.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:44:14.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:44:14.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:44:14.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:44:14.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:44:14.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:44:14.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:44:14.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:44:14.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:44:14.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:44:14.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.ttl: 2592000 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.202+0000 7fd1aaeef880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.203+0000 7fd1aaeef880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.203+0000 7fd1aaeef880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.203+0000 7fd1aaeef880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844654204801, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.203+0000 7fd1aaeef880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.203+0000 7fd1aaeef880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.203+0000 7fd1aaeef880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844654205242, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696844654, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T09:44:14.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.203+0000 7fd1aaeef880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.204+0000 7fd1aaeef880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844654205640, "job": 1, "event": "recovery_finished"} 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.204+0000 7fd1aaeef880 4 rocksdb: DB pointer 0x55e1540f9800 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.204+0000 7fd191cf6700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.204+0000 7fd191cf6700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ** DB Stats ** 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:44:14.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ** Compaction Stats [default] ** 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ** Compaction Stats [default] ** 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:44:14.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Interval compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ** Compaction Stats [default] ** 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ** Compaction Stats [default] ** 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:44:14.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.4 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:44:14.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.204+0000 7fd1aaeef880 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.205+0000 7fd1aaeef880 0 using public_addr v2:172.21.15.110:0/0 -> [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.205+0000 7fd1aaeef880 0 starting mon.b rank -1 at public addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] at bind addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.206+0000 7fd1aaeef880 1 mon.b@-1(???) e0 preinit fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 systemd[1]: Started Ceph mon.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 0 mon.b@-1(synchronizing).mds e1 new map 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: e1 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: legacy client fscid: -1 2023-10-09T09:44:14.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: No filesystems configured 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.223+0000 7fd1944fb700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:44:14.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:12.751067+0000 mgr.y (mgr.14148) 19 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:14.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:12.847577+0000 mon.a (mon.0) 123 : audit [DBG] from='client.? 172.21.15.110:0/1133432248' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:44:14.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:44:14.224+0000 7fd1944fb700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T09:44:14.471 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T09:44:14.471 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T09:44:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:14.232192+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:44:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:14.232383+0000 mon.a (mon.0) 126 : cluster [INF] mon.a calling monitor election 2023-10-09T09:44:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:14.248264+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:14.751486+0000 mgr.y (mgr.14148) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:15.229513+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:16.229642+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:16.230620+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:16.751748+0000 mgr.y (mgr.14148) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:17.229479+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:18.229663+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:18.752094+0000 mgr.y (mgr.14148) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:19.229905+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:19.261490+0000 mon.a (mon.0) 133 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:19.266756+0000 mon.a (mon.0) 134 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0]} removed_ranks: {} 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:19.278824+0000 mon.a (mon.0) 135 : cluster [DBG] fsmap 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:19.278878+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:19.279115+0000 mon.a (mon.0) 137 : cluster [DBG] mgrmap e13: y(active, since 32s) 2023-10-09T09:44:19.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:19.283876+0000 mon.a (mon.0) 138 : cluster [INF] overall HEALTH_OK 2023-10-09T09:44:19.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:19.286608+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:19.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:19.287674+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:19.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:19.288527+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:19.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:19.289264+0000 mgr.y (mgr.14148) 23 : cephadm [INF] Deploying daemon mon.c on smithi012 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:14.232192+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:14.232383+0000 mon.a (mon.0) 126 : cluster [INF] mon.a calling monitor election 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:14.248264+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:14.751486+0000 mgr.y (mgr.14148) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:15.229513+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:16.229642+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:16.230620+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:16.751748+0000 mgr.y (mgr.14148) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:17.229479+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:18.229663+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:18.752094+0000 mgr.y (mgr.14148) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:19.229905+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:19.261490+0000 mon.a (mon.0) 133 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:19.266756+0000 mon.a (mon.0) 134 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0]} removed_ranks: {} 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:19.278824+0000 mon.a (mon.0) 135 : cluster [DBG] fsmap 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:19.278878+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:19.279115+0000 mon.a (mon.0) 137 : cluster [DBG] mgrmap e13: y(active, since 32s) 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:19.283876+0000 mon.a (mon.0) 138 : cluster [INF] overall HEALTH_OK 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:19.286608+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:19.287674+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:19.288527+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:19.289264+0000 mgr.y (mgr.14148) 23 : cephadm [INF] Deploying daemon mon.c on smithi012 2023-10-09T09:44:19.705 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:44:19.705 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":2,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","modified":"2023-10-09T09:44:14.229655Z","created":"2023-10-09T09:42:50.732807Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T09:44:19.707 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 2 2023-10-09T09:44:20.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:19.705002+0000 mon.a (mon.0) 142 : audit [DBG] from='client.? 172.21.15.110:0/2899566968' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:44:20.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:20.230400+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:20.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:19.705002+0000 mon.a (mon.0) 142 : audit [DBG] from='client.? 172.21.15.110:0/2899566968' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:44:20.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:20.230400+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:21.132 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T09:44:21.132 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T09:44:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:20.752384+0000 mgr.y (mgr.14148) 24 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:20.752384+0000 mgr.y (mgr.14148) 24 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:21.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 systemd[1]: Starting Ceph mon.c for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 podman[103008]: 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.913+0000 7f05a5c7b880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.913+0000 7f05a5c7b880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.913+0000 7f05a5c7b880 0 pidfile_write: ignore empty --pid-file 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 bash[103008]: 8ea2dfe53e634657aa10e22ea270e1cd87421070f295a2dcbdf5b140431ba65e 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.917+0000 7f05a5c7b880 0 load: jerasure load: lrc load: isa 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: DB SUMMARY 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-09T09:44:22.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 636 ; 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.env: 0x55cf0aeae080 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.fs: Posix File System 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.info_log: 0x55cf0c283a80 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.statistics: (nil) 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.use_fsync: 0 2023-10-09T09:44:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:44:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.db_log_dir: 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:44:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.write_buffer_manager: 0x55cf0cf7e300 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.unordered_write: 0 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:44:22.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.row_cache: None 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.wal_filter: None 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T09:44:22.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:44:22.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_open_files: -1 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Compression algorithms supported: 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: kZSTD supported: 0 2023-10-09T09:44:22.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:44:22.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.merge_operator: 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_filter: None 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cf0c233d18) 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cache_index_and_filter_blocks: 1 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: pin_top_level_index_and_filter: 1 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: index_type: 0 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: data_block_index_type: 0 2023-10-09T09:44:22.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: index_shortening: 1 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: hash_index_allow_collision: 1 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: checksum: 1 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: no_block_cache: 0 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: block_cache: 0x55cf0c268f10 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: block_cache_name: BinnedLRUCache 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: block_cache_options: 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: capacity : 536870912 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: num_shard_bits : 4 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: strict_capacity_limit : 0 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: high_pri_pool_ratio: 0.000 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: block_cache_compressed: (nil) 2023-10-09T09:44:22.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: persistent_cache: (nil) 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: block_size: 4096 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: block_size_deviation: 10 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: block_restart_interval: 16 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: index_block_restart_interval: 1 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: metadata_block_size: 4096 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: partition_filters: 0 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: use_delta_encoding: 1 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: whole_key_filtering: 1 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: verify_compression: 0 2023-10-09T09:44:22.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: read_amp_bytes_per_bit: 0 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: format_version: 2 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: enable_index_compression: 1 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: block_align: 0 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compression: NoCompression 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.num_levels: 7 2023-10-09T09:44:22.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:44:22.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:44:22.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:44:22.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:44:22.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:44:22.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:44:22.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:44:22.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:44:22.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:44:22.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:44:22.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:44:22.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:44:22.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:44:22.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:44:22.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.ttl: 2592000 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.918+0000 7f05a5c7b880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.919+0000 7f05a5c7b880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.919+0000 7f05a5c7b880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.919+0000 7f05a5c7b880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844661920553, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.919+0000 7f05a5c7b880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.919+0000 7f05a5c7b880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.919+0000 7f05a5c7b880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844661921013, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1562, "table_properties": {"data_size": 648, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696844661, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.919+0000 7f05a5c7b880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.920+0000 7f05a5c7b880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696844661921401, "job": 1, "event": "recovery_finished"} 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.920+0000 7f05a5c7b880 4 rocksdb: DB pointer 0x55cf0c331800 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.920+0000 7f058ca82700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.920+0000 7f058ca82700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ** DB Stats ** 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:44:22.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ** Compaction Stats [default] ** 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ** Compaction Stats [default] ** 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:22.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.201 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ** Compaction Stats [default] ** 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ** Compaction Stats [default] ** 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:44:22.202 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.921+0000 7f05a5c7b880 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-09T09:44:22.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.921+0000 7f05a5c7b880 0 using public_addrv [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.922+0000 7f05a5c7b880 0 starting mon.c rank -1 at public addrs [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] at bind addrs [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 systemd[1]: Started Ceph mon.c for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.923+0000 7f05a5c7b880 1 mon.c@-1(???) e0 preinit fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.938+0000 7f058f287700 0 mon.c@-1(synchronizing).mds e1 new map 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.938+0000 7f058f287700 0 mon.c@-1(synchronizing).mds e1 print_map 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: e1 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: legacy client fscid: -1 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: No filesystems configured 2023-10-09T09:44:22.204 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.938+0000 7f058f287700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.938+0000 7f058f287700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.938+0000 7f058f287700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.938+0000 7f058f287700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.938+0000 7f058f287700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.938+0000 7f058f287700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.939+0000 7f058f287700 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.939+0000 7f058f287700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.939+0000 7f058f287700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.939+0000 7f058f287700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:20.752384+0000 mgr.y (mgr.14148) 24 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:22.205 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:44:21.939+0000 7f058f287700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T09:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout: 2023-10-09T09:44:27.279 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":3,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","modified":"2023-10-09T09:44:21.944768Z","created":"2023-10-09T09:42:50.732807Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3301","nonce":0},{"type":"v1","addr":"172.21.15.12:6790","nonce":0}]},"addr":"172.21.15.12:6790/0","public_addr":"172.21.15.12:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T09:44:27.281 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 3 2023-10-09T09:44:27.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:21.947592+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:44:27.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:21.947779+0000 mon.a (mon.0) 146 : cluster [INF] mon.a calling monitor election 2023-10-09T09:44:27.304 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:21.948181+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T09:44:27.304 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:21.960352+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:27.304 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:21.960543+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.304 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:22.752765+0000 mgr.y (mgr.14148) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:27.304 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:22.944033+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.305 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:23.944014+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.305 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:23.945912+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T09:44:27.305 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:24.753155+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:27.305 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:24.944253+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.305 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:25.944206+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.305 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:26.944589+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.306 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:26.974275+0000 mon.a (mon.0) 154 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:44:27.306 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:26.978306+0000 mon.a (mon.0) 155 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:44:27.306 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:26.990752+0000 mon.a (mon.0) 156 : cluster [DBG] fsmap 2023-10-09T09:44:27.306 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:26.990808+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T09:44:27.306 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:26.991025+0000 mon.a (mon.0) 158 : cluster [DBG] mgrmap e13: y(active, since 40s) 2023-10-09T09:44:27.306 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:26.994623+0000 mon.a (mon.0) 159 : cluster [INF] overall HEALTH_OK 2023-10-09T09:44:27.306 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:26.996976+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:27.307 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:26.998450+0000 mon.a (mon.0) 161 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:21.947592+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:21.947779+0000 mon.a (mon.0) 146 : cluster [INF] mon.a calling monitor election 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:21.948181+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:21.960352+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:21.960543+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:22.752765+0000 mgr.y (mgr.14148) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:22.944033+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:23.944014+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:23.945912+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:24.753155+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:24.944253+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:25.944206+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:26.944589+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:26.974275+0000 mon.a (mon.0) 154 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:26.978306+0000 mon.a (mon.0) 155 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:26.990752+0000 mon.a (mon.0) 156 : cluster [DBG] fsmap 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:26.990808+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T09:44:27.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:21.947592+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:21.947779+0000 mon.a (mon.0) 146 : cluster [INF] mon.a calling monitor election 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:21.948181+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:21.960352+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:21.960543+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:22.752765+0000 mgr.y (mgr.14148) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:22.944033+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:23.944014+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:23.945912+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:24.753155+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:24.944253+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:25.944206+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:26.944589+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:27.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:26.974275+0000 mon.a (mon.0) 154 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:44:27.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:26.978306+0000 mon.a (mon.0) 155 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:44:27.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:26.990752+0000 mon.a (mon.0) 156 : cluster [DBG] fsmap 2023-10-09T09:44:27.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:26.990808+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T09:44:27.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:26.991025+0000 mon.a (mon.0) 158 : cluster [DBG] mgrmap e13: y(active, since 40s) 2023-10-09T09:44:27.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:26.994623+0000 mon.a (mon.0) 159 : cluster [INF] overall HEALTH_OK 2023-10-09T09:44:27.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:26.996976+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:27.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:26.998450+0000 mon.a (mon.0) 161 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:27.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:26.991025+0000 mon.a (mon.0) 158 : cluster [DBG] mgrmap e13: y(active, since 40s) 2023-10-09T09:44:27.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:26.994623+0000 mon.a (mon.0) 159 : cluster [INF] overall HEALTH_OK 2023-10-09T09:44:27.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:26.996976+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:27.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:26.998450+0000 mon.a (mon.0) 161 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:27.702 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T09:44:27.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T09:44:28.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:26.753525+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:27.279241+0000 mon.a (mon.0) 162 : audit [DBG] from='client.? 172.21.15.110:0/4291509039' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:44:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:27.944415+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:28.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:26.753525+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:28.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:27.279241+0000 mon.a (mon.0) 162 : audit [DBG] from='client.? 172.21.15.110:0/4291509039' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:44:28.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:27.944415+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:28.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:26.753525+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:28.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:27.279241+0000 mon.a (mon.0) 162 : audit [DBG] from='client.? 172.21.15.110:0/4291509039' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:44:28.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:27.944415+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:44:29.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.441744+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:28.753966+0000 mgr.y (mgr.14148) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:29.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.791566+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.792144+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:29.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.792781+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:29.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:28.793670+0000 mgr.y (mgr.14148) 29 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:28.793895+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.839660+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.895472+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.898985+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:28.900113+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.900340+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.900992+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:28.901559+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:28.902074+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring daemon mon.a on smithi012 2023-10-09T09:44:29.580 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.441744+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:28.753966+0000 mgr.y (mgr.14148) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.791566+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.792144+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.792781+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:28.793670+0000 mgr.y (mgr.14148) 29 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:28.793895+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.839660+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.895472+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.898985+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:28.900113+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.900340+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:29.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.900992+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:29.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:28.901559+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:29.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:28.902074+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring daemon mon.a on smithi012 2023-10-09T09:44:29.728 INFO:teuthology.orchestra.run.smithi012.stdout:# minimal ceph.conf for 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:44:29.728 INFO:teuthology.orchestra.run.smithi012.stdout:[global] 2023-10-09T09:44:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: fsid = 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:44:29.728 INFO:teuthology.orchestra.run.smithi012.stdout: mon_host = [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] 2023-10-09T09:44:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.441744+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:28.753966+0000 mgr.y (mgr.14148) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.791566+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.792144+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.792781+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:28.793670+0000 mgr.y (mgr.14148) 29 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:28.793895+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.839660+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.895472+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.898985+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:28.900113+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.900340+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.900992+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:28.901559+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:28.902074+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring daemon mon.a on smithi012 2023-10-09T09:44:30.566 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:29.728205+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.12:0/1370751231' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:30.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:30.291559+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:30.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:30.292081+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T09:44:30.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:30.292335+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:30.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:30.293092+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:30.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:30.293792+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:30.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:30.294439+0000 mgr.y (mgr.14148) 34 : cephadm [INF] Reconfiguring daemon mon.c on smithi012 2023-10-09T09:44:30.567 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:29.728205+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.12:0/1370751231' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:30.567 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:30.291559+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:30.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:30.292081+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T09:44:30.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:30.292335+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:30.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:30.293092+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:30.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:30.293792+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:30.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:30.294439+0000 mgr.y (mgr.14148) 34 : cephadm [INF] Reconfiguring daemon mon.c on smithi012 2023-10-09T09:44:30.569 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T09:44:30.569 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:44:30.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T09:44:30.599 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:44:30.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:44:30.674 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:44:30.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T09:44:30.706 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:29.728205+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.12:0/1370751231' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:30.707 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:30.291559+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:30.707 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:30.292081+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T09:44:30.707 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:30.292335+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:30.707 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:30.293092+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:30.707 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:30.293792+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:30.707 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:30.294439+0000 mgr.y (mgr.14148) 34 : cephadm [INF] Reconfiguring daemon mon.c on smithi012 2023-10-09T09:44:30.712 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:44:30.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:44:30.787 INFO:tasks.cephadm:Adding mgr.y on smithi012 2023-10-09T09:44:30.787 INFO:tasks.cephadm:Adding mgr.x on smithi110 2023-10-09T09:44:30.787 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi012=y;smithi110=x' 2023-10-09T09:44:31.643 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:30.754335+0000 mgr.y (mgr.14148) 35 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:31.644 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:31.406747+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:31.644 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:31.407428+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:31.644 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:31.408037+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:31.644 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:31.408612+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:30.754335+0000 mgr.y (mgr.14148) 35 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:31.406747+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:31.407428+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:31.408037+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:31.408612+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:31.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:30.754335+0000 mgr.y (mgr.14148) 35 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:31.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:31.406747+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:31.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:31.407428+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:44:31.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:31.408037+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:44:31.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:31.408612+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:32.533 INFO:teuthology.orchestra.run.smithi110.stdout:Scheduled mgr update... 2023-10-09T09:44:33.177 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:31.407217+0000 mgr.y (mgr.14148) 36 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T09:44:33.177 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:31.409140+0000 mgr.y (mgr.14148) 37 : cephadm [INF] Reconfiguring daemon mon.b on smithi110 2023-10-09T09:44:33.177 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:31.912754+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:33.177 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:32.533381+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:33.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:31.407217+0000 mgr.y (mgr.14148) 36 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T09:44:33.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:31.409140+0000 mgr.y (mgr.14148) 37 : cephadm [INF] Reconfiguring daemon mon.b on smithi110 2023-10-09T09:44:33.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:31.912754+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:33.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:32.533381+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:33.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:31.407217+0000 mgr.y (mgr.14148) 36 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T09:44:33.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:31.409140+0000 mgr.y (mgr.14148) 37 : cephadm [INF] Reconfiguring daemon mon.b on smithi110 2023-10-09T09:44:33.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:31.912754+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:33.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:32.533381+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:33.180 DEBUG:teuthology.orchestra.run.smithi110:mgr.x> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T09:44:33.182 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T09:44:33.182 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:44:33.182 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T09:44:33.205 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T09:44:33.206 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_1 2023-10-09T09:44:33.212 INFO:journalctl@ceph.mgr.x.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:44:33.267 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T09:44:33.268 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T09:44:33.268 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 6h/6d Inode: 390833 Links: 1 2023-10-09T09:44:33.268 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T09:44:33.269 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T09:44:33.269 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2023-10-09 09:44:21.385539294 +0000 2023-10-09T09:44:33.269 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2023-10-09 09:36:40.936424779 +0000 2023-10-09T09:44:33.270 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2023-10-09 09:36:40.936424779 +0000 2023-10-09T09:44:33.270 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: - 2023-10-09T09:44:33.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T09:44:33.354 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2023-10-09T09:44:33.354 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2023-10-09T09:44:33.355 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.00024797 s, 2.1 MB/s 2023-10-09T09:44:33.356 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T09:44:33.377 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_2 2023-10-09T09:44:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T09:44:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T09:44:33.436 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 6h/6d Inode: 391018 Links: 1 2023-10-09T09:44:33.436 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T09:44:33.436 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T09:44:33.436 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2023-10-09 09:44:21.385539294 +0000 2023-10-09T09:44:33.437 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2023-10-09 09:36:41.465420787 +0000 2023-10-09T09:44:33.437 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2023-10-09 09:36:41.465420787 +0000 2023-10-09T09:44:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: - 2023-10-09T09:44:33.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T09:44:33.505 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2023-10-09T09:44:33.506 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2023-10-09T09:44:33.506 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000326734 s, 1.6 MB/s 2023-10-09T09:44:33.507 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T09:44:33.566 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_3 2023-10-09T09:44:33.625 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T09:44:33.626 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T09:44:33.626 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 6h/6d Inode: 391053 Links: 1 2023-10-09T09:44:33.626 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T09:44:33.626 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T09:44:33.626 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2023-10-09 09:44:21.385539294 +0000 2023-10-09T09:44:33.626 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2023-10-09 09:36:42.002416734 +0000 2023-10-09T09:44:33.626 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2023-10-09 09:36:42.002416734 +0000 2023-10-09T09:44:33.626 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: - 2023-10-09T09:44:33.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T09:44:33.694 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2023-10-09T09:44:33.694 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2023-10-09T09:44:33.694 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000230771 s, 2.2 MB/s 2023-10-09T09:44:33.695 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T09:44:33.755 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_4 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 6h/6d Inode: 390117 Links: 1 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2023-10-09 09:44:21.385539294 +0000 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2023-10-09 09:36:42.521412818 +0000 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2023-10-09 09:36:42.521412818 +0000 2023-10-09T09:44:33.815 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: - 2023-10-09T09:44:33.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T09:44:33.883 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2023-10-09T09:44:33.883 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2023-10-09T09:44:33.883 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.00019865 s, 2.6 MB/s 2023-10-09T09:44:33.884 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T09:44:33.944 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:44:33.944 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T09:44:33.961 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T09:44:33.961 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_1 2023-10-09T09:44:34.019 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T09:44:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T09:44:34.020 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 6h/6d Inode: 385926 Links: 1 2023-10-09T09:44:34.020 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T09:44:34.020 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T09:44:34.020 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2023-10-09 09:44:13.682916367 +0000 2023-10-09T09:44:34.020 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2023-10-09 09:36:24.399040527 +0000 2023-10-09T09:44:34.020 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2023-10-09 09:36:24.399040527 +0000 2023-10-09T09:44:34.020 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: - 2023-10-09T09:44:34.020 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T09:44:34.088 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2023-10-09T09:44:34.088 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2023-10-09T09:44:34.088 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000118125 s, 4.3 MB/s 2023-10-09T09:44:34.089 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T09:44:34.103 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:32.528052+0000 mgr.y (mgr.14148) 38 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi012=y;smithi110=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:44:34.103 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:32.529746+0000 mgr.y (mgr.14148) 39 : cephadm [INF] Saving service mgr spec with placement smithi012=y;smithi110=x;count:2 2023-10-09T09:44:34.103 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:32.754711+0000 mgr.y (mgr.14148) 40 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:34.103 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:32.987751+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:34.103 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:32.988863+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:34.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:32.990698+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:34.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:32.991657+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:34.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:32.996076+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:34.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:32.997747+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:44:34.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:33.000521+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T09:44:34.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:33.001377+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:44:34.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:33.002424+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:34.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:33.003203+0000 mgr.y (mgr.14148) 41 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:44:34.121 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_2 2023-10-09T09:44:34.184 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T09:44:34.184 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T09:44:34.184 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 6h/6d Inode: 396315 Links: 1 2023-10-09T09:44:34.184 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T09:44:34.184 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T09:44:34.184 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2023-10-09 09:44:13.682916367 +0000 2023-10-09T09:44:34.185 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2023-10-09 09:36:24.907031643 +0000 2023-10-09T09:44:34.185 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2023-10-09 09:36:24.907031643 +0000 2023-10-09T09:44:34.185 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: - 2023-10-09T09:44:34.185 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T09:44:34.262 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2023-10-09T09:44:34.262 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2023-10-09T09:44:34.262 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000114699 s, 4.5 MB/s 2023-10-09T09:44:34.263 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T09:44:34.322 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_3 2023-10-09T09:44:34.381 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T09:44:34.381 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T09:44:34.381 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 6h/6d Inode: 391727 Links: 1 2023-10-09T09:44:34.382 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T09:44:34.382 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T09:44:34.382 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2023-10-09 09:44:34.316558581 +0000 2023-10-09T09:44:34.382 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2023-10-09 09:36:25.422022636 +0000 2023-10-09T09:44:34.382 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2023-10-09 09:36:25.422022636 +0000 2023-10-09T09:44:34.382 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: - 2023-10-09T09:44:34.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:32.528052+0000 mgr.y (mgr.14148) 38 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi012=y;smithi110=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:32.529746+0000 mgr.y (mgr.14148) 39 : cephadm [INF] Saving service mgr spec with placement smithi012=y;smithi110=x;count:2 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:32.754711+0000 mgr.y (mgr.14148) 40 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:32.987751+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:32.988863+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:32.990698+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:32.991657+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:32.996076+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:32.997747+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:33.000521+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:33.001377+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:33.002424+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:33.003203+0000 mgr.y (mgr.14148) 41 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:32.528052+0000 mgr.y (mgr.14148) 38 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi012=y;smithi110=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:32.529746+0000 mgr.y (mgr.14148) 39 : cephadm [INF] Saving service mgr spec with placement smithi012=y;smithi110=x;count:2 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:32.754711+0000 mgr.y (mgr.14148) 40 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:32.987751+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:32.988863+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:32.990698+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:32.991657+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:32.996076+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:32.997747+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:44:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:33.000521+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T09:44:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:33.001377+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:44:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:33.002424+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:33.003203+0000 mgr.y (mgr.14148) 41 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:44:34.452 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2023-10-09T09:44:34.452 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2023-10-09T09:44:34.452 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 9.8502e-05 s, 5.2 MB/s 2023-10-09T09:44:34.453 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T09:44:34.512 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_4 2023-10-09T09:44:34.570 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T09:44:34.570 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T09:44:34.570 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 6h/6d Inode: 391765 Links: 1 2023-10-09T09:44:34.570 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T09:44:34.570 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T09:44:34.571 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2023-10-09 09:44:34.316558581 +0000 2023-10-09T09:44:34.571 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2023-10-09 09:36:25.916013997 +0000 2023-10-09T09:44:34.571 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2023-10-09 09:36:25.916013997 +0000 2023-10-09T09:44:34.571 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: - 2023-10-09T09:44:34.571 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T09:44:34.634 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:34 smithi110 systemd[1]: Starting Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:44:34.636 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2023-10-09T09:44:34.637 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2023-10-09T09:44:34.637 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000118659 s, 4.3 MB/s 2023-10-09T09:44:34.638 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T09:44:34.695 INFO:tasks.cephadm:Deploying osd.0 on smithi012 with /dev/vg_nvme/lv_4... 2023-10-09T09:44:34.695 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T09:44:34.908 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:34 smithi110 podman[98425]: 2023-10-09T09:44:35.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:34 smithi110 bash[98425]: 2e870acba3a1cd1bb903788ccbbcac8a32c3daf24c070546ba0ae9b61ff21438 2023-10-09T09:44:35.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:34.912+0000 7f17ce25a700 1 -- 172.21.15.110:0/3332248920 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55672bfee4e0 con 0x55672bfc3400 2023-10-09T09:44:35.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:34 smithi110 systemd[1]: Started Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:44:35.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:35.041+0000 7f17de840680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:44:35.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:35.125+0000 7f17de840680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:44:35.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:35.511+0000 7f17de840680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:44:36.301 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:34.755027+0000 mgr.y (mgr.14148) 42 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:36.315 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:35.005550+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:36.315 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:35.007207+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:36.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:34.755027+0000 mgr.y (mgr.14148) 42 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:36.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:35.005550+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:36.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:35.007207+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:36.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:34.755027+0000 mgr.y (mgr.14148) 42 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:36.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:35.005550+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:36.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:35.007207+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:36.523 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T09:44:36.523 INFO:teuthology.orchestra.run.smithi012.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T09:44:36.523 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10+0 records in 2023-10-09T09:44:36.523 INFO:teuthology.orchestra.run.smithi012.stderr:10+0 records out 2023-10-09T09:44:36.523 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111103 s, 944 MB/s 2023-10-09T09:44:36.524 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping successful for: 2023-10-09T09:44:36.596 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:36.300+0000 7f17de840680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:44:36.596 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:36.384+0000 7f17de840680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:44:36.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi012:vg_nvme/lv_4 2023-10-09T09:44:37.652 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:36.450046+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:37.652 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:36.450728+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:36.451531+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:36.456063+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:36.457731+0000 mgr.y (mgr.14148) 43 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:36.458018+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:36.458810+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:36.459457+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:36.460036+0000 mgr.y (mgr.14148) 44 : cephadm [INF] Reconfiguring daemon mgr.y on smithi012 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:36.755471+0000 mgr.y (mgr.14148) 45 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:36.916923+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:37.653 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:36.450046+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:36.450728+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:36.451531+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:36.456063+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:36.457731+0000 mgr.y (mgr.14148) 43 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:36.458018+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:36.458810+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:36.459457+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:36.460036+0000 mgr.y (mgr.14148) 44 : cephadm [INF] Reconfiguring daemon mgr.y on smithi012 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:36.755471+0000 mgr.y (mgr.14148) 45 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:37.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:36.916923+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:37.738 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:36.595+0000 7f17de840680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:37.535+0000 7f17de840680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:37.661+0000 7f17de840680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:37.738+0000 7f17de840680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:36.450046+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:36.450728+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:36.451531+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:36.456063+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:36.457731+0000 mgr.y (mgr.14148) 43 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T09:44:37.739 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:36.458018+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:44:37.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:36.458810+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:44:37.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:36.459457+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:37.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:36.460036+0000 mgr.y (mgr.14148) 44 : cephadm [INF] Reconfiguring daemon mgr.y on smithi012 2023-10-09T09:44:37.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:36.755471+0000 mgr.y (mgr.14148) 45 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:37.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:36.916923+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:38.018 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:37.827+0000 7f17de840680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:44:38.291 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:38.018+0000 7f17de840680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:44:38.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:38.291+0000 7f17de840680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:44:38.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:38.388+0000 7f17de840680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:44:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:37.730914+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:37.731977+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:37.733398+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:37.734022+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:37.738385+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:37.730914+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:37.731977+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:37.733398+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:37.734022+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:37.738385+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:39.119 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:37.730914+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:39.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:37.731977+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:39.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:37.733398+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:39.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:37.734022+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:39.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:37.738385+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:39.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:39.119+0000 7f17de840680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:44:39.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:39.200+0000 7f17de840680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:44:39.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:39.290+0000 7f17de840680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:44:39.737 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:39.461+0000 7f17de840680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:44:39.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:38.755687+0000 mgr.y (mgr.14148) 46 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:39.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:38.930525+0000 mgr.y (mgr.14148) 47 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:44:39.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:38.933302+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:44:39.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:38.935575+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:44:39.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:38.936156+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:39.998 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:39.738+0000 7f17de840680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:44:39.999 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:39.867+0000 7f17de840680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:44:40.023 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:38.755687+0000 mgr.y (mgr.14148) 46 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:40.023 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:38.930525+0000 mgr.y (mgr.14148) 47 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:44:40.023 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:38.933302+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:44:40.023 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:38.935575+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:44:40.023 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:38.936156+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:40.023 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:38.755687+0000 mgr.y (mgr.14148) 46 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:40.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:38.930525+0000 mgr.y (mgr.14148) 47 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:44:40.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:38.933302+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:44:40.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:38.935575+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:44:40.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:38.936156+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:40.248 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:39.997+0000 7f17de840680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:44:40.249 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:40.167+0000 7f17de840680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:44:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:40.262515+0000 mon.a (mon.0) 212 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:44:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:40.264126+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:44:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:40.264523+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:44:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:40.265462+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:44:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:40.265743+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:44:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:40.262515+0000 mon.a (mon.0) 212 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:44:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:40.264126+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:44:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:40.264523+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:44:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:40.265462+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:44:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:40.265743+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:44:41.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:44:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:44:40.248+0000 7f17de840680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:44:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:40.262515+0000 mon.a (mon.0) 212 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:44:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:40.264126+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:44:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:40.264523+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:44:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:40.265462+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:44:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:40.265743+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.? 172.21.15.110:0/3310161896' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:44:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:40.748889+0000 mon.a (mon.0) 217 : cluster [DBG] mgrmap e14: y(active, since 54s), standbys: x 2023-10-09T09:44:42.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:40.749007+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:44:42.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:40.755890+0000 mgr.y (mgr.14148) 48 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:42.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:41.264394+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.12:0/2413987661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]: dispatch 2023-10-09T09:44:42.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:41.264723+0000 mon.a (mon.0) 219 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]: dispatch 2023-10-09T09:44:42.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:41.268054+0000 mon.a (mon.0) 220 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]': finished 2023-10-09T09:44:42.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:41.268142+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T09:44:42.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:41.268267+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:42.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:40.748889+0000 mon.a (mon.0) 217 : cluster [DBG] mgrmap e14: y(active, since 54s), standbys: x 2023-10-09T09:44:42.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:40.749007+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:44:42.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:40.755890+0000 mgr.y (mgr.14148) 48 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:42.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:41.264394+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.12:0/2413987661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]: dispatch 2023-10-09T09:44:42.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:41.264723+0000 mon.a (mon.0) 219 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]: dispatch 2023-10-09T09:44:42.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:41.268054+0000 mon.a (mon.0) 220 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]': finished 2023-10-09T09:44:42.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:41.268142+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T09:44:42.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:41.268267+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:40.748889+0000 mon.a (mon.0) 217 : cluster [DBG] mgrmap e14: y(active, since 54s), standbys: x 2023-10-09T09:44:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:40.749007+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:44:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:40.755890+0000 mgr.y (mgr.14148) 48 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:41.264394+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.12:0/2413987661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]: dispatch 2023-10-09T09:44:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:41.264723+0000 mon.a (mon.0) 219 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]: dispatch 2023-10-09T09:44:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:41.268054+0000 mon.a (mon.0) 220 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "23895d43-58c0-4692-9ca5-d05fc064a71e"}]': finished 2023-10-09T09:44:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:41.268142+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T09:44:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:41.268267+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:43.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:41.863791+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.12:0/2285438960' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:44:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:41.863791+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.12:0/2285438960' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:44:43.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:41.863791+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.12:0/2285438960' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:44:44.136 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:42.756136+0000 mgr.y (mgr.14148) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:44.137 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:42.756136+0000 mgr.y (mgr.14148) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:44.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:42.756136+0000 mgr.y (mgr.14148) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:44.756469+0000 mgr.y (mgr.14148) 50 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:46.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:44.756469+0000 mgr.y (mgr.14148) 50 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:44.756469+0000 mgr.y (mgr.14148) 50 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:47.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:46.765921+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:44:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:46.766609+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:46.765921+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:44:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:46.766609+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:46.765921+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:44:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:46.766609+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:47.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:46.756689+0000 mgr.y (mgr.14148) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:47.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:46.767197+0000 mgr.y (mgr.14148) 52 : cephadm [INF] Deploying daemon osd.0 on smithi012 2023-10-09T09:44:47.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:46.756689+0000 mgr.y (mgr.14148) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:47.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:46.767197+0000 mgr.y (mgr.14148) 52 : cephadm [INF] Deploying daemon osd.0 on smithi012 2023-10-09T09:44:48.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:47 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:46.756689+0000 mgr.y (mgr.14148) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:48.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:47 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:46.767197+0000 mgr.y (mgr.14148) 52 : cephadm [INF] Deploying daemon osd.0 on smithi012 2023-10-09T09:44:50.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:48.756955+0000 mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:50.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:48.756955+0000 mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:50.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:48.756955+0000 mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:52.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:50.757357+0000 mgr.y (mgr.14148) 54 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:52.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:51.169442+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:52.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:51.182038+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:52.287 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:50.757357+0000 mgr.y (mgr.14148) 54 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:52.287 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:51.169442+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:52.288 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:51.182038+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:52.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:50.757357+0000 mgr.y (mgr.14148) 54 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:52.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:51.169442+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:51.182038+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:53.803 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 0 on host 'smithi012' 2023-10-09T09:44:54.116 DEBUG:teuthology.orchestra.run.smithi012:osd.0> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T09:44:54.118 INFO:tasks.cephadm:Deploying osd.1 on smithi012 with /dev/vg_nvme/lv_3... 2023-10-09T09:44:54.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T09:44:54.149 INFO:journalctl@ceph.osd.0.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:44:54.427 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[106439]: debug 2023-10-09T09:44:54.323+0000 7f7255b1c200 -1 osd.0 0 log_to_monitors {default=true} 2023-10-09T09:44:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:52.757622+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:53.212895+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:53.213705+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:53.214548+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:53.219758+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:53.803134+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:53.838187+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:52.757622+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:53.212895+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:53.213705+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:53.214548+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:53.219758+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:53.803134+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:53.838187+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:54.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:52.757622+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:54.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:53.212895+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:53.213705+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:54.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:53.214548+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:54.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:53.219758+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:53.803134+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:54.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:53.838187+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:44:55.543 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:54.327031+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:44:55.543 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:54.327031+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:44:55.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:54.327031+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:44:56.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:54.757848+0000 mgr.y (mgr.14148) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:56.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:55.225497+0000 mon.a (mon.0) 234 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:44:56.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:55.225564+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T09:44:56.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:55.225758+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:56.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:55.226117+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:44:56.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:54.757848+0000 mgr.y (mgr.14148) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:56.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:55.225497+0000 mon.a (mon.0) 234 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:44:56.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:55.225564+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T09:44:56.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:55.225758+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:56.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:55.226117+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:44:56.413 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[106439]: debug 2023-10-09T09:44:56.228+0000 7f724c09d700 -1 osd.0 0 waiting for initial osdmap 2023-10-09T09:44:56.413 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:44:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[106439]: debug 2023-10-09T09:44:56.233+0000 7f7244811700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:44:56.597 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T09:44:56.597 INFO:teuthology.orchestra.run.smithi012.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T09:44:56.597 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10+0 records in 2023-10-09T09:44:56.598 INFO:teuthology.orchestra.run.smithi012.stderr:10+0 records out 2023-10-09T09:44:56.598 INFO:teuthology.orchestra.run.smithi012.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0137825 s, 761 MB/s 2023-10-09T09:44:56.598 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping successful for: 2023-10-09T09:44:56.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:54.757848+0000 mgr.y (mgr.14148) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:56.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:55.225497+0000 mon.a (mon.0) 234 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:44:56.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:55.225564+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T09:44:56.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:55.225758+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:56.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:55.226117+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:44:56.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi012:vg_nvme/lv_3 2023-10-09T09:44:57.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:56.224666+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:44:57.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:56.224770+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T09:44:57.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:56.225159+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:57.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:56.229999+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:57.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:57.044373+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:57.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:57.045293+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:44:57.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:57.050963+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:57.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:57.051837+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:57.485 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:57.052495+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:57.485 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:57.057778+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:57.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:56.224666+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:44:57.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:56.224770+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T09:44:57.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:56.225159+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:57.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:56.229999+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:57.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:57.044373+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:57.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:57.045293+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:44:57.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:57.050963+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:57.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:57.051837+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:57.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:57.052495+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:57.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:57.057778+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:57.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:56.224666+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:44:57.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:56.224770+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T09:44:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:56.225159+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:56.229999+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:57.044373+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:57.045293+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:44:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:57.050963+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:57.051837+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:57.052495+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:44:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:57.057778+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:55.332496+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:55.332545+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:56.758194+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:57.038960+0000 mgr.y (mgr.14148) 58 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:44:57.045876+0000 mgr.y (mgr.14148) 59 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 16149M 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:57.229818+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:57.233963+0000 mon.a (mon.0) 249 : cluster [INF] osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239] boot 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:57.234007+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T09:44:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:57.234424+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:55.332496+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:55.332545+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:56.758194+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:57.038960+0000 mgr.y (mgr.14148) 58 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:44:57.045876+0000 mgr.y (mgr.14148) 59 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 16149M 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:57.229818+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:57.233963+0000 mon.a (mon.0) 249 : cluster [INF] osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239] boot 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:57.234007+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T09:44:58.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:57.234424+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:55.332496+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:44:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:55.332545+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:44:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:56.758194+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T09:44:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:57.038960+0000 mgr.y (mgr.14148) 58 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:44:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:44:57.045876+0000 mgr.y (mgr.14148) 59 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 16149M 2023-10-09T09:44:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:57.229818+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:57.233963+0000 mon.a (mon.0) 249 : cluster [INF] osd.0 [v2:172.21.15.12:6802/3692896239,v1:172.21.15.12:6803/3692896239] boot 2023-10-09T09:44:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:57.234007+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T09:44:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:57.234424+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:44:59.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:58.234059+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T09:44:59.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:58.607755+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:44:59.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:58.610031+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:44:59.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:44:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:58.610610+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:59.582 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:58.234059+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T09:44:59.583 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:58.607755+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:44:59.583 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:58.610031+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:44:59.583 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:44:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:58.610610+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:44:59.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:58.234059+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T09:44:59.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:58.607755+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:44:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:58.610031+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:44:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:44:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:58.610610+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:00.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:44:58.605145+0000 mgr.y (mgr.14148) 60 : audit [DBG] from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:00.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:44:58.758466+0000 mgr.y (mgr.14148) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:00.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:44:58.605145+0000 mgr.y (mgr.14148) 60 : audit [DBG] from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:00.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:44:58.758466+0000 mgr.y (mgr.14148) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:44:58.605145+0000 mgr.y (mgr.14148) 60 : audit [DBG] from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:44:58.758466+0000 mgr.y (mgr.14148) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:00.870581+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.12:0/3995321735' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]: dispatch 2023-10-09T09:45:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:00.870907+0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]: dispatch 2023-10-09T09:45:01.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:00.878365+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]': finished 2023-10-09T09:45:01.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:00.878414+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T09:45:01.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:00.878504+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:00.870581+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.12:0/3995321735' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]: dispatch 2023-10-09T09:45:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:00.870907+0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]: dispatch 2023-10-09T09:45:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:00.878365+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]': finished 2023-10-09T09:45:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:00.878414+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T09:45:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:00.878504+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:01.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:00.870581+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.12:0/3995321735' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]: dispatch 2023-10-09T09:45:01.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:00.870907+0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]: dispatch 2023-10-09T09:45:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:00.878365+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8657aa57-580d-4318-a093-2cc78c0f72a9"}]': finished 2023-10-09T09:45:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:00.878414+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T09:45:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:00.878504+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:02.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:00.758734+0000 mgr.y (mgr.14148) 62 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:02.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:01.485987+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.12:0/2351834707' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:02.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:00.758734+0000 mgr.y (mgr.14148) 62 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:02.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:01.485987+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.12:0/2351834707' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:02.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:00.758734+0000 mgr.y (mgr.14148) 62 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:02.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:01.485987+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.12:0/2351834707' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:03.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:02.759149+0000 mgr.y (mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:03.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:02.759149+0000 mgr.y (mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:03.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:02.759149+0000 mgr.y (mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:06.065 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:04.759505+0000 mgr.y (mgr.14148) 64 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:06.066 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:04.759505+0000 mgr.y (mgr.14148) 64 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:06.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:04.759505+0000 mgr.y (mgr.14148) 64 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:07.132 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:06.524962+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:45:07.132 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:06.525749+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:07.133 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:06.524962+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:45:07.133 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:06.525749+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:07.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:06.524962+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:45:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:06.525749+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:07.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:06.526374+0000 mgr.y (mgr.14148) 65 : cephadm [INF] Deploying daemon osd.1 on smithi012 2023-10-09T09:45:07.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:06.760078+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:07.969 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:06.526374+0000 mgr.y (mgr.14148) 65 : cephadm [INF] Deploying daemon osd.1 on smithi012 2023-10-09T09:45:07.969 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:06.760078+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:08.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:06.526374+0000 mgr.y (mgr.14148) 65 : cephadm [INF] Deploying daemon osd.1 on smithi012 2023-10-09T09:45:08.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:06.760078+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:10.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:08.760380+0000 mgr.y (mgr.14148) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:10.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:08.760380+0000 mgr.y (mgr.14148) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:10.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:08.760380+0000 mgr.y (mgr.14148) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:10.738837+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:10.753550+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:10.760768+0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:10.738837+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:10.753550+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:11.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:10.760768+0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:12.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:10.738837+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:10.753550+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:10.760768+0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:13.475 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 1 on host 'smithi012' 2023-10-09T09:45:13.838 DEBUG:teuthology.orchestra.run.smithi012:osd.1> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T09:45:13.840 INFO:tasks.cephadm:Deploying osd.2 on smithi012 with /dev/vg_nvme/lv_2... 2023-10-09T09:45:13.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T09:45:13.869 INFO:journalctl@ceph.osd.1.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:45:14.124 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:45:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[110447]: debug 2023-10-09T09:45:13.944+0000 7f9cdf20b200 -1 osd.1 0 log_to_monitors {default=true} 2023-10-09T09:45:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:12.761025+0000 mgr.y (mgr.14148) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.124292+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.475058+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.561111+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.561863+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.566083+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.567011+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.567755+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.572486+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.574483+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.948612+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:13.948965+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:12.761025+0000 mgr.y (mgr.14148) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.124292+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.475058+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.561111+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.561863+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.566083+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.567011+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.567755+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.572486+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.574483+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.948612+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:45:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:13.948965+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:45:14.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:12.761025+0000 mgr.y (mgr.14148) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:14.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.124292+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.475058+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.561111+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.561863+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.566083+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.567011+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.567755+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.572486+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.574483+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.948612+0000 mon.c (mon.2) 5 : audit [INF] from='osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:45:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:13.948965+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:45:15.415 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:13.562373+0000 mgr.y (mgr.14148) 70 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 8074M 2023-10-09T09:45:15.415 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:14.624566+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:45:15.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:14.624676+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T09:45:15.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:14.624828+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:15.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:14.625684+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:15.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:14.626061+0000 mon.a (mon.0) 278 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:15.416 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:13.562373+0000 mgr.y (mgr.14148) 70 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 8074M 2023-10-09T09:45:15.416 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:14.624566+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:45:15.416 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:14.624676+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T09:45:15.416 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:14.624828+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:15.417 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:14.625684+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:15.417 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:14.626061+0000 mon.a (mon.0) 278 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:15.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:13.562373+0000 mgr.y (mgr.14148) 70 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 8074M 2023-10-09T09:45:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:14.624566+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:45:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:14.624676+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T09:45:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:14.624828+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:14.625684+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:15.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:14.626061+0000 mon.a (mon.0) 278 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:15.677 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[110447]: debug 2023-10-09T09:45:15.629+0000 7f9cd578c700 -1 osd.1 0 waiting for initial osdmap 2023-10-09T09:45:15.677 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:45:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[110447]: debug 2023-10-09T09:45:15.632+0000 7f9cd1707700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:45:15.975 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T09:45:15.975 INFO:teuthology.orchestra.run.smithi012.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T09:45:15.975 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10+0 records in 2023-10-09T09:45:15.976 INFO:teuthology.orchestra.run.smithi012.stderr:10+0 records out 2023-10-09T09:45:15.976 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121202 s, 865 MB/s 2023-10-09T09:45:15.976 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping successful for: 2023-10-09T09:45:16.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:14.761288+0000 mgr.y (mgr.14148) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:16.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:15.625070+0000 mon.a (mon.0) 279 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:16.330 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:15.625137+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T09:45:16.330 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:15.625402+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:16.330 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:15.629444+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:16.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:14.761288+0000 mgr.y (mgr.14148) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:16.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:15.625070+0000 mon.a (mon.0) 279 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:16.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:15.625137+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T09:45:16.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:15.625402+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:16.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:15.629444+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:16.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi012:vg_nvme/lv_2 2023-10-09T09:45:16.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:14.761288+0000 mgr.y (mgr.14148) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:16.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:15.625070+0000 mon.a (mon.0) 279 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:16.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:15.625137+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T09:45:16.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:15.625402+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:16.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:15.629444+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:14.911852+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:14.911914+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:16.630527+0000 mon.a (mon.0) 283 : cluster [INF] osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467] boot 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:16.630568+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:16.630698+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:16.761581+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:17.010130+0000 mgr.y (mgr.14148) 73 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:17.014569+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:17.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:17.015260+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:17.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:17.015972+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:17.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:17.019964+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:17.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:14.911852+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:17.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:14.911914+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:17.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:16.630527+0000 mon.a (mon.0) 283 : cluster [INF] osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467] boot 2023-10-09T09:45:17.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:16.630568+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T09:45:17.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:16.630698+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:16.761581+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:17.010130+0000 mgr.y (mgr.14148) 73 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:17.014569+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:17.015260+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:17.015972+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:17.019964+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:14.911852+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:14.911914+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:16.630527+0000 mon.a (mon.0) 283 : cluster [INF] osd.1 [v2:172.21.15.12:6810/994418467,v1:172.21.15.12:6811/994418467] boot 2023-10-09T09:45:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:16.630568+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T09:45:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:16.630698+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:45:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:16.761581+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T09:45:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:17.010130+0000 mgr.y (mgr.14148) 73 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:17.014569+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:17.015260+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:17.015972+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:17.019964+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:18.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:17.631443+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T09:45:18.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:18.017307+0000 mgr.y (mgr.14148) 74 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:18.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:18.019883+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:18.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:18.022191+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:18.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:18.022797+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:18.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:17.631443+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T09:45:18.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:18.017307+0000 mgr.y (mgr.14148) 74 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:18.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:18.019883+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:18.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:18.022191+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:18.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:18.022797+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:17.631443+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T09:45:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:18.017307+0000 mgr.y (mgr.14148) 74 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:18.019883+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:18.022191+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:18.022797+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:19.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:18.762067+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:19.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:18.762067+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:18.762067+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:20.287520+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.12:0/3098752179' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]: dispatch 2023-10-09T09:45:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:20.287849+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]: dispatch 2023-10-09T09:45:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:20.291455+0000 mon.a (mon.0) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]': finished 2023-10-09T09:45:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:20.291542+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T09:45:20.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:20.291651+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:20.287520+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.12:0/3098752179' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]: dispatch 2023-10-09T09:45:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:20.287849+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]: dispatch 2023-10-09T09:45:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:20.291455+0000 mon.a (mon.0) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]': finished 2023-10-09T09:45:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:20.291542+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T09:45:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:20.291651+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:20.287520+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.12:0/3098752179' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]: dispatch 2023-10-09T09:45:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:20.287849+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]: dispatch 2023-10-09T09:45:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:20.291455+0000 mon.a (mon.0) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bab5bc5-77e6-4b60-9cd1-511c5d18ff03"}]': finished 2023-10-09T09:45:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:20.291542+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T09:45:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:20.291651+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:21.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:20.762351+0000 mgr.y (mgr.14148) 76 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:21.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:20.896252+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.12:0/2453034278' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:20.762351+0000 mgr.y (mgr.14148) 76 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:20.896252+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.12:0/2453034278' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:21.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:20.762351+0000 mgr.y (mgr.14148) 76 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:21.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:20.896252+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.12:0/2453034278' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:23.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:22.762721+0000 mgr.y (mgr.14148) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:23.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:22.762721+0000 mgr.y (mgr.14148) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:23.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:22.762721+0000 mgr.y (mgr.14148) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:26.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:24.763142+0000 mgr.y (mgr.14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:26.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:24.763142+0000 mgr.y (mgr.14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:26.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:24.763142+0000 mgr.y (mgr.14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:27.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:25.851199+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:45:27.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:25.851911+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:27.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:25.852518+0000 mgr.y (mgr.14148) 79 : cephadm [INF] Deploying daemon osd.2 on smithi012 2023-10-09T09:45:27.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:25.851199+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:45:27.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:25.851911+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:27.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:25.852518+0000 mgr.y (mgr.14148) 79 : cephadm [INF] Deploying daemon osd.2 on smithi012 2023-10-09T09:45:27.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:25.851199+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:45:27.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:25.851911+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:27.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:25.852518+0000 mgr.y (mgr.14148) 79 : cephadm [INF] Deploying daemon osd.2 on smithi012 2023-10-09T09:45:27.960 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:26.763431+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:27.960 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:26.763431+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:26.763431+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:30.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:28.763706+0000 mgr.y (mgr.14148) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:30.119 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:28.763706+0000 mgr.y (mgr.14148) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:28.763706+0000 mgr.y (mgr.14148) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:31.254 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:30.161002+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:31.254 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:30.161224+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:31.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:30.161002+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:31.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:30.161224+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:31.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:30.161002+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:31.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:30.161224+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:32.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:30.763979+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:32.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:30.763979+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:32.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:30.763979+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:32.696 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 2 on host 'smithi012' 2023-10-09T09:45:33.062 DEBUG:teuthology.orchestra.run.smithi012:osd.2> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T09:45:33.064 INFO:tasks.cephadm:Deploying osd.3 on smithi012 with /dev/vg_nvme/lv_1... 2023-10-09T09:45:33.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T09:45:33.099 INFO:journalctl@ceph.osd.2.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:45:33.427 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[114661]: debug 2023-10-09T09:45:33.269+0000 7f3823fc3200 -1 osd.2 0 log_to_monitors {default=true} 2023-10-09T09:45:33.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:32.696311+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:33.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:32.747789+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:33.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:32.766530+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:33.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:33.274048+0000 mon.a (mon.0) 304 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:45:33.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:32.696311+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:33.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:32.747789+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:33.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:32.766530+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:33.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:33.274048+0000 mon.a (mon.0) 304 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:45:34.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:32.696311+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:32.747789+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:32.766530+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:33.274048+0000 mon.a (mon.0) 304 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:45:35.107 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T09:45:35.107 INFO:teuthology.orchestra.run.smithi012.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T09:45:35.107 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10+0 records in 2023-10-09T09:45:35.107 INFO:teuthology.orchestra.run.smithi012.stderr:10+0 records out 2023-10-09T09:45:35.107 INFO:teuthology.orchestra.run.smithi012.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0158022 s, 664 MB/s 2023-10-09T09:45:35.107 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping successful for: 2023-10-09T09:45:35.140 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[114661]: debug 2023-10-09T09:45:34.754+0000 7f381a544700 -1 osd.2 0 waiting for initial osdmap 2023-10-09T09:45:35.140 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[114661]: debug 2023-10-09T09:45:34.759+0000 7f3812cb8700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:45:35.140 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:33.751581+0000 mon.a (mon.0) 305 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:45:35.141 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:33.751665+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T09:45:35.141 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:33.752041+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:35.141 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:33.752456+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:35.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:33.751581+0000 mon.a (mon.0) 305 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:45:35.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:33.751665+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T09:45:35.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:33.752041+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:35.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:33.752456+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:35.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:33.751581+0000 mon.a (mon.0) 305 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:45:35.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:33.751665+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T09:45:35.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:33.752041+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:35.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:33.752456+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:35.459 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi012:vg_nvme/lv_1 2023-10-09T09:45:36.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:34.752522+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:36.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:34.752619+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T09:45:36.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:34.752954+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:34.756404+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:34.766817+0000 mgr.y (mgr.14148) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:36.008 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:34.752522+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:36.008 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:34.752619+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T09:45:36.009 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:34.752954+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.009 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:34.756404+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.009 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:34.766817+0000 mgr.y (mgr.14148) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:36.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:34.752522+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:36.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:34.752619+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T09:45:36.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:34.752954+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:34.756404+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:34.766817+0000 mgr.y (mgr.14148) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T09:45:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:34.267310+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:34.267408+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:35.758124+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:35.762285+0000 mon.a (mon.0) 314 : cluster [INF] osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756] boot 2023-10-09T09:45:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:35.762324+0000 mon.a (mon.0) 315 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T09:45:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:35.762583+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:35.762933+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:45:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:36.124703+0000 mgr.y (mgr.14148) 85 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.223215+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.224291+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.225185+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.225931+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:36.226474+0000 mgr.y (mgr.14148) 86 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 5383M 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.231514+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.232749+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.233513+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.239869+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.242540+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:36.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.244156+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:36.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.244805+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:36.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.250038+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:36.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:34.267310+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:36.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:34.267408+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:36.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:35.758124+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:35.762285+0000 mon.a (mon.0) 314 : cluster [INF] osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756] boot 2023-10-09T09:45:36.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:35.762324+0000 mon.a (mon.0) 315 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T09:45:36.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:35.762583+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:35.762933+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:36.124703+0000 mgr.y (mgr.14148) 85 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.223215+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.224291+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.225185+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.225931+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:36.226474+0000 mgr.y (mgr.14148) 86 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 5383M 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.231514+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.232749+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.233513+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:36.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.239869+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:36.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.242540+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:36.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.244156+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:36.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.244805+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:36.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.250038+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:34.267310+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:34.267408+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:35.758124+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:35.762285+0000 mon.a (mon.0) 314 : cluster [INF] osd.2 [v2:172.21.15.12:6818/979823756,v1:172.21.15.12:6819/979823756] boot 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:35.762324+0000 mon.a (mon.0) 315 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:35.762583+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:35.762933+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:36.124703+0000 mgr.y (mgr.14148) 85 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:37.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.223215+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.224291+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.225185+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.225931+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:36.226474+0000 mgr.y (mgr.14148) 86 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 5383M 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.231514+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.232749+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.233513+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.239869+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.242540+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.244156+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:37.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.244805+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:37.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.250038+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:38.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:36.767258+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v64: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:38.351 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.768041+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T09:45:38.351 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:36.768121+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T09:45:38.351 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:36.769254+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:45:38.351 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:37.033015+0000 mgr.y (mgr.14148) 88 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:38.352 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:37.035607+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:38.352 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:37.037862+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:38.352 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:37.038465+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:38.352 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:36.767258+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v64: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:38.352 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.768041+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T09:45:38.352 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:36.768121+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T09:45:38.352 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:36.769254+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:45:38.352 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:37.033015+0000 mgr.y (mgr.14148) 88 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:38.354 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:37.035607+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:38.354 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:37.037862+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:38.354 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:37.038465+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:38.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:36.767258+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v64: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:38.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.768041+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T09:45:38.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:36.768121+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T09:45:38.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:36.769254+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:45:38.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:37.033015+0000 mgr.y (mgr.14148) 88 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:38.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:37.035607+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:38.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:37.037862+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:38.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:37.038465+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:39.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:37.772957+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:45:39.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:37.773034+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T09:45:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:37.772957+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:45:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:37.773034+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T09:45:39.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:37.772957+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:45:39.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:37.773034+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T09:45:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:38.767591+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v66: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:38.781657+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T09:45:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:39.487204+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.12:0/1083228837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]: dispatch 2023-10-09T09:45:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:39.487567+0000 mon.a (mon.0) 339 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]: dispatch 2023-10-09T09:45:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:39.494069+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]': finished 2023-10-09T09:45:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:39.494115+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T09:45:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:39.494239+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:40.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:38.767591+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v66: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:40.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:38.781657+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T09:45:40.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:39.487204+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.12:0/1083228837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]: dispatch 2023-10-09T09:45:40.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:39.487567+0000 mon.a (mon.0) 339 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]: dispatch 2023-10-09T09:45:40.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:39.494069+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]': finished 2023-10-09T09:45:40.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:39.494115+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T09:45:40.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:39.494239+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:40.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:38.767591+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v66: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:40.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:38.781657+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T09:45:40.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:39.487204+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.12:0/1083228837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]: dispatch 2023-10-09T09:45:40.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:39.487567+0000 mon.a (mon.0) 339 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]: dispatch 2023-10-09T09:45:40.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:39.494069+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bc089815-f2a8-41ef-9b69-505b7b51963e"}]': finished 2023-10-09T09:45:40.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:39.494115+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T09:45:40.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:39.494239+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:41.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:40.104723+0000 mon.a (mon.0) 343 : audit [DBG] from='client.? 172.21.15.12:0/1641883989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:41.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:40.104723+0000 mon.a (mon.0) 343 : audit [DBG] from='client.? 172.21.15.12:0/1641883989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:41.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:40.104723+0000 mon.a (mon.0) 343 : audit [DBG] from='client.? 172.21.15.12:0/1641883989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:40.768064+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v69: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:42.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:40.768064+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v69: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:40.768064+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v69: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:43.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:42.768392+0000 mgr.y (mgr.14148) 91 : cluster [DBG] pgmap v70: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:43.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:42.768392+0000 mgr.y (mgr.14148) 91 : cluster [DBG] pgmap v70: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:43.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:42.768392+0000 mgr.y (mgr.14148) 91 : cluster [DBG] pgmap v70: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:45.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:44.768672+0000 mgr.y (mgr.14148) 92 : cluster [DBG] pgmap v71: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:45.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:45.052326+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:45:45.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:45.052995+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:45.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:45.053558+0000 mgr.y (mgr.14148) 93 : cephadm [INF] Deploying daemon osd.3 on smithi012 2023-10-09T09:45:45.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:44.768672+0000 mgr.y (mgr.14148) 92 : cluster [DBG] pgmap v71: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:45.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:45.052326+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:45:45.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:45.052995+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:45.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:45.053558+0000 mgr.y (mgr.14148) 93 : cephadm [INF] Deploying daemon osd.3 on smithi012 2023-10-09T09:45:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:44.768672+0000 mgr.y (mgr.14148) 92 : cluster [DBG] pgmap v71: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:45.052326+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:45:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:45.052995+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:46.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:45.053558+0000 mgr.y (mgr.14148) 93 : cephadm [INF] Deploying daemon osd.3 on smithi012 2023-10-09T09:45:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:46.769173+0000 mgr.y (mgr.14148) 94 : cluster [DBG] pgmap v72: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:48.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:46.769173+0000 mgr.y (mgr.14148) 94 : cluster [DBG] pgmap v72: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:48.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:47 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:46.769173+0000 mgr.y (mgr.14148) 94 : cluster [DBG] pgmap v72: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:50.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:48.769690+0000 mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:50.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:49.066039+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:50.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:49.091429+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:50.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:48.769690+0000 mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:50.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:49.066039+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:50.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:49.091429+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:50.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:48.769690+0000 mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:50.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:49.066039+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:50.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:49.091429+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:51.711 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 3 on host 'smithi012' 2023-10-09T09:45:52.166 DEBUG:teuthology.orchestra.run.smithi012:osd.3> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T09:45:52.168 INFO:tasks.cephadm:Deploying osd.4 on smithi110 with /dev/vg_nvme/lv_4... 2023-10-09T09:45:52.168 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T09:45:52.197 INFO:journalctl@ceph.osd.3.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:45:52.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:45:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[118796]: debug 2023-10-09T09:45:52.273+0000 7efdbb3bf200 -1 osd.3 0 log_to_monitors {default=true} 2023-10-09T09:45:52.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:50.770016+0000 mgr.y (mgr.14148) 96 : cluster [DBG] pgmap v74: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:52.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:51.710663+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:52.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:51.844543+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:50.770016+0000 mgr.y (mgr.14148) 96 : cluster [DBG] pgmap v74: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:51.710663+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:51.844543+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:52.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:50.770016+0000 mgr.y (mgr.14148) 96 : cluster [DBG] pgmap v74: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:52.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:51.710663+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:51.844543+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:53.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:52.278124+0000 mon.a (mon.0) 350 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:45:53.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:52.278124+0000 mon.a (mon.0) 350 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:45:53.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:52.278124+0000 mon.a (mon.0) 350 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:45:53.960 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T09:45:53.960 INFO:teuthology.orchestra.run.smithi110.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T09:45:53.960 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10+0 records in 2023-10-09T09:45:53.960 INFO:teuthology.orchestra.run.smithi110.stderr:10+0 records out 2023-10-09T09:45:53.960 INFO:teuthology.orchestra.run.smithi110.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00816217 s, 1.3 GB/s 2023-10-09T09:45:53.960 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping successful for: 2023-10-09T09:45:54.338 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:52.770297+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:54.338 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:53.075138+0000 mon.a (mon.0) 351 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:45:54.338 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:53.075187+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T09:45:54.338 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:53.075318+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:54.338 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:53.076062+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:54.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi110:vg_nvme/lv_4 2023-10-09T09:45:54.423 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:52.770297+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:54.423 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:53.075138+0000 mon.a (mon.0) 351 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:45:54.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:53.075187+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T09:45:54.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:53.075318+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:54.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:53.076062+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:54.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:52.770297+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:54.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:53.075138+0000 mon.a (mon.0) 351 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:45:54.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:53.075187+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T09:45:54.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:53.075318+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:54.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:53.076062+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:45:54.425 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[118796]: debug 2023-10-09T09:45:54.080+0000 7efdb1940700 -1 osd.3 0 waiting for initial osdmap 2023-10-09T09:45:54.425 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:45:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[118796]: debug 2023-10-09T09:45:54.088+0000 7efdae0bc700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.077966+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:54.078032+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.078413+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.083593+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.755979+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.756942+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.757584+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.758176+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.348 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.758766+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.763363+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.764246+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:55.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.764886+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:55.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.768745+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.771150+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:55.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.772372+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:55.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.772967+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:55.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:54.776658+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.077966+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:54.078032+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T09:45:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.078413+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.083593+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.755979+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.756942+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.757584+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.758176+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.758766+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.763363+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.764246+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.764886+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.768745+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.771150+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.772372+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.772967+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:54.776658+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.077966+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:54.078032+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.078413+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.083593+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.755979+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.756942+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.757584+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.758176+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.758766+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.763363+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.764246+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:55.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.764886+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:55.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.768745+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:55.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.771150+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:45:55.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.772372+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:55.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.772967+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:45:55.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:54.776658+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:45:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:53.249512+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:53.249586+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:54.750636+0000 mgr.y (mgr.14148) 98 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:45:54.759227+0000 mgr.y (mgr.14148) 99 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 4037M 2023-10-09T09:45:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:54.770524+0000 mgr.y (mgr.14148) 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:55.081528+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:55.084210+0000 mon.a (mon.0) 373 : cluster [INF] osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626] boot 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:55.084241+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:55.084663+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:55.955651+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:55.958659+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:55.959497+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:53.249512+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:53.249586+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:54.750636+0000 mgr.y (mgr.14148) 98 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:56.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:45:54.759227+0000 mgr.y (mgr.14148) 99 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 4037M 2023-10-09T09:45:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:54.770524+0000 mgr.y (mgr.14148) 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:55.081528+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:55.084210+0000 mon.a (mon.0) 373 : cluster [INF] osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626] boot 2023-10-09T09:45:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:55.084241+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T09:45:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:55.084663+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:55.955651+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:55.958659+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:55.959497+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:56.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:53.249512+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:45:56.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:53.249586+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:45:56.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:54.750636+0000 mgr.y (mgr.14148) 98 : cephadm [INF] Detected new or changed devices on smithi012 2023-10-09T09:45:56.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:45:54.759227+0000 mgr.y (mgr.14148) 99 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 4037M 2023-10-09T09:45:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:54.770524+0000 mgr.y (mgr.14148) 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T09:45:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:55.081528+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:55.084210+0000 mon.a (mon.0) 373 : cluster [INF] osd.3 [v2:172.21.15.12:6826/715124626,v1:172.21.15.12:6827/715124626] boot 2023-10-09T09:45:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:55.084241+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T09:45:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:55.084663+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:45:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:55.955651+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:45:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:55.958659+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:45:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:55.959497+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:45:57.310 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:55.952218+0000 mgr.y (mgr.14148) 101 : audit [DBG] from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:57.311 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:56.093983+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T09:45:57.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:55.952218+0000 mgr.y (mgr.14148) 101 : audit [DBG] from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:57.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:56.093983+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T09:45:57.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:55.952218+0000 mgr.y (mgr.14148) 101 : audit [DBG] from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:45:57.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:56.093983+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T09:45:58.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:56.770898+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:45:58.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:56.770898+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:45:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:56.770898+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:45:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:58.166592+0000 mon.b (mon.1) 3 : audit [INF] from='client.? 172.21.15.110:0/3633488526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]: dispatch 2023-10-09T09:45:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:58.167142+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]: dispatch 2023-10-09T09:45:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:58.172769+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]': finished 2023-10-09T09:45:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:58.172865+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T09:45:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:58.173063+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:45:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:45:58.724970+0000 mon.b (mon.1) 4 : audit [DBG] from='client.? 172.21.15.110:0/280288821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:59.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:58.166592+0000 mon.b (mon.1) 3 : audit [INF] from='client.? 172.21.15.110:0/3633488526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]: dispatch 2023-10-09T09:45:59.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:58.167142+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]: dispatch 2023-10-09T09:45:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:58.172769+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]': finished 2023-10-09T09:45:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:58.172865+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T09:45:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:58.173063+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:45:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:45:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:45:58.724970+0000 mon.b (mon.1) 4 : audit [DBG] from='client.? 172.21.15.110:0/280288821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:45:59.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:58.166592+0000 mon.b (mon.1) 3 : audit [INF] from='client.? 172.21.15.110:0/3633488526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]: dispatch 2023-10-09T09:45:59.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:58.167142+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]: dispatch 2023-10-09T09:45:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:58.172769+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad0b6f60-bf01-465b-acee-565d4d4a3df5"}]': finished 2023-10-09T09:45:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:58.172865+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T09:45:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:58.173063+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:45:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:45:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:45:58.724970+0000 mon.b (mon.1) 4 : audit [DBG] from='client.? 172.21.15.110:0/280288821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:00.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:45:58.771336+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:00.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:45:58.771336+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:00.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:45:58.771336+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:02.398 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:00.771813+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:02.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:00.771813+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:02.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:00.771813+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:03.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:02.772259+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:03.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:02.772259+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:03.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:02.772259+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:03.645712+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:46:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:03.646696+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:03.647549+0000 mgr.y (mgr.14148) 106 : cephadm [INF] Deploying daemon osd.4 on smithi110 2023-10-09T09:46:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:03.645712+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:46:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:03.646696+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:03.647549+0000 mgr.y (mgr.14148) 106 : cephadm [INF] Deploying daemon osd.4 on smithi110 2023-10-09T09:46:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:03.645712+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:46:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:03.646696+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:03.647549+0000 mgr.y (mgr.14148) 106 : cephadm [INF] Deploying daemon osd.4 on smithi110 2023-10-09T09:46:05.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:04.772646+0000 mgr.y (mgr.14148) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:04.772646+0000 mgr.y (mgr.14148) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:05.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:04.772646+0000 mgr.y (mgr.14148) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:08.150 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:06.772961+0000 mgr.y (mgr.14148) 108 : cluster [DBG] pgmap v87: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:08.150 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:07.697517+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:08.150 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:07.725277+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:08.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:06.772961+0000 mgr.y (mgr.14148) 108 : cluster [DBG] pgmap v87: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:08.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:07.697517+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:08.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:07.725277+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:06.772961+0000 mgr.y (mgr.14148) 108 : cluster [DBG] pgmap v87: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:07.697517+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:07.725277+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:10.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:08.773336+0000 mgr.y (mgr.14148) 109 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:10.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:09.696415+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:10.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:09.697277+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:10.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:09.698257+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:10.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:09.702926+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:10.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:08.773336+0000 mgr.y (mgr.14148) 109 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:10.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:09.696415+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:10.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:09.697277+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:10.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:09.698257+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:10.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:09.702926+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:10.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:08.773336+0000 mgr.y (mgr.14148) 109 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:10.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:09.696415+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:10.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:09.697277+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:10.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:09.698257+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:10.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:09.702926+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:10.615 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 4 on host 'smithi110' 2023-10-09T09:46:11.019 DEBUG:teuthology.orchestra.run.smithi110:osd.4> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T09:46:11.022 INFO:tasks.cephadm:Deploying osd.5 on smithi110 with /dev/vg_nvme/lv_3... 2023-10-09T09:46:11.022 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T09:46:11.055 INFO:journalctl@ceph.osd.4.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:46:11.854 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:10.614862+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:11.854 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:10.621946+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:11.854 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:10.773643+0000 mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:11.855 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:10.892695+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T09:46:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:10.614862+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:10.621946+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:10.773643+0000 mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:10.892695+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T09:46:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:10.614862+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:10.621946+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:10.773643+0000 mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:10.892695+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T09:46:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:11.620807+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T09:46:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:11.620907+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T09:46:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:11.621270+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:11.621952+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:12.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 09:46:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[101265]: debug 2023-10-09T09:46:12.634+0000 7fd30a987700 -1 osd.4 0 waiting for initial osdmap 2023-10-09T09:46:12.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 09:46:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[101265]: debug 2023-10-09T09:46:12.640+0000 7fd305900700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:46:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:11.620807+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T09:46:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:11.620907+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T09:46:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:11.621270+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:11.621952+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:11.620807+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T09:46:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:11.620907+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T09:46:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:11.621270+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:11.621952+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:13.335 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T09:46:13.335 INFO:teuthology.orchestra.run.smithi110.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T09:46:13.336 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10+0 records in 2023-10-09T09:46:13.336 INFO:teuthology.orchestra.run.smithi110.stderr:10+0 records out 2023-10-09T09:46:13.336 INFO:teuthology.orchestra.run.smithi110.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00957175 s, 1.1 GB/s 2023-10-09T09:46:13.336 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 2023-10-09T09:46:13.336 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping successful for: 2023-10-09T09:46:13.893 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:12.630566+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:13.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:12.630745+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T09:46:13.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:12.631411+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:12.637862+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:12.774088+0000 mgr.y (mgr.14148) 111 : cluster [DBG] pgmap v92: 1 pgs: 1 peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:13.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:13.628636+0000 mon.a (mon.0) 403 : cluster [INF] osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753] boot 2023-10-09T09:46:13.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:13.628675+0000 mon.a (mon.0) 404 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T09:46:13.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:13.629159+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:12.630566+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:12.630745+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T09:46:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:12.631411+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:12.637862+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:12.774088+0000 mgr.y (mgr.14148) 111 : cluster [DBG] pgmap v92: 1 pgs: 1 peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:13.628636+0000 mon.a (mon.0) 403 : cluster [INF] osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753] boot 2023-10-09T09:46:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:13.628675+0000 mon.a (mon.0) 404 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:13.629159+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:12.630566+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:12.630745+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:12.631411+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:12.637862+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:12.774088+0000 mgr.y (mgr.14148) 111 : cluster [DBG] pgmap v92: 1 pgs: 1 peering; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:13.628636+0000 mon.a (mon.0) 403 : cluster [INF] osd.4 [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753] boot 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:13.628675+0000 mon.a (mon.0) 404 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T09:46:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:13.629159+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:46:13.930 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi110:vg_nvme/lv_3 2023-10-09T09:46:14.788 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:11.929955+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:14.788 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:11.930002+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:14.789 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:14.104595+0000 mgr.y (mgr.14148) 112 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:14.789 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:14.110488+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.789 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:14.578940+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:14.789 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:14.586112+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.789 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:14.587218+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:14.789 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:14.588102+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:14.789 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:14.593000+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.789 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:14.627213+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T09:46:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:11.929955+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:11.930002+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:14.104595+0000 mgr.y (mgr.14148) 112 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:14.110488+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:14.578940+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:14.586112+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:14.587218+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:14.588102+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:14.593000+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:14.627213+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:11.929955+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:11.930002+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:14.104595+0000 mgr.y (mgr.14148) 112 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:14.110488+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:14.578940+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:14.586112+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:14.587218+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:14.588102+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:14.593000+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:14.627213+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T09:46:15.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:14.579697+0000 mgr.y (mgr.14148) 113 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 17168M 2023-10-09T09:46:15.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:14.774576+0000 mgr.y (mgr.14148) 114 : cluster [DBG] pgmap v95: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:15.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:15.631599+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T09:46:15.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:14.579697+0000 mgr.y (mgr.14148) 113 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 17168M 2023-10-09T09:46:15.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:14.774576+0000 mgr.y (mgr.14148) 114 : cluster [DBG] pgmap v95: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:15.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:15.631599+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T09:46:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:14.579697+0000 mgr.y (mgr.14148) 113 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 17168M 2023-10-09T09:46:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:14.774576+0000 mgr.y (mgr.14148) 114 : cluster [DBG] pgmap v95: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:15.631599+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T09:46:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:15.667135+0000 mgr.y (mgr.14148) 115 : audit [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:15.670043+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:15.672828+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:15.673652+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:16.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:15.667135+0000 mgr.y (mgr.14148) 115 : audit [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:15.670043+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:15.672828+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:15.673652+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:16.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:15.667135+0000 mgr.y (mgr.14148) 115 : audit [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:16.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:15.670043+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:16.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:15.672828+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:16.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:15.673652+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:17.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:16.775095+0000 mgr.y (mgr.14148) 116 : cluster [DBG] pgmap v97: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:17.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:16.775095+0000 mgr.y (mgr.14148) 116 : cluster [DBG] pgmap v97: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:16.775095+0000 mgr.y (mgr.14148) 116 : cluster [DBG] pgmap v97: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:17.763247+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.110:0/3287610673' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]: dispatch 2023-10-09T09:46:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:17.763800+0000 mon.a (mon.0) 417 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]: dispatch 2023-10-09T09:46:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:17.768379+0000 mon.a (mon.0) 418 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]': finished 2023-10-09T09:46:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:17.768492+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T09:46:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:17.768674+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:18.354699+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.110:0/1498895159' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:17.763247+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.110:0/3287610673' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]: dispatch 2023-10-09T09:46:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:17.763800+0000 mon.a (mon.0) 417 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]: dispatch 2023-10-09T09:46:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:17.768379+0000 mon.a (mon.0) 418 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]': finished 2023-10-09T09:46:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:17.768492+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T09:46:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:17.768674+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:18.354699+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.110:0/1498895159' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:17.763247+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.110:0/3287610673' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]: dispatch 2023-10-09T09:46:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:17.763800+0000 mon.a (mon.0) 417 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]: dispatch 2023-10-09T09:46:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:17.768379+0000 mon.a (mon.0) 418 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63ba8578-42c2-4852-acac-e616de4ccd47"}]': finished 2023-10-09T09:46:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:17.768492+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T09:46:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:17.768674+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:18.354699+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.110:0/1498895159' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:19.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:18.775585+0000 mgr.y (mgr.14148) 117 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:19.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:18.775585+0000 mgr.y (mgr.14148) 117 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:18.775585+0000 mgr.y (mgr.14148) 117 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:21.979 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:20.775988+0000 mgr.y (mgr.14148) 118 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:22.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:20.775988+0000 mgr.y (mgr.14148) 118 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:22.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:20.775988+0000 mgr.y (mgr.14148) 118 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:23.899 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:22.776459+0000 mgr.y (mgr.14148) 119 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:23.899 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:23.140653+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:46:23.899 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:23.141611+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:23.899 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:23.142505+0000 mgr.y (mgr.14148) 120 : cephadm [INF] Deploying daemon osd.5 on smithi110 2023-10-09T09:46:23.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:22.776459+0000 mgr.y (mgr.14148) 119 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:23.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:23.140653+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:46:23.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:23.141611+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:23.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:23.142505+0000 mgr.y (mgr.14148) 120 : cephadm [INF] Deploying daemon osd.5 on smithi110 2023-10-09T09:46:23.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:22.776459+0000 mgr.y (mgr.14148) 119 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:23.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:23.140653+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:46:23.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:23.141611+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:23.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:23.142505+0000 mgr.y (mgr.14148) 120 : cephadm [INF] Deploying daemon osd.5 on smithi110 2023-10-09T09:46:26.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:24.776916+0000 mgr.y (mgr.14148) 121 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:26.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:24.776916+0000 mgr.y (mgr.14148) 121 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:24.776916+0000 mgr.y (mgr.14148) 121 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:28.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:26.777365+0000 mgr.y (mgr.14148) 122 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:27.108653+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:27.143672+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:26.777365+0000 mgr.y (mgr.14148) 122 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:27.108653+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:27.143672+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:28.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:26.777365+0000 mgr.y (mgr.14148) 122 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:28.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:27.108653+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:28.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:27.143672+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:29.628 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 5 on host 'smithi110' 2023-10-09T09:46:30.000 DEBUG:teuthology.orchestra.run.smithi110:osd.5> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T09:46:30.003 INFO:tasks.cephadm:Deploying osd.6 on smithi110 with /dev/vg_nvme/lv_2... 2023-10-09T09:46:30.003 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T09:46:30.031 INFO:journalctl@ceph.osd.5.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:46:30.328 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:28.777834+0000 mgr.y (mgr.14148) 123 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:30.328 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:29.167672+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.328 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:29.168607+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:30.328 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:29.169569+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:30.329 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:29.177071+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.329 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:29.627230+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.329 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:29.644644+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:30.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:28.777834+0000 mgr.y (mgr.14148) 123 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:30.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:29.167672+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:29.168607+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:29.169569+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:29.177071+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:29.627230+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:29.644644+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:28.777834+0000 mgr.y (mgr.14148) 123 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:29.167672+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:29.168607+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:30.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:29.169569+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:30.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:29.177071+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:29.627230+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:30.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:29.644644+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:30.627 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 09:46:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T09:46:30.326+0000 7f2249a5a200 -1 osd.5 0 log_to_monitors {default=true} 2023-10-09T09:46:31.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:30.331873+0000 mon.b (mon.1) 7 : audit [INF] from='osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T09:46:31.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:30.332267+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T09:46:31.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:30.331873+0000 mon.b (mon.1) 7 : audit [INF] from='osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T09:46:31.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:30.332267+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T09:46:31.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:30.331873+0000 mon.b (mon.1) 7 : audit [INF] from='osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T09:46:31.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:30.332267+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T09:46:32.221 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T09:46:32.221 INFO:teuthology.orchestra.run.smithi110.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T09:46:32.221 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10+0 records in 2023-10-09T09:46:32.221 INFO:teuthology.orchestra.run.smithi110.stderr:10+0 records out 2023-10-09T09:46:32.221 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00996821 s, 1.1 GB/s 2023-10-09T09:46:32.221 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping successful for: 2023-10-09T09:46:32.442 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 09:46:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T09:46:32.182+0000 7f223ffdb700 -1 osd.5 0 waiting for initial osdmap 2023-10-09T09:46:32.443 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 09:46:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T09:46:32.187+0000 7f223bf56700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:46:32.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:30.778261+0000 mgr.y (mgr.14148) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:32.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:31.176787+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T09:46:32.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:31.176875+0000 mon.a (mon.0) 433 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T09:46:32.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:31.177028+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:32.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:31.177459+0000 mon.b (mon.1) 8 : audit [INF] from='osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:32.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:31.177906+0000 mon.a (mon.0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:32.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:30.778261+0000 mgr.y (mgr.14148) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:32.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:31.176787+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T09:46:32.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:31.176875+0000 mon.a (mon.0) 433 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T09:46:32.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:31.177028+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:32.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:31.177459+0000 mon.b (mon.1) 8 : audit [INF] from='osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:32.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:31.177906+0000 mon.a (mon.0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:32.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:30.778261+0000 mgr.y (mgr.14148) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:32.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:31.176787+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T09:46:32.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:31.176875+0000 mon.a (mon.0) 433 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T09:46:32.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:31.177028+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:32.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:31.177459+0000 mon.b (mon.1) 8 : audit [INF] from='osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:32.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:31.177906+0000 mon.a (mon.0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:32.763 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi110:vg_nvme/lv_2 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:32.177450+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:32.177599+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:32.178102+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:32.183866+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:32.640635+0000 mon.a (mon.0) 440 : cluster [INF] osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369] boot 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:32.640721+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:32.641108+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:33.122613+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:33.123713+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:33.124717+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:33.129841+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:33.130891+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:33.131756+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:33.136516+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:33.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:32.177450+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:33.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:32.177599+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T09:46:33.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:32.178102+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:32.183866+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:32.640635+0000 mon.a (mon.0) 440 : cluster [INF] osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369] boot 2023-10-09T09:46:33.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:32.640721+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T09:46:33.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:32.641108+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:33.122613+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:33.123713+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:33.124717+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:33.129841+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:33.130891+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:33.131756+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:33.136516+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:32.177450+0000 mon.a (mon.0) 436 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:32.177599+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T09:46:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:32.178102+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:32.183866+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:32.640635+0000 mon.a (mon.0) 440 : cluster [INF] osd.5 [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369] boot 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:32.640721+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:32.641108+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:33.122613+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:33.123713+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:33.124717+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:33.129841+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:33.130891+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:33.131756+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:33.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:33.136516+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:34.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:31.363972+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:34.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:31.364014+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:32.778736+0000 mgr.y (mgr.14148) 125 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:33.115943+0000 mgr.y (mgr.14148) 126 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:33.125433+0000 mgr.y (mgr.14148) 127 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 8584M 2023-10-09T09:46:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:33.649780+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T09:46:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:31.363972+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:31.364014+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:32.778736+0000 mgr.y (mgr.14148) 125 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:33.115943+0000 mgr.y (mgr.14148) 126 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:33.125433+0000 mgr.y (mgr.14148) 127 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 8584M 2023-10-09T09:46:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:33.649780+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T09:46:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:31.363972+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:31.364014+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:32.778736+0000 mgr.y (mgr.14148) 125 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T09:46:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:33.115943+0000 mgr.y (mgr.14148) 126 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:33.125433+0000 mgr.y (mgr.14148) 127 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 8584M 2023-10-09T09:46:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:33.649780+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T09:46:35.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:34.307954+0000 mgr.y (mgr.14148) 128 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:35.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:34.310973+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:35.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:34.313866+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:35.378 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:34.314745+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:35.378 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:34.653306+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T09:46:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:34.307954+0000 mgr.y (mgr.14148) 128 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:34.310973+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:34.313866+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:34.314745+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:34.653306+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T09:46:35.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:34.307954+0000 mgr.y (mgr.14148) 128 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:34.310973+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:34.313866+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:34.314745+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:34.653306+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T09:46:36.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:34.779187+0000 mgr.y (mgr.14148) 129 : cluster [DBG] pgmap v112: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:36.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:34.779187+0000 mgr.y (mgr.14148) 129 : cluster [DBG] pgmap v112: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:36.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:34.779187+0000 mgr.y (mgr.14148) 129 : cluster [DBG] pgmap v112: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:37.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:36.479410+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.110:0/2756151316' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]: dispatch 2023-10-09T09:46:37.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:36.479872+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]: dispatch 2023-10-09T09:46:37.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:36.485201+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]': finished 2023-10-09T09:46:37.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:36.485299+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T09:46:37.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:36.485477+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:37.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:37.071550+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.110:0/2874362567' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:36.479410+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.110:0/2756151316' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]: dispatch 2023-10-09T09:46:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:36.479872+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]: dispatch 2023-10-09T09:46:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:36.485201+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]': finished 2023-10-09T09:46:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:36.485299+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T09:46:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:36.485477+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:37.071550+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.110:0/2874362567' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:36.479410+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.110:0/2756151316' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]: dispatch 2023-10-09T09:46:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:36.479872+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]: dispatch 2023-10-09T09:46:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:36.485201+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "00113d3e-cc98-49fe-ba99-7172c2c8c80b"}]': finished 2023-10-09T09:46:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:36.485299+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T09:46:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:36.485477+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:37.071550+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.110:0/2874362567' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:38.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:36.779719+0000 mgr.y (mgr.14148) 130 : cluster [DBG] pgmap v114: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:38.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:36.779719+0000 mgr.y (mgr.14148) 130 : cluster [DBG] pgmap v114: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:38.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:36.779719+0000 mgr.y (mgr.14148) 130 : cluster [DBG] pgmap v114: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:40.590 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:38.780153+0000 mgr.y (mgr.14148) 131 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:40.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:38.780153+0000 mgr.y (mgr.14148) 131 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:40.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:38.780153+0000 mgr.y (mgr.14148) 131 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:40.780677+0000 mgr.y (mgr.14148) 132 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:42.097209+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:46:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:42.098171+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:42.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:40.780677+0000 mgr.y (mgr.14148) 132 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:42.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:42.097209+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:46:42.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:42.098171+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:40.780677+0000 mgr.y (mgr.14148) 132 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:42.097209+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:46:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:42.098171+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:43.441 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:42.098969+0000 mgr.y (mgr.14148) 133 : cephadm [INF] Deploying daemon osd.6 on smithi110 2023-10-09T09:46:43.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:42.098969+0000 mgr.y (mgr.14148) 133 : cephadm [INF] Deploying daemon osd.6 on smithi110 2023-10-09T09:46:43.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:42.098969+0000 mgr.y (mgr.14148) 133 : cephadm [INF] Deploying daemon osd.6 on smithi110 2023-10-09T09:46:44.351 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:42.780979+0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:44.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:42.780979+0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:44.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:42.780979+0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:46.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:44.781399+0000 mgr.y (mgr.14148) 135 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:46.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:46.051097+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:46.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:46.087637+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:46.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:44.781399+0000 mgr.y (mgr.14148) 135 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:46.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:46.051097+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:46.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:46.087637+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:46.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:44.781399+0000 mgr.y (mgr.14148) 135 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:46.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:46.051097+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:46.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:46.087637+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:48.517 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:46.781856+0000 mgr.y (mgr.14148) 136 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:48.517 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:48.161048+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:48.517 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:48.161943+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:48.517 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:48.163026+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:48.517 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:48.170468+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:46.781856+0000 mgr.y (mgr.14148) 136 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:48.161048+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:48.161943+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:48.163026+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:48.170468+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:48.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:46.781856+0000 mgr.y (mgr.14148) 136 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:48.161048+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:48.161943+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:48.163026+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:48.170468+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:48.766 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 6 on host 'smithi110' 2023-10-09T09:46:49.139 DEBUG:teuthology.orchestra.run.smithi110:osd.6> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T09:46:49.141 INFO:tasks.cephadm:Deploying osd.7 on smithi110 with /dev/vg_nvme/lv_1... 2023-10-09T09:46:49.141 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T09:46:49.170 INFO:journalctl@ceph.osd.6.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:46:49.442 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 09:46:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T09:46:49.269+0000 7f7cbfd39200 -1 osd.6 0 log_to_monitors {default=true} 2023-10-09T09:46:50.150 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:48.765519+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:50.151 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:48.782378+0000 mgr.y (mgr.14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:50.151 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:48.788737+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:50.151 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:49.272846+0000 mon.b (mon.1) 11 : audit [INF] from='osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T09:46:50.151 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:49.273205+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T09:46:50.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:48.765519+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:50.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:48.782378+0000 mgr.y (mgr.14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:50.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:48.788737+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:50.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:49.272846+0000 mon.b (mon.1) 11 : audit [INF] from='osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T09:46:50.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:49.273205+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T09:46:50.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:48.765519+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:50.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:48.782378+0000 mgr.y (mgr.14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:50.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:48.788737+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:46:50.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:49.272846+0000 mon.b (mon.1) 11 : audit [INF] from='osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T09:46:50.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:49.273205+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T09:46:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:49.773089+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T09:46:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:49.773170+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T09:46:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:49.773487+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:49.773705+0000 mon.b (mon.1) 12 : audit [INF] from='osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:49.774224+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:49.773089+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T09:46:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:49.773170+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T09:46:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:49.773487+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:49.773705+0000 mon.b (mon.1) 12 : audit [INF] from='osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:49.774224+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:49.773089+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T09:46:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:49.773170+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T09:46:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:49.773487+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:49.773705+0000 mon.b (mon.1) 12 : audit [INF] from='osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:49.774224+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:46:51.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 09:46:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T09:46:50.781+0000 7f7cb62ba700 -1 osd.6 0 waiting for initial osdmap 2023-10-09T09:46:51.194 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 09:46:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T09:46:50.787+0000 7f7cb2a36700 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:46:51.255 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T09:46:51.255 INFO:teuthology.orchestra.run.smithi110.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T09:46:51.255 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10+0 records in 2023-10-09T09:46:51.255 INFO:teuthology.orchestra.run.smithi110.stderr:10+0 records out 2023-10-09T09:46:51.255 INFO:teuthology.orchestra.run.smithi110.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00946123 s, 1.1 GB/s 2023-10-09T09:46:51.255 INFO:teuthology.orchestra.run.smithi110.stderr:--> Zapping successful for: 2023-10-09T09:46:51.672 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi110:vg_nvme/lv_1 2023-10-09T09:46:51.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:50.778012+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:51.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:50.778137+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T09:46:51.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:50.778605+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:51.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:50.782501+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:51.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:50.782723+0000 mgr.y (mgr.14148) 138 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:51.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:51.776492+0000 mon.a (mon.0) 478 : cluster [INF] osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135] boot 2023-10-09T09:46:51.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:51.776554+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T09:46:51.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:51.776739+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:52.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:50.778012+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:50.778137+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T09:46:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:50.778605+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:50.782501+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:50.782723+0000 mgr.y (mgr.14148) 138 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:51.776492+0000 mon.a (mon.0) 478 : cluster [INF] osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135] boot 2023-10-09T09:46:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:51.776554+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T09:46:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:51.776739+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:50.778012+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:46:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:50.778137+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T09:46:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:50.778605+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:50.782501+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:50.782723+0000 mgr.y (mgr.14148) 138 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T09:46:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:51.776492+0000 mon.a (mon.0) 478 : cluster [INF] osd.6 [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135] boot 2023-10-09T09:46:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:51.776554+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T09:46:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:51.776739+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:46:53.046 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:50.227691+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:53.046 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:50.227776+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:53.046 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:52.356704+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.047 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:52.358091+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.047 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:52.359118+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.047 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:52.360078+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.047 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:52.364833+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.047 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:52.365955+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:53.047 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:52.366862+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:53.047 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:52.371441+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.047 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:52.647449+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:50.227691+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:50.227776+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:52.356704+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:52.358091+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:52.359118+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:52.360078+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:52.364833+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:52.365955+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:52.366862+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:52.371441+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:52.647449+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:50.227691+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:50.227776+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:52.356704+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:52.358091+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:52.359118+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:52.360078+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:46:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:52.364833+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:52.365955+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:53.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:52.366862+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:46:53.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:52.371441+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:46:53.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:52.647449+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T09:46:54.138 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:52.351582+0000 mgr.y (mgr.14148) 139 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:54.138 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:46:52.360764+0000 mgr.y (mgr.14148) 140 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 5722M 2023-10-09T09:46:54.139 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:52.783124+0000 mgr.y (mgr.14148) 141 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:46:54.139 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:53.363378+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:54.139 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:53.366345+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:54.139 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:53.367141+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:54.139 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:53.651738+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T09:46:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:52.351582+0000 mgr.y (mgr.14148) 139 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:46:52.360764+0000 mgr.y (mgr.14148) 140 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 5722M 2023-10-09T09:46:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:52.783124+0000 mgr.y (mgr.14148) 141 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:46:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:53.363378+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:53.366345+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:53.367141+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:53.651738+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T09:46:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:52.351582+0000 mgr.y (mgr.14148) 139 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:46:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:46:52.360764+0000 mgr.y (mgr.14148) 140 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 5722M 2023-10-09T09:46:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:52.783124+0000 mgr.y (mgr.14148) 141 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:46:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:53.363378+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T09:46:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:53.366345+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T09:46:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:53.367141+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:46:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:53.651738+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T09:46:55.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:53.360408+0000 mgr.y (mgr.14148) 142 : audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:55.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:53.360408+0000 mgr.y (mgr.14148) 142 : audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:53.360408+0000 mgr.y (mgr.14148) 142 : audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:46:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:54.783635+0000 mgr.y (mgr.14148) 143 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:46:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:55.567117+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.110:0/1256523997' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]: dispatch 2023-10-09T09:46:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:55.567700+0000 mon.a (mon.0) 494 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]: dispatch 2023-10-09T09:46:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:55.575628+0000 mon.a (mon.0) 495 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]': finished 2023-10-09T09:46:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:55.575727+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T09:46:56.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:55.575902+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:46:56.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:54.783635+0000 mgr.y (mgr.14148) 143 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:46:56.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:55.567117+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.110:0/1256523997' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]: dispatch 2023-10-09T09:46:56.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:55.567700+0000 mon.a (mon.0) 494 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]: dispatch 2023-10-09T09:46:56.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:55.575628+0000 mon.a (mon.0) 495 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]': finished 2023-10-09T09:46:56.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:55.575727+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T09:46:56.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:55.575902+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:46:56.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:54.783635+0000 mgr.y (mgr.14148) 143 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:46:56.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:55.567117+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.110:0/1256523997' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]: dispatch 2023-10-09T09:46:56.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:55.567700+0000 mon.a (mon.0) 494 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]: dispatch 2023-10-09T09:46:56.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:55.575628+0000 mon.a (mon.0) 495 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a34ee707-c52a-48c8-be23-cce0e1cc0801"}]': finished 2023-10-09T09:46:56.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:55.575727+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T09:46:56.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:55.575902+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:46:57.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:46:56.148672+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.110:0/4121161940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:57.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:46:56.148672+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.110:0/4121161940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:46:56.148672+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.110:0/4121161940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T09:46:58.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:56.784040+0000 mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:46:58.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:56.784040+0000 mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:46:58.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:56.784040+0000 mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:00.105 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:46:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:46:58.784485+0000 mgr.y (mgr.14148) 145 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:00.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:46:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:46:58.784485+0000 mgr.y (mgr.14148) 145 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:00.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:46:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:46:58.784485+0000 mgr.y (mgr.14148) 145 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:01.915 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:00.784800+0000 mgr.y (mgr.14148) 146 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:01.916 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:01.052512+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:47:01.916 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:01.053562+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:01.916 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:01.054436+0000 mgr.y (mgr.14148) 147 : cephadm [INF] Deploying daemon osd.7 on smithi110 2023-10-09T09:47:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:00.784800+0000 mgr.y (mgr.14148) 146 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:01.052512+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:47:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:01.053562+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:01.054436+0000 mgr.y (mgr.14148) 147 : cephadm [INF] Deploying daemon osd.7 on smithi110 2023-10-09T09:47:02.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:00.784800+0000 mgr.y (mgr.14148) 146 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:01.052512+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:47:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:01.053562+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:01.054436+0000 mgr.y (mgr.14148) 147 : cephadm [INF] Deploying daemon osd.7 on smithi110 2023-10-09T09:47:03.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:02.785191+0000 mgr.y (mgr.14148) 148 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:03.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:02.785191+0000 mgr.y (mgr.14148) 148 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:03.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:02.785191+0000 mgr.y (mgr.14148) 148 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:06.315 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:04.785633+0000 mgr.y (mgr.14148) 149 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:06.315 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:05.229871+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:06.315 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:05.273066+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:04.785633+0000 mgr.y (mgr.14148) 149 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:05.229871+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:05.273066+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:06.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:04.785633+0000 mgr.y (mgr.14148) 149 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:06.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:05.229871+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:06.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:05.273066+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:07.733 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 7 on host 'smithi110' 2023-10-09T09:47:08.084 DEBUG:teuthology.orchestra.run.smithi110:osd.7> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T09:47:08.086 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T09:47:08.086 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T09:47:08.115 INFO:journalctl@ceph.osd.7.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:47:08.351 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:06.786130+0000 mgr.y (mgr.14148) 150 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:08.352 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:07.687616+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.352 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:07.688576+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:08.352 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:07.689571+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:08.352 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:07.696790+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.352 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:07.732387+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.352 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:07.732735+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:08.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:06.786130+0000 mgr.y (mgr.14148) 150 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:08.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:07.687616+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:07.688576+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:08.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:07.689571+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:08.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:07.696790+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:07.732387+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:07.732735+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:08.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:06.786130+0000 mgr.y (mgr.14148) 150 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:08.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:07.687616+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:07.688576+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:08.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:07.689571+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:08.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:07.696790+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:07.732387+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:08.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:07.732735+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:08.604 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 09:47:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T09:47:08.350+0000 7f193e509200 -1 osd.7 0 log_to_monitors {default=true} 2023-10-09T09:47:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:08.354625+0000 mon.b (mon.1) 15 : audit [INF] from='osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:47:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:08.354953+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:47:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:08.354625+0000 mon.b (mon.1) 15 : audit [INF] from='osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:47:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:08.354953+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:47:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:08.354625+0000 mon.b (mon.1) 15 : audit [INF] from='osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:47:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:08.354953+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T09:47:09.706 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:47:10.065 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":46,"num_osds":8,"num_up_osds":7,"osd_up_since":1696844811,"num_in_osds":8,"osd_in_since":1696844815,"num_remapped_pgs":0} 2023-10-09T09:47:10.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:08.786496+0000 mgr.y (mgr.14148) 151 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:10.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:09.244706+0000 mon.a (mon.0) 509 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T09:47:10.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:09.244772+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T09:47:10.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:09.244894+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:10.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:09.245393+0000 mon.b (mon.1) 16 : audit [INF] from='osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:47:10.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:09.245710+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:47:10.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:09.706904+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.12:0/255558736' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:47:10.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:08.786496+0000 mgr.y (mgr.14148) 151 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:10.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:09.244706+0000 mon.a (mon.0) 509 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T09:47:10.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:09.244772+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T09:47:10.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:09.244894+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:10.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:09.245393+0000 mon.b (mon.1) 16 : audit [INF] from='osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:47:10.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:09.245710+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:47:10.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:09.706904+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.12:0/255558736' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:47:10.442 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T09:47:10.251+0000 7f1934a8a700 -1 osd.7 0 waiting for initial osdmap 2023-10-09T09:47:10.442 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T09:47:10.257+0000 7f192fa03700 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:47:10.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:08.786496+0000 mgr.y (mgr.14148) 151 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:10.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:09.244706+0000 mon.a (mon.0) 509 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T09:47:10.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:09.244772+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T09:47:10.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:09.244894+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:10.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:09.245393+0000 mon.b (mon.1) 16 : audit [INF] from='osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:47:10.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:09.245710+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T09:47:10.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:09.706904+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.12:0/255558736' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:47:11.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T09:47:11.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.247779+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:47:11.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:10.247887+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T09:47:11.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.248413+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:11.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.253171+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:11.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.453385+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.454423+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.455189+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.455896+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.456564+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.463155+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.464089+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.464852+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:10.469843+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:11.243944+0000 mon.a (mon.0) 526 : cluster [INF] osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104] boot 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:11.243993+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T09:47:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:11.244179+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.247779+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:47:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:10.247887+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T09:47:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.248413+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.253171+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.453385+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.454423+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.455189+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.455896+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.456564+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.463155+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.464089+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:11.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.464852+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:11.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:10.469843+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:11.243944+0000 mon.a (mon.0) 526 : cluster [INF] osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104] boot 2023-10-09T09:47:11.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:11.243993+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T09:47:11.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:11.244179+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:11.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.247779+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:10.247887+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.248413+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.253171+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.453385+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.454423+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.455189+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.455896+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.456564+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.463155+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.464089+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:11.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.464852+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:11.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:10.469843+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:11.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:11.243944+0000 mon.a (mon.0) 526 : cluster [INF] osd.7 [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104] boot 2023-10-09T09:47:11.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:11.243993+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T09:47:11.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:11.244179+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:12.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:09.341600+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:47:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:09.341649+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:47:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:10.446196+0000 mgr.y (mgr.14148) 152 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:47:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:10.457098+0000 mgr.y (mgr.14148) 153 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 4292M 2023-10-09T09:47:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:10.786928+0000 mgr.y (mgr.14148) 154 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:12.245209+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T09:47:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:09.341600+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:47:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:09.341649+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:47:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:10.446196+0000 mgr.y (mgr.14148) 152 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:47:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:10.457098+0000 mgr.y (mgr.14148) 153 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 4292M 2023-10-09T09:47:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:10.786928+0000 mgr.y (mgr.14148) 154 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:12.245209+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T09:47:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:09.341600+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T09:47:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:09.341649+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T09:47:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:10.446196+0000 mgr.y (mgr.14148) 152 : cephadm [INF] Detected new or changed devices on smithi110 2023-10-09T09:47:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:10.457098+0000 mgr.y (mgr.14148) 153 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 4292M 2023-10-09T09:47:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:10.786928+0000 mgr.y (mgr.14148) 154 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T09:47:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:12.245209+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T09:47:12.698 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:47:13.139 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":49,"num_osds":8,"num_up_osds":8,"osd_up_since":1696844831,"num_in_osds":8,"osd_in_since":1696844815,"num_remapped_pgs":0} 2023-10-09T09:47:13.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:47:13.745 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:13.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:12.698500+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.12:0/78528772' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:47:13.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:12.787277+0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:13.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:13.248267+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T09:47:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:12.698500+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.12:0/78528772' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:47:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:12.787277+0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:13.248267+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T09:47:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:12.698500+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.12:0/78528772' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T09:47:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:12.787277+0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:13.248267+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T09:47:15.052 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:47:15.052 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":50,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T09:47:13.242809+0000","last_up_change":"2023-10-09T09:47:11.240268+0000","last_in_change":"2023-10-09T09:46:55.568500+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6803","nonce":3692896239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6805","nonce":3692896239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6809","nonce":3692896239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6807","nonce":3692896239}]},"public_addr":"172.21.15.12:6803/3692896239","cluster_addr":"172.21.15.12:6805/3692896239","heartbeat_back_addr":"172.21.15.12:6809/3692896239","heartbeat_front_addr":"172.21.15.12:6807/3692896239","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6811","nonce":994418467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6813","nonce":994418467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6817","nonce":994418467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6815","nonce":994418467}]},"public_addr":"172.21.15.12:6811/994418467","cluster_addr":"172.21.15.12:6813/994418467","heartbeat_back_addr":"172.21.15.12:6817/994418467","heartbeat_front_addr":"172.21.15.12:6815/994418467","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6819","nonce":979823756}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6821","nonce":979823756}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6825","nonce":979823756}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6823","nonce":979823756}]},"public_addr":"172.21.15.12:6819/979823756","cluster_addr":"172.21.15.12:6821/979823756","heartbeat_back_addr":"172.21.15.12:6825/979823756","heartbeat_front_addr":"172.21.15.12:6823/979823756","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6827","nonce":715124626}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6829","nonce":715124626}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6833","nonce":715124626}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6831","nonce":715124626}]},"public_addr":"172.21.15.12:6827/715124626","cluster_addr":"172.21.15.12:6829/715124626","heartbeat_back_addr":"172.21.15.12:6833/715124626","heartbeat_front_addr":"172.21.15.12:6831/715124626","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6801","nonce":3628101753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6803","nonce":3628101753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6807","nonce":3628101753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6805","nonce":3628101753}]},"public_addr":"172.21.15.110:6801/3628101753","cluster_addr":"172.21.15.110:6803/3628101753","heartbeat_back_addr":"172.21.15.110:6807/3628101753","heartbeat_front_addr":"172.21.15.110:6805/3628101753","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:47:15.474 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-10-09T09:45:36.218461+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-10-09T09:47:15.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-10-09T09:47:16.079 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:16.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:14.787582+0000 mgr.y (mgr.14148) 156 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:16.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:15.051966+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.12:0/3127873427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:16.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:14.787582+0000 mgr.y (mgr.14148) 156 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:16.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:15.051966+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.12:0/3127873427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:16.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:14.787582+0000 mgr.y (mgr.14148) 156 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:16.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:15.051966+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.12:0/3127873427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:17.333 INFO:teuthology.orchestra.run.smithi012.stdout:pg_num: 1 2023-10-09T09:47:17.799 INFO:tasks.cephadm:Adding prometheus.a on smithi110 2023-10-09T09:47:17.800 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi110=a' 2023-10-09T09:47:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:16.787901+0000 mgr.y (mgr.14148) 157 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:17.333182+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.12:0/2346138569' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T09:47:18.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:16.787901+0000 mgr.y (mgr.14148) 157 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:18.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:17.333182+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.12:0/2346138569' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T09:47:18.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:16.787901+0000 mgr.y (mgr.14148) 157 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:18.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:17.333182+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.12:0/2346138569' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T09:47:19.462 INFO:teuthology.orchestra.run.smithi110.stdout:Scheduled prometheus update... 2023-10-09T09:47:19.861 DEBUG:teuthology.orchestra.run.smithi110:prometheus.a> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T09:47:19.865 INFO:tasks.cephadm:Adding node-exporter.a on smithi012 2023-10-09T09:47:19.865 INFO:tasks.cephadm:Adding node-exporter.b on smithi110 2023-10-09T09:47:19.865 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi012=a;smithi110=b' 2023-10-09T09:47:19.894 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:47:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:18.788302+0000 mgr.y (mgr.14148) 158 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:19.461542+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:19.504427+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:20.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:18.788302+0000 mgr.y (mgr.14148) 158 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:20.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:19.461542+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:20.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:19.504427+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:18.788302+0000 mgr.y (mgr.14148) 158 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:19.461542+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:19.504427+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:20.966 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:19.455568+0000 mgr.y (mgr.14148) 159 : audit [DBG] from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:20.966 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:19.457271+0000 mgr.y (mgr.14148) 160 : cephadm [INF] Saving service prometheus spec with placement smithi110=a;count:1 2023-10-09T09:47:20.966 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:19.920852+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:20.966 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:19.921894+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:20.966 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:19.927448+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:20.967 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:19.933954+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T09:47:21.126 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:19.455568+0000 mgr.y (mgr.14148) 159 : audit [DBG] from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:21.126 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:19.457271+0000 mgr.y (mgr.14148) 160 : cephadm [INF] Saving service prometheus spec with placement smithi110=a;count:1 2023-10-09T09:47:21.126 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:19.920852+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:21.126 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:19.921894+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:21.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:19.927448+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:21.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:19.933954+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T09:47:21.127 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ignoring --setuser ceph since I am not root 2023-10-09T09:47:21.127 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ignoring --setgroup ceph since I am not root 2023-10-09T09:47:21.127 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:20.979+0000 7fd1ccb1e700 1 -- 172.21.15.12:0/2895782362 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x565103cec4e0 con 0x565103cc2c00 2023-10-09T09:47:21.127 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:19.455568+0000 mgr.y (mgr.14148) 159 : audit [DBG] from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:21.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:19.457271+0000 mgr.y (mgr.14148) 160 : cephadm [INF] Saving service prometheus spec with placement smithi110=a;count:1 2023-10-09T09:47:21.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:19.920852+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:21.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:19.921894+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:21.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:19.927448+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' 2023-10-09T09:47:21.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:19.933954+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T09:47:21.217 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ignoring --setuser ceph since I am not root 2023-10-09T09:47:21.217 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ignoring --setgroup ceph since I am not root 2023-10-09T09:47:21.217 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:20.971+0000 7fcad2887700 1 -- 172.21.15.110:0/2262235628 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e6954ca4e0 con 0x55e6954a1000 2023-10-09T09:47:21.217 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:21.098+0000 7fcae2a4f680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:47:21.217 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:21.187+0000 7fcae2a4f680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:47:21.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:21.125+0000 7fd1dcce6680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:47:21.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:21.223+0000 7fd1dcce6680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:47:21.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:21.670+0000 7fd1dcce6680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:47:21.931 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:21.570+0000 7fcae2a4f680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:47:22.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:20.936970+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T09:47:22.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:20.937039+0000 mon.a (mon.0) 538 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2023-10-09T09:47:22.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:20.936970+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T09:47:22.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:20.937039+0000 mon.a (mon.0) 538 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2023-10-09T09:47:22.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:20.936970+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14148 172.21.15.12:0/760109364' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T09:47:22.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:20.937039+0000 mon.a (mon.0) 538 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2023-10-09T09:47:22.598 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:22.304+0000 7fcae2a4f680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:47:22.598 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:22.388+0000 7fcae2a4f680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:47:22.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:22.599+0000 7fd1dcce6680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:47:22.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:22.695+0000 7fd1dcce6680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:47:22.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:22.597+0000 7fcae2a4f680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:47:23.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:22.943+0000 7fd1dcce6680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:47:23.817 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:23.523+0000 7fcae2a4f680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:47:23.817 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:23.649+0000 7fcae2a4f680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:47:23.818 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:23.726+0000 7fcae2a4f680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:47:24.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:23.816+0000 7fcae2a4f680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:47:24.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:24.009+0000 7fcae2a4f680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:47:24.329 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:24.070+0000 7fd1dcce6680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:47:24.329 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:24.233+0000 7fd1dcce6680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:47:24.657 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:24.329+0000 7fd1dcce6680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:47:24.657 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:24.439+0000 7fd1dcce6680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:47:24.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:24.287+0000 7fcae2a4f680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:47:24.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:24.384+0000 7fcae2a4f680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:47:24.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:24.657+0000 7fd1dcce6680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:47:25.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:25.022+0000 7fd1dcce6680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:47:25.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:25.139+0000 7fd1dcce6680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:47:25.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:25.100+0000 7fcae2a4f680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:47:25.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:25.182+0000 7fcae2a4f680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:47:25.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:25.274+0000 7fcae2a4f680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:47:25.702 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:25.447+0000 7fcae2a4f680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:47:25.963 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:25.701+0000 7fcae2a4f680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:47:25.963 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:25.830+0000 7fcae2a4f680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:47:26.216 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:25.962+0000 7fcae2a4f680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:47:26.216 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:26.131+0000 7fcae2a4f680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:47:26.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:26.229092+0000 mon.a (mon.0) 539 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:47:26.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:26.229186+0000 mon.a (mon.0) 540 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:47:26.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:26.230436+0000 mon.b (mon.1) 17 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:47:26.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:26.230958+0000 mon.b (mon.1) 18 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:47:26.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:26.232744+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:47:26.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:26.233674+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:47:26.288 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:26.005+0000 7fd1dcce6680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:47:26.288 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:26.104+0000 7fd1dcce6680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:47:26.288 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:26.219+0000 7fd1dcce6680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:47:26.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:26.431+0000 7fd1dcce6680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:47:26.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:26.229092+0000 mon.a (mon.0) 539 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:47:26.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:26.229186+0000 mon.a (mon.0) 540 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:47:26.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:26.230436+0000 mon.b (mon.1) 17 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:47:26.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:26.230958+0000 mon.b (mon.1) 18 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:47:26.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:26.232744+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:47:26.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:26.233674+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:47:26.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:26.229092+0000 mon.a (mon.0) 539 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:47:26.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:26.229186+0000 mon.a (mon.0) 540 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:47:26.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:26.230436+0000 mon.b (mon.1) 17 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:47:26.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:26.230958+0000 mon.b (mon.1) 18 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:47:26.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:26.232744+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:47:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:26.233674+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.110:0/567719059' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:47:26.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: debug 2023-10-09T09:47:26.214+0000 7fcae2a4f680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:47:26.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: [09/Oct/2023:09:47:26] ENGINE Bus STARTING 2023-10-09T09:47:26.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: CherryPy Checker: 2023-10-09T09:47:26.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: The Application mounted at '' has an empty config. 2023-10-09T09:47:26.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: 2023-10-09T09:47:26.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: [09/Oct/2023:09:47:26] ENGINE Serving on http://:::9283 2023-10-09T09:47:26.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: [09/Oct/2023:09:47:26] ENGINE Bus STARTED 2023-10-09T09:47:27.045 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:26.716+0000 7fd1dcce6680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:47:27.045 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:26.877+0000 7fd1dcce6680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:47:27.362 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:26.285657+0000 mon.a (mon.0) 541 : cluster [DBG] mgrmap e16: y(active, since 3m), standbys: x 2023-10-09T09:47:27.363 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:27.045+0000 7fd1dcce6680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:47:27.363 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:27.260+0000 7fd1dcce6680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:47:27.363 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:26.285657+0000 mon.a (mon.0) 541 : cluster [DBG] mgrmap e16: y(active, since 3m), standbys: x 2023-10-09T09:47:27.562 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:26.285657+0000 mon.a (mon.0) 541 : cluster [DBG] mgrmap e16: y(active, since 3m), standbys: x 2023-10-09T09:47:27.647 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:47:27.362+0000 7fd1dcce6680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:47:27.647 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:47:27] ENGINE Bus STARTING 2023-10-09T09:47:27.648 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [80B blob data] 2023-10-09T09:47:27.648 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [79B blob data] 2023-10-09T09:47:27.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: CherryPy Checker: 2023-10-09T09:47:27.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: The Application mounted at '' has an empty config. 2023-10-09T09:47:27.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: 2023-10-09T09:47:27.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:47:27] ENGINE Serving on http://:::9283 2023-10-09T09:47:27.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:47:27] ENGINE Bus STARTED 2023-10-09T09:47:28.400 INFO:teuthology.orchestra.run.smithi110.stdout:Scheduled node-exporter update... 2023-10-09T09:47:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:27.376323+0000 mon.a (mon.0) 542 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:47:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:27.376743+0000 mon.a (mon.0) 543 : cluster [INF] Activating manager daemon y 2023-10-09T09:47:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:27.384816+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T09:47:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:27.385215+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e17: y(active, starting, since 0.00860665s), standbys: x 2023-10-09T09:47:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.386586+0000 mon.a (mon.0) 546 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:47:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.386862+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:47:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.387044+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:47:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.387984+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.388194+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.388470+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.388819+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.389122+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.389419+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.389660+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.389895+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.390075+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.390323+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.390539+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:47:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.390717+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:47:28.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.391124+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:47:28.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:27.393982+0000 mon.a (mon.0) 562 : cluster [INF] Manager daemon y is now available 2023-10-09T09:47:28.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.524658+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:28.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.526166+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:28.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.563456+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:28.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.574508+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:47:28.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:27.583000+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:47:28.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:27.376323+0000 mon.a (mon.0) 542 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:27.376743+0000 mon.a (mon.0) 543 : cluster [INF] Activating manager daemon y 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:27.384816+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:27.385215+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e17: y(active, starting, since 0.00860665s), standbys: x 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.386586+0000 mon.a (mon.0) 546 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.386862+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.387044+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.387984+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.388194+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.388470+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:47:28.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.388819+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:47:28.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.389122+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:47:28.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.389419+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:47:28.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.389660+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:47:28.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.389895+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:47:28.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.390075+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:47:28.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.390323+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:28.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.390539+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:47:28.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.390717+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:47:28.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.391124+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:47:28.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:27.393982+0000 mon.a (mon.0) 562 : cluster [INF] Manager daemon y is now available 2023-10-09T09:47:28.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.524658+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:28.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.526166+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:28.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.563456+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:28.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.574508+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:47:28.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:27.583000+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:47:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:27.376323+0000 mon.a (mon.0) 542 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:47:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:27.376743+0000 mon.a (mon.0) 543 : cluster [INF] Activating manager daemon y 2023-10-09T09:47:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:27.384816+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T09:47:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:27.385215+0000 mon.a (mon.0) 545 : cluster [DBG] mgrmap e17: y(active, starting, since 0.00860665s), standbys: x 2023-10-09T09:47:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.386586+0000 mon.a (mon.0) 546 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:47:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.386862+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:47:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.387044+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:47:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.387984+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.388194+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.388470+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.388819+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.389122+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.389419+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.389660+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.389895+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.390075+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:47:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.390323+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.390539+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.390717+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.391124+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:27.393982+0000 mon.a (mon.0) 562 : cluster [INF] Manager daemon y is now available 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.524658+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.526166+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.563456+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.574508+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:47:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:27.583000+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:47:28.955 DEBUG:teuthology.orchestra.run.smithi012:node-exporter.a> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T09:47:28.958 DEBUG:teuthology.orchestra.run.smithi110:node-exporter.b> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T09:47:28.960 INFO:tasks.cephadm:Adding alertmanager.a on smithi012 2023-10-09T09:47:28.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi012=a' 2023-10-09T09:47:28.990 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:47:28.991 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:47:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:28.392149+0000 mon.a (mon.0) 568 : cluster [DBG] mgrmap e18: y(active, since 1.01554s), standbys: x 2023-10-09T09:47:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:28.396967+0000 mgr.y (mgr.14427) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi012=a;smithi110=b;count:2 2023-10-09T09:47:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:28.399906+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:28.392149+0000 mon.a (mon.0) 568 : cluster [DBG] mgrmap e18: y(active, since 1.01554s), standbys: x 2023-10-09T09:47:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:28.396967+0000 mgr.y (mgr.14427) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi012=a;smithi110=b;count:2 2023-10-09T09:47:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:28.399906+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:29.690 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:28.392149+0000 mon.a (mon.0) 568 : cluster [DBG] mgrmap e18: y(active, since 1.01554s), standbys: x 2023-10-09T09:47:29.690 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:28.396967+0000 mgr.y (mgr.14427) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi012=a;smithi110=b;count:2 2023-10-09T09:47:29.690 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:28.399906+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:30.617 INFO:teuthology.orchestra.run.smithi110.stdout:Scheduled alertmanager update... 2023-10-09T09:47:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:29.389360+0000 mgr.y (mgr.14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.066403+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.366364+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:30.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:29.389360+0000 mgr.y (mgr.14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:30.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.066403+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:30.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.366364+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:30.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:29.389360+0000 mgr.y (mgr.14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:30.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.066403+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:30.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.366364+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:30.945 DEBUG:teuthology.orchestra.run.smithi012:alertmanager.a> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T09:47:30.948 INFO:tasks.cephadm:Adding grafana.a on smithi110 2023-10-09T09:47:30.948 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi110=a' 2023-10-09T09:47:30.978 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T09:47:31.673 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:30.395825+0000 mon.a (mon.0) 572 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T09:47:31.673 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.612666+0000 mgr.y (mgr.14427) 5 : audit [DBG] from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:31.673 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:30.614002+0000 mgr.y (mgr.14427) 6 : cephadm [INF] Saving service alertmanager spec with placement smithi012=a;count:1 2023-10-09T09:47:31.673 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.617473+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.673 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.763630+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:31.673 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.764468+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:31.673 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:30.765691+0000 mgr.y (mgr.14427) 7 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:47:31.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:30.765957+0000 mgr.y (mgr.14427) 8 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:47:31.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:30.783291+0000 mgr.y (mgr.14427) 9 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:47:31.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:30.799205+0000 mgr.y (mgr.14427) 10 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:47:31.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.813189+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.837422+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:30.844471+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:30.862670+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Deploying daemon prometheus.a on smithi110 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:30.395825+0000 mon.a (mon.0) 572 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.612666+0000 mgr.y (mgr.14427) 5 : audit [DBG] from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:30.614002+0000 mgr.y (mgr.14427) 6 : cephadm [INF] Saving service alertmanager spec with placement smithi012=a;count:1 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.617473+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.763630+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.764468+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:30.765691+0000 mgr.y (mgr.14427) 7 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:30.765957+0000 mgr.y (mgr.14427) 8 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:30.783291+0000 mgr.y (mgr.14427) 9 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:47:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:30.799205+0000 mgr.y (mgr.14427) 10 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.813189+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.837422+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:30.844471+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:30.862670+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Deploying daemon prometheus.a on smithi110 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:30.395825+0000 mon.a (mon.0) 572 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.612666+0000 mgr.y (mgr.14427) 5 : audit [DBG] from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:30.614002+0000 mgr.y (mgr.14427) 6 : cephadm [INF] Saving service alertmanager spec with placement smithi012=a;count:1 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.617473+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.763630+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.764468+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:47:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:30.765691+0000 mgr.y (mgr.14427) 7 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:47:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:30.765957+0000 mgr.y (mgr.14427) 8 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:47:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:30.783291+0000 mgr.y (mgr.14427) 9 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:47:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:30.799205+0000 mgr.y (mgr.14427) 10 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:47:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.813189+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.837422+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:30.844471+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:30.862670+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Deploying daemon prometheus.a on smithi110 2023-10-09T09:47:32.678 INFO:teuthology.orchestra.run.smithi110.stdout:Scheduled grafana update... 2023-10-09T09:47:33.141 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:31.389719+0000 mgr.y (mgr.14427) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:33.141 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:32.677359+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:33.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:31.389719+0000 mgr.y (mgr.14427) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:33.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:32.677359+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:33.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:31.389719+0000 mgr.y (mgr.14427) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:33.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:32.677359+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:33.404 DEBUG:teuthology.orchestra.run.smithi110:grafana.a> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T09:47:33.406 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T09:47:33.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T09:47:33.439 INFO:journalctl@ceph.grafana.a.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T09:47:34.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:32.668595+0000 mgr.y (mgr.14427) 13 : audit [DBG] from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:34.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:32.670182+0000 mgr.y (mgr.14427) 14 : cephadm [INF] Saving service grafana spec with placement smithi110=a;count:1 2023-10-09T09:47:34.170 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:32.668595+0000 mgr.y (mgr.14427) 13 : audit [DBG] from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:34.170 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:32.670182+0000 mgr.y (mgr.14427) 14 : cephadm [INF] Saving service grafana spec with placement smithi110=a;count:1 2023-10-09T09:47:34.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:32.668595+0000 mgr.y (mgr.14427) 13 : audit [DBG] from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi110=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:47:34.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:32.670182+0000 mgr.y (mgr.14427) 14 : cephadm [INF] Saving service grafana spec with placement smithi110=a;count:1 2023-10-09T09:47:35.136 INFO:teuthology.orchestra.run.smithi012.stdout:[client.0] 2023-10-09T09:47:35.136 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQA3zCNle/e7BxAAsC6oeQB/fCosH8LlXpRc9Q== 2023-10-09T09:47:35.158 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:33.390274+0000 mgr.y (mgr.14427) 15 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:35.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:33.390274+0000 mgr.y (mgr.14427) 15 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:35.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:33.390274+0000 mgr.y (mgr.14427) 15 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:35.464 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:47:35.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T09:47:35.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T09:47:35.506 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T09:47:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:35.129244+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.12:0/3674726168' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:35.129608+0000 mon.a (mon.0) 580 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:35.135253+0000 mon.a (mon.0) 581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:47:36.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:35.129244+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.12:0/3674726168' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:36.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:35.129608+0000 mon.a (mon.0) 580 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:36.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:35.135253+0000 mon.a (mon.0) 581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:47:36.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:35.129244+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.12:0/3674726168' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:36.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:35.129608+0000 mon.a (mon.0) 580 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:36.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:35.135253+0000 mon.a (mon.0) 581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:47:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:35.390761+0000 mgr.y (mgr.14427) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:35.390761+0000 mgr.y (mgr.14427) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:37.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:35.390761+0000 mgr.y (mgr.14427) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:38.460 INFO:teuthology.orchestra.run.smithi110.stdout:[client.1] 2023-10-09T09:47:38.460 INFO:teuthology.orchestra.run.smithi110.stdout: key = AQA6zCNlujg2GxAAOmG19WlX/U6I3Wyz5b0xaA== 2023-10-09T09:47:38.827 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2023-10-09T09:47:38.828 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T09:47:38.828 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T09:47:38.876 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T09:47:38.876 DEBUG:teuthology.orchestra.run.smithi012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T09:47:39.136 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:37.391188+0000 mgr.y (mgr.14427) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:39.136 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:38.455633+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.110:0/35955630' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:39.136 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:38.456336+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:39.136 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:38.460501+0000 mon.a (mon.0) 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:47:39.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:37.391188+0000 mgr.y (mgr.14427) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:39.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:38.455633+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.110:0/35955630' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:39.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:38.456336+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:39.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:38.460501+0000 mon.a (mon.0) 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:47:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:37.391188+0000 mgr.y (mgr.14427) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:38.455633+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.110:0/35955630' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:38.456336+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T09:47:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:38.460501+0000 mon.a (mon.0) 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T09:47:39.218 INFO:teuthology.misc.health.smithi012.stdout: 2023-10-09T09:47:39.219 INFO:teuthology.misc.health.smithi012.stdout:{"epoch":51,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T09:47:27.376419+0000","last_up_change":"2023-10-09T09:47:11.240268+0000","last_in_change":"2023-10-09T09:46:55.568500+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6803","nonce":3692896239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6805","nonce":3692896239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6809","nonce":3692896239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6807","nonce":3692896239}]},"public_addr":"172.21.15.12:6803/3692896239","cluster_addr":"172.21.15.12:6805/3692896239","heartbeat_back_addr":"172.21.15.12:6809/3692896239","heartbeat_front_addr":"172.21.15.12:6807/3692896239","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6811","nonce":994418467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6813","nonce":994418467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6817","nonce":994418467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6815","nonce":994418467}]},"public_addr":"172.21.15.12:6811/994418467","cluster_addr":"172.21.15.12:6813/994418467","heartbeat_back_addr":"172.21.15.12:6817/994418467","heartbeat_front_addr":"172.21.15.12:6815/994418467","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6819","nonce":979823756}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6821","nonce":979823756}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6825","nonce":979823756}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6823","nonce":979823756}]},"public_addr":"172.21.15.12:6819/979823756","cluster_addr":"172.21.15.12:6821/979823756","heartbeat_back_addr":"172.21.15.12:6825/979823756","heartbeat_front_addr":"172.21.15.12:6823/979823756","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6827","nonce":715124626}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6829","nonce":715124626}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6833","nonce":715124626}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6831","nonce":715124626}]},"public_addr":"172.21.15.12:6827/715124626","cluster_addr":"172.21.15.12:6829/715124626","heartbeat_back_addr":"172.21.15.12:6833/715124626","heartbeat_front_addr":"172.21.15.12:6831/715124626","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6801","nonce":3628101753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6803","nonce":3628101753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6807","nonce":3628101753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6805","nonce":3628101753}]},"public_addr":"172.21.15.110:6801/3628101753","cluster_addr":"172.21.15.110:6803/3628101753","heartbeat_back_addr":"172.21.15.110:6807/3628101753","heartbeat_front_addr":"172.21.15.110:6805/3628101753","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:47:39.231 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T09:47:39.231 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T09:47:39.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T09:47:39.693 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 systemd[1]: Starting Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:47:40.006 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:39.218424+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.12:0/2570602839' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:40.006 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 podman[117086]: 2023-10-09T09:47:40.006 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 bash[117086]: 3a29aae6016195610290b44554b7fb861e0f464069aad3bc3e497393e01e8adc 2023-10-09T09:47:40.006 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.923Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.923Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.923Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi110 (none))" 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.923Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.923Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.924Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.924Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.926Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.928Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.928Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.241µs 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.928Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T09:47:40.007 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.928Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-10-09T09:47:40.008 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.928Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=34.724µs wal_replay_duration=144.21µs total_replay_duration=234.565µs 2023-10-09T09:47:40.008 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.929Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T09:47:40.008 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.929Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T09:47:40.008 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.929Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T09:47:40.008 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.944Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.849203ms db_storage=605ns remote_storage=1.126µs web_handler=172ns query_engine=552ns scrape=452.258µs scrape_sd=17.106µs notify=883ns notify_sd=1.221µs rules=14.203465ms 2023-10-09T09:47:40.008 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:47:39.944Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T09:47:40.133 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:39.218424+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.12:0/2570602839' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:40.133 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:39.218424+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.12:0/2570602839' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:40.442 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:47:40 smithi110 systemd[1]: Started Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:47:41.064 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:39.391536+0000 mgr.y (mgr.14427) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:41.064 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:40.064544+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:41.064 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:40 smithi012 systemd[1]: Starting Ceph node-exporter.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:47:41.065 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:39.391536+0000 mgr.y (mgr.14427) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:41.065 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:40.064544+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:41.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:39.391536+0000 mgr.y (mgr.14427) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:41.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:40.064544+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:41.426 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:41 smithi012 bash[122502]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-10-09T09:47:41.886 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'rbd' created 2023-10-09T09:47:41.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:40.066294+0000 mgr.y (mgr.14427) 19 : cephadm [INF] Deploying daemon node-exporter.a on smithi012 2023-10-09T09:47:41.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:41.177053+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.12:0/3331721528' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:47:41.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:41.177390+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:47:41.910 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:41 smithi012 bash[122502]: Getting image source signatures 2023-10-09T09:47:41.911 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:41 smithi012 bash[122502]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-10-09T09:47:41.911 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:41 smithi012 bash[122502]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-10-09T09:47:41.911 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:41 smithi012 bash[122502]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-10-09T09:47:41.911 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:41 smithi012 bash[122502]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-10-09T09:47:41.911 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:41 smithi012 bash[122502]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-10-09T09:47:41.911 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:41 smithi012 bash[122502]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-10-09T09:47:41.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:40.066294+0000 mgr.y (mgr.14427) 19 : cephadm [INF] Deploying daemon node-exporter.a on smithi012 2023-10-09T09:47:41.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:41.177053+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.12:0/3331721528' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:47:41.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:41.177390+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:47:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:40.066294+0000 mgr.y (mgr.14427) 19 : cephadm [INF] Deploying daemon node-exporter.a on smithi012 2023-10-09T09:47:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:41.177053+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.12:0/3331721528' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:47:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:41.177390+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T09:47:42.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T09:47:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:41.391842+0000 mgr.y (mgr.14427) 20 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:41.882409+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T09:47:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:41.882505+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T09:47:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:41.883034+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:47:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:41.884807+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:47:43.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:42.569419+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:43.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:41.391842+0000 mgr.y (mgr.14427) 20 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:43.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:41.882409+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T09:47:43.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:41.882505+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T09:47:43.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:41.883034+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:47:43.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:41.884807+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:47:43.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:42.569419+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:43.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:41.391842+0000 mgr.y (mgr.14427) 20 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:43.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:41.882409+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T09:47:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:41.882505+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T09:47:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:41.883034+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T09:47:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:41.884807+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T09:47:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:42.569419+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:43.547 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:43 smithi012 bash[122502]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-10-09T09:47:43.547 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:43 smithi012 bash[122502]: Writing manifest to image destination 2023-10-09T09:47:43.547 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:43 smithi012 bash[122502]: Storing signatures 2023-10-09T09:47:43.891 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:42.883877+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:47:43.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:42.883956+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T09:47:43.892 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:42.883877+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:47:43.892 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:42.883956+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T09:47:44.177 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:43 smithi012 podman[122502]: 2023-10-09T09:47:44.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:42.883877+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T09:47:44.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:42.883956+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T09:47:44.581 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.249Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T09:47:44.581 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.249Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T09:47:44.581 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T09:47:44.581 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T09:47:44.582 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.250Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T09:47:44.583 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T09:47:44.584 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[122691]: ts=2023-10-09T09:47:44.251Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 bash[122502]: 8a39bf7d1c98fb70abbdaa16dee5ee25c2be183b3ec9187bebcc4c83a0e5121f 2023-10-09T09:47:44.585 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:47:44 smithi012 systemd[1]: Started Ceph node-exporter.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:47:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:43.392148+0000 mgr.y (mgr.14427) 21 : cluster [DBG] pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:43.893451+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T09:47:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:44.483363+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:43.392148+0000 mgr.y (mgr.14427) 21 : cluster [DBG] pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:43.893451+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T09:47:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:44.483363+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:43.392148+0000 mgr.y (mgr.14427) 21 : cluster [DBG] pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:43.893451+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T09:47:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:44.483363+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:45.193 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:45 smithi110 systemd[1]: Starting Ceph node-exporter.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:47:45.692 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:45 smithi110 bash[117300]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-10-09T09:47:45.908 INFO:teuthology.orchestra.run.smithi012.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T09:47:46.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:44.484244+0000 mgr.y (mgr.14427) 22 : cephadm [INF] Deploying daemon node-exporter.b on smithi110 2023-10-09T09:47:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:44.980596+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.12:0/2585737029' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:47:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:44.484244+0000 mgr.y (mgr.14427) 22 : cephadm [INF] Deploying daemon node-exporter.b on smithi110 2023-10-09T09:47:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:44.980596+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.12:0/2585737029' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:47:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:44.484244+0000 mgr.y (mgr.14427) 22 : cephadm [INF] Deploying daemon node-exporter.b on smithi110 2023-10-09T09:47:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:44.980596+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.12:0/2585737029' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:47:46.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:47:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:47:46.193 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:46 smithi110 bash[117300]: Getting image source signatures 2023-10-09T09:47:46.193 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:46 smithi110 bash[117300]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-10-09T09:47:46.193 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:46 smithi110 bash[117300]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-10-09T09:47:46.193 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:46 smithi110 bash[117300]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-10-09T09:47:46.193 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:46 smithi110 bash[117300]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-10-09T09:47:46.193 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:46 smithi110 bash[117300]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-10-09T09:47:46.193 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:46 smithi110 bash[117300]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-10-09T09:47:46.259 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T09:47:46.259 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T09:47:46.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T09:47:46.867 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:47.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:45.392469+0000 mgr.y (mgr.14427) 23 : cluster [DBG] pgmap v15: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:45.903927+0000 mon.a (mon.0) 596 : audit [INF] from='client.? 172.21.15.12:0/2585737029' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:47:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:45.904020+0000 mon.a (mon.0) 597 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T09:47:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:45.392469+0000 mgr.y (mgr.14427) 23 : cluster [DBG] pgmap v15: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:45.903927+0000 mon.a (mon.0) 596 : audit [INF] from='client.? 172.21.15.12:0/2585737029' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:47:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:45.904020+0000 mon.a (mon.0) 597 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T09:47:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:45.392469+0000 mgr.y (mgr.14427) 23 : cluster [DBG] pgmap v15: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:45.903927+0000 mon.a (mon.0) 596 : audit [INF] from='client.? 172.21.15.12:0/2585737029' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:47:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:45.904020+0000 mon.a (mon.0) 597 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T09:47:47.942 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:47 smithi110 bash[117300]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-10-09T09:47:47.942 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:47 smithi110 bash[117300]: Writing manifest to image destination 2023-10-09T09:47:47.942 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:47 smithi110 bash[117300]: Storing signatures 2023-10-09T09:47:48.216 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:47:48.426 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 podman[117300]: 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.339Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.339Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.339Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.339Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T09:47:48.427 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T09:47:48.428 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T09:47:48.429 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T09:47:48.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T09:47:48.431 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T09:47:48.431 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T09:47:48.431 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T09:47:48.431 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[117352]: ts=2023-10-09T09:47:48.340Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T09:47:48.431 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 bash[117300]: 3f4a30ff217b420f1721348869a9ef4ddb3a2cfd21b0163574bf9d1fc893dc5c 2023-10-09T09:47:48.692 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:47:48 smithi110 systemd[1]: Started Ceph node-exporter.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:47:48.783 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":19,"active_gid":14427,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":1925191424},{"type":"v1","addr":"172.21.15.12:6801","nonce":1925191424}]},"active_addr":"172.21.15.12:6801/1925191424","active_change":"2023-10-09T09:47:27.376594+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24328,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2293469660}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":928078357}]}]} 2023-10-09T09:47:48.787 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T09:47:48.787 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T09:47:48.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:47:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:47.392782+0000 mgr.y (mgr.14427) 24 : cluster [DBG] pgmap v17: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:48.211093+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.12:0/1137026071' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:47:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:48.482981+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:49.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:47.392782+0000 mgr.y (mgr.14427) 24 : cluster [DBG] pgmap v17: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:49.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:48.211093+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.12:0/1137026071' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:47:49.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:48.482981+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:47.392782+0000 mgr.y (mgr.14427) 24 : cluster [DBG] pgmap v17: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:48.211093+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.12:0/1137026071' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:47:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:48.482981+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:49.397 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:49.910 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:47:48.492269+0000 mgr.y (mgr.14427) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi012 2023-10-09T09:47:49.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:47:48.492269+0000 mgr.y (mgr.14427) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi012 2023-10-09T09:47:50.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:47:48.492269+0000 mgr.y (mgr.14427) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi012 2023-10-09T09:47:50.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:49.393226+0000 mgr.y (mgr.14427) 26 : cluster [DBG] pgmap v18: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:51.013 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:47:51.014 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":55,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T09:47:45.891946+0000","last_up_change":"2023-10-09T09:47:11.240268+0000","last_in_change":"2023-10-09T09:46:55.568500+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:47:41.614981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6803","nonce":3692896239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6805","nonce":3692896239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6809","nonce":3692896239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6807","nonce":3692896239}]},"public_addr":"172.21.15.12:6803/3692896239","cluster_addr":"172.21.15.12:6805/3692896239","heartbeat_back_addr":"172.21.15.12:6809/3692896239","heartbeat_front_addr":"172.21.15.12:6807/3692896239","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6811","nonce":994418467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6813","nonce":994418467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6817","nonce":994418467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6815","nonce":994418467}]},"public_addr":"172.21.15.12:6811/994418467","cluster_addr":"172.21.15.12:6813/994418467","heartbeat_back_addr":"172.21.15.12:6817/994418467","heartbeat_front_addr":"172.21.15.12:6815/994418467","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6819","nonce":979823756}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6821","nonce":979823756}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6825","nonce":979823756}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6823","nonce":979823756}]},"public_addr":"172.21.15.12:6819/979823756","cluster_addr":"172.21.15.12:6821/979823756","heartbeat_back_addr":"172.21.15.12:6825/979823756","heartbeat_front_addr":"172.21.15.12:6823/979823756","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6827","nonce":715124626}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6829","nonce":715124626}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6833","nonce":715124626}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6831","nonce":715124626}]},"public_addr":"172.21.15.12:6827/715124626","cluster_addr":"172.21.15.12:6829/715124626","heartbeat_back_addr":"172.21.15.12:6833/715124626","heartbeat_front_addr":"172.21.15.12:6831/715124626","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6801","nonce":3628101753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6803","nonce":3628101753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6807","nonce":3628101753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6805","nonce":3628101753}]},"public_addr":"172.21.15.110:6801/3628101753","cluster_addr":"172.21.15.110:6803/3628101753","heartbeat_back_addr":"172.21.15.110:6807/3628101753","heartbeat_front_addr":"172.21.15.110:6805/3628101753","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:47:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:49.393226+0000 mgr.y (mgr.14427) 26 : cluster [DBG] pgmap v18: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:51.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:49.393226+0000 mgr.y (mgr.14427) 26 : cluster [DBG] pgmap v18: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:52.030 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T09:47:52.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:47:52.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:51.012898+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.12:0/3131082275' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:51.012898+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.12:0/3131082275' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:52.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:51.012898+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.12:0/3131082275' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:51.393540+0000 mgr.y (mgr.14427) 27 : cluster [DBG] pgmap v19: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:52.579755+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:53.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:51.393540+0000 mgr.y (mgr.14427) 27 : cluster [DBG] pgmap v19: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:53.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:52.579755+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:53.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:51.393540+0000 mgr.y (mgr.14427) 27 : cluster [DBG] pgmap v19: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:53.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:52.579755+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:53.418 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:54.167 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:47:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:47:54] "GET /metrics HTTP/1.1" 200 155645 "" "Prometheus/2.33.4" 2023-10-09T09:47:54.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:54 smithi012 systemd[1]: Starting Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:47:55.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:53.393877+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:55.009 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:53.393877+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:53.393877+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:55.344 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:47:55.344 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":55,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T09:47:45.891946+0000","last_up_change":"2023-10-09T09:47:11.240268+0000","last_in_change":"2023-10-09T09:46:55.568500+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:47:41.614981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6803","nonce":3692896239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6805","nonce":3692896239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6809","nonce":3692896239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6807","nonce":3692896239}]},"public_addr":"172.21.15.12:6803/3692896239","cluster_addr":"172.21.15.12:6805/3692896239","heartbeat_back_addr":"172.21.15.12:6809/3692896239","heartbeat_front_addr":"172.21.15.12:6807/3692896239","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6811","nonce":994418467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6813","nonce":994418467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6817","nonce":994418467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6815","nonce":994418467}]},"public_addr":"172.21.15.12:6811/994418467","cluster_addr":"172.21.15.12:6813/994418467","heartbeat_back_addr":"172.21.15.12:6817/994418467","heartbeat_front_addr":"172.21.15.12:6815/994418467","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6819","nonce":979823756}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6821","nonce":979823756}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6825","nonce":979823756}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6823","nonce":979823756}]},"public_addr":"172.21.15.12:6819/979823756","cluster_addr":"172.21.15.12:6821/979823756","heartbeat_back_addr":"172.21.15.12:6825/979823756","heartbeat_front_addr":"172.21.15.12:6823/979823756","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6827","nonce":715124626}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6829","nonce":715124626}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6833","nonce":715124626}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6831","nonce":715124626}]},"public_addr":"172.21.15.12:6827/715124626","cluster_addr":"172.21.15.12:6829/715124626","heartbeat_back_addr":"172.21.15.12:6833/715124626","heartbeat_front_addr":"172.21.15.12:6831/715124626","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6801","nonce":3628101753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6803","nonce":3628101753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6807","nonce":3628101753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6805","nonce":3628101753}]},"public_addr":"172.21.15.110:6801/3628101753","cluster_addr":"172.21.15.110:6803/3628101753","heartbeat_back_addr":"172.21.15.110:6807/3628101753","heartbeat_front_addr":"172.21.15.110:6805/3628101753","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:47:55.419 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 podman[123735]: 2023-10-09T09:47:55.419 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:55.387Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T09:47:55.419 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:55.387Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T09:47:55.420 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:55.388Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.12 port=9094 2023-10-09T09:47:55.420 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:55.390Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T09:47:55.676 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 bash[123735]: 28647345eb0e80186ddad00a19f1039a936c3a376047ed87b491b3f233fbb4f3 2023-10-09T09:47:55.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:55.424Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:47:55.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:55.424Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:47:55.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:55.427Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T09:47:55.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:55.427Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T09:47:55.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:55 smithi012 systemd[1]: Started Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:47:55.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T09:47:55.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T09:47:55.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T09:47:55.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T09:47:55.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T09:47:55.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T09:47:55.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T09:47:55.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T09:47:56.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:47:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:47:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:47:56.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:55.342852+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.12:0/1723785441' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:56.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:55.537867+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:56.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:55.539069+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:56.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:55.342852+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.12:0/1723785441' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:55.537867+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:55.539069+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:56.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:55.342852+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.12:0/1723785441' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:47:56.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:55.537867+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:56.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:55.539069+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:47:56.644 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:56.673 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:56.800 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:56.970 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:57.125 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:57.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:55.394207+0000 mgr.y (mgr.14427) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:57.127 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:55.394207+0000 mgr.y (mgr.14427) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:57.172 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:57.172 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:57.185 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:47:57.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:55.394207+0000 mgr.y (mgr.14427) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:57.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:47:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:47:57.390Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000144983s 2023-10-09T09:47:58.867 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:57.394638+0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:58.867 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:57.589486+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:58.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:57.394638+0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:58.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:57.589486+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:57.394638+0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:47:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:57.589486+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:47:59.865 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:47:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:47:58.863471+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:00.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:47:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:47:58.863471+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:47:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:47:58.863471+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:00.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:47:59.395105+0000 mgr.y (mgr.14427) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:00.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:47:59.395105+0000 mgr.y (mgr.14427) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:47:59.395105+0000 mgr.y (mgr.14427) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:01.539 INFO:teuthology.orchestra.run.smithi012.stdout:206158430219 2023-10-09T09:48:01.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T09:48:02.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:01.395442+0000 mgr.y (mgr.14427) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:02.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:01.395442+0000 mgr.y (mgr.14427) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:01.395442+0000 mgr.y (mgr.14427) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:03.216 INFO:teuthology.orchestra.run.smithi012.stdout:77309411358 2023-10-09T09:48:03.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T09:48:03.365 INFO:teuthology.orchestra.run.smithi012.stdout:154618822675 2023-10-09T09:48:03.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T09:48:03.724 INFO:teuthology.orchestra.run.smithi012.stdout:128849018903 2023-10-09T09:48:03.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T09:48:04.317 INFO:teuthology.orchestra.run.smithi012.stdout:180388626447 2023-10-09T09:48:04.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T09:48:04.346 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:04] "GET /metrics HTTP/1.1" 200 155640 "" "Prometheus/2.33.4" 2023-10-09T09:48:04.783 INFO:teuthology.orchestra.run.smithi012.stdout:107374182426 2023-10-09T09:48:04.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T09:48:04.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:03.395825+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:03.395825+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:05.154 INFO:teuthology.orchestra.run.smithi012.stdout:34359738407 2023-10-09T09:48:05.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T09:48:05.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:03.395825+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:05.473 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:05.490 INFO:teuthology.orchestra.run.smithi012.stdout:55834574883 2023-10-09T09:48:05.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T09:48:05.676 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:48:05.392Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002330767s 2023-10-09T09:48:06.074 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:06.074 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:06.075 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:06.169 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:06.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:48:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:05.396170+0000 mgr.y (mgr.14427) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.412612+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.413558+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.414269+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.419473+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.615574+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.624909+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.627873+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:48:06.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.635172+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:05.396170+0000 mgr.y (mgr.14427) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.412612+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.413558+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.414269+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.419473+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.615574+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.624909+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.627873+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:48:06.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.635172+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:06.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:06] ENGINE Bus STOPPING 2023-10-09T09:48:06.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:48:06.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:06] ENGINE Bus STOPPED 2023-10-09T09:48:06.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:06] ENGINE Bus STARTING 2023-10-09T09:48:06.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:06] ENGINE Serving on http://:::9283 2023-10-09T09:48:06.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:06] ENGINE Bus STARTED 2023-10-09T09:48:07.061 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:07.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:05.396170+0000 mgr.y (mgr.14427) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:07.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.412612+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:07.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.413558+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.414269+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.419473+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.615574+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.624909+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.627873+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:48:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.635172+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:07.863 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:08.144 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:06.628278+0000 mgr.y (mgr.14427) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:48:08.144 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:48:06.647373+0000 mgr.y (mgr.14427) 36 : cephadm [INF] Deploying daemon grafana.a on smithi110 2023-10-09T09:48:08.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:06.628278+0000 mgr.y (mgr.14427) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:48:08.145 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:48:06.647373+0000 mgr.y (mgr.14427) 36 : cephadm [INF] Deploying daemon grafana.a on smithi110 2023-10-09T09:48:08.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:06.628278+0000 mgr.y (mgr.14427) 35 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T09:48:08.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:48:06.647373+0000 mgr.y (mgr.14427) 36 : cephadm [INF] Deploying daemon grafana.a on smithi110 2023-10-09T09:48:08.745 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:07.396537+0000 mgr.y (mgr.14427) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:09.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:08.818978+0000 mon.a (mon.0) 612 : audit [DBG] from='client.? 172.21.15.12:0/1626049286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:48:09.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:07.396537+0000 mgr.y (mgr.14427) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:09.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:08.818978+0000 mon.a (mon.0) 612 : audit [DBG] from='client.? 172.21.15.12:0/1626049286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:48:09.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:07.396537+0000 mgr.y (mgr.14427) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:09.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:08.818978+0000 mon.a (mon.0) 612 : audit [DBG] from='client.? 172.21.15.12:0/1626049286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:48:09.511 INFO:teuthology.orchestra.run.smithi012.stdout:154618822676 2023-10-09T09:48:09.584 INFO:teuthology.orchestra.run.smithi012.stdout:206158430220 2023-10-09T09:48:10.155 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:09.511630+0000 mon.a (mon.0) 613 : audit [DBG] from='client.? 172.21.15.12:0/3163181305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:48:10.156 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:09.511630+0000 mon.a (mon.0) 613 : audit [DBG] from='client.? 172.21.15.12:0/3163181305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:48:10.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:09.511630+0000 mon.a (mon.0) 613 : audit [DBG] from='client.? 172.21.15.12:0/3163181305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:48:10.218 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822675 got 154618822676 for osd.5 2023-10-09T09:48:10.219 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:11.001 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:09.396913+0000 mgr.y (mgr.14427) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:11.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:09.396913+0000 mgr.y (mgr.14427) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:11.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:09.396913+0000 mgr.y (mgr.14427) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:11.575 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430219 got 206158430220 for osd.7 2023-10-09T09:48:11.575 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:11.805 INFO:teuthology.orchestra.run.smithi012.stdout:180388626449 2023-10-09T09:48:12.157 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:11.805467+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.12:0/2546213873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:48:12.157 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:11.805467+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.12:0/2546213873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:48:12.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:11.805467+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.12:0/2546213873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:48:12.215 INFO:teuthology.orchestra.run.smithi012.stdout:128849018905 2023-10-09T09:48:12.531 INFO:teuthology.orchestra.run.smithi012.stdout:34359738408 2023-10-09T09:48:12.749 INFO:teuthology.orchestra.run.smithi012.stdout:77309411360 2023-10-09T09:48:12.948 INFO:teuthology.orchestra.run.smithi012.stdout:107374182427 2023-10-09T09:48:12.978 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018903 got 128849018905 for osd.4 2023-10-09T09:48:12.978 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:13.100 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:11.397222+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:13.101 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:12.215542+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.12:0/1708326216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:48:13.101 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:12.531388+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.12:0/3037531545' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:48:13.101 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:12.672489+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.12:0/2402547084' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:48:13.101 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:11.397222+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:13.101 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:12.215542+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.12:0/1708326216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:48:13.101 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:12.531388+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.12:0/3037531545' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:48:13.101 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:12.672489+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.12:0/2402547084' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:48:13.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:11.397222+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:13.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:12.215542+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.12:0/1708326216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:48:13.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:12.531388+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.12:0/3037531545' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:48:13.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:12.672489+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.12:0/2402547084' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:48:13.295 INFO:teuthology.orchestra.run.smithi012.stdout:55834574884 2023-10-09T09:48:13.384 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411358 got 77309411360 for osd.2 2023-10-09T09:48:13.384 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:13.926 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626447 got 180388626449 for osd.6 2023-10-09T09:48:13.926 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:14.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:12.948442+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.12:0/2721683553' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:48:14.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:13.295928+0000 mon.a (mon.0) 614 : audit [DBG] from='client.? 172.21.15.12:0/4136387483' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:48:14.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:14] "GET /metrics HTTP/1.1" 200 155647 "" "Prometheus/2.33.4" 2023-10-09T09:48:14.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:12.948442+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.12:0/2721683553' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:48:14.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:13.295928+0000 mon.a (mon.0) 614 : audit [DBG] from='client.? 172.21.15.12:0/4136387483' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:48:14.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:12.948442+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.12:0/2721683553' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:48:14.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:13.295928+0000 mon.a (mon.0) 614 : audit [DBG] from='client.? 172.21.15.12:0/4136387483' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:48:14.194 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574883 got 55834574884 for osd.1 2023-10-09T09:48:14.195 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:14.290 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738407 got 34359738408 for osd.0 2023-10-09T09:48:14.291 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:14.457 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182426 got 107374182427 for osd.3 2023-10-09T09:48:14.457 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:14.458 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T09:48:14.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:48:15.063 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:15.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:13.397537+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:13.397537+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:15.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:13.397537+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:16.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:48:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:48:16.679 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:48:16.680 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:48:17.021 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:15.397816+0000 mgr.y (mgr.14427) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:17.021 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:15.397816+0000 mgr.y (mgr.14427) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:17.081 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":31,"stamp":"2023-10-09T09:48:15.397678+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380984,"kb_used_data":4216,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580232,"statfs":{"total":767960285184,"available":765522157568,"internally_reserved":0,"allocated":4317184,"data_stored":1763490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.860240+0000","last_change":"2023-10-09T09:47:42.887390+0000","last_active":"2023-10-09T09:47:44.860240+0000","last_peered":"2023-10-09T09:47:44.860240+0000","last_clean":"2023-10-09T09:47:44.860240+0000","last_became_active":"2023-10-09T09:47:42.887040+0000","last_became_peered":"2023-10-09T09:47:42.887040+0000","last_unstale":"2023-10-09T09:47:44.860240+0000","last_undegraded":"2023-10-09T09:47:44.860240+0000","last_fullsized":"2023-10-09T09:47:44.860240+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:43.892596+0000","last_change":"2023-10-09T09:47:42.989781+0000","last_active":"2023-10-09T09:47:43.892596+0000","last_peered":"2023-10-09T09:47:43.892596+0000","last_clean":"2023-10-09T09:47:43.892596+0000","last_became_active":"2023-10-09T09:47:42.989461+0000","last_became_peered":"2023-10-09T09:47:42.989461+0000","last_unstale":"2023-10-09T09:47:43.892596+0000","last_undegraded":"2023-10-09T09:47:43.892596+0000","last_fullsized":"2023-10-09T09:47:43.892596+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.360003+0000","last_change":"2023-10-09T09:47:42.989936+0000","last_active":"2023-10-09T09:47:44.360003+0000","last_peered":"2023-10-09T09:47:44.360003+0000","last_clean":"2023-10-09T09:47:44.360003+0000","last_became_active":"2023-10-09T09:47:42.989558+0000","last_became_peered":"2023-10-09T09:47:42.989558+0000","last_unstale":"2023-10-09T09:47:44.360003+0000","last_undegraded":"2023-10-09T09:47:44.360003+0000","last_fullsized":"2023-10-09T09:47:44.360003+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.359982+0000","last_change":"2023-10-09T09:47:42.989937+0000","last_active":"2023-10-09T09:47:44.359982+0000","last_peered":"2023-10-09T09:47:44.359982+0000","last_clean":"2023-10-09T09:47:44.359982+0000","last_became_active":"2023-10-09T09:47:42.989371+0000","last_became_peered":"2023-10-09T09:47:42.989371+0000","last_unstale":"2023-10-09T09:47:44.359982+0000","last_undegraded":"2023-10-09T09:47:44.359982+0000","last_fullsized":"2023-10-09T09:47:44.359982+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.360035+0000","last_change":"2023-10-09T09:47:13.676606+0000","last_active":"2023-10-09T09:47:44.360035+0000","last_peered":"2023-10-09T09:47:44.360035+0000","last_clean":"2023-10-09T09:47:44.360035+0000","last_became_active":"2023-10-09T09:47:13.676235+0000","last_became_peered":"2023-10-09T09:47:13.676235+0000","last_unstale":"2023-10-09T09:47:44.360035+0000","last_undegraded":"2023-10-09T09:47:44.360035+0000","last_fullsized":"2023-10-09T09:47:44.360035+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:43.892041+0000","last_change":"2023-10-09T09:47:42.887560+0000","last_active":"2023-10-09T09:47:43.892041+0000","last_peered":"2023-10-09T09:47:43.892041+0000","last_clean":"2023-10-09T09:47:43.892041+0000","last_became_active":"2023-10-09T09:47:42.887101+0000","last_became_peered":"2023-10-09T09:47:42.887101+0000","last_unstale":"2023-10-09T09:47:43.892041+0000","last_undegraded":"2023-10-09T09:47:43.892041+0000","last_fullsized":"2023-10-09T09:47:43.892041+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.860301+0000","last_change":"2023-10-09T09:47:42.989695+0000","last_active":"2023-10-09T09:47:44.860301+0000","last_peered":"2023-10-09T09:47:44.860301+0000","last_clean":"2023-10-09T09:47:44.860301+0000","last_became_active":"2023-10-09T09:47:42.989373+0000","last_became_peered":"2023-10-09T09:47:42.989373+0000","last_unstale":"2023-10-09T09:47:44.860301+0000","last_undegraded":"2023-10-09T09:47:44.860301+0000","last_fullsized":"2023-10-09T09:47:44.860301+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:43.892054+0000","last_change":"2023-10-09T09:47:42.888479+0000","last_active":"2023-10-09T09:47:43.892054+0000","last_peered":"2023-10-09T09:47:43.892054+0000","last_clean":"2023-10-09T09:47:43.892054+0000","last_became_active":"2023-10-09T09:47:42.888097+0000","last_became_peered":"2023-10-09T09:47:42.888097+0000","last_unstale":"2023-10-09T09:47:43.892054+0000","last_undegraded":"2023-10-09T09:47:43.892054+0000","last_fullsized":"2023-10-09T09:47:43.892054+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:43.892574+0000","last_change":"2023-10-09T09:47:42.989707+0000","last_active":"2023-10-09T09:47:43.892574+0000","last_peered":"2023-10-09T09:47:43.892574+0000","last_clean":"2023-10-09T09:47:43.892574+0000","last_became_active":"2023-10-09T09:47:42.989360+0000","last_became_peered":"2023-10-09T09:47:42.989360+0000","last_unstale":"2023-10-09T09:47:43.892574+0000","last_undegraded":"2023-10-09T09:47:43.892574+0000","last_fullsized":"2023-10-09T09:47:43.892574+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430222,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.64400000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.625}]},{"osd":2,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.496}]},{"osd":3,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.55800000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.48899999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.437},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.57399999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.44800000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626450,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.51900000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.39500000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.45700000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.436},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.50800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48199999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574885,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.34399999999999997,"15min":0.34000000000000002},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.51700000000000002,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.35399999999999998,"15min":0.34100000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.40999999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:47:38 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.38500000000000001,"15min":0.38200000000000001},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.42099999999999999,"15min":0.41499999999999998},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.51000000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.46800000000000003,"15min":0.46600000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64800000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.36499999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1870000000000001,"5min":1.1870000000000001,"15min":1.1870000000000001},"last":0.498}]},{"osd":5,"last update":"Mon Oct 9 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.44}]},{"osd":6,"last update":"Mon Oct 9 09:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.55600000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52200000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738409,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.33500000000000002,"15min":0.32100000000000001},"min":{"1min":0.30499999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.34200000000000003,"15min":0.32400000000000001},"min":{"1min":0.311,"5min":0.219,"15min":0.219},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.32200000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:47:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.436,"15min":0.42699999999999999},"min":{"1min":0.35599999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.40000000000000002,"15min":0.39400000000000002},"min":{"1min":0.34000000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.41099999999999998,"15min":0.40200000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.48499999999999999,"15min":0.47599999999999998},"min":{"1min":0.38300000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.29199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.59899999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.57499999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411361,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:38 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.34899999999999998,"15min":0.34100000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.373,"15min":0.36699999999999999},"min":{"1min":0.28499999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.27700000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:47:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.40799999999999997,"15min":0.39600000000000002},"min":{"1min":0.33000000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.33700000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:47:58 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.53900000000000003,"15min":0.48799999999999999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.4450000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.502,"15min":0.502},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.45500000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:47:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.436},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.47599999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:47:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":0.504},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":6.4089999999999998,"5min":6.4089999999999998,"15min":6.4089999999999998},"last":0.48699999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57599999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182428,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.41299999999999998,"15min":0.41899999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57099999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.442,"15min":0.45500000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.51100000000000001,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.433}]},{"osd":1,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.434,"15min":0.44800000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.48999999999999999,"5min":0.623,"15min":0.623},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.628,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.495,"15min":0.51100000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.53300000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.48599999999999999,"15min":0.499},"min":{"1min":0.34699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.5,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.5}]},{"osd":4,"last update":"Mon Oct 9 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.625}]},{"osd":6,"last update":"Mon Oct 9 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66100000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018906,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.47299999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.629}]},{"osd":2,"last update":"Mon Oct 9 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.46800000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:47:35 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.35399999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:47:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.33400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.378}]}]},{"osd":5,"up_from":36,"seq":154618822678,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.68700000000000006}]},{"osd":1,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.435}]},{"osd":2,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.39400000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.371},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.313}]},{"osd":6,"last update":"Mon Oct 9 09:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.41999999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.441},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.41599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:48:17.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:48:17.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:15.397816+0000 mgr.y (mgr.14427) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:17.689 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:18.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:16.675055+0000 mgr.y (mgr.14427) 42 : audit [DBG] from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:48:18.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:16.675055+0000 mgr.y (mgr.14427) 42 : audit [DBG] from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:48:18.184 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:16.675055+0000 mgr.y (mgr.14427) 42 : audit [DBG] from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:48:18.443 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 systemd[1]: Starting Ceph grafana.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:48:18.825 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 podman[118091]: 2023-10-09T09:48:18.825 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 bash[118091]: b2e250c8bb7589ddb63ba48a4c51df45d4529d69ac64ad6cd31185fa12b63e47 2023-10-09T09:48:18.825 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T09:48:18.826 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T09:48:18.826 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T09:48:18.826 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T09:48:18.826 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T09:48:18.826 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T09:48:18.826 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T09:48:18.827 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T09:48:18.827 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T09:48:18.827 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T09:48:18.827 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T09:48:18.827 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T09:48:18.828 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T09:48:18.828 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T09:48:18.828 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T09:48:18.828 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T09:48:18.828 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T09:48:18.828 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-10-09T09:48:18.829 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-10-09T09:48:18.829 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-10-09T09:48:18.829 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-10-09T09:48:18.829 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-10-09T09:48:18.829 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-10-09T09:48:18.829 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-10-09T09:48:18.830 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-10-09T09:48:18.830 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-10-09T09:48:18.830 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-10-09T09:48:18.830 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T09:48:18.830 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-10-09T09:48:18.830 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-10-09T09:48:18.831 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-10-09T09:48:18.831 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-10-09T09:48:18.831 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-10-09T09:48:18.831 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-10-09T09:48:18.831 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-10-09T09:48:18.831 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-10-09T09:48:18.832 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-10-09T09:48:18.832 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-10-09T09:48:18.832 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T09:48:18.832 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-10-09T09:48:18.832 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-10-09T09:48:18.833 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-10-09T09:48:18.833 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-10-09T09:48:18.833 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-10-09T09:48:18.833 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-10-09T09:48:18.833 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-10-09T09:48:18.833 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T09:48:18.834 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-10-09T09:48:18.834 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-10-09T09:48:18.834 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-10-09T09:48:18.834 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-10-09T09:48:18.834 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-10-09T09:48:18.834 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-10-09T09:48:18.835 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-10-09T09:48:18.835 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T09:48:18.835 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-10-09T09:48:18.835 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-10-09T09:48:18.835 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-10-09T09:48:18.836 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-10-09T09:48:18.836 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-10-09T09:48:18.836 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-10-09T09:48:18.836 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T09:48:18.836 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-10-09T09:48:18.836 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-10-09T09:48:18.837 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-10-09T09:48:18.838 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-10-09T09:48:18.839 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-10-09T09:48:18.840 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-10-09T09:48:18.841 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-10-09T09:48:18.841 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-10-09T09:48:18.841 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T09:48:18.841 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-10-09T09:48:18.842 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-10-09T09:48:18.843 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-10-09T09:48:18.844 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-10-09T09:48:18.845 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T09:48:18.846 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T09:48:19.054 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:48:19.055 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:48:19.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:17.398320+0000 mgr.y (mgr.14427) 43 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-10-09T09:48:19.194 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-10-09T09:48:19.195 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T09:48:19.196 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-10-09T09:48:19.197 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-10-09T09:48:19.198 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-10-09T09:48:19.199 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-10-09T09:48:19.200 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T09:48:19.201 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-10-09T09:48:19.202 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 systemd[1]: Started Ceph grafana.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T09:48:19.203 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T09:48:19.204 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-10-09T09:48:19.205 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-10-09T09:48:19.206 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T09:48:19.207 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T09:48:19.208 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-10-09T09:48:19.209 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-10-09T09:48:19.210 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T09:48:19.211 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-10-09T09:48:19.212 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-10-09T09:48:19.213 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-10-09T09:48:19.214 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=167.553583ms 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:18+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:19+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:19+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:19+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T09:48:19.215 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:19+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T09:48:19.216 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:48:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:48:19+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T09:48:19.223 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:17.398320+0000 mgr.y (mgr.14427) 43 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:19.224 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:17.398320+0000 mgr.y (mgr.14427) 43 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:19.522 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":32,"stamp":"2023-10-09T09:48:17.398072+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380984,"kb_used_data":4216,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580232,"statfs":{"total":767960285184,"available":765522157568,"internally_reserved":0,"allocated":4317184,"data_stored":1763490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.860240+0000","last_change":"2023-10-09T09:47:42.887390+0000","last_active":"2023-10-09T09:47:44.860240+0000","last_peered":"2023-10-09T09:47:44.860240+0000","last_clean":"2023-10-09T09:47:44.860240+0000","last_became_active":"2023-10-09T09:47:42.887040+0000","last_became_peered":"2023-10-09T09:47:42.887040+0000","last_unstale":"2023-10-09T09:47:44.860240+0000","last_undegraded":"2023-10-09T09:47:44.860240+0000","last_fullsized":"2023-10-09T09:47:44.860240+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:43.892596+0000","last_change":"2023-10-09T09:47:42.989781+0000","last_active":"2023-10-09T09:47:43.892596+0000","last_peered":"2023-10-09T09:47:43.892596+0000","last_clean":"2023-10-09T09:47:43.892596+0000","last_became_active":"2023-10-09T09:47:42.989461+0000","last_became_peered":"2023-10-09T09:47:42.989461+0000","last_unstale":"2023-10-09T09:47:43.892596+0000","last_undegraded":"2023-10-09T09:47:43.892596+0000","last_fullsized":"2023-10-09T09:47:43.892596+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.360003+0000","last_change":"2023-10-09T09:47:42.989936+0000","last_active":"2023-10-09T09:47:44.360003+0000","last_peered":"2023-10-09T09:47:44.360003+0000","last_clean":"2023-10-09T09:47:44.360003+0000","last_became_active":"2023-10-09T09:47:42.989558+0000","last_became_peered":"2023-10-09T09:47:42.989558+0000","last_unstale":"2023-10-09T09:47:44.360003+0000","last_undegraded":"2023-10-09T09:47:44.360003+0000","last_fullsized":"2023-10-09T09:47:44.360003+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.359982+0000","last_change":"2023-10-09T09:47:42.989937+0000","last_active":"2023-10-09T09:47:44.359982+0000","last_peered":"2023-10-09T09:47:44.359982+0000","last_clean":"2023-10-09T09:47:44.359982+0000","last_became_active":"2023-10-09T09:47:42.989371+0000","last_became_peered":"2023-10-09T09:47:42.989371+0000","last_unstale":"2023-10-09T09:47:44.359982+0000","last_undegraded":"2023-10-09T09:47:44.359982+0000","last_fullsized":"2023-10-09T09:47:44.359982+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.360035+0000","last_change":"2023-10-09T09:47:13.676606+0000","last_active":"2023-10-09T09:47:44.360035+0000","last_peered":"2023-10-09T09:47:44.360035+0000","last_clean":"2023-10-09T09:47:44.360035+0000","last_became_active":"2023-10-09T09:47:13.676235+0000","last_became_peered":"2023-10-09T09:47:13.676235+0000","last_unstale":"2023-10-09T09:47:44.360035+0000","last_undegraded":"2023-10-09T09:47:44.360035+0000","last_fullsized":"2023-10-09T09:47:44.360035+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:43.892041+0000","last_change":"2023-10-09T09:47:42.887560+0000","last_active":"2023-10-09T09:47:43.892041+0000","last_peered":"2023-10-09T09:47:43.892041+0000","last_clean":"2023-10-09T09:47:43.892041+0000","last_became_active":"2023-10-09T09:47:42.887101+0000","last_became_peered":"2023-10-09T09:47:42.887101+0000","last_unstale":"2023-10-09T09:47:43.892041+0000","last_undegraded":"2023-10-09T09:47:43.892041+0000","last_fullsized":"2023-10-09T09:47:43.892041+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:44.860301+0000","last_change":"2023-10-09T09:47:42.989695+0000","last_active":"2023-10-09T09:47:44.860301+0000","last_peered":"2023-10-09T09:47:44.860301+0000","last_clean":"2023-10-09T09:47:44.860301+0000","last_became_active":"2023-10-09T09:47:42.989373+0000","last_became_peered":"2023-10-09T09:47:42.989373+0000","last_unstale":"2023-10-09T09:47:44.860301+0000","last_undegraded":"2023-10-09T09:47:44.860301+0000","last_fullsized":"2023-10-09T09:47:44.860301+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:43.892054+0000","last_change":"2023-10-09T09:47:42.888479+0000","last_active":"2023-10-09T09:47:43.892054+0000","last_peered":"2023-10-09T09:47:43.892054+0000","last_clean":"2023-10-09T09:47:43.892054+0000","last_became_active":"2023-10-09T09:47:42.888097+0000","last_became_peered":"2023-10-09T09:47:42.888097+0000","last_unstale":"2023-10-09T09:47:43.892054+0000","last_undegraded":"2023-10-09T09:47:43.892054+0000","last_fullsized":"2023-10-09T09:47:43.892054+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T09:47:43.892574+0000","last_change":"2023-10-09T09:47:42.989707+0000","last_active":"2023-10-09T09:47:43.892574+0000","last_peered":"2023-10-09T09:47:43.892574+0000","last_clean":"2023-10-09T09:47:43.892574+0000","last_became_active":"2023-10-09T09:47:42.989360+0000","last_became_peered":"2023-10-09T09:47:42.989360+0000","last_unstale":"2023-10-09T09:47:43.892574+0000","last_undegraded":"2023-10-09T09:47:43.892574+0000","last_fullsized":"2023-10-09T09:47:43.892574+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430222,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.64400000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.625}]},{"osd":2,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.496}]},{"osd":3,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.55800000000000005}]},{"osd":4,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.48899999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.437},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.57399999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.44800000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626450,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.51900000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.39500000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.45700000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:47:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.436},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.50800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48199999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574885,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:18 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.34399999999999997,"15min":0.34000000000000002},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.51700000000000002,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.35399999999999998,"15min":0.34100000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.40999999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:47:38 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.38500000000000001,"15min":0.38200000000000001},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.42099999999999999,"15min":0.41499999999999998},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.51000000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.46800000000000003,"15min":0.46600000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64800000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.36499999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1870000000000001,"5min":1.1870000000000001,"15min":1.1870000000000001},"last":0.498}]},{"osd":5,"last update":"Mon Oct 9 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.44}]},{"osd":6,"last update":"Mon Oct 9 09:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.55600000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52200000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738409,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.33500000000000002,"15min":0.32100000000000001},"min":{"1min":0.30499999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.34200000000000003,"15min":0.32400000000000001},"min":{"1min":0.311,"5min":0.219,"15min":0.219},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.32200000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:47:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.436,"15min":0.42699999999999999},"min":{"1min":0.35599999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.40000000000000002,"15min":0.39400000000000002},"min":{"1min":0.34000000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.41099999999999998,"15min":0.40200000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.48499999999999999,"15min":0.47599999999999998},"min":{"1min":0.38300000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.29199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:47:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.59899999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.57499999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411361,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:38 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.34899999999999998,"15min":0.34100000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.373,"15min":0.36699999999999999},"min":{"1min":0.28499999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.27700000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:47:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.40799999999999997,"15min":0.39600000000000002},"min":{"1min":0.33000000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.33700000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:47:58 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.53900000000000003,"15min":0.48799999999999999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.4450000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.502,"15min":0.502},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.45500000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:47:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.436},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.47599999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:47:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":0.504},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":6.4089999999999998,"5min":6.4089999999999998,"15min":6.4089999999999998},"last":0.48699999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57599999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182428,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.41299999999999998,"15min":0.41899999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57099999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.442,"15min":0.45500000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.51100000000000001,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.433}]},{"osd":1,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.434,"15min":0.44800000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.48999999999999999,"5min":0.623,"15min":0.623},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.628,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.495,"15min":0.51100000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.53300000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.48599999999999999,"15min":0.499},"min":{"1min":0.34699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.5,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.5}]},{"osd":4,"last update":"Mon Oct 9 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.625}]},{"osd":6,"last update":"Mon Oct 9 09:47:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66100000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018906,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.47299999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.629}]},{"osd":2,"last update":"Mon Oct 9 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.46800000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:47:35 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.35399999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:47:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.33400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.378}]}]},{"osd":5,"up_from":36,"seq":154618822678,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.68700000000000006}]},{"osd":1,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.435}]},{"osd":2,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.39400000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.371},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.313}]},{"osd":6,"last update":"Mon Oct 9 09:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.41999999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.441},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.41599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:48:19.524 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T09:48:19.525 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T09:48:19.525 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T09:48:19.525 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T09:48:20.133 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:18.936851+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:18.938445+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:20.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:18.936851+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:20.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:18.938445+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:20.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:18.936851+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:20.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:18.938445+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:21.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:19.050276+0000 mgr.y (mgr.14427) 44 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:48:21.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:19.398660+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:21.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:19.050276+0000 mgr.y (mgr.14427) 44 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:48:21.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:19.398660+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:19.050276+0000 mgr.y (mgr.14427) 44 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:48:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:19.398660+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:21.646 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:48:22.063 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T09:48:22.063 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T09:48:22.063 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T09:48:22.063 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T09:48:22.073 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi012.front.sepia.ceph.com 2023-10-09T09:48:22.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T09:48:22.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:21.645411+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.12:0/1592526512' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:48:22.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:21.645411+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.12:0/1592526512' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:48:22.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:21.645411+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.12:0/1592526512' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:48:23.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:21.399012+0000 mgr.y (mgr.14427) 46 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:23.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:22.606632+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:23.161 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:21.399012+0000 mgr.y (mgr.14427) 46 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:23.162 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:22.606632+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:23.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:21.399012+0000 mgr.y (mgr.14427) 46 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:22.606632+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.364 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:24] "GET /metrics HTTP/1.1" 200 155647 "" "Prometheus/2.33.4" 2023-10-09T09:48:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:23.194690+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:23.492637+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:23.493551+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:24.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:23.494352+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:23.499230+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:23.194690+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:23.492637+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:23.493551+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:24.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:23.494352+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:24.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:23.499230+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:23.194690+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:23.492637+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:24.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:23.493551+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:23.494352+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:23.499230+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:23.399376+0000 mgr.y (mgr.14427) 47 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:48:23.502535+0000 mgr.y (mgr.14427) 48 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:48:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:48:23.506365+0000 mgr.y (mgr.14427) 49 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi012 2023-10-09T09:48:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:23.908027+0000 mgr.y (mgr.14427) 50 : audit [DBG] from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:48:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:23.908550+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T09:48:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:24.501677+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T09:48:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:24.501748+0000 mon.a (mon.0) 626 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T09:48:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:24.505493+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T09:48:25.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:23.399376+0000 mgr.y (mgr.14427) 47 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:25.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:48:23.502535+0000 mgr.y (mgr.14427) 48 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:48:25.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:48:23.506365+0000 mgr.y (mgr.14427) 49 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi012 2023-10-09T09:48:25.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:23.908027+0000 mgr.y (mgr.14427) 50 : audit [DBG] from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:48:25.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:23.908550+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T09:48:25.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:24.501677+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T09:48:25.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:24.501748+0000 mon.a (mon.0) 626 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T09:48:25.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:24.505493+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T09:48:25.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:23.399376+0000 mgr.y (mgr.14427) 47 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:25.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:48:23.502535+0000 mgr.y (mgr.14427) 48 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:48:25.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:48:23.506365+0000 mgr.y (mgr.14427) 49 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi012 2023-10-09T09:48:25.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:23.908027+0000 mgr.y (mgr.14427) 50 : audit [DBG] from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:48:25.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:23.908550+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T09:48:25.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:24.501677+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T09:48:25.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:24.501748+0000 mon.a (mon.0) 626 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T09:48:25.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:24.505493+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T09:48:25.883 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:25 smithi012 systemd[1]: Stopping Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:48:25.884 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:25 smithi012 bash[129106]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager.a" found: no such container 2023-10-09T09:48:25.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:48:25.507+0000 7f784c2e7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T09:48:25.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T09:48:26.178 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[123798]: level=info ts=2023-10-09T09:48:26.114Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:48:26.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:48:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:48:26.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:25.399728+0000 mgr.y (mgr.14427) 51 : cluster [DBG] pgmap v37: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:26.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:25.505180+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T09:48:26.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:25.505297+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:25.507671+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:25.508480+0000 mon.a (mon.0) 631 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:25.508505+0000 mon.a (mon.0) 632 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:25.518864+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:25.519011+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:25.519100+0000 mon.a (mon.0) 635 : cluster [DBG] fsmap foo:0 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:48:25.525359+0000 mgr.y (mgr.14427) 52 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:25.531531+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:25.399728+0000 mgr.y (mgr.14427) 51 : cluster [DBG] pgmap v37: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:26.780 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:25.505180+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:25.505297+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:25.507671+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:25.508480+0000 mon.a (mon.0) 631 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:25.508505+0000 mon.a (mon.0) 632 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:25.518864+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:25.519011+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:25.519100+0000 mon.a (mon.0) 635 : cluster [DBG] fsmap foo:0 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:48:25.525359+0000 mgr.y (mgr.14427) 52 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T09:48:26.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:25.531531+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:25.399728+0000 mgr.y (mgr.14427) 51 : cluster [DBG] pgmap v37: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:25.505180+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:25.505297+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:25.507671+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:25.508480+0000 mon.a (mon.0) 631 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:25.508505+0000 mon.a (mon.0) 632 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:25.518864+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:25.519011+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:25.519100+0000 mon.a (mon.0) 635 : cluster [DBG] fsmap foo:0 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:48:25.525359+0000 mgr.y (mgr.14427) 52 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T09:48:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:25.531531+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:27.105 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:26 smithi012 bash[129191]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T09:48:27.426 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T09:48:27.427 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 systemd[1]: Stopped Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:48:27.427 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 systemd[1]: Starting Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:48:27.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:26.517865+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T09:48:27.780 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:26.517865+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T09:48:27.780 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 podman[129388]: 2023-10-09T09:48:27.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:26.517865+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T09:48:28.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:27.905Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T09:48:28.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:27.905Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T09:48:28.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:27.906Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.12 port=9094 2023-10-09T09:48:28.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:27.908Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T09:48:28.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:27.941Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:48:28.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:27.941Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:48:28.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:27.944Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T09:48:28.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:27.944Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T09:48:28.178 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:28 smithi012 bash[129388]: 2bd91ea7a5f065b9fc5256ff6de12d610c8ac1acf5741652f8dba866fe127141 2023-10-09T09:48:28.178 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:28 smithi012 systemd[1]: Started Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:48:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:27.400065+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v41: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:27.526683+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T09:48:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:27.573075+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:28.090828+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:28.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:27.400065+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v41: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:28.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:27.526683+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T09:48:28.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:27.573075+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:28.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:28.090828+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:28.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T09:48:28.835 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:27.400065+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v41: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:28.835 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:27.526683+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T09:48:28.835 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:27.573075+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:28.835 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:28.090828+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:29.630 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:48:28.091809+0000 mgr.y (mgr.14427) 54 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:48:29.630 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:48:28.095372+0000 mgr.y (mgr.14427) 55 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:48:29.630 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:48:28.091809+0000 mgr.y (mgr.14427) 54 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:48:29.631 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:48:28.095372+0000 mgr.y (mgr.14427) 55 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:48:29.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:48:28.091809+0000 mgr.y (mgr.14427) 54 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:48:29.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:48:28.095372+0000 mgr.y (mgr.14427) 55 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:48:30.177 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:29.909Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00070602s 2023-10-09T09:48:30.443 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 systemd[1]: Stopping Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:48:30.443 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 bash[118796]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus.a" found: no such container 2023-10-09T09:48:30.736 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:29.400592+0000 mgr.y (mgr.14427) 56 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:30.737 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:48:30.737 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T09:48:30.737 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T09:48:30.737 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T09:48:30.737 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T09:48:30.737 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T09:48:30.738 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T09:48:30.738 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T09:48:30.738 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T09:48:30.738 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T09:48:30.738 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T09:48:30.738 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[117102]: ts=2023-10-09T09:48:30.476Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T09:48:30.820 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:29.400592+0000 mgr.y (mgr.14427) 56 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:30.820 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:29.400592+0000 mgr.y (mgr.14427) 56 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:30.824 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T09:48:30.832 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T09:48:30.833 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T09:48:30.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T09:48:31.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 bash[118837]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:48:31.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T09:48:31.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 systemd[1]: Stopped Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:48:31.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:30 smithi110 systemd[1]: Starting Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:48:31.438 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:31.500 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 podman[118960]: 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 bash[118960]: 4318ed52adb72496e7e2de82fee1a43c2e900a924a2eb753bd41ad435977a4ab 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.453Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.453Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.453Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi110 (none))" 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.454Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.454Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.455Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.456Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.457Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T09:48:31.501 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.459Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T09:48:31.502 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.459Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.695µs 2023-10-09T09:48:31.502 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.459Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T09:48:31.882 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STOPPING 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STOPPED 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STARTING 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Serving on http://:::9283 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STARTED 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STOPPING 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STOPPED 2023-10-09T09:48:31.883 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STARTING 2023-10-09T09:48:31.884 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Serving on http://:::9283 2023-10-09T09:48:31.884 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STARTED 2023-10-09T09:48:31.884 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STOPPING 2023-10-09T09:48:31.884 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:48:31.884 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STOPPED 2023-10-09T09:48:31.884 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STARTING 2023-10-09T09:48:31.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 systemd[1]: Started Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:48:31.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.625Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T09:48:31.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.625Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T09:48:31.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.625Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=39.85µs wal_replay_duration=166.542525ms total_replay_duration=166.608302ms 2023-10-09T09:48:31.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.627Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T09:48:31.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.627Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T09:48:31.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.627Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T09:48:31.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.644Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.781811ms db_storage=795ns remote_storage=1.15µs web_handler=298ns query_engine=657ns scrape=586.097µs scrape_sd=26.73µs notify=19.407µs notify_sd=6.269µs rules=15.809777ms 2023-10-09T09:48:31.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:48:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:48:31.644Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T09:48:32.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Serving on http://:::9283 2023-10-09T09:48:32.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:48:31] ENGINE Bus STARTED 2023-10-09T09:48:32.878 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:48:32.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:31.401026+0000 mgr.y (mgr.14427) 57 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:32.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.526040+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.529568+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:48:32.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.529975+0000 mgr.y (mgr.14427) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:48:32.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.530834+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2023-10-09T09:48:32.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.531117+0000 mgr.y (mgr.14427) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.541807+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.546585+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.547018+0000 mgr.y (mgr.14427) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.551461+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi110.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.551869+0000 mgr.y (mgr.14427) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi110.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.558158+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.562913+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.563358+0000 mgr.y (mgr.14427) 62 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:48:32.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.568922+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi110.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:48:32.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:31.401026+0000 mgr.y (mgr.14427) 57 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:32.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.526040+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.529568+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:48:32.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.529975+0000 mgr.y (mgr.14427) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:48:32.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.530834+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2023-10-09T09:48:32.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.531117+0000 mgr.y (mgr.14427) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2023-10-09T09:48:32.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.541807+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.546585+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:48:32.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.547018+0000 mgr.y (mgr.14427) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:48:32.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.551461+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi110.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:48:32.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.551869+0000 mgr.y (mgr.14427) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi110.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:48:32.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.558158+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.562913+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:48:32.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.563358+0000 mgr.y (mgr.14427) 62 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:48:32.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.568922+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi110.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:48:32.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.569292+0000 mgr.y (mgr.14427) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi110.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:48:32.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.575850+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:31.579744+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:32.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.569292+0000 mgr.y (mgr.14427) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi110.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:48:32.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.575850+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:31.579744+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:31.401026+0000 mgr.y (mgr.14427) 57 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.526040+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.529568+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:48:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.529975+0000 mgr.y (mgr.14427) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:48:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.530834+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2023-10-09T09:48:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.531117+0000 mgr.y (mgr.14427) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2023-10-09T09:48:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.541807+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.546585+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.547018+0000 mgr.y (mgr.14427) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.551461+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi110.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.551869+0000 mgr.y (mgr.14427) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi110.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.558158+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.562913+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.563358+0000 mgr.y (mgr.14427) 62 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.568922+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi110.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.569292+0000 mgr.y (mgr.14427) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi110.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T09:48:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.575850+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:32.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:31.579744+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:33.476 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":19,"active_gid":14427,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":1925191424},{"type":"v1","addr":"172.21.15.12:6801","nonce":1925191424}]},"active_addr":"172.21.15.12:6801/1925191424","active_change":"2023-10-09T09:47:27.376594+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24328,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2293469660}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":928078357}]}]} 2023-10-09T09:48:33.481 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T09:48:33.482 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T09:48:33.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:48:33.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:32.873003+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.12:0/3625096358' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:48:33.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:32.873003+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.12:0/3625096358' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:48:33.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:32.873003+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.12:0/3625096358' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:48:34.160 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:34.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:33.401477+0000 mgr.y (mgr.14427) 64 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:34.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:33.401477+0000 mgr.y (mgr.14427) 64 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:33.401477+0000 mgr.y (mgr.14427) 64 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:35.652 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:48:35.652 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":60,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T09:48:27.514758+0000","last_up_change":"2023-10-09T09:47:11.240268+0000","last_in_change":"2023-10-09T09:46:55.568500+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:47:41.614981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:48:24.361405+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:48:24.948128+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6803","nonce":3692896239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6805","nonce":3692896239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6809","nonce":3692896239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6807","nonce":3692896239}]},"public_addr":"172.21.15.12:6803/3692896239","cluster_addr":"172.21.15.12:6805/3692896239","heartbeat_back_addr":"172.21.15.12:6809/3692896239","heartbeat_front_addr":"172.21.15.12:6807/3692896239","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6811","nonce":994418467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6813","nonce":994418467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6817","nonce":994418467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6815","nonce":994418467}]},"public_addr":"172.21.15.12:6811/994418467","cluster_addr":"172.21.15.12:6813/994418467","heartbeat_back_addr":"172.21.15.12:6817/994418467","heartbeat_front_addr":"172.21.15.12:6815/994418467","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6819","nonce":979823756}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6821","nonce":979823756}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6825","nonce":979823756}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6823","nonce":979823756}]},"public_addr":"172.21.15.12:6819/979823756","cluster_addr":"172.21.15.12:6821/979823756","heartbeat_back_addr":"172.21.15.12:6825/979823756","heartbeat_front_addr":"172.21.15.12:6823/979823756","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6827","nonce":715124626}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6829","nonce":715124626}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6833","nonce":715124626}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6831","nonce":715124626}]},"public_addr":"172.21.15.12:6827/715124626","cluster_addr":"172.21.15.12:6829/715124626","heartbeat_back_addr":"172.21.15.12:6833/715124626","heartbeat_front_addr":"172.21.15.12:6831/715124626","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6801","nonce":3628101753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6803","nonce":3628101753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6807","nonce":3628101753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6805","nonce":3628101753}]},"public_addr":"172.21.15.110:6801/3628101753","cluster_addr":"172.21.15.110:6803/3628101753","heartbeat_back_addr":"172.21.15.110:6807/3628101753","heartbeat_front_addr":"172.21.15.110:6805/3628101753","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:48:36.281 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T09:48:36.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:48:36.901 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:35.401784+0000 mgr.y (mgr.14427) 65 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:35.621608+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:35.651370+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.12:0/1783557030' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:35.992754+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:36.451981+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:36.452765+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:36.458633+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:36.467552+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:36.468442+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:36.472649+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:48:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:36.473487+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:35.401784+0000 mgr.y (mgr.14427) 65 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:35.621608+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:35.651370+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.12:0/1783557030' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:35.992754+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:36.451981+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:36.452765+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:36.458633+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:36.467552+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:36.468442+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:48:36.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:36.472649+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:48:36.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:36.473487+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:35.401784+0000 mgr.y (mgr.14427) 65 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:35.621608+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:35.651370+0000 mon.a (mon.0) 654 : audit [DBG] from='client.? 172.21.15.12:0/1783557030' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:35.992754+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:36.451981+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:36.452765+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:36.458633+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:36.467552+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:36.468442+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:36.472649+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:48:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:36.473487+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:37.840 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:48:36.474115+0000 mgr.y (mgr.14427) 66 : cephadm [INF] Deploying daemon mds.foo.smithi012.mbnrgj on smithi012 2023-10-09T09:48:37.840 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:48:36.474115+0000 mgr.y (mgr.14427) 66 : cephadm [INF] Deploying daemon mds.foo.smithi012.mbnrgj on smithi012 2023-10-09T09:48:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:48:36.474115+0000 mgr.y (mgr.14427) 66 : cephadm [INF] Deploying daemon mds.foo.smithi012.mbnrgj on smithi012 2023-10-09T09:48:38.114 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:48:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:48:37.911Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003207004s 2023-10-09T09:48:38.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:37.402219+0000 mgr.y (mgr.14427) 67 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:38.716 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:37.402219+0000 mgr.y (mgr.14427) 67 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:37.402219+0000 mgr.y (mgr.14427) 67 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:39.447 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:48:39.447 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":60,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T09:48:27.514758+0000","last_up_change":"2023-10-09T09:47:11.240268+0000","last_in_change":"2023-10-09T09:46:55.568500+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:47:41.614981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:48:24.361405+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:48:24.948128+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6803","nonce":3692896239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6805","nonce":3692896239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6809","nonce":3692896239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3692896239},{"type":"v1","addr":"172.21.15.12:6807","nonce":3692896239}]},"public_addr":"172.21.15.12:6803/3692896239","cluster_addr":"172.21.15.12:6805/3692896239","heartbeat_back_addr":"172.21.15.12:6809/3692896239","heartbeat_front_addr":"172.21.15.12:6807/3692896239","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6811","nonce":994418467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6813","nonce":994418467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6817","nonce":994418467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":994418467},{"type":"v1","addr":"172.21.15.12:6815","nonce":994418467}]},"public_addr":"172.21.15.12:6811/994418467","cluster_addr":"172.21.15.12:6813/994418467","heartbeat_back_addr":"172.21.15.12:6817/994418467","heartbeat_front_addr":"172.21.15.12:6815/994418467","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6819","nonce":979823756}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6821","nonce":979823756}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6825","nonce":979823756}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":979823756},{"type":"v1","addr":"172.21.15.12:6823","nonce":979823756}]},"public_addr":"172.21.15.12:6819/979823756","cluster_addr":"172.21.15.12:6821/979823756","heartbeat_back_addr":"172.21.15.12:6825/979823756","heartbeat_front_addr":"172.21.15.12:6823/979823756","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6827","nonce":715124626}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6829","nonce":715124626}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6833","nonce":715124626}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":715124626},{"type":"v1","addr":"172.21.15.12:6831","nonce":715124626}]},"public_addr":"172.21.15.12:6827/715124626","cluster_addr":"172.21.15.12:6829/715124626","heartbeat_back_addr":"172.21.15.12:6833/715124626","heartbeat_front_addr":"172.21.15.12:6831/715124626","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6801","nonce":3628101753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6803","nonce":3628101753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6807","nonce":3628101753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6805","nonce":3628101753}]},"public_addr":"172.21.15.110:6801/3628101753","cluster_addr":"172.21.15.110:6803/3628101753","heartbeat_back_addr":"172.21.15.110:6807/3628101753","heartbeat_front_addr":"172.21.15.110:6805/3628101753","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:48:39.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T09:48:39.838 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T09:48:39.838 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T09:48:39.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T09:48:39.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T09:48:39.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T09:48:39.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T09:48:39.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T09:48:40.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:39.046350+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:40.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:39.047434+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:48:40.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:39.050711+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:48:40.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:39.051449+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:40.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:48:39.052037+0000 mgr.y (mgr.14427) 68 : cephadm [INF] Deploying daemon mds.foo.smithi110.sgosap on smithi110 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.402713+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:39.446776+0000 mon.a (mon.0) 667 : audit [DBG] from='client.? 172.21.15.12:0/2606513025' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.667360+0000 mon.a (mon.0) 668 : cluster [DBG] mds.? [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] up:boot 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.667426+0000 mon.a (mon.0) 669 : cluster [INF] daemon mds.foo.smithi012.mbnrgj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.667501+0000 mon.a (mon.0) 670 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.667527+0000 mon.a (mon.0) 671 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.667552+0000 mon.a (mon.0) 672 : cluster [INF] Cluster is now healthy 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.667750+0000 mon.a (mon.0) 673 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.670799+0000 mon.a (mon.0) 674 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:creating} 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:39.670990+0000 mon.a (mon.0) 675 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:48:40.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:39.682999+0000 mon.a (mon.0) 676 : cluster [INF] daemon mds.foo.smithi012.mbnrgj is now active in filesystem foo as rank 0 2023-10-09T09:48:40.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:39.046350+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:40.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:39.047434+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:48:40.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:39.050711+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:48:40.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:39.051449+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:40.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:48:39.052037+0000 mgr.y (mgr.14427) 68 : cephadm [INF] Deploying daemon mds.foo.smithi110.sgosap on smithi110 2023-10-09T09:48:40.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.402713+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:40.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:39.446776+0000 mon.a (mon.0) 667 : audit [DBG] from='client.? 172.21.15.12:0/2606513025' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.667360+0000 mon.a (mon.0) 668 : cluster [DBG] mds.? [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] up:boot 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.667426+0000 mon.a (mon.0) 669 : cluster [INF] daemon mds.foo.smithi012.mbnrgj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.667501+0000 mon.a (mon.0) 670 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.667527+0000 mon.a (mon.0) 671 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.667552+0000 mon.a (mon.0) 672 : cluster [INF] Cluster is now healthy 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.667750+0000 mon.a (mon.0) 673 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.670799+0000 mon.a (mon.0) 674 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:creating} 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:39.670990+0000 mon.a (mon.0) 675 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:48:40.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:39.682999+0000 mon.a (mon.0) 676 : cluster [INF] daemon mds.foo.smithi012.mbnrgj is now active in filesystem foo as rank 0 2023-10-09T09:48:40.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:39.046350+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:40.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:39.047434+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:48:40.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:39.050711+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T09:48:40.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:39.051449+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:40.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:48:39.052037+0000 mgr.y (mgr.14427) 68 : cephadm [INF] Deploying daemon mds.foo.smithi110.sgosap on smithi110 2023-10-09T09:48:40.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.402713+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:40.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:39.446776+0000 mon.a (mon.0) 667 : audit [DBG] from='client.? 172.21.15.12:0/2606513025' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.667360+0000 mon.a (mon.0) 668 : cluster [DBG] mds.? [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] up:boot 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.667426+0000 mon.a (mon.0) 669 : cluster [INF] daemon mds.foo.smithi012.mbnrgj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.667501+0000 mon.a (mon.0) 670 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.667527+0000 mon.a (mon.0) 671 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.667552+0000 mon.a (mon.0) 672 : cluster [INF] Cluster is now healthy 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.667750+0000 mon.a (mon.0) 673 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.670799+0000 mon.a (mon.0) 674 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:creating} 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:39.670990+0000 mon.a (mon.0) 675 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:48:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:39.682999+0000 mon.a (mon.0) 676 : cluster [INF] daemon mds.foo.smithi012.mbnrgj is now active in filesystem foo as rank 0 2023-10-09T09:48:40.655 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:40.809 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:40.935 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:40.936 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:40.936 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:40.978 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:41.079 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:41.230 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:41.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:40.673089+0000 mon.a (mon.0) 677 : cluster [DBG] mds.? [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] up:active 2023-10-09T09:48:41.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:40.673143+0000 mon.a (mon.0) 678 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 2023-10-09T09:48:41.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:41.057860+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:41.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:41.059405+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:41.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:40.673089+0000 mon.a (mon.0) 677 : cluster [DBG] mds.? [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] up:active 2023-10-09T09:48:41.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:40.673143+0000 mon.a (mon.0) 678 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 2023-10-09T09:48:41.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:41.057860+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:41.781 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:41.059405+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:41.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:40.673089+0000 mon.a (mon.0) 677 : cluster [DBG] mds.? [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] up:active 2023-10-09T09:48:41.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:40.673143+0000 mon.a (mon.0) 678 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 2023-10-09T09:48:41.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:41.057860+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:41.059405+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:41.403056+0000 mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:41.678453+0000 mon.a (mon.0) 681 : cluster [DBG] mds.? [v2:172.21.15.110:6832/704760757,v1:172.21.15.110:6833/704760757] up:boot 2023-10-09T09:48:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:41.679906+0000 mon.a (mon.0) 682 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:48:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:41.680001+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:48:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:41.682143+0000 mon.a (mon.0) 684 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:48:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:42.571707+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:42.619748+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:43.029 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:41.403056+0000 mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:43.029 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:41.678453+0000 mon.a (mon.0) 681 : cluster [DBG] mds.? [v2:172.21.15.110:6832/704760757,v1:172.21.15.110:6833/704760757] up:boot 2023-10-09T09:48:43.030 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:41.679906+0000 mon.a (mon.0) 682 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:48:43.030 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:41.680001+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:48:43.030 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:41.682143+0000 mon.a (mon.0) 684 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:48:43.030 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:42.571707+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:43.030 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:42.619748+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:43.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:41.403056+0000 mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:43.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:41.678453+0000 mon.a (mon.0) 681 : cluster [DBG] mds.? [v2:172.21.15.110:6832/704760757,v1:172.21.15.110:6833/704760757] up:boot 2023-10-09T09:48:43.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:41.679906+0000 mon.a (mon.0) 682 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:48:43.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:41.680001+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:48:43.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:41.682143+0000 mon.a (mon.0) 684 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:48:43.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:42.571707+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:43.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:42.619748+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:44.287 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:44] "GET /metrics HTTP/1.1" 200 191112 "" "Prometheus/2.33.4" 2023-10-09T09:48:44.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:43.403690+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:44.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:43.403690+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:43.403690+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:46.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:45.053471+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:45.404231+0000 mgr.y (mgr.14427) 72 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:45.053471+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:45.404231+0000 mgr.y (mgr.14427) 72 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:45.053471+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:45.404231+0000 mgr.y (mgr.14427) 72 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:46.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:48:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:48:46.430 INFO:teuthology.orchestra.run.smithi012.stdout:55834574892 2023-10-09T09:48:46.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T09:48:46.880 INFO:teuthology.orchestra.run.smithi012.stdout:154618822685 2023-10-09T09:48:46.881 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T09:48:46.947 INFO:teuthology.orchestra.run.smithi012.stdout:206158430229 2023-10-09T09:48:46.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T09:48:47.542 INFO:teuthology.orchestra.run.smithi012.stdout:128849018913 2023-10-09T09:48:47.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T09:48:47.567 INFO:teuthology.orchestra.run.smithi012.stdout:34359738416 2023-10-09T09:48:47.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T09:48:48.766 INFO:teuthology.orchestra.run.smithi012.stdout:77309411368 2023-10-09T09:48:48.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T09:48:48.768 INFO:teuthology.orchestra.run.smithi012.stdout:180388626457 2023-10-09T09:48:48.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T09:48:48.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:47.404622+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:48.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:47.404622+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:47.404622+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:49.469 INFO:teuthology.orchestra.run.smithi012.stdout:107374182435 2023-10-09T09:48:49.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T09:48:49.549 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:49.677 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:49.688 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:49.856 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:49.866 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:50.047 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:50.279 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:50.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:49.405257+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:50.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:49.405257+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:49.405257+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:51.476 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:52.861 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:51.405600+0000 mgr.y (mgr.14427) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:52.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:51.405600+0000 mgr.y (mgr.14427) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:51.405600+0000 mgr.y (mgr.14427) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:53.751 INFO:teuthology.orchestra.run.smithi012.stdout:206158430231 2023-10-09T09:48:54.304 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:54] "GET /metrics HTTP/1.1" 200 191112 "" "Prometheus/2.33.4" 2023-10-09T09:48:54.397 INFO:teuthology.orchestra.run.smithi012.stdout:128849018915 2023-10-09T09:48:54.420 INFO:teuthology.orchestra.run.smithi012.stdout:55834574894 2023-10-09T09:48:54.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:53.406083+0000 mgr.y (mgr.14427) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:54.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:53.751722+0000 mon.a (mon.0) 688 : audit [DBG] from='client.? 172.21.15.12:0/1409047627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:48:54.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:54.397210+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.12:0/1587288396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:48:54.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:54.420242+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.12:0/3371206366' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:48:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:53.406083+0000 mgr.y (mgr.14427) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:53.751722+0000 mon.a (mon.0) 688 : audit [DBG] from='client.? 172.21.15.12:0/1409047627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:48:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:54.397210+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.12:0/1587288396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:48:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:54.420242+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.12:0/3371206366' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:48:54.942 INFO:teuthology.orchestra.run.smithi012.stdout:180388626459 2023-10-09T09:48:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:53.406083+0000 mgr.y (mgr.14427) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:53.751722+0000 mon.a (mon.0) 688 : audit [DBG] from='client.? 172.21.15.12:0/1409047627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:48:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:54.397210+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.12:0/1587288396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:48:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:54.420242+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.12:0/3371206366' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:48:55.469 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430229 got 206158430231 for osd.7 2023-10-09T09:48:55.469 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:55.604 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018913 got 128849018915 for osd.4 2023-10-09T09:48:55.604 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:55.721 INFO:teuthology.orchestra.run.smithi012.stdout:107374182437 2023-10-09T09:48:55.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:54.942431+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.12:0/806125175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:48:56.164 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:54.942431+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.12:0/806125175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:48:56.166 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574894 for osd.1 2023-10-09T09:48:56.320 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:56.321 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:54.942431+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.12:0/806125175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:48:56.321 INFO:teuthology.orchestra.run.smithi012.stdout:154618822687 2023-10-09T09:48:56.321 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:48:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:48:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:48:56.727 INFO:teuthology.orchestra.run.smithi012.stdout:34359738418 2023-10-09T09:48:56.779 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626457 got 180388626459 for osd.6 2023-10-09T09:48:57.141 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:57.142 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:55.406427+0000 mgr.y (mgr.14427) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:57.143 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:55.721744+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.12:0/3106216964' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:48:57.143 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:56.308792+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.12:0/3492130325' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:48:57.144 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182435 got 107374182437 for osd.3 2023-10-09T09:48:57.144 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:55.406427+0000 mgr.y (mgr.14427) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:57.145 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:55.721744+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.12:0/3106216964' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:48:57.145 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:56.308792+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.12:0/3492130325' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:48:57.145 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:55.406427+0000 mgr.y (mgr.14427) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T09:48:57.145 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:55.721744+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.12:0/3106216964' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:48:57.146 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:56.308792+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.12:0/3492130325' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:48:57.146 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:57.146 INFO:teuthology.orchestra.run.smithi012.stdout:77309411370 2023-10-09T09:48:57.458 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411370 for osd.2 2023-10-09T09:48:57.459 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:57.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:56.727815+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.12:0/2119017364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:48:57.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:56.839006+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.12:0/2603571882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:48:57.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:56.727815+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.12:0/2119017364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:48:57.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:56.839006+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.12:0/2603571882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:48:57.938 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738418 for osd.0 2023-10-09T09:48:57.939 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:57.939 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822685 got 154618822687 for osd.5 2023-10-09T09:48:57.939 DEBUG:teuthology.parallel:result is None 2023-10-09T09:48:57.939 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T09:48:57.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:48:57.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:56.727815+0000 mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.12:0/2119017364' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:48:57.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:56.839006+0000 mon.a (mon.0) 692 : audit [DBG] from='client.? 172.21.15.12:0/2603571882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:48:58.603 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:48:58.996 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:57.406737+0000 mgr.y (mgr.14427) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:58.996 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:58.306693+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:58.997 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:58.307536+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:58.997 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:58.308257+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:58.997 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:58.314789+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:58.997 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:58.319469+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:58.997 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:57.406737+0000 mgr.y (mgr.14427) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:58.997 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:58.306693+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:58.997 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:58.307536+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:58.997 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:58.308257+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:58.998 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:58.314789+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:58.998 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:48:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:58.319469+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:59.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:57.406737+0000 mgr.y (mgr.14427) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:48:59.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:58.306693+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:59.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:58.307536+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:48:59.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:58.308257+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:48:59.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:58.314789+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:48:59.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:48:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:58.319469+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:48:59.977 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:48:59.978 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:49:00.422 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":58,"stamp":"2023-10-09T09:48:59.406944+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381560,"kb_used_data":4792,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579656,"statfs":{"total":767960285184,"available":765521567744,"internally_reserved":0,"allocated":4907008,"data_stored":2055400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002532"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528779+0000","last_change":"2023-10-09T09:48:26.527913+0000","last_active":"2023-10-09T09:48:26.528779+0000","last_peered":"2023-10-09T09:48:26.528779+0000","last_clean":"2023-10-09T09:48:26.528779+0000","last_became_active":"2023-10-09T09:48:26.527556+0000","last_became_peered":"2023-10-09T09:48:26.527556+0000","last_unstale":"2023-10-09T09:48:26.528779+0000","last_undegraded":"2023-10-09T09:48:26.528779+0000","last_fullsized":"2023-10-09T09:48:26.528779+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.681440+0000","last_change":"2023-10-09T09:48:25.517901+0000","last_active":"2023-10-09T09:48:39.681440+0000","last_peered":"2023-10-09T09:48:39.681440+0000","last_clean":"2023-10-09T09:48:39.681440+0000","last_became_active":"2023-10-09T09:48:25.517587+0000","last_became_peered":"2023-10-09T09:48:25.517587+0000","last_unstale":"2023-10-09T09:48:39.681440+0000","last_undegraded":"2023-10-09T09:48:39.681440+0000","last_fullsized":"2023-10-09T09:48:39.681440+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.531851+0000","last_change":"2023-10-09T09:48:26.530667+0000","last_active":"2023-10-09T09:48:26.531851+0000","last_peered":"2023-10-09T09:48:26.531851+0000","last_clean":"2023-10-09T09:48:26.531851+0000","last_became_active":"2023-10-09T09:48:26.530391+0000","last_became_peered":"2023-10-09T09:48:26.530391+0000","last_unstale":"2023-10-09T09:48:26.531851+0000","last_undegraded":"2023-10-09T09:48:26.531851+0000","last_fullsized":"2023-10-09T09:48:26.531851+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.750869+0000","last_change":"2023-10-09T09:48:25.516399+0000","last_active":"2023-10-09T09:48:27.750869+0000","last_peered":"2023-10-09T09:48:27.750869+0000","last_clean":"2023-10-09T09:48:27.750869+0000","last_became_active":"2023-10-09T09:48:25.516062+0000","last_became_peered":"2023-10-09T09:48:25.516062+0000","last_unstale":"2023-10-09T09:48:27.750869+0000","last_undegraded":"2023-10-09T09:48:27.750869+0000","last_fullsized":"2023-10-09T09:48:27.750869+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.531427+0000","last_change":"2023-10-09T09:48:26.531123+0000","last_active":"2023-10-09T09:48:26.531427+0000","last_peered":"2023-10-09T09:48:26.531427+0000","last_clean":"2023-10-09T09:48:26.531427+0000","last_became_active":"2023-10-09T09:48:26.530886+0000","last_became_peered":"2023-10-09T09:48:26.530886+0000","last_unstale":"2023-10-09T09:48:26.531427+0000","last_undegraded":"2023-10-09T09:48:26.531427+0000","last_fullsized":"2023-10-09T09:48:26.531427+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680942+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T09:48:39.680942+0000","last_peered":"2023-10-09T09:48:39.680942+0000","last_clean":"2023-10-09T09:48:39.680942+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T09:48:39.680942+0000","last_undegraded":"2023-10-09T09:48:39.680942+0000","last_fullsized":"2023-10-09T09:48:39.680942+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.530719+0000","last_change":"2023-10-09T09:48:26.530435+0000","last_active":"2023-10-09T09:48:26.530719+0000","last_peered":"2023-10-09T09:48:26.530719+0000","last_clean":"2023-10-09T09:48:26.530719+0000","last_became_active":"2023-10-09T09:48:26.530151+0000","last_became_peered":"2023-10-09T09:48:26.530151+0000","last_unstale":"2023-10-09T09:48:26.530719+0000","last_undegraded":"2023-10-09T09:48:26.530719+0000","last_fullsized":"2023-10-09T09:48:26.530719+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680933+0000","last_change":"2023-10-09T09:48:25.517291+0000","last_active":"2023-10-09T09:48:39.680933+0000","last_peered":"2023-10-09T09:48:39.680933+0000","last_clean":"2023-10-09T09:48:39.680933+0000","last_became_active":"2023-10-09T09:48:25.515878+0000","last_became_peered":"2023-10-09T09:48:25.515878+0000","last_unstale":"2023-10-09T09:48:39.680933+0000","last_undegraded":"2023-10-09T09:48:39.680933+0000","last_fullsized":"2023-10-09T09:48:39.680933+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.522734+0000","last_change":"2023-10-09T09:48:26.521866+0000","last_active":"2023-10-09T09:48:26.522734+0000","last_peered":"2023-10-09T09:48:26.522734+0000","last_clean":"2023-10-09T09:48:26.522734+0000","last_became_active":"2023-10-09T09:48:26.521510+0000","last_became_peered":"2023-10-09T09:48:26.521510+0000","last_unstale":"2023-10-09T09:48:26.522734+0000","last_undegraded":"2023-10-09T09:48:26.522734+0000","last_fullsized":"2023-10-09T09:48:26.522734+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.750152+0000","last_change":"2023-10-09T09:48:25.514721+0000","last_active":"2023-10-09T09:48:27.750152+0000","last_peered":"2023-10-09T09:48:27.750152+0000","last_clean":"2023-10-09T09:48:27.750152+0000","last_became_active":"2023-10-09T09:48:25.514406+0000","last_became_peered":"2023-10-09T09:48:25.514406+0000","last_unstale":"2023-10-09T09:48:27.750152+0000","last_undegraded":"2023-10-09T09:48:27.750152+0000","last_fullsized":"2023-10-09T09:48:27.750152+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.526027+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T09:48:26.526027+0000","last_peered":"2023-10-09T09:48:26.526027+0000","last_clean":"2023-10-09T09:48:26.526027+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T09:48:26.526027+0000","last_undegraded":"2023-10-09T09:48:26.526027+0000","last_fullsized":"2023-10-09T09:48:26.526027+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.750581+0000","last_change":"2023-10-09T09:48:25.524325+0000","last_active":"2023-10-09T09:48:27.750581+0000","last_peered":"2023-10-09T09:48:27.750581+0000","last_clean":"2023-10-09T09:48:27.750581+0000","last_became_active":"2023-10-09T09:48:25.523933+0000","last_became_peered":"2023-10-09T09:48:25.523933+0000","last_unstale":"2023-10-09T09:48:27.750581+0000","last_undegraded":"2023-10-09T09:48:27.750581+0000","last_fullsized":"2023-10-09T09:48:27.750581+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.516987+0000","last_change":"2023-10-09T09:47:42.887390+0000","last_active":"2023-10-09T09:48:26.516987+0000","last_peered":"2023-10-09T09:48:26.516987+0000","last_clean":"2023-10-09T09:48:26.516987+0000","last_became_active":"2023-10-09T09:47:42.887040+0000","last_became_peered":"2023-10-09T09:47:42.887040+0000","last_unstale":"2023-10-09T09:48:26.516987+0000","last_undegraded":"2023-10-09T09:48:26.516987+0000","last_fullsized":"2023-10-09T09:48:26.516987+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.156209+0000","last_change":"2023-10-09T09:48:27.155326+0000","last_active":"2023-10-09T09:48:27.156209+0000","last_peered":"2023-10-09T09:48:27.156209+0000","last_clean":"2023-10-09T09:48:27.156209+0000","last_became_active":"2023-10-09T09:48:27.155012+0000","last_became_peered":"2023-10-09T09:48:27.155012+0000","last_unstale":"2023-10-09T09:48:27.156209+0000","last_undegraded":"2023-10-09T09:48:27.156209+0000","last_fullsized":"2023-10-09T09:48:27.156209+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.519267+0000","last_change":"2023-10-09T09:47:42.989781+0000","last_active":"2023-10-09T09:48:26.519267+0000","last_peered":"2023-10-09T09:48:26.519267+0000","last_clean":"2023-10-09T09:48:26.519267+0000","last_became_active":"2023-10-09T09:47:42.989461+0000","last_became_peered":"2023-10-09T09:47:42.989461+0000","last_unstale":"2023-10-09T09:48:26.519267+0000","last_undegraded":"2023-10-09T09:48:26.519267+0000","last_fullsized":"2023-10-09T09:48:26.519267+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680602+0000","last_change":"2023-10-09T09:48:25.526649+0000","last_active":"2023-10-09T09:48:39.680602+0000","last_peered":"2023-10-09T09:48:39.680602+0000","last_clean":"2023-10-09T09:48:39.680602+0000","last_became_active":"2023-10-09T09:48:25.526348+0000","last_became_peered":"2023-10-09T09:48:25.526348+0000","last_unstale":"2023-10-09T09:48:39.680602+0000","last_undegraded":"2023-10-09T09:48:39.680602+0000","last_fullsized":"2023-10-09T09:48:39.680602+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154079+0000","last_change":"2023-10-09T09:48:27.152984+0000","last_active":"2023-10-09T09:48:27.154079+0000","last_peered":"2023-10-09T09:48:27.154079+0000","last_clean":"2023-10-09T09:48:27.154079+0000","last_became_active":"2023-10-09T09:48:27.152645+0000","last_became_peered":"2023-10-09T09:48:27.152645+0000","last_unstale":"2023-10-09T09:48:27.154079+0000","last_undegraded":"2023-10-09T09:48:27.154079+0000","last_fullsized":"2023-10-09T09:48:27.154079+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.518198+0000","last_change":"2023-10-09T09:48:25.529731+0000","last_active":"2023-10-09T09:48:26.518198+0000","last_peered":"2023-10-09T09:48:26.518198+0000","last_clean":"2023-10-09T09:48:26.518198+0000","last_became_active":"2023-10-09T09:48:25.529437+0000","last_became_peered":"2023-10-09T09:48:25.529437+0000","last_unstale":"2023-10-09T09:48:26.518198+0000","last_undegraded":"2023-10-09T09:48:26.518198+0000","last_fullsized":"2023-10-09T09:48:26.518198+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.153008+0000","last_change":"2023-10-09T09:47:42.989936+0000","last_active":"2023-10-09T09:48:27.153008+0000","last_peered":"2023-10-09T09:48:27.153008+0000","last_clean":"2023-10-09T09:48:27.153008+0000","last_became_active":"2023-10-09T09:47:42.989558+0000","last_became_peered":"2023-10-09T09:47:42.989558+0000","last_unstale":"2023-10-09T09:48:27.153008+0000","last_undegraded":"2023-10-09T09:48:27.153008+0000","last_fullsized":"2023-10-09T09:48:27.153008+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.530822+0000","last_change":"2023-10-09T09:48:26.529333+0000","last_active":"2023-10-09T09:48:26.530822+0000","last_peered":"2023-10-09T09:48:26.530822+0000","last_clean":"2023-10-09T09:48:26.530822+0000","last_became_active":"2023-10-09T09:48:26.529008+0000","last_became_peered":"2023-10-09T09:48:26.529008+0000","last_unstale":"2023-10-09T09:48:26.530822+0000","last_undegraded":"2023-10-09T09:48:26.530822+0000","last_fullsized":"2023-10-09T09:48:26.530822+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.523953+0000","last_change":"2023-10-09T09:48:25.521961+0000","last_active":"2023-10-09T09:48:26.523953+0000","last_peered":"2023-10-09T09:48:26.523953+0000","last_clean":"2023-10-09T09:48:26.523953+0000","last_became_active":"2023-10-09T09:48:25.520782+0000","last_became_peered":"2023-10-09T09:48:25.520782+0000","last_unstale":"2023-10-09T09:48:26.523953+0000","last_undegraded":"2023-10-09T09:48:26.523953+0000","last_fullsized":"2023-10-09T09:48:26.523953+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.152682+0000","last_change":"2023-10-09T09:47:42.989937+0000","last_active":"2023-10-09T09:48:27.152682+0000","last_peered":"2023-10-09T09:48:27.152682+0000","last_clean":"2023-10-09T09:48:27.152682+0000","last_became_active":"2023-10-09T09:47:42.989371+0000","last_became_peered":"2023-10-09T09:47:42.989371+0000","last_unstale":"2023-10-09T09:48:27.152682+0000","last_undegraded":"2023-10-09T09:48:27.152682+0000","last_fullsized":"2023-10-09T09:48:27.152682+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154458+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T09:48:27.154458+0000","last_peered":"2023-10-09T09:48:27.154458+0000","last_clean":"2023-10-09T09:48:27.154458+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T09:48:27.154458+0000","last_undegraded":"2023-10-09T09:48:27.154458+0000","last_fullsized":"2023-10-09T09:48:27.154458+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.522340+0000","last_change":"2023-10-09T09:48:25.525392+0000","last_active":"2023-10-09T09:48:26.522340+0000","last_peered":"2023-10-09T09:48:26.522340+0000","last_clean":"2023-10-09T09:48:26.522340+0000","last_became_active":"2023-10-09T09:48:25.525042+0000","last_became_peered":"2023-10-09T09:48:25.525042+0000","last_unstale":"2023-10-09T09:48:26.522340+0000","last_undegraded":"2023-10-09T09:48:26.522340+0000","last_fullsized":"2023-10-09T09:48:26.522340+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.152425+0000","last_change":"2023-10-09T09:47:13.676606+0000","last_active":"2023-10-09T09:48:27.152425+0000","last_peered":"2023-10-09T09:48:27.152425+0000","last_clean":"2023-10-09T09:48:27.152425+0000","last_became_active":"2023-10-09T09:47:13.676235+0000","last_became_peered":"2023-10-09T09:47:13.676235+0000","last_unstale":"2023-10-09T09:48:27.152425+0000","last_undegraded":"2023-10-09T09:48:27.152425+0000","last_fullsized":"2023-10-09T09:48:27.152425+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521157+0000","last_change":"2023-10-09T09:47:42.887560+0000","last_active":"2023-10-09T09:48:26.521157+0000","last_peered":"2023-10-09T09:48:26.521157+0000","last_clean":"2023-10-09T09:48:26.521157+0000","last_became_active":"2023-10-09T09:47:42.887101+0000","last_became_peered":"2023-10-09T09:47:42.887101+0000","last_unstale":"2023-10-09T09:48:26.521157+0000","last_undegraded":"2023-10-09T09:48:26.521157+0000","last_fullsized":"2023-10-09T09:48:26.521157+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.525879+0000","last_change":"2023-10-09T09:48:26.525332+0000","last_active":"2023-10-09T09:48:26.525879+0000","last_peered":"2023-10-09T09:48:26.525879+0000","last_clean":"2023-10-09T09:48:26.525879+0000","last_became_active":"2023-10-09T09:48:26.525037+0000","last_became_peered":"2023-10-09T09:48:26.525037+0000","last_unstale":"2023-10-09T09:48:26.525879+0000","last_undegraded":"2023-10-09T09:48:26.525879+0000","last_fullsized":"2023-10-09T09:48:26.525879+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.518860+0000","last_change":"2023-10-09T09:48:25.512889+0000","last_active":"2023-10-09T09:48:26.518860+0000","last_peered":"2023-10-09T09:48:26.518860+0000","last_clean":"2023-10-09T09:48:26.518860+0000","last_became_active":"2023-10-09T09:48:25.512548+0000","last_became_peered":"2023-10-09T09:48:25.512548+0000","last_unstale":"2023-10-09T09:48:26.518860+0000","last_undegraded":"2023-10-09T09:48:26.518860+0000","last_fullsized":"2023-10-09T09:48:26.518860+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.516324+0000","last_change":"2023-10-09T09:47:42.989695+0000","last_active":"2023-10-09T09:48:26.516324+0000","last_peered":"2023-10-09T09:48:26.516324+0000","last_clean":"2023-10-09T09:48:26.516324+0000","last_became_active":"2023-10-09T09:47:42.989373+0000","last_became_peered":"2023-10-09T09:47:42.989373+0000","last_unstale":"2023-10-09T09:48:26.516324+0000","last_undegraded":"2023-10-09T09:48:26.516324+0000","last_fullsized":"2023-10-09T09:48:26.516324+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.523074+0000","last_change":"2023-10-09T09:48:26.522502+0000","last_active":"2023-10-09T09:48:26.523074+0000","last_peered":"2023-10-09T09:48:26.523074+0000","last_clean":"2023-10-09T09:48:26.523074+0000","last_became_active":"2023-10-09T09:48:26.522158+0000","last_became_peered":"2023-10-09T09:48:26.522158+0000","last_unstale":"2023-10-09T09:48:26.523074+0000","last_undegraded":"2023-10-09T09:48:26.523074+0000","last_fullsized":"2023-10-09T09:48:26.523074+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.522022+0000","last_change":"2023-10-09T09:48:25.519131+0000","last_active":"2023-10-09T09:48:26.522022+0000","last_peered":"2023-10-09T09:48:26.522022+0000","last_clean":"2023-10-09T09:48:26.522022+0000","last_became_active":"2023-10-09T09:48:25.518279+0000","last_became_peered":"2023-10-09T09:48:25.518279+0000","last_unstale":"2023-10-09T09:48:26.522022+0000","last_undegraded":"2023-10-09T09:48:26.522022+0000","last_fullsized":"2023-10-09T09:48:26.522022+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521595+0000","last_change":"2023-10-09T09:47:42.888479+0000","last_active":"2023-10-09T09:48:26.521595+0000","last_peered":"2023-10-09T09:48:26.521595+0000","last_clean":"2023-10-09T09:48:26.521595+0000","last_became_active":"2023-10-09T09:47:42.888097+0000","last_became_peered":"2023-10-09T09:47:42.888097+0000","last_unstale":"2023-10-09T09:48:26.521595+0000","last_undegraded":"2023-10-09T09:48:26.521595+0000","last_fullsized":"2023-10-09T09:48:26.521595+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.527356+0000","last_change":"2023-10-09T09:48:26.526917+0000","last_active":"2023-10-09T09:48:26.527356+0000","last_peered":"2023-10-09T09:48:26.527356+0000","last_clean":"2023-10-09T09:48:26.527356+0000","last_became_active":"2023-10-09T09:48:26.526416+0000","last_became_peered":"2023-10-09T09:48:26.526416+0000","last_unstale":"2023-10-09T09:48:26.527356+0000","last_undegraded":"2023-10-09T09:48:26.527356+0000","last_fullsized":"2023-10-09T09:48:26.527356+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.523305+0000","last_change":"2023-10-09T09:48:25.522731+0000","last_active":"2023-10-09T09:48:26.523305+0000","last_peered":"2023-10-09T09:48:26.523305+0000","last_clean":"2023-10-09T09:48:26.523305+0000","last_became_active":"2023-10-09T09:48:25.522394+0000","last_became_peered":"2023-10-09T09:48:25.522394+0000","last_unstale":"2023-10-09T09:48:26.523305+0000","last_undegraded":"2023-10-09T09:48:26.523305+0000","last_fullsized":"2023-10-09T09:48:26.523305+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.518083+0000","last_change":"2023-10-09T09:47:42.989707+0000","last_active":"2023-10-09T09:48:26.518083+0000","last_peered":"2023-10-09T09:48:26.518083+0000","last_clean":"2023-10-09T09:48:26.518083+0000","last_became_active":"2023-10-09T09:47:42.989360+0000","last_became_peered":"2023-10-09T09:47:42.989360+0000","last_unstale":"2023-10-09T09:48:26.518083+0000","last_undegraded":"2023-10-09T09:48:26.518083+0000","last_fullsized":"2023-10-09T09:48:26.518083+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.529290+0000","last_change":"2023-10-09T09:48:26.528797+0000","last_active":"2023-10-09T09:48:26.529290+0000","last_peered":"2023-10-09T09:48:26.529290+0000","last_clean":"2023-10-09T09:48:26.529290+0000","last_became_active":"2023-10-09T09:48:26.528493+0000","last_became_peered":"2023-10-09T09:48:26.528493+0000","last_unstale":"2023-10-09T09:48:26.529290+0000","last_undegraded":"2023-10-09T09:48:26.529290+0000","last_fullsized":"2023-10-09T09:48:26.529290+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.520803+0000","last_change":"2023-10-09T09:48:25.530209+0000","last_active":"2023-10-09T09:48:26.520803+0000","last_peered":"2023-10-09T09:48:26.520803+0000","last_clean":"2023-10-09T09:48:26.520803+0000","last_became_active":"2023-10-09T09:48:25.529904+0000","last_became_peered":"2023-10-09T09:48:25.529904+0000","last_unstale":"2023-10-09T09:48:26.520803+0000","last_undegraded":"2023-10-09T09:48:26.520803+0000","last_fullsized":"2023-10-09T09:48:26.520803+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528188+0000","last_change":"2023-10-09T09:48:26.527282+0000","last_active":"2023-10-09T09:48:26.528188+0000","last_peered":"2023-10-09T09:48:26.528188+0000","last_clean":"2023-10-09T09:48:26.528188+0000","last_became_active":"2023-10-09T09:48:26.526970+0000","last_became_peered":"2023-10-09T09:48:26.526970+0000","last_unstale":"2023-10-09T09:48:26.528188+0000","last_undegraded":"2023-10-09T09:48:26.528188+0000","last_fullsized":"2023-10-09T09:48:26.528188+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521147+0000","last_change":"2023-10-09T09:48:25.516918+0000","last_active":"2023-10-09T09:48:26.521147+0000","last_peered":"2023-10-09T09:48:26.521147+0000","last_clean":"2023-10-09T09:48:26.521147+0000","last_became_active":"2023-10-09T09:48:25.516410+0000","last_became_peered":"2023-10-09T09:48:25.516410+0000","last_unstale":"2023-10-09T09:48:26.521147+0000","last_undegraded":"2023-10-09T09:48:26.521147+0000","last_fullsized":"2023-10-09T09:48:26.521147+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.530167+0000","last_change":"2023-10-09T09:48:26.529940+0000","last_active":"2023-10-09T09:48:26.530167+0000","last_peered":"2023-10-09T09:48:26.530167+0000","last_clean":"2023-10-09T09:48:26.530167+0000","last_became_active":"2023-10-09T09:48:26.529684+0000","last_became_peered":"2023-10-09T09:48:26.529684+0000","last_unstale":"2023-10-09T09:48:26.530167+0000","last_undegraded":"2023-10-09T09:48:26.530167+0000","last_fullsized":"2023-10-09T09:48:26.530167+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.516871+0000","last_change":"2023-10-09T09:48:25.517709+0000","last_active":"2023-10-09T09:48:26.516871+0000","last_peered":"2023-10-09T09:48:26.516871+0000","last_clean":"2023-10-09T09:48:26.516871+0000","last_became_active":"2023-10-09T09:48:25.517245+0000","last_became_peered":"2023-10-09T09:48:25.517245+0000","last_unstale":"2023-10-09T09:48:26.516871+0000","last_undegraded":"2023-10-09T09:48:26.516871+0000","last_fullsized":"2023-10-09T09:48:26.516871+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528690+0000","last_change":"2023-10-09T09:48:26.528068+0000","last_active":"2023-10-09T09:48:26.528690+0000","last_peered":"2023-10-09T09:48:26.528690+0000","last_clean":"2023-10-09T09:48:26.528690+0000","last_became_active":"2023-10-09T09:48:26.527657+0000","last_became_peered":"2023-10-09T09:48:26.527657+0000","last_unstale":"2023-10-09T09:48:26.528690+0000","last_undegraded":"2023-10-09T09:48:26.528690+0000","last_fullsized":"2023-10-09T09:48:26.528690+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680734+0000","last_change":"2023-10-09T09:48:25.526760+0000","last_active":"2023-10-09T09:48:39.680734+0000","last_peered":"2023-10-09T09:48:39.680734+0000","last_clean":"2023-10-09T09:48:39.680734+0000","last_became_active":"2023-10-09T09:48:25.526411+0000","last_became_peered":"2023-10-09T09:48:25.526411+0000","last_unstale":"2023-10-09T09:48:39.680734+0000","last_undegraded":"2023-10-09T09:48:39.680734+0000","last_fullsized":"2023-10-09T09:48:39.680734+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.530751+0000","last_change":"2023-10-09T09:48:26.529463+0000","last_active":"2023-10-09T09:48:26.530751+0000","last_peered":"2023-10-09T09:48:26.530751+0000","last_clean":"2023-10-09T09:48:26.530751+0000","last_became_active":"2023-10-09T09:48:26.529152+0000","last_became_peered":"2023-10-09T09:48:26.529152+0000","last_unstale":"2023-10-09T09:48:26.530751+0000","last_undegraded":"2023-10-09T09:48:26.530751+0000","last_fullsized":"2023-10-09T09:48:26.530751+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521887+0000","last_change":"2023-10-09T09:48:25.523527+0000","last_active":"2023-10-09T09:48:26.521887+0000","last_peered":"2023-10-09T09:48:26.521887+0000","last_clean":"2023-10-09T09:48:26.521887+0000","last_became_active":"2023-10-09T09:48:25.523043+0000","last_became_peered":"2023-10-09T09:48:25.523043+0000","last_unstale":"2023-10-09T09:48:26.521887+0000","last_undegraded":"2023-10-09T09:48:26.521887+0000","last_fullsized":"2023-10-09T09:48:26.521887+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.153857+0000","last_change":"2023-10-09T09:48:27.153033+0000","last_active":"2023-10-09T09:48:27.153857+0000","last_peered":"2023-10-09T09:48:27.153857+0000","last_clean":"2023-10-09T09:48:27.153857+0000","last_became_active":"2023-10-09T09:48:27.152705+0000","last_became_peered":"2023-10-09T09:48:27.152705+0000","last_unstale":"2023-10-09T09:48:27.153857+0000","last_undegraded":"2023-10-09T09:48:27.153857+0000","last_fullsized":"2023-10-09T09:48:27.153857+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.751066+0000","last_change":"2023-10-09T09:48:25.514756+0000","last_active":"2023-10-09T09:48:27.751066+0000","last_peered":"2023-10-09T09:48:27.751066+0000","last_clean":"2023-10-09T09:48:27.751066+0000","last_became_active":"2023-10-09T09:48:25.514387+0000","last_became_peered":"2023-10-09T09:48:25.514387+0000","last_unstale":"2023-10-09T09:48:27.751066+0000","last_undegraded":"2023-10-09T09:48:27.751066+0000","last_fullsized":"2023-10-09T09:48:27.751066+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.529169+0000","last_change":"2023-10-09T09:48:26.528888+0000","last_active":"2023-10-09T09:48:26.529169+0000","last_peered":"2023-10-09T09:48:26.529169+0000","last_clean":"2023-10-09T09:48:26.529169+0000","last_became_active":"2023-10-09T09:48:26.528596+0000","last_became_peered":"2023-10-09T09:48:26.528596+0000","last_unstale":"2023-10-09T09:48:26.529169+0000","last_undegraded":"2023-10-09T09:48:26.529169+0000","last_fullsized":"2023-10-09T09:48:26.529169+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680156+0000","last_change":"2023-10-09T09:48:25.520127+0000","last_active":"2023-10-09T09:48:39.680156+0000","last_peered":"2023-10-09T09:48:39.680156+0000","last_clean":"2023-10-09T09:48:39.680156+0000","last_became_active":"2023-10-09T09:48:25.519822+0000","last_became_peered":"2023-10-09T09:48:25.519822+0000","last_unstale":"2023-10-09T09:48:39.680156+0000","last_undegraded":"2023-10-09T09:48:39.680156+0000","last_fullsized":"2023-10-09T09:48:39.680156+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.155537+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T09:48:27.155537+0000","last_peered":"2023-10-09T09:48:27.155537+0000","last_clean":"2023-10-09T09:48:27.155537+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T09:48:27.155537+0000","last_undegraded":"2023-10-09T09:48:27.155537+0000","last_fullsized":"2023-10-09T09:48:27.155537+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.750907+0000","last_change":"2023-10-09T09:48:25.524829+0000","last_active":"2023-10-09T09:48:27.750907+0000","last_peered":"2023-10-09T09:48:27.750907+0000","last_clean":"2023-10-09T09:48:27.750907+0000","last_became_active":"2023-10-09T09:48:25.524492+0000","last_became_peered":"2023-10-09T09:48:25.524492+0000","last_unstale":"2023-10-09T09:48:27.750907+0000","last_undegraded":"2023-10-09T09:48:27.750907+0000","last_fullsized":"2023-10-09T09:48:27.750907+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154597+0000","last_change":"2023-10-09T09:48:27.153176+0000","last_active":"2023-10-09T09:48:27.154597+0000","last_peered":"2023-10-09T09:48:27.154597+0000","last_clean":"2023-10-09T09:48:27.154597+0000","last_became_active":"2023-10-09T09:48:27.152826+0000","last_became_peered":"2023-10-09T09:48:27.152826+0000","last_unstale":"2023-10-09T09:48:27.154597+0000","last_undegraded":"2023-10-09T09:48:27.154597+0000","last_fullsized":"2023-10-09T09:48:27.154597+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.681842+0000","last_change":"2023-10-09T09:48:25.513226+0000","last_active":"2023-10-09T09:48:39.681842+0000","last_peered":"2023-10-09T09:48:39.681842+0000","last_clean":"2023-10-09T09:48:39.681842+0000","last_became_active":"2023-10-09T09:48:25.512767+0000","last_became_peered":"2023-10-09T09:48:25.512767+0000","last_unstale":"2023-10-09T09:48:39.681842+0000","last_undegraded":"2023-10-09T09:48:39.681842+0000","last_fullsized":"2023-10-09T09:48:39.681842+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154127+0000","last_change":"2023-10-09T09:48:27.153722+0000","last_active":"2023-10-09T09:48:27.154127+0000","last_peered":"2023-10-09T09:48:27.154127+0000","last_clean":"2023-10-09T09:48:27.154127+0000","last_became_active":"2023-10-09T09:48:27.153307+0000","last_became_peered":"2023-10-09T09:48:27.153307+0000","last_unstale":"2023-10-09T09:48:27.154127+0000","last_undegraded":"2023-10-09T09:48:27.154127+0000","last_fullsized":"2023-10-09T09:48:27.154127+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.152505+0000","last_change":"2023-10-09T09:48:25.523230+0000","last_active":"2023-10-09T09:48:27.152505+0000","last_peered":"2023-10-09T09:48:27.152505+0000","last_clean":"2023-10-09T09:48:27.152505+0000","last_became_active":"2023-10-09T09:48:25.522608+0000","last_became_peered":"2023-10-09T09:48:25.522608+0000","last_unstale":"2023-10-09T09:48:27.152505+0000","last_undegraded":"2023-10-09T09:48:27.152505+0000","last_fullsized":"2023-10-09T09:48:27.152505+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.155504+0000","last_change":"2023-10-09T09:48:27.155121+0000","last_active":"2023-10-09T09:48:27.155504+0000","last_peered":"2023-10-09T09:48:27.155504+0000","last_clean":"2023-10-09T09:48:27.155504+0000","last_became_active":"2023-10-09T09:48:27.154733+0000","last_became_peered":"2023-10-09T09:48:27.154733+0000","last_unstale":"2023-10-09T09:48:27.155504+0000","last_undegraded":"2023-10-09T09:48:27.155504+0000","last_fullsized":"2023-10-09T09:48:27.155504+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.527696+0000","last_change":"2023-10-09T09:48:25.519906+0000","last_active":"2023-10-09T09:48:26.527696+0000","last_peered":"2023-10-09T09:48:26.527696+0000","last_clean":"2023-10-09T09:48:26.527696+0000","last_became_active":"2023-10-09T09:48:25.519595+0000","last_became_peered":"2023-10-09T09:48:25.519595+0000","last_unstale":"2023-10-09T09:48:26.527696+0000","last_undegraded":"2023-10-09T09:48:26.527696+0000","last_fullsized":"2023-10-09T09:48:26.527696+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154630+0000","last_change":"2023-10-09T09:48:27.153292+0000","last_active":"2023-10-09T09:48:27.154630+0000","last_peered":"2023-10-09T09:48:27.154630+0000","last_clean":"2023-10-09T09:48:27.154630+0000","last_became_active":"2023-10-09T09:48:27.152784+0000","last_became_peered":"2023-10-09T09:48:27.152784+0000","last_unstale":"2023-10-09T09:48:27.154630+0000","last_undegraded":"2023-10-09T09:48:27.154630+0000","last_fullsized":"2023-10-09T09:48:27.154630+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680160+0000","last_change":"2023-10-09T09:48:25.520564+0000","last_active":"2023-10-09T09:48:39.680160+0000","last_peered":"2023-10-09T09:48:39.680160+0000","last_clean":"2023-10-09T09:48:39.680160+0000","last_became_active":"2023-10-09T09:48:25.520267+0000","last_became_peered":"2023-10-09T09:48:25.520267+0000","last_unstale":"2023-10-09T09:48:39.680160+0000","last_undegraded":"2023-10-09T09:48:39.680160+0000","last_fullsized":"2023-10-09T09:48:39.680160+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528864+0000","last_change":"2023-10-09T09:48:26.528227+0000","last_active":"2023-10-09T09:48:26.528864+0000","last_peered":"2023-10-09T09:48:26.528864+0000","last_clean":"2023-10-09T09:48:26.528864+0000","last_became_active":"2023-10-09T09:48:26.527937+0000","last_became_peered":"2023-10-09T09:48:26.527937+0000","last_unstale":"2023-10-09T09:48:26.528864+0000","last_undegraded":"2023-10-09T09:48:26.528864+0000","last_fullsized":"2023-10-09T09:48:26.528864+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.681570+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T09:48:39.681570+0000","last_peered":"2023-10-09T09:48:39.681570+0000","last_clean":"2023-10-09T09:48:39.681570+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T09:48:39.681570+0000","last_undegraded":"2023-10-09T09:48:39.681570+0000","last_fullsized":"2023-10-09T09:48:39.681570+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.527244+0000","last_change":"2023-10-09T09:48:26.526722+0000","last_active":"2023-10-09T09:48:26.527244+0000","last_peered":"2023-10-09T09:48:26.527244+0000","last_clean":"2023-10-09T09:48:26.527244+0000","last_became_active":"2023-10-09T09:48:26.525824+0000","last_became_peered":"2023-10-09T09:48:26.525824+0000","last_unstale":"2023-10-09T09:48:26.527244+0000","last_undegraded":"2023-10-09T09:48:26.527244+0000","last_fullsized":"2023-10-09T09:48:26.527244+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680820+0000","last_change":"2023-10-09T09:48:25.522979+0000","last_active":"2023-10-09T09:48:39.680820+0000","last_peered":"2023-10-09T09:48:39.680820+0000","last_clean":"2023-10-09T09:48:39.680820+0000","last_became_active":"2023-10-09T09:48:25.522649+0000","last_became_peered":"2023-10-09T09:48:25.522649+0000","last_unstale":"2023-10-09T09:48:39.680820+0000","last_undegraded":"2023-10-09T09:48:39.680820+0000","last_fullsized":"2023-10-09T09:48:39.680820+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.156140+0000","last_change":"2023-10-09T09:48:27.155564+0000","last_active":"2023-10-09T09:48:27.156140+0000","last_peered":"2023-10-09T09:48:27.156140+0000","last_clean":"2023-10-09T09:48:27.156140+0000","last_became_active":"2023-10-09T09:48:27.153001+0000","last_became_peered":"2023-10-09T09:48:27.153001+0000","last_unstale":"2023-10-09T09:48:27.156140+0000","last_undegraded":"2023-10-09T09:48:27.156140+0000","last_fullsized":"2023-10-09T09:48:27.156140+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521046+0000","last_change":"2023-10-09T09:48:25.514371+0000","last_active":"2023-10-09T09:48:26.521046+0000","last_peered":"2023-10-09T09:48:26.521046+0000","last_clean":"2023-10-09T09:48:26.521046+0000","last_became_active":"2023-10-09T09:48:25.514036+0000","last_became_peered":"2023-10-09T09:48:25.514036+0000","last_unstale":"2023-10-09T09:48:26.521046+0000","last_undegraded":"2023-10-09T09:48:26.521046+0000","last_fullsized":"2023-10-09T09:48:26.521046+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528669+0000","last_change":"2023-10-09T09:48:26.528344+0000","last_active":"2023-10-09T09:48:26.528669+0000","last_peered":"2023-10-09T09:48:26.528669+0000","last_clean":"2023-10-09T09:48:26.528669+0000","last_became_active":"2023-10-09T09:48:26.528003+0000","last_became_peered":"2023-10-09T09:48:26.528003+0000","last_unstale":"2023-10-09T09:48:26.528669+0000","last_undegraded":"2023-10-09T09:48:26.528669+0000","last_fullsized":"2023-10-09T09:48:26.528669+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528059+0000","last_change":"2023-10-09T09:48:25.522120+0000","last_active":"2023-10-09T09:48:26.528059+0000","last_peered":"2023-10-09T09:48:26.528059+0000","last_clean":"2023-10-09T09:48:26.528059+0000","last_became_active":"2023-10-09T09:48:25.521819+0000","last_became_peered":"2023-10-09T09:48:25.521819+0000","last_unstale":"2023-10-09T09:48:26.528059+0000","last_undegraded":"2023-10-09T09:48:26.528059+0000","last_fullsized":"2023-10-09T09:48:26.528059+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.526076+0000","last_change":"2023-10-09T09:48:26.524019+0000","last_active":"2023-10-09T09:48:26.526076+0000","last_peered":"2023-10-09T09:48:26.526076+0000","last_clean":"2023-10-09T09:48:26.526076+0000","last_became_active":"2023-10-09T09:48:26.523487+0000","last_became_peered":"2023-10-09T09:48:26.523487+0000","last_unstale":"2023-10-09T09:48:26.526076+0000","last_undegraded":"2023-10-09T09:48:26.526076+0000","last_fullsized":"2023-10-09T09:48:26.526076+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.522720+0000","last_change":"2023-10-09T09:48:25.529519+0000","last_active":"2023-10-09T09:48:26.522720+0000","last_peered":"2023-10-09T09:48:26.522720+0000","last_clean":"2023-10-09T09:48:26.522720+0000","last_became_active":"2023-10-09T09:48:25.529197+0000","last_became_peered":"2023-10-09T09:48:25.529197+0000","last_unstale":"2023-10-09T09:48:26.522720+0000","last_undegraded":"2023-10-09T09:48:26.522720+0000","last_fullsized":"2023-10-09T09:48:26.522720+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154107+0000","last_change":"2023-10-09T09:48:27.153631+0000","last_active":"2023-10-09T09:48:27.154107+0000","last_peered":"2023-10-09T09:48:27.154107+0000","last_clean":"2023-10-09T09:48:27.154107+0000","last_became_active":"2023-10-09T09:48:27.153342+0000","last_became_peered":"2023-10-09T09:48:27.153342+0000","last_unstale":"2023-10-09T09:48:27.154107+0000","last_undegraded":"2023-10-09T09:48:27.154107+0000","last_fullsized":"2023-10-09T09:48:27.154107+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.517867+0000","last_change":"2023-10-09T09:48:25.528225+0000","last_active":"2023-10-09T09:48:26.517867+0000","last_peered":"2023-10-09T09:48:26.517867+0000","last_clean":"2023-10-09T09:48:26.517867+0000","last_became_active":"2023-10-09T09:48:25.527905+0000","last_became_peered":"2023-10-09T09:48:25.527905+0000","last_unstale":"2023-10-09T09:48:26.517867+0000","last_undegraded":"2023-10-09T09:48:26.517867+0000","last_fullsized":"2023-10-09T09:48:26.517867+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.531886+0000","last_change":"2023-10-09T09:48:26.531031+0000","last_active":"2023-10-09T09:48:26.531886+0000","last_peered":"2023-10-09T09:48:26.531886+0000","last_clean":"2023-10-09T09:48:26.531886+0000","last_became_active":"2023-10-09T09:48:26.530702+0000","last_became_peered":"2023-10-09T09:48:26.530702+0000","last_unstale":"2023-10-09T09:48:26.531886+0000","last_undegraded":"2023-10-09T09:48:26.531886+0000","last_fullsized":"2023-10-09T09:48:26.531886+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.751065+0000","last_change":"2023-10-09T09:48:25.520986+0000","last_active":"2023-10-09T09:48:27.751065+0000","last_peered":"2023-10-09T09:48:27.751065+0000","last_clean":"2023-10-09T09:48:27.751065+0000","last_became_active":"2023-10-09T09:48:25.520654+0000","last_became_peered":"2023-10-09T09:48:25.520654+0000","last_unstale":"2023-10-09T09:48:27.751065+0000","last_undegraded":"2023-10-09T09:48:27.751065+0000","last_fullsized":"2023-10-09T09:48:27.751065+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430232,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":257891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.498}]},{"osd":1,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.46100000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.436},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.61799999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.63600000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.53400000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.57699999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626460,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":257430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.49199999999999999,"15min":0.49299999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.504,"15min":0.504},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.501}]},{"osd":1,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.73999999999999999,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.51200000000000001,"15min":0.51800000000000002},"min":{"1min":0.35799999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.63600000000000001,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.41499999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.38300000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76300000000000001,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.51900000000000002,"15min":0.51600000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.40200000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.55200000000000005,"15min":0.54200000000000004},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.40100000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.54900000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.44,"15min":0.434},"min":{"1min":0.31,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60499999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.48699999999999999,"15min":0.48299999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.74099999999999999,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.60699999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.77300000000000002,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.501,"15min":0.495},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.55900000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.58099999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.35699999999999998,"15min":0.34499999999999997},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.36699999999999999,"15min":0.34499999999999997},"min":{"1min":0.255,"5min":0.223,"15min":0.223},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.28499999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:48:42 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.52800000000000002,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.376},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.41799999999999998,"15min":0.41399999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.59499999999999997,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.33100000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.46800000000000003,"15min":0.46600000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64800000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.63,"15min":0.63400000000000001},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.89100000000000001,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.63,"15min":0.63600000000000001},"min":{"1min":0.45300000000000001,"5min":0.439,"15min":0.439},"max":{"1min":0.88200000000000001,"5min":1.1870000000000001,"15min":1.1870000000000001},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 09:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.433,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.876,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.58799999999999997,"15min":0.59199999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.80200000000000005,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.61099999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.61799999999999999,"15min":0.62},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80900000000000005,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60899999999999999,"15min":0.61499999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83299999999999996,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.69399999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.54600000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.34599999999999997,"15min":0.32500000000000001},"min":{"1min":0.245,"5min":0.193,"15min":0.193},"max":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.436},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.37,"15min":0.33300000000000002},"min":{"1min":0.29999999999999999,"5min":0.219,"15min":0.219},"max":{"1min":0.63800000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:48:43 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.42799999999999999,"15min":0.42399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.60399999999999998,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.39500000000000002,"15min":0.39200000000000002},"min":{"1min":0.246,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.48399999999999999,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.32300000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.41099999999999998,"15min":0.40200000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.48499999999999999,"15min":0.47599999999999998},"min":{"1min":0.38300000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.495}]},{"osd":4,"last update":"Mon Oct 9 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.629,"15min":0.63100000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.873,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.63600000000000001,"15min":0.64000000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.81200000000000006,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.65100000000000002,"15min":0.66300000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.874,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63300000000000001,"15min":0.64200000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.78000000000000003,"5min":0.87,"15min":0.87},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:48:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.67000000000000004,"15min":0.68999999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.69299999999999995,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.67700000000000005,"15min":0.68999999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83999999999999997,"5min":1.01,"15min":1.01},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.625}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.42799999999999999,"15min":0.36799999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":6.3810000000000002,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.38600000000000001,"15min":0.371},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.831}]},{"osd":1,"last update":"Mon Oct 9 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.48999999999999999,"15min":0.42299999999999999},"min":{"1min":0.26000000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.4729999999999999,"5min":6.4729999999999999,"15min":6.4729999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.437,"15min":0.42399999999999999},"min":{"1min":0.28599999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"last":0.47199999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:47:58 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.53900000000000003,"15min":0.48799999999999999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.4450000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":1.9390000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.502,"15min":0.502},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":1.95}]},{"osd":4,"last update":"Mon Oct 9 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.59499999999999997,"15min":0.58899999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":2.1509999999999998},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.68400000000000005,"15min":0.63600000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":6.2919999999999998,"5min":6.2919999999999998,"15min":6.2919999999999998},"last":0.77900000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:48:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.68000000000000005,"15min":0.63200000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.3490000000000002,"5min":6.3490000000000002,"15min":6.3490000000000002},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.623,"15min":0.61799999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":2.1800000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:47:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":2.1989999999999998},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":6.4089999999999998,"5min":6.4089999999999998,"15min":6.4089999999999998},"last":0.81499999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.7090000000000001,"5min":1.7090000000000001,"15min":1.7090000000000001},"last":2.105}]}]},{"osd":3,"up_from":25,"seq":107374182438,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.41299999999999998,"15min":0.41899999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57099999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.442,"15min":0.45500000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.51100000000000001,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.30199999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.434,"15min":0.44800000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.48999999999999999,"5min":0.623,"15min":0.623},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.628,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.36199999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.495,"15min":0.51100000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.53300000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.441},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.48599999999999999,"15min":0.499},"min":{"1min":0.34699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.5,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.39500000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.60799999999999998,"15min":0.61799999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.79000000000000004,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.69999999999999996,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:48:34 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.63400000000000001,"15min":0.64400000000000002},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84899999999999998,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60099999999999998,"15min":0.60699999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.755,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.74099999999999999,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59599999999999997,"15min":0.59999999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.81599999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 09:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.70499999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018916,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":257821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.40799999999999997,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.69899999999999995,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.58599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.69899999999999995,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.54800000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56000000000000005,"15min":0.56799999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67500000000000004,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.56200000000000006,"15min":0.56399999999999995},"min":{"1min":0.42499999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67800000000000005,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.68799999999999994,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.76400000000000001,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.66000000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.72699999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.626},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.59199999999999997,"15min":0.60099999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.67100000000000004,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.61699999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.29999999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.432,"15min":0.42999999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.42899999999999999,"15min":0.42299999999999999},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.67400000000000004}]}]},{"osd":5,"up_from":36,"seq":154618822688,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.52300000000000002,"15min":0.53100000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.67200000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53000000000000003,"15min":0.53300000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69199999999999995,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.45900000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.52600000000000002,"15min":0.53200000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.64600000000000002,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51500000000000001,"15min":0.52100000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.66600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.41999999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54500000000000004,"15min":0.55100000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.39000000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.65800000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":2.9319999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69599999999999995,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.59399999999999997,"15min":0.58199999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.637,"5min":2.637,"15min":2.637},"last":0.39900000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.41799999999999998,"15min":0.40600000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.503},{"interface":"front","average":{"1min":0.501,"5min":0.439,"15min":0.42899999999999999},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56699999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.48499999999999999,"15min":0.47899999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.628},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.49399999999999999,"15min":0.48199999999999998},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.53500000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.58199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:49:00.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:49:01.033 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:01.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:48:59.407307+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:01.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:48:59.965563+0000 mgr.y (mgr.14427) 80 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:49:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:48:59.407307+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:48:59.965563+0000 mgr.y (mgr.14427) 80 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:49:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:48:59.407307+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:48:59.965563+0000 mgr.y (mgr.14427) 80 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:49:02.386 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:49:02.387 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:49:02.860 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":59,"stamp":"2023-10-09T09:49:01.407483+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381560,"kb_used_data":4792,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579656,"statfs":{"total":767960285184,"available":765521567744,"internally_reserved":0,"allocated":4907008,"data_stored":2055400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002680"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528779+0000","last_change":"2023-10-09T09:48:26.527913+0000","last_active":"2023-10-09T09:48:26.528779+0000","last_peered":"2023-10-09T09:48:26.528779+0000","last_clean":"2023-10-09T09:48:26.528779+0000","last_became_active":"2023-10-09T09:48:26.527556+0000","last_became_peered":"2023-10-09T09:48:26.527556+0000","last_unstale":"2023-10-09T09:48:26.528779+0000","last_undegraded":"2023-10-09T09:48:26.528779+0000","last_fullsized":"2023-10-09T09:48:26.528779+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.681440+0000","last_change":"2023-10-09T09:48:25.517901+0000","last_active":"2023-10-09T09:48:39.681440+0000","last_peered":"2023-10-09T09:48:39.681440+0000","last_clean":"2023-10-09T09:48:39.681440+0000","last_became_active":"2023-10-09T09:48:25.517587+0000","last_became_peered":"2023-10-09T09:48:25.517587+0000","last_unstale":"2023-10-09T09:48:39.681440+0000","last_undegraded":"2023-10-09T09:48:39.681440+0000","last_fullsized":"2023-10-09T09:48:39.681440+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.531851+0000","last_change":"2023-10-09T09:48:26.530667+0000","last_active":"2023-10-09T09:48:26.531851+0000","last_peered":"2023-10-09T09:48:26.531851+0000","last_clean":"2023-10-09T09:48:26.531851+0000","last_became_active":"2023-10-09T09:48:26.530391+0000","last_became_peered":"2023-10-09T09:48:26.530391+0000","last_unstale":"2023-10-09T09:48:26.531851+0000","last_undegraded":"2023-10-09T09:48:26.531851+0000","last_fullsized":"2023-10-09T09:48:26.531851+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.750869+0000","last_change":"2023-10-09T09:48:25.516399+0000","last_active":"2023-10-09T09:48:27.750869+0000","last_peered":"2023-10-09T09:48:27.750869+0000","last_clean":"2023-10-09T09:48:27.750869+0000","last_became_active":"2023-10-09T09:48:25.516062+0000","last_became_peered":"2023-10-09T09:48:25.516062+0000","last_unstale":"2023-10-09T09:48:27.750869+0000","last_undegraded":"2023-10-09T09:48:27.750869+0000","last_fullsized":"2023-10-09T09:48:27.750869+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.531427+0000","last_change":"2023-10-09T09:48:26.531123+0000","last_active":"2023-10-09T09:48:26.531427+0000","last_peered":"2023-10-09T09:48:26.531427+0000","last_clean":"2023-10-09T09:48:26.531427+0000","last_became_active":"2023-10-09T09:48:26.530886+0000","last_became_peered":"2023-10-09T09:48:26.530886+0000","last_unstale":"2023-10-09T09:48:26.531427+0000","last_undegraded":"2023-10-09T09:48:26.531427+0000","last_fullsized":"2023-10-09T09:48:26.531427+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680942+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T09:48:39.680942+0000","last_peered":"2023-10-09T09:48:39.680942+0000","last_clean":"2023-10-09T09:48:39.680942+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T09:48:39.680942+0000","last_undegraded":"2023-10-09T09:48:39.680942+0000","last_fullsized":"2023-10-09T09:48:39.680942+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.530719+0000","last_change":"2023-10-09T09:48:26.530435+0000","last_active":"2023-10-09T09:48:26.530719+0000","last_peered":"2023-10-09T09:48:26.530719+0000","last_clean":"2023-10-09T09:48:26.530719+0000","last_became_active":"2023-10-09T09:48:26.530151+0000","last_became_peered":"2023-10-09T09:48:26.530151+0000","last_unstale":"2023-10-09T09:48:26.530719+0000","last_undegraded":"2023-10-09T09:48:26.530719+0000","last_fullsized":"2023-10-09T09:48:26.530719+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680933+0000","last_change":"2023-10-09T09:48:25.517291+0000","last_active":"2023-10-09T09:48:39.680933+0000","last_peered":"2023-10-09T09:48:39.680933+0000","last_clean":"2023-10-09T09:48:39.680933+0000","last_became_active":"2023-10-09T09:48:25.515878+0000","last_became_peered":"2023-10-09T09:48:25.515878+0000","last_unstale":"2023-10-09T09:48:39.680933+0000","last_undegraded":"2023-10-09T09:48:39.680933+0000","last_fullsized":"2023-10-09T09:48:39.680933+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.522734+0000","last_change":"2023-10-09T09:48:26.521866+0000","last_active":"2023-10-09T09:48:26.522734+0000","last_peered":"2023-10-09T09:48:26.522734+0000","last_clean":"2023-10-09T09:48:26.522734+0000","last_became_active":"2023-10-09T09:48:26.521510+0000","last_became_peered":"2023-10-09T09:48:26.521510+0000","last_unstale":"2023-10-09T09:48:26.522734+0000","last_undegraded":"2023-10-09T09:48:26.522734+0000","last_fullsized":"2023-10-09T09:48:26.522734+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.750152+0000","last_change":"2023-10-09T09:48:25.514721+0000","last_active":"2023-10-09T09:48:27.750152+0000","last_peered":"2023-10-09T09:48:27.750152+0000","last_clean":"2023-10-09T09:48:27.750152+0000","last_became_active":"2023-10-09T09:48:25.514406+0000","last_became_peered":"2023-10-09T09:48:25.514406+0000","last_unstale":"2023-10-09T09:48:27.750152+0000","last_undegraded":"2023-10-09T09:48:27.750152+0000","last_fullsized":"2023-10-09T09:48:27.750152+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.526027+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T09:48:26.526027+0000","last_peered":"2023-10-09T09:48:26.526027+0000","last_clean":"2023-10-09T09:48:26.526027+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T09:48:26.526027+0000","last_undegraded":"2023-10-09T09:48:26.526027+0000","last_fullsized":"2023-10-09T09:48:26.526027+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.750581+0000","last_change":"2023-10-09T09:48:25.524325+0000","last_active":"2023-10-09T09:48:27.750581+0000","last_peered":"2023-10-09T09:48:27.750581+0000","last_clean":"2023-10-09T09:48:27.750581+0000","last_became_active":"2023-10-09T09:48:25.523933+0000","last_became_peered":"2023-10-09T09:48:25.523933+0000","last_unstale":"2023-10-09T09:48:27.750581+0000","last_undegraded":"2023-10-09T09:48:27.750581+0000","last_fullsized":"2023-10-09T09:48:27.750581+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.516987+0000","last_change":"2023-10-09T09:47:42.887390+0000","last_active":"2023-10-09T09:48:26.516987+0000","last_peered":"2023-10-09T09:48:26.516987+0000","last_clean":"2023-10-09T09:48:26.516987+0000","last_became_active":"2023-10-09T09:47:42.887040+0000","last_became_peered":"2023-10-09T09:47:42.887040+0000","last_unstale":"2023-10-09T09:48:26.516987+0000","last_undegraded":"2023-10-09T09:48:26.516987+0000","last_fullsized":"2023-10-09T09:48:26.516987+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.156209+0000","last_change":"2023-10-09T09:48:27.155326+0000","last_active":"2023-10-09T09:48:27.156209+0000","last_peered":"2023-10-09T09:48:27.156209+0000","last_clean":"2023-10-09T09:48:27.156209+0000","last_became_active":"2023-10-09T09:48:27.155012+0000","last_became_peered":"2023-10-09T09:48:27.155012+0000","last_unstale":"2023-10-09T09:48:27.156209+0000","last_undegraded":"2023-10-09T09:48:27.156209+0000","last_fullsized":"2023-10-09T09:48:27.156209+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.519267+0000","last_change":"2023-10-09T09:47:42.989781+0000","last_active":"2023-10-09T09:48:26.519267+0000","last_peered":"2023-10-09T09:48:26.519267+0000","last_clean":"2023-10-09T09:48:26.519267+0000","last_became_active":"2023-10-09T09:47:42.989461+0000","last_became_peered":"2023-10-09T09:47:42.989461+0000","last_unstale":"2023-10-09T09:48:26.519267+0000","last_undegraded":"2023-10-09T09:48:26.519267+0000","last_fullsized":"2023-10-09T09:48:26.519267+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680602+0000","last_change":"2023-10-09T09:48:25.526649+0000","last_active":"2023-10-09T09:48:39.680602+0000","last_peered":"2023-10-09T09:48:39.680602+0000","last_clean":"2023-10-09T09:48:39.680602+0000","last_became_active":"2023-10-09T09:48:25.526348+0000","last_became_peered":"2023-10-09T09:48:25.526348+0000","last_unstale":"2023-10-09T09:48:39.680602+0000","last_undegraded":"2023-10-09T09:48:39.680602+0000","last_fullsized":"2023-10-09T09:48:39.680602+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154079+0000","last_change":"2023-10-09T09:48:27.152984+0000","last_active":"2023-10-09T09:48:27.154079+0000","last_peered":"2023-10-09T09:48:27.154079+0000","last_clean":"2023-10-09T09:48:27.154079+0000","last_became_active":"2023-10-09T09:48:27.152645+0000","last_became_peered":"2023-10-09T09:48:27.152645+0000","last_unstale":"2023-10-09T09:48:27.154079+0000","last_undegraded":"2023-10-09T09:48:27.154079+0000","last_fullsized":"2023-10-09T09:48:27.154079+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.518198+0000","last_change":"2023-10-09T09:48:25.529731+0000","last_active":"2023-10-09T09:48:26.518198+0000","last_peered":"2023-10-09T09:48:26.518198+0000","last_clean":"2023-10-09T09:48:26.518198+0000","last_became_active":"2023-10-09T09:48:25.529437+0000","last_became_peered":"2023-10-09T09:48:25.529437+0000","last_unstale":"2023-10-09T09:48:26.518198+0000","last_undegraded":"2023-10-09T09:48:26.518198+0000","last_fullsized":"2023-10-09T09:48:26.518198+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.153008+0000","last_change":"2023-10-09T09:47:42.989936+0000","last_active":"2023-10-09T09:48:27.153008+0000","last_peered":"2023-10-09T09:48:27.153008+0000","last_clean":"2023-10-09T09:48:27.153008+0000","last_became_active":"2023-10-09T09:47:42.989558+0000","last_became_peered":"2023-10-09T09:47:42.989558+0000","last_unstale":"2023-10-09T09:48:27.153008+0000","last_undegraded":"2023-10-09T09:48:27.153008+0000","last_fullsized":"2023-10-09T09:48:27.153008+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.530822+0000","last_change":"2023-10-09T09:48:26.529333+0000","last_active":"2023-10-09T09:48:26.530822+0000","last_peered":"2023-10-09T09:48:26.530822+0000","last_clean":"2023-10-09T09:48:26.530822+0000","last_became_active":"2023-10-09T09:48:26.529008+0000","last_became_peered":"2023-10-09T09:48:26.529008+0000","last_unstale":"2023-10-09T09:48:26.530822+0000","last_undegraded":"2023-10-09T09:48:26.530822+0000","last_fullsized":"2023-10-09T09:48:26.530822+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.523953+0000","last_change":"2023-10-09T09:48:25.521961+0000","last_active":"2023-10-09T09:48:26.523953+0000","last_peered":"2023-10-09T09:48:26.523953+0000","last_clean":"2023-10-09T09:48:26.523953+0000","last_became_active":"2023-10-09T09:48:25.520782+0000","last_became_peered":"2023-10-09T09:48:25.520782+0000","last_unstale":"2023-10-09T09:48:26.523953+0000","last_undegraded":"2023-10-09T09:48:26.523953+0000","last_fullsized":"2023-10-09T09:48:26.523953+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.152682+0000","last_change":"2023-10-09T09:47:42.989937+0000","last_active":"2023-10-09T09:48:27.152682+0000","last_peered":"2023-10-09T09:48:27.152682+0000","last_clean":"2023-10-09T09:48:27.152682+0000","last_became_active":"2023-10-09T09:47:42.989371+0000","last_became_peered":"2023-10-09T09:47:42.989371+0000","last_unstale":"2023-10-09T09:48:27.152682+0000","last_undegraded":"2023-10-09T09:48:27.152682+0000","last_fullsized":"2023-10-09T09:48:27.152682+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154458+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T09:48:27.154458+0000","last_peered":"2023-10-09T09:48:27.154458+0000","last_clean":"2023-10-09T09:48:27.154458+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T09:48:27.154458+0000","last_undegraded":"2023-10-09T09:48:27.154458+0000","last_fullsized":"2023-10-09T09:48:27.154458+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.522340+0000","last_change":"2023-10-09T09:48:25.525392+0000","last_active":"2023-10-09T09:48:26.522340+0000","last_peered":"2023-10-09T09:48:26.522340+0000","last_clean":"2023-10-09T09:48:26.522340+0000","last_became_active":"2023-10-09T09:48:25.525042+0000","last_became_peered":"2023-10-09T09:48:25.525042+0000","last_unstale":"2023-10-09T09:48:26.522340+0000","last_undegraded":"2023-10-09T09:48:26.522340+0000","last_fullsized":"2023-10-09T09:48:26.522340+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.152425+0000","last_change":"2023-10-09T09:47:13.676606+0000","last_active":"2023-10-09T09:48:27.152425+0000","last_peered":"2023-10-09T09:48:27.152425+0000","last_clean":"2023-10-09T09:48:27.152425+0000","last_became_active":"2023-10-09T09:47:13.676235+0000","last_became_peered":"2023-10-09T09:47:13.676235+0000","last_unstale":"2023-10-09T09:48:27.152425+0000","last_undegraded":"2023-10-09T09:48:27.152425+0000","last_fullsized":"2023-10-09T09:48:27.152425+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521157+0000","last_change":"2023-10-09T09:47:42.887560+0000","last_active":"2023-10-09T09:48:26.521157+0000","last_peered":"2023-10-09T09:48:26.521157+0000","last_clean":"2023-10-09T09:48:26.521157+0000","last_became_active":"2023-10-09T09:47:42.887101+0000","last_became_peered":"2023-10-09T09:47:42.887101+0000","last_unstale":"2023-10-09T09:48:26.521157+0000","last_undegraded":"2023-10-09T09:48:26.521157+0000","last_fullsized":"2023-10-09T09:48:26.521157+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.525879+0000","last_change":"2023-10-09T09:48:26.525332+0000","last_active":"2023-10-09T09:48:26.525879+0000","last_peered":"2023-10-09T09:48:26.525879+0000","last_clean":"2023-10-09T09:48:26.525879+0000","last_became_active":"2023-10-09T09:48:26.525037+0000","last_became_peered":"2023-10-09T09:48:26.525037+0000","last_unstale":"2023-10-09T09:48:26.525879+0000","last_undegraded":"2023-10-09T09:48:26.525879+0000","last_fullsized":"2023-10-09T09:48:26.525879+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.518860+0000","last_change":"2023-10-09T09:48:25.512889+0000","last_active":"2023-10-09T09:48:26.518860+0000","last_peered":"2023-10-09T09:48:26.518860+0000","last_clean":"2023-10-09T09:48:26.518860+0000","last_became_active":"2023-10-09T09:48:25.512548+0000","last_became_peered":"2023-10-09T09:48:25.512548+0000","last_unstale":"2023-10-09T09:48:26.518860+0000","last_undegraded":"2023-10-09T09:48:26.518860+0000","last_fullsized":"2023-10-09T09:48:26.518860+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.516324+0000","last_change":"2023-10-09T09:47:42.989695+0000","last_active":"2023-10-09T09:48:26.516324+0000","last_peered":"2023-10-09T09:48:26.516324+0000","last_clean":"2023-10-09T09:48:26.516324+0000","last_became_active":"2023-10-09T09:47:42.989373+0000","last_became_peered":"2023-10-09T09:47:42.989373+0000","last_unstale":"2023-10-09T09:48:26.516324+0000","last_undegraded":"2023-10-09T09:48:26.516324+0000","last_fullsized":"2023-10-09T09:48:26.516324+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.523074+0000","last_change":"2023-10-09T09:48:26.522502+0000","last_active":"2023-10-09T09:48:26.523074+0000","last_peered":"2023-10-09T09:48:26.523074+0000","last_clean":"2023-10-09T09:48:26.523074+0000","last_became_active":"2023-10-09T09:48:26.522158+0000","last_became_peered":"2023-10-09T09:48:26.522158+0000","last_unstale":"2023-10-09T09:48:26.523074+0000","last_undegraded":"2023-10-09T09:48:26.523074+0000","last_fullsized":"2023-10-09T09:48:26.523074+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.522022+0000","last_change":"2023-10-09T09:48:25.519131+0000","last_active":"2023-10-09T09:48:26.522022+0000","last_peered":"2023-10-09T09:48:26.522022+0000","last_clean":"2023-10-09T09:48:26.522022+0000","last_became_active":"2023-10-09T09:48:25.518279+0000","last_became_peered":"2023-10-09T09:48:25.518279+0000","last_unstale":"2023-10-09T09:48:26.522022+0000","last_undegraded":"2023-10-09T09:48:26.522022+0000","last_fullsized":"2023-10-09T09:48:26.522022+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521595+0000","last_change":"2023-10-09T09:47:42.888479+0000","last_active":"2023-10-09T09:48:26.521595+0000","last_peered":"2023-10-09T09:48:26.521595+0000","last_clean":"2023-10-09T09:48:26.521595+0000","last_became_active":"2023-10-09T09:47:42.888097+0000","last_became_peered":"2023-10-09T09:47:42.888097+0000","last_unstale":"2023-10-09T09:48:26.521595+0000","last_undegraded":"2023-10-09T09:48:26.521595+0000","last_fullsized":"2023-10-09T09:48:26.521595+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.527356+0000","last_change":"2023-10-09T09:48:26.526917+0000","last_active":"2023-10-09T09:48:26.527356+0000","last_peered":"2023-10-09T09:48:26.527356+0000","last_clean":"2023-10-09T09:48:26.527356+0000","last_became_active":"2023-10-09T09:48:26.526416+0000","last_became_peered":"2023-10-09T09:48:26.526416+0000","last_unstale":"2023-10-09T09:48:26.527356+0000","last_undegraded":"2023-10-09T09:48:26.527356+0000","last_fullsized":"2023-10-09T09:48:26.527356+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.523305+0000","last_change":"2023-10-09T09:48:25.522731+0000","last_active":"2023-10-09T09:48:26.523305+0000","last_peered":"2023-10-09T09:48:26.523305+0000","last_clean":"2023-10-09T09:48:26.523305+0000","last_became_active":"2023-10-09T09:48:25.522394+0000","last_became_peered":"2023-10-09T09:48:25.522394+0000","last_unstale":"2023-10-09T09:48:26.523305+0000","last_undegraded":"2023-10-09T09:48:26.523305+0000","last_fullsized":"2023-10-09T09:48:26.523305+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.518083+0000","last_change":"2023-10-09T09:47:42.989707+0000","last_active":"2023-10-09T09:48:26.518083+0000","last_peered":"2023-10-09T09:48:26.518083+0000","last_clean":"2023-10-09T09:48:26.518083+0000","last_became_active":"2023-10-09T09:47:42.989360+0000","last_became_peered":"2023-10-09T09:47:42.989360+0000","last_unstale":"2023-10-09T09:48:26.518083+0000","last_undegraded":"2023-10-09T09:48:26.518083+0000","last_fullsized":"2023-10-09T09:48:26.518083+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.529290+0000","last_change":"2023-10-09T09:48:26.528797+0000","last_active":"2023-10-09T09:48:26.529290+0000","last_peered":"2023-10-09T09:48:26.529290+0000","last_clean":"2023-10-09T09:48:26.529290+0000","last_became_active":"2023-10-09T09:48:26.528493+0000","last_became_peered":"2023-10-09T09:48:26.528493+0000","last_unstale":"2023-10-09T09:48:26.529290+0000","last_undegraded":"2023-10-09T09:48:26.529290+0000","last_fullsized":"2023-10-09T09:48:26.529290+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.520803+0000","last_change":"2023-10-09T09:48:25.530209+0000","last_active":"2023-10-09T09:48:26.520803+0000","last_peered":"2023-10-09T09:48:26.520803+0000","last_clean":"2023-10-09T09:48:26.520803+0000","last_became_active":"2023-10-09T09:48:25.529904+0000","last_became_peered":"2023-10-09T09:48:25.529904+0000","last_unstale":"2023-10-09T09:48:26.520803+0000","last_undegraded":"2023-10-09T09:48:26.520803+0000","last_fullsized":"2023-10-09T09:48:26.520803+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528188+0000","last_change":"2023-10-09T09:48:26.527282+0000","last_active":"2023-10-09T09:48:26.528188+0000","last_peered":"2023-10-09T09:48:26.528188+0000","last_clean":"2023-10-09T09:48:26.528188+0000","last_became_active":"2023-10-09T09:48:26.526970+0000","last_became_peered":"2023-10-09T09:48:26.526970+0000","last_unstale":"2023-10-09T09:48:26.528188+0000","last_undegraded":"2023-10-09T09:48:26.528188+0000","last_fullsized":"2023-10-09T09:48:26.528188+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521147+0000","last_change":"2023-10-09T09:48:25.516918+0000","last_active":"2023-10-09T09:48:26.521147+0000","last_peered":"2023-10-09T09:48:26.521147+0000","last_clean":"2023-10-09T09:48:26.521147+0000","last_became_active":"2023-10-09T09:48:25.516410+0000","last_became_peered":"2023-10-09T09:48:25.516410+0000","last_unstale":"2023-10-09T09:48:26.521147+0000","last_undegraded":"2023-10-09T09:48:26.521147+0000","last_fullsized":"2023-10-09T09:48:26.521147+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.530167+0000","last_change":"2023-10-09T09:48:26.529940+0000","last_active":"2023-10-09T09:48:26.530167+0000","last_peered":"2023-10-09T09:48:26.530167+0000","last_clean":"2023-10-09T09:48:26.530167+0000","last_became_active":"2023-10-09T09:48:26.529684+0000","last_became_peered":"2023-10-09T09:48:26.529684+0000","last_unstale":"2023-10-09T09:48:26.530167+0000","last_undegraded":"2023-10-09T09:48:26.530167+0000","last_fullsized":"2023-10-09T09:48:26.530167+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.516871+0000","last_change":"2023-10-09T09:48:25.517709+0000","last_active":"2023-10-09T09:48:26.516871+0000","last_peered":"2023-10-09T09:48:26.516871+0000","last_clean":"2023-10-09T09:48:26.516871+0000","last_became_active":"2023-10-09T09:48:25.517245+0000","last_became_peered":"2023-10-09T09:48:25.517245+0000","last_unstale":"2023-10-09T09:48:26.516871+0000","last_undegraded":"2023-10-09T09:48:26.516871+0000","last_fullsized":"2023-10-09T09:48:26.516871+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528690+0000","last_change":"2023-10-09T09:48:26.528068+0000","last_active":"2023-10-09T09:48:26.528690+0000","last_peered":"2023-10-09T09:48:26.528690+0000","last_clean":"2023-10-09T09:48:26.528690+0000","last_became_active":"2023-10-09T09:48:26.527657+0000","last_became_peered":"2023-10-09T09:48:26.527657+0000","last_unstale":"2023-10-09T09:48:26.528690+0000","last_undegraded":"2023-10-09T09:48:26.528690+0000","last_fullsized":"2023-10-09T09:48:26.528690+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680734+0000","last_change":"2023-10-09T09:48:25.526760+0000","last_active":"2023-10-09T09:48:39.680734+0000","last_peered":"2023-10-09T09:48:39.680734+0000","last_clean":"2023-10-09T09:48:39.680734+0000","last_became_active":"2023-10-09T09:48:25.526411+0000","last_became_peered":"2023-10-09T09:48:25.526411+0000","last_unstale":"2023-10-09T09:48:39.680734+0000","last_undegraded":"2023-10-09T09:48:39.680734+0000","last_fullsized":"2023-10-09T09:48:39.680734+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.530751+0000","last_change":"2023-10-09T09:48:26.529463+0000","last_active":"2023-10-09T09:48:26.530751+0000","last_peered":"2023-10-09T09:48:26.530751+0000","last_clean":"2023-10-09T09:48:26.530751+0000","last_became_active":"2023-10-09T09:48:26.529152+0000","last_became_peered":"2023-10-09T09:48:26.529152+0000","last_unstale":"2023-10-09T09:48:26.530751+0000","last_undegraded":"2023-10-09T09:48:26.530751+0000","last_fullsized":"2023-10-09T09:48:26.530751+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521887+0000","last_change":"2023-10-09T09:48:25.523527+0000","last_active":"2023-10-09T09:48:26.521887+0000","last_peered":"2023-10-09T09:48:26.521887+0000","last_clean":"2023-10-09T09:48:26.521887+0000","last_became_active":"2023-10-09T09:48:25.523043+0000","last_became_peered":"2023-10-09T09:48:25.523043+0000","last_unstale":"2023-10-09T09:48:26.521887+0000","last_undegraded":"2023-10-09T09:48:26.521887+0000","last_fullsized":"2023-10-09T09:48:26.521887+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.153857+0000","last_change":"2023-10-09T09:48:27.153033+0000","last_active":"2023-10-09T09:48:27.153857+0000","last_peered":"2023-10-09T09:48:27.153857+0000","last_clean":"2023-10-09T09:48:27.153857+0000","last_became_active":"2023-10-09T09:48:27.152705+0000","last_became_peered":"2023-10-09T09:48:27.152705+0000","last_unstale":"2023-10-09T09:48:27.153857+0000","last_undegraded":"2023-10-09T09:48:27.153857+0000","last_fullsized":"2023-10-09T09:48:27.153857+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.751066+0000","last_change":"2023-10-09T09:48:25.514756+0000","last_active":"2023-10-09T09:48:27.751066+0000","last_peered":"2023-10-09T09:48:27.751066+0000","last_clean":"2023-10-09T09:48:27.751066+0000","last_became_active":"2023-10-09T09:48:25.514387+0000","last_became_peered":"2023-10-09T09:48:25.514387+0000","last_unstale":"2023-10-09T09:48:27.751066+0000","last_undegraded":"2023-10-09T09:48:27.751066+0000","last_fullsized":"2023-10-09T09:48:27.751066+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.529169+0000","last_change":"2023-10-09T09:48:26.528888+0000","last_active":"2023-10-09T09:48:26.529169+0000","last_peered":"2023-10-09T09:48:26.529169+0000","last_clean":"2023-10-09T09:48:26.529169+0000","last_became_active":"2023-10-09T09:48:26.528596+0000","last_became_peered":"2023-10-09T09:48:26.528596+0000","last_unstale":"2023-10-09T09:48:26.529169+0000","last_undegraded":"2023-10-09T09:48:26.529169+0000","last_fullsized":"2023-10-09T09:48:26.529169+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680156+0000","last_change":"2023-10-09T09:48:25.520127+0000","last_active":"2023-10-09T09:48:39.680156+0000","last_peered":"2023-10-09T09:48:39.680156+0000","last_clean":"2023-10-09T09:48:39.680156+0000","last_became_active":"2023-10-09T09:48:25.519822+0000","last_became_peered":"2023-10-09T09:48:25.519822+0000","last_unstale":"2023-10-09T09:48:39.680156+0000","last_undegraded":"2023-10-09T09:48:39.680156+0000","last_fullsized":"2023-10-09T09:48:39.680156+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.155537+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T09:48:27.155537+0000","last_peered":"2023-10-09T09:48:27.155537+0000","last_clean":"2023-10-09T09:48:27.155537+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T09:48:27.155537+0000","last_undegraded":"2023-10-09T09:48:27.155537+0000","last_fullsized":"2023-10-09T09:48:27.155537+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.750907+0000","last_change":"2023-10-09T09:48:25.524829+0000","last_active":"2023-10-09T09:48:27.750907+0000","last_peered":"2023-10-09T09:48:27.750907+0000","last_clean":"2023-10-09T09:48:27.750907+0000","last_became_active":"2023-10-09T09:48:25.524492+0000","last_became_peered":"2023-10-09T09:48:25.524492+0000","last_unstale":"2023-10-09T09:48:27.750907+0000","last_undegraded":"2023-10-09T09:48:27.750907+0000","last_fullsized":"2023-10-09T09:48:27.750907+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154597+0000","last_change":"2023-10-09T09:48:27.153176+0000","last_active":"2023-10-09T09:48:27.154597+0000","last_peered":"2023-10-09T09:48:27.154597+0000","last_clean":"2023-10-09T09:48:27.154597+0000","last_became_active":"2023-10-09T09:48:27.152826+0000","last_became_peered":"2023-10-09T09:48:27.152826+0000","last_unstale":"2023-10-09T09:48:27.154597+0000","last_undegraded":"2023-10-09T09:48:27.154597+0000","last_fullsized":"2023-10-09T09:48:27.154597+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.681842+0000","last_change":"2023-10-09T09:48:25.513226+0000","last_active":"2023-10-09T09:48:39.681842+0000","last_peered":"2023-10-09T09:48:39.681842+0000","last_clean":"2023-10-09T09:48:39.681842+0000","last_became_active":"2023-10-09T09:48:25.512767+0000","last_became_peered":"2023-10-09T09:48:25.512767+0000","last_unstale":"2023-10-09T09:48:39.681842+0000","last_undegraded":"2023-10-09T09:48:39.681842+0000","last_fullsized":"2023-10-09T09:48:39.681842+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154127+0000","last_change":"2023-10-09T09:48:27.153722+0000","last_active":"2023-10-09T09:48:27.154127+0000","last_peered":"2023-10-09T09:48:27.154127+0000","last_clean":"2023-10-09T09:48:27.154127+0000","last_became_active":"2023-10-09T09:48:27.153307+0000","last_became_peered":"2023-10-09T09:48:27.153307+0000","last_unstale":"2023-10-09T09:48:27.154127+0000","last_undegraded":"2023-10-09T09:48:27.154127+0000","last_fullsized":"2023-10-09T09:48:27.154127+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.152505+0000","last_change":"2023-10-09T09:48:25.523230+0000","last_active":"2023-10-09T09:48:27.152505+0000","last_peered":"2023-10-09T09:48:27.152505+0000","last_clean":"2023-10-09T09:48:27.152505+0000","last_became_active":"2023-10-09T09:48:25.522608+0000","last_became_peered":"2023-10-09T09:48:25.522608+0000","last_unstale":"2023-10-09T09:48:27.152505+0000","last_undegraded":"2023-10-09T09:48:27.152505+0000","last_fullsized":"2023-10-09T09:48:27.152505+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.155504+0000","last_change":"2023-10-09T09:48:27.155121+0000","last_active":"2023-10-09T09:48:27.155504+0000","last_peered":"2023-10-09T09:48:27.155504+0000","last_clean":"2023-10-09T09:48:27.155504+0000","last_became_active":"2023-10-09T09:48:27.154733+0000","last_became_peered":"2023-10-09T09:48:27.154733+0000","last_unstale":"2023-10-09T09:48:27.155504+0000","last_undegraded":"2023-10-09T09:48:27.155504+0000","last_fullsized":"2023-10-09T09:48:27.155504+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.527696+0000","last_change":"2023-10-09T09:48:25.519906+0000","last_active":"2023-10-09T09:48:26.527696+0000","last_peered":"2023-10-09T09:48:26.527696+0000","last_clean":"2023-10-09T09:48:26.527696+0000","last_became_active":"2023-10-09T09:48:25.519595+0000","last_became_peered":"2023-10-09T09:48:25.519595+0000","last_unstale":"2023-10-09T09:48:26.527696+0000","last_undegraded":"2023-10-09T09:48:26.527696+0000","last_fullsized":"2023-10-09T09:48:26.527696+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154630+0000","last_change":"2023-10-09T09:48:27.153292+0000","last_active":"2023-10-09T09:48:27.154630+0000","last_peered":"2023-10-09T09:48:27.154630+0000","last_clean":"2023-10-09T09:48:27.154630+0000","last_became_active":"2023-10-09T09:48:27.152784+0000","last_became_peered":"2023-10-09T09:48:27.152784+0000","last_unstale":"2023-10-09T09:48:27.154630+0000","last_undegraded":"2023-10-09T09:48:27.154630+0000","last_fullsized":"2023-10-09T09:48:27.154630+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680160+0000","last_change":"2023-10-09T09:48:25.520564+0000","last_active":"2023-10-09T09:48:39.680160+0000","last_peered":"2023-10-09T09:48:39.680160+0000","last_clean":"2023-10-09T09:48:39.680160+0000","last_became_active":"2023-10-09T09:48:25.520267+0000","last_became_peered":"2023-10-09T09:48:25.520267+0000","last_unstale":"2023-10-09T09:48:39.680160+0000","last_undegraded":"2023-10-09T09:48:39.680160+0000","last_fullsized":"2023-10-09T09:48:39.680160+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528864+0000","last_change":"2023-10-09T09:48:26.528227+0000","last_active":"2023-10-09T09:48:26.528864+0000","last_peered":"2023-10-09T09:48:26.528864+0000","last_clean":"2023-10-09T09:48:26.528864+0000","last_became_active":"2023-10-09T09:48:26.527937+0000","last_became_peered":"2023-10-09T09:48:26.527937+0000","last_unstale":"2023-10-09T09:48:26.528864+0000","last_undegraded":"2023-10-09T09:48:26.528864+0000","last_fullsized":"2023-10-09T09:48:26.528864+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.681570+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T09:48:39.681570+0000","last_peered":"2023-10-09T09:48:39.681570+0000","last_clean":"2023-10-09T09:48:39.681570+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T09:48:39.681570+0000","last_undegraded":"2023-10-09T09:48:39.681570+0000","last_fullsized":"2023-10-09T09:48:39.681570+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.527244+0000","last_change":"2023-10-09T09:48:26.526722+0000","last_active":"2023-10-09T09:48:26.527244+0000","last_peered":"2023-10-09T09:48:26.527244+0000","last_clean":"2023-10-09T09:48:26.527244+0000","last_became_active":"2023-10-09T09:48:26.525824+0000","last_became_peered":"2023-10-09T09:48:26.525824+0000","last_unstale":"2023-10-09T09:48:26.527244+0000","last_undegraded":"2023-10-09T09:48:26.527244+0000","last_fullsized":"2023-10-09T09:48:26.527244+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:39.680820+0000","last_change":"2023-10-09T09:48:25.522979+0000","last_active":"2023-10-09T09:48:39.680820+0000","last_peered":"2023-10-09T09:48:39.680820+0000","last_clean":"2023-10-09T09:48:39.680820+0000","last_became_active":"2023-10-09T09:48:25.522649+0000","last_became_peered":"2023-10-09T09:48:25.522649+0000","last_unstale":"2023-10-09T09:48:39.680820+0000","last_undegraded":"2023-10-09T09:48:39.680820+0000","last_fullsized":"2023-10-09T09:48:39.680820+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.156140+0000","last_change":"2023-10-09T09:48:27.155564+0000","last_active":"2023-10-09T09:48:27.156140+0000","last_peered":"2023-10-09T09:48:27.156140+0000","last_clean":"2023-10-09T09:48:27.156140+0000","last_became_active":"2023-10-09T09:48:27.153001+0000","last_became_peered":"2023-10-09T09:48:27.153001+0000","last_unstale":"2023-10-09T09:48:27.156140+0000","last_undegraded":"2023-10-09T09:48:27.156140+0000","last_fullsized":"2023-10-09T09:48:27.156140+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.521046+0000","last_change":"2023-10-09T09:48:25.514371+0000","last_active":"2023-10-09T09:48:26.521046+0000","last_peered":"2023-10-09T09:48:26.521046+0000","last_clean":"2023-10-09T09:48:26.521046+0000","last_became_active":"2023-10-09T09:48:25.514036+0000","last_became_peered":"2023-10-09T09:48:25.514036+0000","last_unstale":"2023-10-09T09:48:26.521046+0000","last_undegraded":"2023-10-09T09:48:26.521046+0000","last_fullsized":"2023-10-09T09:48:26.521046+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528669+0000","last_change":"2023-10-09T09:48:26.528344+0000","last_active":"2023-10-09T09:48:26.528669+0000","last_peered":"2023-10-09T09:48:26.528669+0000","last_clean":"2023-10-09T09:48:26.528669+0000","last_became_active":"2023-10-09T09:48:26.528003+0000","last_became_peered":"2023-10-09T09:48:26.528003+0000","last_unstale":"2023-10-09T09:48:26.528669+0000","last_undegraded":"2023-10-09T09:48:26.528669+0000","last_fullsized":"2023-10-09T09:48:26.528669+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.528059+0000","last_change":"2023-10-09T09:48:25.522120+0000","last_active":"2023-10-09T09:48:26.528059+0000","last_peered":"2023-10-09T09:48:26.528059+0000","last_clean":"2023-10-09T09:48:26.528059+0000","last_became_active":"2023-10-09T09:48:25.521819+0000","last_became_peered":"2023-10-09T09:48:25.521819+0000","last_unstale":"2023-10-09T09:48:26.528059+0000","last_undegraded":"2023-10-09T09:48:26.528059+0000","last_fullsized":"2023-10-09T09:48:26.528059+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.526076+0000","last_change":"2023-10-09T09:48:26.524019+0000","last_active":"2023-10-09T09:48:26.526076+0000","last_peered":"2023-10-09T09:48:26.526076+0000","last_clean":"2023-10-09T09:48:26.526076+0000","last_became_active":"2023-10-09T09:48:26.523487+0000","last_became_peered":"2023-10-09T09:48:26.523487+0000","last_unstale":"2023-10-09T09:48:26.526076+0000","last_undegraded":"2023-10-09T09:48:26.526076+0000","last_fullsized":"2023-10-09T09:48:26.526076+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.522720+0000","last_change":"2023-10-09T09:48:25.529519+0000","last_active":"2023-10-09T09:48:26.522720+0000","last_peered":"2023-10-09T09:48:26.522720+0000","last_clean":"2023-10-09T09:48:26.522720+0000","last_became_active":"2023-10-09T09:48:25.529197+0000","last_became_peered":"2023-10-09T09:48:25.529197+0000","last_unstale":"2023-10-09T09:48:26.522720+0000","last_undegraded":"2023-10-09T09:48:26.522720+0000","last_fullsized":"2023-10-09T09:48:26.522720+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.154107+0000","last_change":"2023-10-09T09:48:27.153631+0000","last_active":"2023-10-09T09:48:27.154107+0000","last_peered":"2023-10-09T09:48:27.154107+0000","last_clean":"2023-10-09T09:48:27.154107+0000","last_became_active":"2023-10-09T09:48:27.153342+0000","last_became_peered":"2023-10-09T09:48:27.153342+0000","last_unstale":"2023-10-09T09:48:27.154107+0000","last_undegraded":"2023-10-09T09:48:27.154107+0000","last_fullsized":"2023-10-09T09:48:27.154107+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.517867+0000","last_change":"2023-10-09T09:48:25.528225+0000","last_active":"2023-10-09T09:48:26.517867+0000","last_peered":"2023-10-09T09:48:26.517867+0000","last_clean":"2023-10-09T09:48:26.517867+0000","last_became_active":"2023-10-09T09:48:25.527905+0000","last_became_peered":"2023-10-09T09:48:25.527905+0000","last_unstale":"2023-10-09T09:48:26.517867+0000","last_undegraded":"2023-10-09T09:48:26.517867+0000","last_fullsized":"2023-10-09T09:48:26.517867+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T09:48:26.531886+0000","last_change":"2023-10-09T09:48:26.531031+0000","last_active":"2023-10-09T09:48:26.531886+0000","last_peered":"2023-10-09T09:48:26.531886+0000","last_clean":"2023-10-09T09:48:26.531886+0000","last_became_active":"2023-10-09T09:48:26.530702+0000","last_became_peered":"2023-10-09T09:48:26.530702+0000","last_unstale":"2023-10-09T09:48:26.531886+0000","last_undegraded":"2023-10-09T09:48:26.531886+0000","last_fullsized":"2023-10-09T09:48:26.531886+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-09T09:48:27.751065+0000","last_change":"2023-10-09T09:48:25.520986+0000","last_active":"2023-10-09T09:48:27.751065+0000","last_peered":"2023-10-09T09:48:27.751065+0000","last_clean":"2023-10-09T09:48:27.751065+0000","last_became_active":"2023-10-09T09:48:25.520654+0000","last_became_peered":"2023-10-09T09:48:25.520654+0000","last_unstale":"2023-10-09T09:48:27.751065+0000","last_undegraded":"2023-10-09T09:48:27.751065+0000","last_fullsized":"2023-10-09T09:48:27.751065+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430232,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":257891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.498}]},{"osd":1,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.46100000000000002}]},{"osd":2,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.436},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.61799999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.63600000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.53400000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.57699999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626460,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":257430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.49199999999999999,"15min":0.49299999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.504,"15min":0.504},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.501}]},{"osd":1,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.73999999999999999,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.51200000000000001,"15min":0.51800000000000002},"min":{"1min":0.35799999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.63600000000000001,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.41499999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.38300000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76300000000000001,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.51900000000000002,"15min":0.51600000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.40200000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.55200000000000005,"15min":0.54200000000000004},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.40100000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.54900000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.44,"15min":0.434},"min":{"1min":0.31,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60499999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.48699999999999999,"15min":0.48299999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.74099999999999999,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.60699999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.77300000000000002,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.501,"15min":0.495},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.55900000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.58099999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.35699999999999998,"15min":0.34499999999999997},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.36699999999999999,"15min":0.34499999999999997},"min":{"1min":0.255,"5min":0.223,"15min":0.223},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.28499999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:48:42 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.52800000000000002,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.376},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.41799999999999998,"15min":0.41399999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.59499999999999997,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.33100000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.46800000000000003,"15min":0.46600000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.64800000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.63,"15min":0.63400000000000001},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.89100000000000001,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.63,"15min":0.63600000000000001},"min":{"1min":0.45300000000000001,"5min":0.439,"15min":0.439},"max":{"1min":0.88200000000000001,"5min":1.1870000000000001,"15min":1.1870000000000001},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 09:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.433,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.876,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.58799999999999997,"15min":0.59199999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.80200000000000005,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.61099999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.61799999999999999,"15min":0.62},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80900000000000005,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60899999999999999,"15min":0.61499999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83299999999999996,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.69399999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.54600000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.34599999999999997,"15min":0.32500000000000001},"min":{"1min":0.245,"5min":0.193,"15min":0.193},"max":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.436},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.37,"15min":0.33300000000000002},"min":{"1min":0.29999999999999999,"5min":0.219,"15min":0.219},"max":{"1min":0.63800000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:48:43 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.42799999999999999,"15min":0.42399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.60399999999999998,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.39500000000000002,"15min":0.39200000000000002},"min":{"1min":0.246,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.48399999999999999,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.32300000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.41099999999999998,"15min":0.40200000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.48499999999999999,"15min":0.47599999999999998},"min":{"1min":0.38300000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.495}]},{"osd":4,"last update":"Mon Oct 9 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.629,"15min":0.63100000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.873,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.63600000000000001,"15min":0.64000000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.81200000000000006,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.65100000000000002,"15min":0.66300000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.874,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63300000000000001,"15min":0.64200000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.78000000000000003,"5min":0.87,"15min":0.87},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 09:48:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.67000000000000004,"15min":0.68999999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.69299999999999995,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.67700000000000005,"15min":0.68999999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83999999999999997,"5min":1.01,"15min":1.01},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.625}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.42799999999999999,"15min":0.36799999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":6.3810000000000002,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.38600000000000001,"15min":0.371},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.831}]},{"osd":1,"last update":"Mon Oct 9 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.48999999999999999,"15min":0.42299999999999999},"min":{"1min":0.26000000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.4729999999999999,"5min":6.4729999999999999,"15min":6.4729999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.437,"15min":0.42399999999999999},"min":{"1min":0.28599999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"last":0.47199999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:47:58 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.53900000000000003,"15min":0.48799999999999999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.4450000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":1.9390000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.502,"15min":0.502},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":1.95}]},{"osd":4,"last update":"Mon Oct 9 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.59499999999999997,"15min":0.58899999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":2.1509999999999998},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.68400000000000005,"15min":0.63600000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":6.2919999999999998,"5min":6.2919999999999998,"15min":6.2919999999999998},"last":0.77900000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:48:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.68000000000000005,"15min":0.63200000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.3490000000000002,"5min":6.3490000000000002,"15min":6.3490000000000002},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.623,"15min":0.61799999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":2.1800000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:47:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":2.1989999999999998},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":6.4089999999999998,"5min":6.4089999999999998,"15min":6.4089999999999998},"last":0.81499999999999995}]},{"osd":7,"last update":"Mon Oct 9 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.7090000000000001,"5min":1.7090000000000001,"15min":1.7090000000000001},"last":2.105}]}]},{"osd":3,"up_from":25,"seq":107374182438,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.41299999999999998,"15min":0.41899999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57099999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.442,"15min":0.45500000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.51100000000000001,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.30199999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.434,"15min":0.44800000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.48999999999999999,"5min":0.623,"15min":0.623},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.628,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.36199999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.495,"15min":0.51100000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.53300000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.441},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.48599999999999999,"15min":0.499},"min":{"1min":0.34699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.5,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.39500000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.60799999999999998,"15min":0.61799999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.79000000000000004,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.69999999999999996,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:48:34 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.63400000000000001,"15min":0.64400000000000002},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84899999999999998,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60099999999999998,"15min":0.60699999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.755,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.74099999999999999,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59599999999999997,"15min":0.59999999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.81599999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 09:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.70499999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018916,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":257821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.40799999999999997,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.69899999999999995,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.58599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.69899999999999995,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.54800000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56000000000000005,"15min":0.56799999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67500000000000004,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.56200000000000006,"15min":0.56399999999999995},"min":{"1min":0.42499999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67800000000000005,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.68799999999999994,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.76400000000000001,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.66000000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.72699999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.626},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.59199999999999997,"15min":0.60099999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.67100000000000004,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.61699999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:48:39 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.29999999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.432,"15min":0.42999999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.42899999999999999,"15min":0.42299999999999999},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.67400000000000004}]}]},{"osd":5,"up_from":36,"seq":154618822688,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.52300000000000002,"15min":0.53100000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.67200000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53000000000000003,"15min":0.53300000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69199999999999995,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.45900000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.52600000000000002,"15min":0.53200000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.64600000000000002,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51500000000000001,"15min":0.52100000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.66600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.41999999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54500000000000004,"15min":0.55100000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.39000000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.65800000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":2.9319999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69599999999999995,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.59399999999999997,"15min":0.58199999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.637,"5min":2.637,"15min":2.637},"last":0.39900000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.41799999999999998,"15min":0.40600000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.503},{"interface":"front","average":{"1min":0.501,"5min":0.439,"15min":0.42899999999999999},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56699999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.48499999999999999,"15min":0.47899999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.628},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.49399999999999999,"15min":0.48199999999999998},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.53500000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:48:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.58199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:49:02.866 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T09:49:02.866 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T09:49:02.867 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T09:49:02.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T09:49:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:01.407642+0000 mgr.y (mgr.14427) 81 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:02.502135+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:02.503077+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:02.503880+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:02.510538+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:03.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:01.407642+0000 mgr.y (mgr.14427) 81 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:02.502135+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:02.503077+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:02.503880+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:02.510538+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:01.407642+0000 mgr.y (mgr.14427) 81 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:02.502135+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:02.503077+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:02.503880+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:02.510538+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:03.474 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:49:03.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:02.374727+0000 mgr.y (mgr.14427) 82 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:49:03.955 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:02.374727+0000 mgr.y (mgr.14427) 82 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:49:04.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:02.374727+0000 mgr.y (mgr.14427) 82 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:49:04.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:04] "GET /metrics HTTP/1.1" 200 191456 "" "Prometheus/2.33.4" 2023-10-09T09:49:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:49:04.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:03.408128+0000 mgr.y (mgr.14427) 83 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:04.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:03.408128+0000 mgr.y (mgr.14427) 83 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:05.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:03.408128+0000 mgr.y (mgr.14427) 83 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:05.306 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T09:49:05.307 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T09:49:05.307 INFO:teuthology.run_tasks:Running task print... 2023-10-09T09:49:05.364 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T09:49:05.364 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T09:49:05.385 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T09:49:05.385 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T09:49:05.386 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T09:49:05.390 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T09:49:05.391 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T09:49:05.391 DEBUG:teuthology.orchestra.run.smithi012:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:49:05.392 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T09:49:05.392 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T09:49:05.392 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi012.front.sepia.ceph.com 2023-10-09T09:49:05.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T09:49:05.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T09:49:05.412 INFO:teuthology.orchestra.run.smithi012.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T09:49:05.413 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:49:05.472 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T09:49:05.472 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T09:49:05.532 INFO:tasks.workunit:timeout=3h 2023-10-09T09:49:05.532 INFO:tasks.workunit:cleanup=True 2023-10-09T09:49:05.533 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:49:05.590 INFO:tasks.workunit.client.0.smithi012.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T09:49:05.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:04.915031+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.12:0/4279702637' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:49:05.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:04.915031+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.12:0/4279702637' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:49:06.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:04.915031+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.12:0/4279702637' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:49:06.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:49:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:49:07.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:05.408469+0000 mgr.y (mgr.14427) 84 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:07.087 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:07] ENGINE Bus STOPPING 2023-10-09T09:49:07.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:05.408469+0000 mgr.y (mgr.14427) 84 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:07.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:05.408469+0000 mgr.y (mgr.14427) 84 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:07.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:49:07.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:07] ENGINE Bus STOPPED 2023-10-09T09:49:07.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:07] ENGINE Bus STARTING 2023-10-09T09:49:07.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:07] ENGINE Serving on http://:::9283 2023-10-09T09:49:07.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:07] ENGINE Bus STARTED 2023-10-09T09:49:07.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T09:49:08.341 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:07.081797+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T09:49:08.341 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:07.138335+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:08.341 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:07.139842+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:07.140626+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:07.147852+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:07.409006+0000 mgr.y (mgr.14427) 85 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:07.081797+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:07.138335+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:07.139842+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:07.140626+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:07.147852+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:08.342 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:07.409006+0000 mgr.y (mgr.14427) 85 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:07.081797+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T09:49:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:07.138335+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:07.139842+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:07.140626+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:07.147852+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:07.409006+0000 mgr.y (mgr.14427) 85 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:09.747 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:09] ENGINE Bus STOPPING 2023-10-09T09:49:09.747 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:49:09.747 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:09] ENGINE Bus STOPPED 2023-10-09T09:49:09.747 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:09] ENGINE Bus STARTING 2023-10-09T09:49:09.747 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:09] ENGINE Serving on http://:::9283 2023-10-09T09:49:09.747 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:49:09] ENGINE Bus STARTED 2023-10-09T09:49:10.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T09:49:10.584 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T09:49:10.628 INFO:tasks.workunit.client.0.smithi012.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T09:49:10.630 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:49:10.630 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T09:49:10.725 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:09.409536+0000 mgr.y (mgr.14427) 86 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:10.725 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:09.470459+0000 mon.a (mon.0) 707 : audit [INF] from='client.? 172.21.15.12:0/2381603979' entity='client.admin' 2023-10-09T09:49:10.725 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:09.520535+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:10.725 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:09.521960+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:10.725 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:09.522594+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:10.725 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:09.529380+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:10.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:09.409536+0000 mgr.y (mgr.14427) 86 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:10.726 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:09.470459+0000 mon.a (mon.0) 707 : audit [INF] from='client.? 172.21.15.12:0/2381603979' entity='client.admin' 2023-10-09T09:49:10.726 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:09.520535+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:10.726 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:09.521960+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:10.726 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:09.522594+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:10.726 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:09.529380+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:10.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:09.409536+0000 mgr.y (mgr.14427) 86 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:09.470459+0000 mon.a (mon.0) 707 : audit [INF] from='client.? 172.21.15.12:0/2381603979' entity='client.admin' 2023-10-09T09:49:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:09.520535+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:09.521960+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:09.522594+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:09.529380+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:11.734 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T09:49:11.800 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T09:49:11.839 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T09:49:11.841 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:49:11.841 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T09:49:11.879 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T09:49:11.882 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T09:49:11.882 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T09:49:11.900 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T09:49:11.901 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T09:49:11.901 DEBUG:teuthology.orchestra.run.smithi012:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T09:49:11.972 INFO:tasks.workunit.client.0.smithi012.stdout:Running main() from gmock_main.cc 2023-10-09T09:49:11.972 INFO:tasks.workunit.client.0.smithi012.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T09:49:11.972 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] Global test environment set-up. 2023-10-09T09:49:11.972 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T09:49:12.428 INFO:teuthology.orchestra.run.smithi012.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:49:12.788 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:11.409911+0000 mgr.y (mgr.14427) 87 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:12.788 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:12.428537+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:12.788 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:12.453012+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:12.788 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:12.454566+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:12.789 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:12.455159+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:12.789 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:12.461877+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:12.789 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:11.409911+0000 mgr.y (mgr.14427) 87 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:12.789 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:12.428537+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:12.789 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:12.453012+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:12.789 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:12.454566+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:12.789 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:12.455159+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:12.789 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:12.461877+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:12.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:49:12.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:11.409911+0000 mgr.y (mgr.14427) 87 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:12.428537+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:12.453012+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:12.454566+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:49:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:12.455159+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:49:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:12.461877+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:13.484 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T09:49:13.502 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_all_features (19 ms) 2023-10-09T09:49:13.502 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T09:49:13.705 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.copyup (203 ms) 2023-10-09T09:49:13.706 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T09:49:13.719 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.sparse_copyup (14 ms) 2023-10-09T09:49:13.719 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T09:49:13.729 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_and_set_id (10 ms) 2023-10-09T09:49:13.730 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T09:49:13.741 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2023-10-09T09:49:13.741 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T09:49:13.775 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.directory_methods (34 ms) 2023-10-09T09:49:13.775 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.create 2023-10-09T09:49:13.791 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.create (17 ms) 2023-10-09T09:49:13.792 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T09:49:13.793 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-10-09T09:49:13.793 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T09:49:13.795 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T09:49:13.795 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T09:49:13.797 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-10-09T09:49:13.797 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T09:49:13.798 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-10-09T09:49:13.799 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T09:49:13.800 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-10-09T09:49:13.800 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T09:49:13.803 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-10-09T09:49:13.803 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T09:49:13.808 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2023-10-09T09:49:13.808 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T09:49:13.813 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-10-09T09:49:13.814 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T09:49:13.837 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.protection_status (23 ms) 2023-10-09T09:49:13.837 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T09:49:13.845 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapshot_limits (9 ms) 2023-10-09T09:49:13.846 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T09:49:13.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:12.143877+0000 mgr.y (mgr.14427) 88 : audit [DBG] from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:49:13.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:49:12.144617+0000 mgr.y (mgr.14427) 89 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:49:13.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:49:12.466220+0000 mgr.y (mgr.14427) 90 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:49:13.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:12.482488+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T09:49:13.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:12.488472+0000 mon.a (mon.0) 718 : audit [INF] from='client.? 172.21.15.12:0/2055226150' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-136433-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:49:13.879 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:12.143877+0000 mgr.y (mgr.14427) 88 : audit [DBG] from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:49:13.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:49:12.144617+0000 mgr.y (mgr.14427) 89 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:49:13.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:49:12.466220+0000 mgr.y (mgr.14427) 90 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:49:13.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:12.482488+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T09:49:13.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:12.488472+0000 mon.a (mon.0) 718 : audit [INF] from='client.? 172.21.15.12:0/2055226150' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-136433-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:49:13.902 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.parents_v1 (56 ms) 2023-10-09T09:49:13.902 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T09:49:13.937 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.parents_v2 (36 ms) 2023-10-09T09:49:13.938 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T09:49:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:12.143877+0000 mgr.y (mgr.14427) 88 : audit [DBG] from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:49:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:49:12.144617+0000 mgr.y (mgr.14427) 89 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:49:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:49:12.466220+0000 mgr.y (mgr.14427) 90 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T09:49:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:12.482488+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T09:49:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:12.488472+0000 mon.a (mon.0) 718 : audit [INF] from='client.? 172.21.15.12:0/2055226150' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-136433-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:49:13.959 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2023-10-09T09:49:13.959 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T09:49:13.963 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-10-09T09:49:13.963 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T09:49:13.972 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-10-09T09:49:13.972 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T09:49:13.980 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_save (8 ms) 2023-10-09T09:49:13.980 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T09:49:13.985 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-10-09T09:49:13.985 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T09:49:13.991 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2023-10-09T09:49:13.991 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T09:49:13.991 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T09:49:13.991 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T09:49:13.995 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2023-10-09T09:49:13.995 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T09:49:13.999 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2023-10-09T09:49:13.999 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T09:49:14.004 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.flags (4 ms) 2023-10-09T09:49:14.004 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T09:49:14.029 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.metadata (26 ms) 2023-10-09T09:49:14.029 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T09:49:14.037 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-10-09T09:49:14.037 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T09:49:14.074 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror (38 ms) 2023-10-09T09:49:14.074 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T09:49:14.093 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image (19 ms) 2023-10-09T09:49:14.093 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T09:49:14.170 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:14] "GET /metrics HTTP/1.1" 200 193875 "" "Prometheus/2.33.4" 2023-10-09T09:49:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:13.410445+0000 mgr.y (mgr.14427) 91 : cluster [DBG] pgmap v66: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:13.483343+0000 mon.a (mon.0) 719 : audit [INF] from='client.? 172.21.15.12:0/2055226150' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-136433-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:49:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:13.483434+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T09:49:14.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:13.410445+0000 mgr.y (mgr.14427) 91 : cluster [DBG] pgmap v66: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:14.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:13.483343+0000 mon.a (mon.0) 719 : audit [INF] from='client.? 172.21.15.12:0/2055226150' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-136433-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:49:14.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:13.483434+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T09:49:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:13.410445+0000 mgr.y (mgr.14427) 91 : cluster [DBG] pgmap v66: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:13.483343+0000 mon.a (mon.0) 719 : audit [INF] from='client.? 172.21.15.12:0/2055226150' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-136433-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T09:49:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:13.483434+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T09:49:15.265 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:49:15.265 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (47s) 16s ago 79s 18.3M - 0.23.0 ba2b418f427c 2bd91ea7a5f0 2023-10-09T09:49:15.265 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi110 *:3000 running (56s) 12s ago 56s 48.9M - 8.3.5 dad864ee21e9 b2e250c8bb75 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.mbnrgj smithi012 running (36s) 16s ago 36s 14.0M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5be3d4216704 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi110.sgosap smithi110 running (34s) 12s ago 34s 13.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f17cc81b8a5e 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi110 *:8443 running (4m) 12s ago 4m 392M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 2e870acba3a1 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:9283 running (6m) 16s ago 6m 447M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 bcfbb2bc52d4 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (6m) 16s ago 6m 101M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 3c03dd9903d9 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi110 running (5m) 12s ago 5m 102M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 99ff116de7cf 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (4m) 16s ago 4m 94.7M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8ea2dfe53e63 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (91s) 16s ago 90s 13.6M - 1.3.1 1dbe0e931976 8a39bf7d1c98 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi110 *:9100 running (87s) 12s ago 86s 21.6M - 1.3.1 1dbe0e931976 3f4a30ff217b 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (4m) 16s ago 4m 46.7M 4037M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 40f45dc46a61 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (4m) 16s ago 4m 47.1M 4037M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e2afac5c73f9 2023-10-09T09:49:15.266 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (3m) 16s ago 3m 44.2M 4037M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 71decd1a84e3 2023-10-09T09:49:15.267 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (3m) 16s ago 3m 45.5M 4037M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8587e6142c61 2023-10-09T09:49:15.267 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi110 running (3m) 12s ago 3m 48.3M 4292M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 18029670387d 2023-10-09T09:49:15.267 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi110 running (2m) 12s ago 2m 47.1M 4292M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c927f6e714d1 2023-10-09T09:49:15.267 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi110 running (2m) 12s ago 2m 45.0M 4292M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b96408c9e315 2023-10-09T09:49:15.267 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi110 running (2m) 12s ago 2m 45.4M 4292M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 ac326d7f40d1 2023-10-09T09:49:15.267 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi110 *:9095 running (44s) 12s ago 95s 43.6M - 2.33.4 514e6a882f6e 4318ed52adb7 2023-10-09T09:49:15.649 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:14.493118+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T09:49:15.652 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:14.493118+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T09:49:15.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T09:49:15.886 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:14.493118+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T09:49:16.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:49:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:49:16.561 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:15.246962+0000 mgr.y (mgr.14427) 92 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:49:16.561 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:15.410806+0000 mgr.y (mgr.14427) 93 : cluster [DBG] pgmap v69: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:16.562 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:15.246962+0000 mgr.y (mgr.14427) 92 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:49:16.562 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:15.410806+0000 mgr.y (mgr.14427) 93 : cluster [DBG] pgmap v69: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:16.571 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image_status (2478 ms) 2023-10-09T09:49:16.571 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T09:49:16.660 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image_map (88 ms) 2023-10-09T09:49:16.660 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T09:49:16.669 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_instances (10 ms) 2023-10-09T09:49:16.669 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T09:49:16.683 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_snapshot (14 ms) 2023-10-09T09:49:16.683 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T09:49:16.685 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-09T09:49:16.685 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T09:49:16.687 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T09:49:16.688 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T09:49:16.690 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T09:49:16.690 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T09:49:16.696 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-10-09T09:49:16.697 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T09:49:16.700 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-10-09T09:49:16.700 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T09:49:16.704 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2023-10-09T09:49:16.704 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T09:49:16.706 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-10-09T09:49:16.706 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T09:49:16.709 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-09T09:49:16.709 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T09:49:16.714 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-10-09T09:49:16.714 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T09:49:16.718 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-10-09T09:49:16.718 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T09:49:16.721 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-10-09T09:49:16.721 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T09:49:16.724 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-10-09T09:49:16.725 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T09:49:16.727 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2023-10-09T09:49:16.727 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T09:49:16.729 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (1 ms) 2023-10-09T09:49:16.729 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T09:49:16.731 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T09:49:16.731 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T09:49:16.734 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-09T09:49:16.734 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T09:49:16.736 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T09:49:16.736 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T09:49:16.739 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-10-09T09:49:16.739 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T09:49:16.742 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2023-10-09T09:49:16.742 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T09:49:16.759 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (17 ms) 2023-10-09T09:49:16.759 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T09:49:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:15.246962+0000 mgr.y (mgr.14427) 92 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:49:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:15.410806+0000 mgr.y (mgr.14427) 93 : cluster [DBG] pgmap v69: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:16.958 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (198 ms) 2023-10-09T09:49:16.958 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T09:49:16.961 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-10-09T09:49:16.962 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T09:49:16.964 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T09:49:16.964 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T09:49:16.972 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2023-10-09T09:49:16.972 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T09:49:16.979 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-10-09T09:49:16.979 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T09:49:17.001 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_parent (22 ms) 2023-10-09T09:49:17.001 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T09:49:17.010 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-10-09T09:49:17.010 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T09:49:17.028 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_child (18 ms) 2023-10-09T09:49:17.029 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T09:49:17.036 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.namespace_methods (8 ms) 2023-10-09T09:49:17.036 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T09:49:17.047 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.migration (11 ms) 2023-10-09T09:49:17.047 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T09:49:17.058 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2023-10-09T09:49:17.058 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T09:49:17.192 INFO:teuthology.orchestra.run.smithi012.stdout:wait for minority of mons to upgrade 2023-10-09T09:49:17.601 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T09:49:18.524 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1467 ms) 2023-10-09T09:49:18.524 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T09:49:18.632 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.sparsify (108 ms) 2023-10-09T09:49:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:17.411155+0000 mgr.y (mgr.14427) 94 : cluster [DBG] pgmap v70: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:17.515757+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T09:49:18.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:17.411155+0000 mgr.y (mgr.14427) 94 : cluster [DBG] pgmap v70: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:18.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:17.515757+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T09:49:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:17.411155+0000 mgr.y (mgr.14427) 94 : cluster [DBG] pgmap v70: 105 pgs: 32 creating+peering, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:17.515757+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T09:49:19.545 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] 70 tests from TestClsRbd (5149 ms total) 2023-10-09T09:49:19.545 INFO:tasks.workunit.client.0.smithi012.stdout: 2023-10-09T09:49:19.546 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] Global test environment tear-down 2023-10-09T09:49:19.546 INFO:tasks.workunit.client.0.smithi012.stdout:[==========] 70 tests from 1 test suite ran. (7571 ms total) 2023-10-09T09:49:19.546 INFO:tasks.workunit.client.0.smithi012.stdout:[ PASSED ] 70 tests. 2023-10-09T09:49:19.547 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T09:49:19.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T09:49:19.580 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T09:49:19.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T09:49:19.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:18.522067+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T09:49:19.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:19.443994+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.12:0/1426909515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:19.828 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:18.522067+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T09:49:19.828 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:19.443994+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.12:0/1426909515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:19.829 DEBUG:teuthology.parallel:result is None 2023-10-09T09:49:19.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:49:19.857 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:49:19.858 DEBUG:teuthology.orchestra.run.smithi012:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T09:49:19.926 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T09:49:19.926 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T09:49:19.926 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T09:49:19.926 DEBUG:teuthology.parallel:result is None 2023-10-09T09:49:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:18.522067+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T09:49:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:19.443994+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.12:0/1426909515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:20.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:19.411508+0000 mgr.y (mgr.14427) 95 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 2.7 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.4 MiB/s wr, 2.48k op/s 2023-10-09T09:49:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:19.538840+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T09:49:20.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:19.411508+0000 mgr.y (mgr.14427) 95 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 2.7 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.4 MiB/s wr, 2.48k op/s 2023-10-09T09:49:20.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:19.538840+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T09:49:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:19.411508+0000 mgr.y (mgr.14427) 95 : cluster [DBG] pgmap v73: 105 pgs: 105 active+clean; 2.7 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.4 MiB/s wr, 2.48k op/s 2023-10-09T09:49:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:19.538840+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T09:49:22.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:21.411841+0000 mgr.y (mgr.14427) 96 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:22.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:21.829328+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.12:0/210454193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:22.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:21.411841+0000 mgr.y (mgr.14427) 96 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:22.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:21.829328+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.12:0/210454193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:22.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:21.411841+0000 mgr.y (mgr.14427) 96 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:22.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:21.829328+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.12:0/210454193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:24.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:24] "GET /metrics HTTP/1.1" 200 193875 "" "Prometheus/2.33.4" 2023-10-09T09:49:24.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:23.412297+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:24.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:24.216988+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.12:0/3852936777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:24.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:23.412297+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:24.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:24.216988+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.12:0/3852936777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:23.412297+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:24.216988+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.12:0/3852936777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:26.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:49:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:49:26.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:25.412611+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:26.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:26.609658+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.12:0/500169706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:26.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:25.412611+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:26.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:26.609658+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.12:0/500169706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:25.412611+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:26.609658+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.12:0/500169706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:27.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:27.400211+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-10-09T09:49:27.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:27.400422+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T09:49:27.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:27.400597+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T09:49:28.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:27.400211+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-10-09T09:49:28.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:27.400422+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T09:49:28.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:27.400597+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T09:49:28.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:27.400211+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-10-09T09:49:28.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:27.400422+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T09:49:28.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:27.400597+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T09:49:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:27.412945+0000 mgr.y (mgr.14427) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:27.683593+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-10-09T09:49:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:27.683662+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T09:49:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:27.683730+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T09:49:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:27.683815+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T09:49:29.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:27.412945+0000 mgr.y (mgr.14427) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:27.683593+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:27.683662+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:27.683730+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:27.683815+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:27.412945+0000 mgr.y (mgr.14427) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:27.683593+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:27.683662+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:27.683730+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T09:49:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:27.683815+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T09:49:30.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:28.694287+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T09:49:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:29.035269+0000 mon.a (mon.0) 737 : audit [DBG] from='client.? 172.21.15.12:0/1518264654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:28.694287+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T09:49:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:29.035269+0000 mon.a (mon.0) 737 : audit [DBG] from='client.? 172.21.15.12:0/1518264654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:28.694287+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T09:49:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:29.035269+0000 mon.a (mon.0) 737 : audit [DBG] from='client.? 172.21.15.12:0/1518264654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:31.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:29.413476+0000 mgr.y (mgr.14427) 100 : cluster [DBG] pgmap v81: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:31.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:29.413476+0000 mgr.y (mgr.14427) 100 : cluster [DBG] pgmap v81: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:29.413476+0000 mgr.y (mgr.14427) 100 : cluster [DBG] pgmap v81: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:32.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:31.435413+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.12:0/2374345738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:32.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:31.435413+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.12:0/2374345738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:32.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:31.435413+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.12:0/2374345738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:33.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:31.413857+0000 mgr.y (mgr.14427) 101 : cluster [DBG] pgmap v82: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:33.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:31.413857+0000 mgr.y (mgr.14427) 101 : cluster [DBG] pgmap v82: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:31.413857+0000 mgr.y (mgr.14427) 101 : cluster [DBG] pgmap v82: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:34.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:34] "GET /metrics HTTP/1.1" 200 192381 "" "Prometheus/2.33.4" 2023-10-09T09:49:35.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:33.414398+0000 mgr.y (mgr.14427) 102 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:35.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:33.824322+0000 mon.a (mon.0) 738 : audit [DBG] from='client.? 172.21.15.12:0/1084097434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:35.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:33.414398+0000 mgr.y (mgr.14427) 102 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:35.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:33.824322+0000 mon.a (mon.0) 738 : audit [DBG] from='client.? 172.21.15.12:0/1084097434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:35.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:33.414398+0000 mgr.y (mgr.14427) 102 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:35.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:33.824322+0000 mon.a (mon.0) 738 : audit [DBG] from='client.? 172.21.15.12:0/1084097434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:36.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:49:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:49:37.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:35.414741+0000 mgr.y (mgr.14427) 103 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:36.183267+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.12:0/4159955580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:37.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:35.414741+0000 mgr.y (mgr.14427) 103 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:37.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:36.183267+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.12:0/4159955580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:37.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:35.414741+0000 mgr.y (mgr.14427) 103 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:37.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:36.183267+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.12:0/4159955580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:37.415078+0000 mgr.y (mgr.14427) 104 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:38.557378+0000 mon.a (mon.0) 739 : audit [DBG] from='client.? 172.21.15.12:0/2084706812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:38.614841+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:38.616147+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:38.622932+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:37.415078+0000 mgr.y (mgr.14427) 104 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:38.557378+0000 mon.a (mon.0) 739 : audit [DBG] from='client.? 172.21.15.12:0/2084706812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:38.614841+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:38.616147+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:38.622932+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:39.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:37.415078+0000 mgr.y (mgr.14427) 104 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:39.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:38.557378+0000 mon.a (mon.0) 739 : audit [DBG] from='client.? 172.21.15.12:0/2084706812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:39.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:38.614841+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:39.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:38.616147+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:49:39.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:38.622932+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:49:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:49:38.615120+0000 mgr.y (mgr.14427) 105 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T09:49:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:49:38.615192+0000 mgr.y (mgr.14427) 106 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T09:49:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:49:38.623539+0000 mgr.y (mgr.14427) 107 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T09:49:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:49:38.990724+0000 mgr.y (mgr.14427) 108 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi110 2023-10-09T09:49:40.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:49:38.615120+0000 mgr.y (mgr.14427) 105 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T09:49:40.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:49:38.615192+0000 mgr.y (mgr.14427) 106 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T09:49:40.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:49:38.623539+0000 mgr.y (mgr.14427) 107 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T09:49:40.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:49:38.990724+0000 mgr.y (mgr.14427) 108 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi110 2023-10-09T09:49:40.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:49:38.615120+0000 mgr.y (mgr.14427) 105 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T09:49:40.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:49:38.615192+0000 mgr.y (mgr.14427) 106 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T09:49:40.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:49:38.623539+0000 mgr.y (mgr.14427) 107 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T09:49:40.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:49:38.990724+0000 mgr.y (mgr.14427) 108 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi110 2023-10-09T09:49:41.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:39.415818+0000 mgr.y (mgr.14427) 109 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:41.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:39.415818+0000 mgr.y (mgr.14427) 109 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:41.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:39.415818+0000 mgr.y (mgr.14427) 109 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:42.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:40.925685+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.12:0/2774612719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:42.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:40.925685+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.12:0/2774612719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:40.925685+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.12:0/2774612719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:43.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:41.416294+0000 mgr.y (mgr.14427) 110 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:41.416294+0000 mgr.y (mgr.14427) 110 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:43.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:41.416294+0000 mgr.y (mgr.14427) 110 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:44.078 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:43.293534+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.12:0/1549833519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:44.078 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:43.293534+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.12:0/1549833519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:44.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:43.293534+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.12:0/1549833519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:44.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:44] "GET /metrics HTTP/1.1" 200 192397 "" "Prometheus/2.33.4" 2023-10-09T09:49:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:43.416828+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:43.416828+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:45.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:43.416828+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:45.668177+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.12:0/1553431699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:45.668177+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.12:0/1553431699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:46.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:45.668177+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.12:0/1553431699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:46.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:49:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:49:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:45.417141+0000 mgr.y (mgr.14427) 112 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:45.417141+0000 mgr.y (mgr.14427) 112 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:45.417141+0000 mgr.y (mgr.14427) 112 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:47.417780+0000 mgr.y (mgr.14427) 113 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:49.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:48.040126+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.12:0/2003332157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:49.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:47.417780+0000 mgr.y (mgr.14427) 113 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:49.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:48.040126+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.12:0/2003332157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:47.417780+0000 mgr.y (mgr.14427) 113 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:48.040126+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.12:0/2003332157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:51.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:49.418552+0000 mgr.y (mgr.14427) 114 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:50.408457+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.12:0/1211221692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:49.418552+0000 mgr.y (mgr.14427) 114 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:50.408457+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.12:0/1211221692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:51.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:49.418552+0000 mgr.y (mgr.14427) 114 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:51.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:50.408457+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.12:0/1211221692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:53.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:51.418927+0000 mgr.y (mgr.14427) 115 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:53.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:51.418927+0000 mgr.y (mgr.14427) 115 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:53.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:51.418927+0000 mgr.y (mgr.14427) 115 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:54.079 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:52.779345+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.12:0/1482629152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:54.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:52.779345+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.12:0/1482629152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:52.779345+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.12:0/1482629152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:54.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:49:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:54] "GET /metrics HTTP/1.1" 200 192397 "" "Prometheus/2.33.4" 2023-10-09T09:49:55.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:53.419726+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:55.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:53.419726+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:53.419726+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:56.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:55.147032+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.12:0/3435580065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:56.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:55.147032+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.12:0/3435580065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:56.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:55.147032+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.12:0/3435580065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:56.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:49:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[98502]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:49:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:49:57.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:55.420117+0000 mgr.y (mgr.14427) 117 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:57.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:55.420117+0000 mgr.y (mgr.14427) 117 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:57.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:55.420117+0000 mgr.y (mgr.14427) 117 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:58.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:57.516939+0000 mon.a (mon.0) 745 : audit [DBG] from='client.? 172.21.15.12:0/2657515492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:58.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:57.516939+0000 mon.a (mon.0) 745 : audit [DBG] from='client.? 172.21.15.12:0/2657515492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:58.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:57.516939+0000 mon.a (mon.0) 745 : audit [DBG] from='client.? 172.21.15.12:0/2657515492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:49:59.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:49:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:57.420468+0000 mgr.y (mgr.14427) 118 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:59.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:49:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:57.420468+0000 mgr.y (mgr.14427) 118 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:49:59.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:49:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:57.420468+0000 mgr.y (mgr.14427) 118 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:01.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:49:59.421309+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:01.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:49:59.888489+0000 mon.a (mon.0) 746 : audit [DBG] from='client.? 172.21.15.12:0/1500329561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:01.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:00.000169+0000 mon.a (mon.0) 747 : cluster [INF] overall HEALTH_OK 2023-10-09T09:50:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:49:59.421309+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:49:59.888489+0000 mon.a (mon.0) 746 : audit [DBG] from='client.? 172.21.15.12:0/1500329561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:00.000169+0000 mon.a (mon.0) 747 : cluster [INF] overall HEALTH_OK 2023-10-09T09:50:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:49:59.421309+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:49:59.888489+0000 mon.a (mon.0) 746 : audit [DBG] from='client.? 172.21.15.12:0/1500329561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:00.000169+0000 mon.a (mon.0) 747 : cluster [INF] overall HEALTH_OK 2023-10-09T09:50:03.176 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:01.421817+0000 mgr.y (mgr.14427) 120 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:03.177 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:02.254610+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.12:0/2000614946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:01.421817+0000 mgr.y (mgr.14427) 120 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:02.254610+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.12:0/2000614946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:03.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:01.421817+0000 mgr.y (mgr.14427) 120 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:02.254610+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.12:0/2000614946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:04.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:50:04] "GET /metrics HTTP/1.1" 200 192391 "" "Prometheus/2.33.4" 2023-10-09T09:50:04.884 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:03.422382+0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:04.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:04.064123+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:04.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:04.065609+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:50:04.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:04.066619+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:50:04.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:04.067631+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:04.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:04.625291+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.12:0/129073046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:05.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:03.422382+0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:05.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:04.064123+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:05.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:04.065609+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:50:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:04.066619+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:50:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:04.067631+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:04.625291+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.12:0/129073046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:05.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:03.422382+0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:05.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:04.064123+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:04.065609+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:50:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:04.066619+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:50:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:04.067631+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:04.625291+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.12:0/129073046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:05.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:05 smithi110 systemd[1]: Stopping Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:05.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:05 smithi110 bash[121128]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr.x" found: no such container 2023-10-09T09:50:05.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:04.057382+0000 mgr.y (mgr.14427) 122 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:50:05.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:04.068521+0000 mgr.y (mgr.14427) 123 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:50:05.943 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:05 smithi110 bash[121170]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T09:50:05.943 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:05 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:50:06.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:04.057382+0000 mgr.y (mgr.14427) 122 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:50:06.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:04.068521+0000 mgr.y (mgr.14427) 123 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:50:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:04.057382+0000 mgr.y (mgr.14427) 122 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:50:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:04.068521+0000 mgr.y (mgr.14427) 123 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:50:06.439 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T09:50:06.439 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 systemd[1]: Stopped Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:06.439 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 systemd[1]: Starting Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:06.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 podman[121319]: 2023-10-09T09:50:07.021 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:05.422940+0000 mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:07.022 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:06.781556+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:07.022 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:06.782421+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:07.022 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 bash[121319]: 40ae6b98943cb5e406ae2de5270f5d602e1e2086a3dee8fbbfb4e577f2ebd937 2023-10-09T09:50:07.022 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:06.706+0000 7fa2c791a700 1 -- 172.21.15.110:0/3704208145 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5647c91291e0 con 0x5647c90fec00 2023-10-09T09:50:07.022 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 systemd[1]: Started Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:07.022 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:06.830+0000 7fa2d1a68200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:50:07.022 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:06.922+0000 7fa2d1a68200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:50:07.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:05.422940+0000 mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:07.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:06.781556+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:07.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:06.782421+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:07.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:05.422940+0000 mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:07.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:06.781556+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:07.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:06.782421+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:07.299 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:07.017+0000 7fa2d1a68200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:50:07.299 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:07.111+0000 7fa2d1a68200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:50:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:07.001811+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.12:0/3062317880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:07.943 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:07.297+0000 7fa2d1a68200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:50:07.943 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:07.677+0000 7fa2d1a68200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:50:07.943 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:07.821+0000 7fa2d1a68200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:50:07.943 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:07.914+0000 7fa2d1a68200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:50:08.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:07.001811+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.12:0/3062317880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:07.001811+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.12:0/3062317880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:08.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:08.014+0000 7fa2d1a68200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:50:08.803 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:08.475+0000 7fa2d1a68200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:50:09.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:07.423325+0000 mgr.y (mgr.14427) 125 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:09.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:07.423325+0000 mgr.y (mgr.14427) 125 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:09.179 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:07.423325+0000 mgr.y (mgr.14427) 125 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:09.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:09.178+0000 7fa2d1a68200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:50:09.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:09.263+0000 7fa2d1a68200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:50:10.090 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:09.369200+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.12:0/4236420669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:10.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:09.369200+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.12:0/4236420669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:10.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:09.369200+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.12:0/4236420669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:10.364 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:10.088+0000 7fa2d1a68200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:50:10.364 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:10.186+0000 7fa2d1a68200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:50:10.659 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:10.363+0000 7fa2d1a68200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:50:10.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:09.423928+0000 mgr.y (mgr.14427) 126 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:10.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:10.658+0000 7fa2d1a68200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:50:10.943 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:10.893+0000 7fa2d1a68200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:50:11.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:09.423928+0000 mgr.y (mgr.14427) 126 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:11.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:09.423928+0000 mgr.y (mgr.14427) 126 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:11.340 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:11.058+0000 7fa2d1a68200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:50:11.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:11 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:11.339+0000 7fa2d1a68200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:50:12.326 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:12.074+0000 7fa2d1a68200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:50:12.326 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:12.204+0000 7fa2d1a68200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:50:12.574 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:12.285+0000 7fd1ca319700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T09:50:12.574 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ignoring --setuser ceph since I am not root 2023-10-09T09:50:12.574 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ignoring --setgroup ceph since I am not root 2023-10-09T09:50:12.574 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:12.330+0000 7f8820ea9700 1 -- 172.21.15.12:0/2142664144 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555f0a1624e0 con 0x555f0a139400 2023-10-09T09:50:12.574 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:12.473+0000 7f8831071680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:50:12.575 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:12.572+0000 7f8831071680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:50:12.635 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:12.407+0000 7fa2d1a68200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:50:12.635 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:12.521+0000 7fa2d1a68200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:50:12.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:11.424337+0000 mgr.y (mgr.14427) 127 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:11.633999+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:11.745679+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.12:0/435397200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:11.849935+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:12.262981+0000 mon.a (mon.0) 758 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:12.263660+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:12.270548+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:12.275753+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:12.280062+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:12.288696+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T09:50:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:12.288904+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:12.288953+0000 mon.a (mon.0) 765 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00848827s) 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:11.424337+0000 mgr.y (mgr.14427) 127 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:11.633999+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:11.745679+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.12:0/435397200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:11.849935+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:12.262981+0000 mon.a (mon.0) 758 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:12.263660+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:12.270548+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:12.275753+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:12.280062+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T09:50:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:12.288696+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T09:50:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:12.288904+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T09:50:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:12.288953+0000 mon.a (mon.0) 765 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00848827s) 2023-10-09T09:50:12.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:12.743+0000 7fa2d1a68200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:50:12.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:11.424337+0000 mgr.y (mgr.14427) 127 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:12.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:11.633999+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:11.745679+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.12:0/435397200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:12.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:11.849935+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:12.262981+0000 mon.a (mon.0) 758 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:12.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:12.263660+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:12.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:12.270548+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' 2023-10-09T09:50:12.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:12.275753+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:12.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:12.280062+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T09:50:12.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:12.288696+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T09:50:12.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:12.288904+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14427 172.21.15.12:0/1290340819' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T09:50:12.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:12.288953+0000 mon.a (mon.0) 765 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00848827s) 2023-10-09T09:50:13.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:13.027+0000 7f8831071680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.601+0000 7fa2d1a68200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE Bus STARTING 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: CherryPy Checker: 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: The Application mounted at '' has an empty config. 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE Serving on http://:::9283 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE Bus STARTED 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE Bus STOPPING 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:50:13.889 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE Bus STOPPED 2023-10-09T09:50:13.890 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE Bus STARTING 2023-10-09T09:50:13.890 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:13.616420+0000 mon.a (mon.0) 766 : cluster [INF] Active manager daemon x restarted 2023-10-09T09:50:13.890 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:13.616792+0000 mon.a (mon.0) 767 : cluster [INF] Activating manager daemon x 2023-10-09T09:50:13.890 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.617974+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:50:13.890 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:13.622083+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T09:50:13.890 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:13.622341+0000 mon.a (mon.0) 769 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0057215s) 2023-10-09T09:50:13.890 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.623740+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:50:13.890 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.623896+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.624091+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.624279+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.624481+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.624729+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.624975+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.625199+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.625456+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.625706+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.625961+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.626208+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.626468+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:50:13.891 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.626715+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:50:13.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.626970+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:50:13.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.627220+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:50:13.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.627517+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:50:13.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.627999+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:50:13.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.628224+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:50:13.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.629504+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:50:13.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.629800+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:50:13.950 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:13.616420+0000 mon.a (mon.0) 766 : cluster [INF] Active manager daemon x restarted 2023-10-09T09:50:13.950 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:13.616792+0000 mon.a (mon.0) 767 : cluster [INF] Activating manager daemon x 2023-10-09T09:50:13.950 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.617974+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:50:13.950 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:13.622083+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T09:50:13.950 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:13.622341+0000 mon.a (mon.0) 769 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0057215s) 2023-10-09T09:50:13.950 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.623740+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.623896+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.624091+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.624279+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.624481+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.624729+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.624975+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.625199+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.625456+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.625706+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:50:13.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.625961+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.626208+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.626468+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.626715+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.626970+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.627220+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.627517+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.627999+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.628224+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.629504+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:50:13.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.629800+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:50:13.953 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:13.616420+0000 mon.a (mon.0) 766 : cluster [INF] Active manager daemon x restarted 2023-10-09T09:50:13.953 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:13.616792+0000 mon.a (mon.0) 767 : cluster [INF] Activating manager daemon x 2023-10-09T09:50:13.953 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.617974+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:50:13.953 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:13.622083+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:13.622341+0000 mon.a (mon.0) 769 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0057215s) 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.623740+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.623896+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.624091+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.624279+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.624481+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.624729+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.624975+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.625199+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:50:13.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.625456+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.625706+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.625961+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.626208+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.626468+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.626715+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.626970+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.627220+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.627517+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.627999+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:50:13.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.628224+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:50:13.956 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.629504+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:50:13.956 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.629800+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:50:14.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa232ee1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa232ee1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa232ee1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa232ee1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa232ee1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa22fedb700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa22fedb700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa22fedb700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa22fedb700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:50:13.890+0000 7fa22fedb700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:50:14.194 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE Serving on http://:::9283 2023-10-09T09:50:14.194 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:13] ENGINE Bus STARTED 2023-10-09T09:50:14.340 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:13.982+0000 7f8831071680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:50:14.340 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:14.079+0000 7f8831071680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:50:14.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:14.339+0000 7f8831071680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:50:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:13.786071+0000 mon.a (mon.0) 770 : cluster [INF] Manager daemon x is now available 2023-10-09T09:50:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.807340+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.811811+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.814082+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.822515+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.823093+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.833827+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.842066+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.842496+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.863284+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.864143+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:13.866981+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:14.138533+0000 mon.a (mon.0) 776 : audit [DBG] from='client.? 172.21.15.12:0/3556217087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:13.786071+0000 mon.a (mon.0) 770 : cluster [INF] Manager daemon x is now available 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.807340+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.811811+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.814082+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.822515+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.823093+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.833827+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.842066+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:50:15.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.842496+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.863284+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.864143+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:13.866981+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:14.138533+0000 mon.a (mon.0) 776 : audit [DBG] from='client.? 172.21.15.12:0/3556217087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:13.786071+0000 mon.a (mon.0) 770 : cluster [INF] Manager daemon x is now available 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.807340+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.811811+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.814082+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:50:15.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.822515+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:15.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.823093+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:15.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.833827+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:15.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.842066+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:50:15.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.842496+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:50:15.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.863284+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:50:15.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.864143+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T09:50:15.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:13.866981+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:15.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:14.138533+0000 mon.a (mon.0) 776 : audit [DBG] from='client.? 172.21.15.12:0/3556217087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:15.769 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:14.214387+0000 mgr.x (mgr.24778) 1 : cephadm [INF] Deploying cephadm binary to smithi110 2023-10-09T09:50:15.769 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:14.224542+0000 mgr.x (mgr.24778) 2 : cephadm [INF] Deploying cephadm binary to smithi012 2023-10-09T09:50:15.769 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:14.702171+0000 mon.a (mon.0) 777 : cluster [DBG] mgrmap e22: x(active, since 1.08555s) 2023-10-09T09:50:15.769 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:14.810429+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:15.769 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:14.810493+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T09:50:15.770 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:14.826642+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:15.770 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:14.827095+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:15.770 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:15.099519+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:15.770 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:15.114069+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:15.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:14.214387+0000 mgr.x (mgr.24778) 1 : cephadm [INF] Deploying cephadm binary to smithi110 2023-10-09T09:50:15.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:14.224542+0000 mgr.x (mgr.24778) 2 : cephadm [INF] Deploying cephadm binary to smithi012 2023-10-09T09:50:15.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:14.702171+0000 mon.a (mon.0) 777 : cluster [DBG] mgrmap e22: x(active, since 1.08555s) 2023-10-09T09:50:15.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:14.810429+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:15.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:14.810493+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T09:50:15.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:14.826642+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:15.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:14.827095+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:15.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:15.099519+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:15.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:15.114069+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:15.771 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:15.501+0000 7f8831071680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:50:15.771 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:15.668+0000 7f8831071680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:50:15.771 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:15.768+0000 7f8831071680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:50:16.121 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:15.889+0000 7f8831071680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:50:16.122 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:16.120+0000 7f8831071680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:50:16.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:50:15] "GET /metrics HTTP/1.1" 200 31030 "" "Prometheus/2.33.4" 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:14.214387+0000 mgr.x (mgr.24778) 1 : cephadm [INF] Deploying cephadm binary to smithi110 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:14.224542+0000 mgr.x (mgr.24778) 2 : cephadm [INF] Deploying cephadm binary to smithi012 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:14.702171+0000 mon.a (mon.0) 777 : cluster [DBG] mgrmap e22: x(active, since 1.08555s) 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:14.810429+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:14.810493+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:14.826642+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:14.827095+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:15.099519+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:16.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:15.114069+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:14.722735+0000 mgr.x (mgr.24778) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:15.627259+0000 mgr.x (mgr.24778) 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:15.641566+0000 mgr.x (mgr.24778) 5 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Bus STARTING 2023-10-09T09:50:16.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:15.812993+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:16.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:15.813070+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T09:50:16.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:16.122713+0000 mon.a (mon.0) 785 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T09:50:16.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:16.576637+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.12:0/3459938380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:16.928 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:16.528+0000 7f8831071680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:50:16.928 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:16.655+0000 7f8831071680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:50:16.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:14.722735+0000 mgr.x (mgr.24778) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:16.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:15.627259+0000 mgr.x (mgr.24778) 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:16.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:15.641566+0000 mgr.x (mgr.24778) 5 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Bus STARTING 2023-10-09T09:50:16.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:15.812993+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:16.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:15.813070+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T09:50:16.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:16.122713+0000 mon.a (mon.0) 785 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T09:50:16.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:16.576637+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.12:0/3459938380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:17.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:14.722735+0000 mgr.x (mgr.24778) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:17.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:15.627259+0000 mgr.x (mgr.24778) 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:17.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:15.641566+0000 mgr.x (mgr.24778) 5 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Bus STARTING 2023-10-09T09:50:17.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:15.812993+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:50:17.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:15.813070+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T09:50:17.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:16.122713+0000 mon.a (mon.0) 785 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T09:50:17.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:16.576637+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.12:0/3459938380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:17.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:17.586+0000 7f8831071680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:50:17.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:17.691+0000 7f8831071680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:50:17.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:17.815+0000 7f8831071680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:50:17.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:15.758033+0000 mgr.x (mgr.24778) 6 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Serving on http://172.21.15.110:8765 2023-10-09T09:50:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:15.874091+0000 mgr.x (mgr.24778) 7 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Serving on https://172.21.15.110:7150 2023-10-09T09:50:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:15.874294+0000 mgr.x (mgr.24778) 8 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Bus STARTED 2023-10-09T09:50:17.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:16.832198+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:17.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:15.758033+0000 mgr.x (mgr.24778) 6 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Serving on http://172.21.15.110:8765 2023-10-09T09:50:17.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:15.874091+0000 mgr.x (mgr.24778) 7 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Serving on https://172.21.15.110:7150 2023-10-09T09:50:17.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:15.874294+0000 mgr.x (mgr.24778) 8 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Bus STARTED 2023-10-09T09:50:17.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:16.832198+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:18.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:15.758033+0000 mgr.x (mgr.24778) 6 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Serving on http://172.21.15.110:8765 2023-10-09T09:50:18.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:15.874091+0000 mgr.x (mgr.24778) 7 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Serving on https://172.21.15.110:7150 2023-10-09T09:50:18.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:15.874294+0000 mgr.x (mgr.24778) 8 : cephadm [INF] [09/Oct/2023:09:50:15] ENGINE Bus STARTED 2023-10-09T09:50:18.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:16.832198+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:18.361 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:18.057+0000 7f8831071680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:50:18.361 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:18.360+0000 7f8831071680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:50:18.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:18.531+0000 7f8831071680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:50:19.068 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:18.719+0000 7f8831071680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:50:19.068 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:18.952+0000 7f8831071680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:50:19.068 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:17.627811+0000 mgr.x (mgr.24778) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:19.068 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:17.835624+0000 mon.a (mon.0) 788 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-10-09T09:50:19.069 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:17.627811+0000 mgr.x (mgr.24778) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:19.069 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:17.835624+0000 mon.a (mon.0) 788 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-10-09T09:50:19.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:17.627811+0000 mgr.x (mgr.24778) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:19.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:17.835624+0000 mon.a (mon.0) 788 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-10-09T09:50:19.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: debug 2023-10-09T09:50:19.066+0000 7f8831071680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:50:19.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:50:19] ENGINE Bus STARTING 2023-10-09T09:50:19.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: CherryPy Checker: 2023-10-09T09:50:19.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: The Application mounted at '' has an empty config. 2023-10-09T09:50:19.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: 2023-10-09T09:50:19.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:50:19] ENGINE Serving on http://:::9283 2023-10-09T09:50:19.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: [09/Oct/2023:09:50:19] ENGINE Bus STARTED 2023-10-09T09:50:20.176 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:18.997636+0000 mon.a (mon.0) 789 : audit [DBG] from='client.? 172.21.15.12:0/1364970175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:20.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:19.081472+0000 mon.a (mon.0) 790 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:50:20.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:19.084331+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:50:20.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:19.085794+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:50:20.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:19.088416+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:50:20.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:19.088838+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:50:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:18.997636+0000 mon.a (mon.0) 789 : audit [DBG] from='client.? 172.21.15.12:0/1364970175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:19.081472+0000 mon.a (mon.0) 790 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:50:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:19.084331+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:50:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:19.085794+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:50:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:19.088416+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:50:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:19.088838+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:50:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:18.997636+0000 mon.a (mon.0) 789 : audit [DBG] from='client.? 172.21.15.12:0/1364970175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:19.081472+0000 mon.a (mon.0) 790 : cluster [DBG] Standby manager daemon y started 2023-10-09T09:50:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:19.084331+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:50:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:19.085794+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:50:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:19.088416+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:50:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:19.088838+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.? 172.21.15.12:0/3671084177' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:50:21.176 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:19.628156+0000 mgr.x (mgr.24778) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:19.715765+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:19.716063+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e25: x(active, since 6s), standbys: y 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:19.783250+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:19.788157+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:20.146066+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:20.452747+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:20.461338+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:20.675464+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:20.682629+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:19.628156+0000 mgr.x (mgr.24778) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:19.715765+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:19.716063+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e25: x(active, since 6s), standbys: y 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:19.783250+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:19.788157+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:20.146066+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:20.452747+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:20.461338+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:20.675464+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:20.682629+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:19.628156+0000 mgr.x (mgr.24778) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:19.715765+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:50:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:19.716063+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e25: x(active, since 6s), standbys: y 2023-10-09T09:50:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:19.783250+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:19.788157+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:20.146066+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:20.452747+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:20.461338+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:20.675464+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:20.682629+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:22.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:21.296654+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:22.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:21.303016+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:22.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:21.403664+0000 mon.a (mon.0) 805 : audit [DBG] from='client.? 172.21.15.12:0/1408959191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:22.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:21.707690+0000 mon.a (mon.0) 806 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:22.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:21.296654+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:22.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:21.303016+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:22.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:21.403664+0000 mon.a (mon.0) 805 : audit [DBG] from='client.? 172.21.15.12:0/1408959191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:22.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:21.707690+0000 mon.a (mon.0) 806 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:23.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:21.296654+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:23.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:21.303016+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:23.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:21.403664+0000 mon.a (mon.0) 805 : audit [DBG] from='client.? 172.21.15.12:0/1408959191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:23.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:21.707690+0000 mon.a (mon.0) 806 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:24.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:50:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:50:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:21.628677+0000 mgr.x (mgr.24778) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 236 KiB/s wr, 31 op/s 2023-10-09T09:50:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:21.707953+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:21.708141+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:21.708296+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:22.613864+0000 mon.a (mon.0) 807 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:22.615530+0000 mon.b (mon.1) 25 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:22.615674+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:22.620656+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:22.620835+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:23.134963+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:21.628677+0000 mgr.x (mgr.24778) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 236 KiB/s wr, 31 op/s 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:21.707953+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:21.708141+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:21.708296+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:22.613864+0000 mon.a (mon.0) 807 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:22.615530+0000 mon.b (mon.1) 25 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:22.615674+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:22.620656+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:24.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:22.620835+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:24.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:23.134963+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:24.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:21.628677+0000 mgr.x (mgr.24778) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 236 KiB/s wr, 31 op/s 2023-10-09T09:50:24.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:21.707953+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:24.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:21.708141+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:21.708296+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:22.613864+0000 mon.a (mon.0) 807 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:22.615530+0000 mon.b (mon.1) 25 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:22.615674+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:22.620656+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:22.620835+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:23.134963+0000 mon.b (mon.1) 26 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:23.142573+0000 mon.c (mon.2) 72 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:23.629042+0000 mgr.x (mgr.24778) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T09:50:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:23.800725+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.12:0/3529803449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.035482+0000 mon.c (mon.2) 73 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.038540+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:25.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.038711+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.038827+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.047042+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.050208+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.055794+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.058614+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.059012+0000 mon.c (mon.2) 77 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.059281+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.060188+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.060358+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.060445+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:23.142573+0000 mon.c (mon.2) 72 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:23.629042+0000 mgr.x (mgr.24778) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:23.800725+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.12:0/3529803449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.035482+0000 mon.c (mon.2) 73 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.038540+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.038711+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.038827+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.047042+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.050208+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.055794+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.058614+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.059012+0000 mon.c (mon.2) 77 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.059281+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.060188+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.060358+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.060445+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.060785+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.061173+0000 mon.c (mon.2) 80 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.061376+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.061453+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.061692+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.062267+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.062459+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.062474+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.062689+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.063217+0000 mgr.x (mgr.24778) 13 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 2501M 2023-10-09T09:50:25.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.063513+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.063957+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.066624+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.068108+0000 mgr.x (mgr.24778) 14 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 2500M 2023-10-09T09:50:25.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.071519+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.072708+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:24.073438+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.074440+0000 mgr.x (mgr.24778) 15 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.074615+0000 mgr.x (mgr.24778) 16 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.338793+0000 mgr.x (mgr.24778) 17 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.338959+0000 mgr.x (mgr.24778) 18 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.603126+0000 mgr.x (mgr.24778) 19 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.603301+0000 mgr.x (mgr.24778) 20 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.036929+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.060785+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.061173+0000 mon.c (mon.2) 80 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.061376+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.061453+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.061692+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.062267+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.062459+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.062474+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.062689+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.063217+0000 mgr.x (mgr.24778) 13 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 2501M 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.063513+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.063957+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.066624+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.068108+0000 mgr.x (mgr.24778) 14 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 2500M 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.071519+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.072708+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:24.073438+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.074440+0000 mgr.x (mgr.24778) 15 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.074615+0000 mgr.x (mgr.24778) 16 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.338793+0000 mgr.x (mgr.24778) 17 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.338959+0000 mgr.x (mgr.24778) 18 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.603126+0000 mgr.x (mgr.24778) 19 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.603301+0000 mgr.x (mgr.24778) 20 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:50:25.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.036929+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:23.142573+0000 mon.c (mon.2) 72 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:23.629042+0000 mgr.x (mgr.24778) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:23.800725+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.12:0/3529803449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.035482+0000 mon.c (mon.2) 73 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.038540+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.038711+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.038827+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.047042+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.050208+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.055794+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.058614+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.059012+0000 mon.c (mon.2) 77 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.059281+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.060188+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.060358+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.060445+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.060785+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.061173+0000 mon.c (mon.2) 80 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.061376+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.061453+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.061692+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.062267+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.062459+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.062474+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.062689+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.063217+0000 mgr.x (mgr.24778) 13 : cephadm [INF] Adjusting osd_memory_target on smithi012 to 2501M 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.063513+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.063957+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.066624+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.068108+0000 mgr.x (mgr.24778) 14 : cephadm [INF] Adjusting osd_memory_target on smithi110 to 2500M 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.071519+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.072708+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:50:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:24.073438+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:50:25.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.074440+0000 mgr.x (mgr.24778) 15 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:50:25.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.074615+0000 mgr.x (mgr.24778) 16 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:50:25.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.338793+0000 mgr.x (mgr.24778) 17 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:50:25.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.338959+0000 mgr.x (mgr.24778) 18 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:50:25.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.603126+0000 mgr.x (mgr.24778) 19 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:50:25.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.603301+0000 mgr.x (mgr.24778) 20 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:50:25.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.036929+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:25.931 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:25 smithi012 systemd[1]: Stopping Ceph node-exporter.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:25.931 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:25 smithi012 bash[140962]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter.a" found: no such container 2023-10-09T09:50:26.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:50:25] "GET /metrics HTTP/1.1" 200 31030 "" "Prometheus/2.33.4" 2023-10-09T09:50:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.817638+0000 mgr.x (mgr.24778) 21 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:50:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:24.867727+0000 mgr.x (mgr.24778) 22 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:50:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.042731+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.087118+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.095174+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.102027+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.173006+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.176063+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.179067+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:25.181989+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:25.182926+0000 mgr.x (mgr.24778) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T09:50:26.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:25.183275+0000 mgr.x (mgr.24778) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi012 2023-10-09T09:50:26.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:25.629785+0000 mgr.x (mgr.24778) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-09T09:50:26.367 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 bash[141005]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T09:50:26.367 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:50:26.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.817638+0000 mgr.x (mgr.24778) 21 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:50:26.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:24.867727+0000 mgr.x (mgr.24778) 22 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:50:26.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.042731+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.087118+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.095174+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.102027+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.173006+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.176063+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.179067+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:25.181989+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:25.182926+0000 mgr.x (mgr.24778) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T09:50:26.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:25.183275+0000 mgr.x (mgr.24778) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi012 2023-10-09T09:50:26.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:25.629785+0000 mgr.x (mgr.24778) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-09T09:50:26.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.817638+0000 mgr.x (mgr.24778) 21 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:50:26.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:24.867727+0000 mgr.x (mgr.24778) 22 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:50:26.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.042731+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.087118+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.095174+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.102027+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.173006+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.176063+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.179067+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:25.181989+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:26.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:25.182926+0000 mgr.x (mgr.24778) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T09:50:26.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:25.183275+0000 mgr.x (mgr.24778) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi012 2023-10-09T09:50:26.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:25.629785+0000 mgr.x (mgr.24778) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-09T09:50:26.640 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T09:50:26.641 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 systemd[1]: Stopped Ceph node-exporter.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:26.641 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 systemd[1]: Starting Ceph node-exporter.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:26.902 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 podman[141151]: 2023-10-09T09:50:26.902 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.882Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T09:50:26.902 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.882Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.882Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.882Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T09:50:26.903 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T09:50:26.904 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T09:50:26.905 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T09:50:26.906 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T09:50:26.907 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T09:50:26.907 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.883Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T09:50:26.907 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.884Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T09:50:26.907 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a[141166]: ts=2023-10-09T09:50:26.884Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T09:50:26.907 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 bash[141151]: c780854e434e2b4e40b33484c5dd8a9ba691db227c3da7d6bfaf6198e2c7c697 2023-10-09T09:50:27.177 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 09:50:26 smithi012 systemd[1]: Started Ceph node-exporter.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:26.219649+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.12:0/1102894324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:26.976692+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:26.982195+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:27.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:26.219649+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.12:0/1102894324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:27.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:26.976692+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:27.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:26.982195+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:26.219649+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.12:0/1102894324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:26.976692+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:26.982195+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:27.001989+0000 mgr.x (mgr.24778) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:50:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:27.012848+0000 mgr.x (mgr.24778) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi012 2023-10-09T09:50:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:27.630380+0000 mgr.x (mgr.24778) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-09T09:50:29.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:27.001989+0000 mgr.x (mgr.24778) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:50:29.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:27.012848+0000 mgr.x (mgr.24778) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi012 2023-10-09T09:50:29.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:27.630380+0000 mgr.x (mgr.24778) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-09T09:50:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:27.001989+0000 mgr.x (mgr.24778) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T09:50:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:27.012848+0000 mgr.x (mgr.24778) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi012 2023-10-09T09:50:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:27.630380+0000 mgr.x (mgr.24778) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-09T09:50:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:28.708789+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.12:0/945464773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:28.834251+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:30.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:28.708789+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.12:0/945464773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:28.834251+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:28.708789+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.12:0/945464773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:28.834251+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:30.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:29.630771+0000 mgr.x (mgr.24778) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T09:50:30.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:29.630771+0000 mgr.x (mgr.24778) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T09:50:30.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:29.630771+0000 mgr.x (mgr.24778) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T09:50:31.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:31.106936+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.12:0/1688801603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:31.831 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:31 smithi012 systemd[1]: Stopping Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:31.832 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:31.106936+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.12:0/1688801603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:32.124 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:31 smithi012 bash[141534]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager.a" found: no such container 2023-10-09T09:50:32.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:31.106936+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.12:0/1688801603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:32.426 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[129425]: level=info ts=2023-10-09T09:50:32.124Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:50:32.684 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:32 smithi012 bash[141576]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T09:50:32.685 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:32 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T09:50:32.685 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:32 smithi012 systemd[1]: Stopped Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:32.685 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:32 smithi012 systemd[1]: Starting Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:32.692 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.12:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:50:32.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:31.631536+0000 mgr.x (mgr.24778) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T09:50:32.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:31.631536+0000 mgr.x (mgr.24778) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T09:50:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:31.631536+0000 mgr.x (mgr.24778) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T09:50:33.254 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:32 smithi012 podman[141702]: 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:33.254Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:33.254Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:33.255Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.12 port=9094 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:33.256Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 bash[141702]: f37e3c8f7f6635b17bac2b03df648a8fe9a5edeac632bd4ae4627f41ec2d5024 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:33.292Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:33.292Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:33.294Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:33.295Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T09:50:33.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:33 smithi012 systemd[1]: Started Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:34.397 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[96945]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:50:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:50:34.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:33.398120+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:33.403238+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:33.441706+0000 mgr.x (mgr.24778) 31 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:50:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:33.532926+0000 mon.a (mon.0) 837 : audit [DBG] from='client.? 172.21.15.12:0/927758528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:33.631869+0000 mgr.x (mgr.24778) 32 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T09:50:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:33.665393+0000 mgr.x (mgr.24778) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:50:34.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:33.398120+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:34.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:33.403238+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:34.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:33.441706+0000 mgr.x (mgr.24778) 31 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:50:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:33.532926+0000 mon.a (mon.0) 837 : audit [DBG] from='client.? 172.21.15.12:0/927758528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:33.631869+0000 mgr.x (mgr.24778) 32 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T09:50:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:33.665393+0000 mgr.x (mgr.24778) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:50:34.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:33.398120+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:34.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:33.403238+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:34.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:33.441706+0000 mgr.x (mgr.24778) 31 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:50:34.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:33.532926+0000 mon.a (mon.0) 837 : audit [DBG] from='client.? 172.21.15.12:0/927758528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:34.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:33.631869+0000 mgr.x (mgr.24778) 32 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T09:50:34.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:33.665393+0000 mgr.x (mgr.24778) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:50:35.676 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:35.257Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000888617s 2023-10-09T09:50:36.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:50:35] "GET /metrics HTTP/1.1" 200 33864 "" "Prometheus/2.33.4" 2023-10-09T09:50:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:35.632331+0000 mgr.x (mgr.24778) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T09:50:36.966 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:35.944294+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.12:0/2864059438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:35.632331+0000 mgr.x (mgr.24778) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T09:50:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:35.944294+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.12:0/2864059438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:37.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:35.632331+0000 mgr.x (mgr.24778) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T09:50:37.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:35.944294+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.12:0/2864059438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:39.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:37.632793+0000 mgr.x (mgr.24778) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:39.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:38.348544+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.12:0/3039687576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:37.632793+0000 mgr.x (mgr.24778) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:38.348544+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.12:0/3039687576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:39.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:37.632793+0000 mgr.x (mgr.24778) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:39.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:38.348544+0000 mon.a (mon.0) 838 : audit [DBG] from='client.? 172.21.15.12:0/3039687576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:41.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:39.633154+0000 mgr.x (mgr.24778) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:41.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:39.633154+0000 mgr.x (mgr.24778) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:41.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:39.633154+0000 mgr.x (mgr.24778) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:41.954 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:40.750971+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.12:0/928692663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:42.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:40.750971+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.12:0/928692663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:42.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:40.750971+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.12:0/928692663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:42.581 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.12:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:50:42.878 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:41.633762+0000 mgr.x (mgr.24778) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:43.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:41.633762+0000 mgr.x (mgr.24778) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:41.633762+0000 mgr.x (mgr.24778) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:43.443 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 systemd[1]: Stopping Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:43.443 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 bash[125299]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus.a" found: no such container 2023-10-09T09:50:43.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 09:50:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T09:50:43.259Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003038618s 2023-10-09T09:50:43.852 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:43.152452+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.12:0/2484639979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:43.852 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:50:43.852 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.542Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.543Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T09:50:43.853 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.543Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T09:50:43.854 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[118976]: ts=2023-10-09T09:50:43.543Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T09:50:43.854 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:43 smithi110 bash[125342]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:50:44.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:43.152452+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.12:0/2484639979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:44.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:43.152452+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.12:0/2484639979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:44.305 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T09:50:44.305 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 systemd[1]: Stopped Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:44.305 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 systemd[1]: Starting Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:44.560 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 podman[125464]: 2023-10-09T09:50:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:43.634056+0000 mgr.x (mgr.24778) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:43.834673+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:44.644488+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:44.650298+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:44.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.560Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T09:50:44.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.560Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.560Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi110 (none))" 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.560Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.560Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.562Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.562Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.563Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.566Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.566Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.617µs 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.566Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T09:50:44.944 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 bash[125464]: 4473ddd68c7ad123a91d32cf1b2767752bef440a74268f960ab8009f05d145c2 2023-10-09T09:50:44.945 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 systemd[1]: Started Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:44.945 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.732Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T09:50:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:43.634056+0000 mgr.x (mgr.24778) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:43.834673+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:44.644488+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:44.650298+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:43.634056+0000 mgr.x (mgr.24778) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:43.834673+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:50:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:44.644488+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:44.650298+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:45.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:45 smithi110 systemd[1]: Stopping Ceph node-exporter.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:45.430 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:45 smithi110 bash[125611]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter.b" found: no such container 2023-10-09T09:50:45.430 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.972Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T09:50:45.430 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.972Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T09:50:45.430 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.972Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.323µs wal_replay_duration=405.894903ms total_replay_duration=405.939751ms 2023-10-09T09:50:45.430 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.975Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T09:50:45.430 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.975Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T09:50:45.431 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.975Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T09:50:45.431 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.992Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.927357ms db_storage=773ns remote_storage=1.304µs web_handler=468ns query_engine=629ns scrape=209.835µs scrape_sd=123.169µs notify=34.385µs notify_sd=20.711µs rules=16.295166ms 2023-10-09T09:50:45.431 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:44.992Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T09:50:45.692 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:45 smithi110 bash[125654]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T09:50:45.693 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:45 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:50:46.101 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:44.657736+0000 mgr.x (mgr.24778) 39 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:50:46.101 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:44.658220+0000 mgr.x (mgr.24778) 40 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi110 2023-10-09T09:50:46.102 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:45.560467+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.12:0/2771555645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:46.102 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:45 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T09:50:46.102 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:45 smithi110 systemd[1]: Stopped Ceph node-exporter.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:46.102 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:45 smithi110 systemd[1]: Starting Ceph node-exporter.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:44.657736+0000 mgr.x (mgr.24778) 39 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:50:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:44.658220+0000 mgr.x (mgr.24778) 40 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi110 2023-10-09T09:50:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:45.560467+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.12:0/2771555645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:44.657736+0000 mgr.x (mgr.24778) 39 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:50:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:44.658220+0000 mgr.x (mgr.24778) 40 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi110 2023-10-09T09:50:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:45.560467+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.12:0/2771555645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:46.364 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 podman[125775]: 2023-10-09T09:50:46.693 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.364Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T09:50:46.693 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.364Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T09:50:46.693 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.364Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T09:50:46.693 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.364Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T09:50:46.693 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.365Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.365Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.365Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T09:50:46.694 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T09:50:46.695 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T09:50:46.696 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T09:50:46.696 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T09:50:46.696 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T09:50:46.696 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T09:50:46.696 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T09:50:46.696 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T09:50:46.696 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T09:50:46.696 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T09:50:46.697 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T09:50:46.698 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T09:50:46.698 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T09:50:46.698 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T09:50:46.698 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T09:50:46.698 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T09:50:46.698 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T09:50:46.698 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T09:50:46.699 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b[125790]: ts=2023-10-09T09:50:46.366Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T09:50:46.699 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 bash[125775]: c0d7506efa6481bec8f4b5adb97c120454b87a5d3ffe11f5b3ba6bed14528df6 2023-10-09T09:50:46.699 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 09:50:46 smithi110 systemd[1]: Started Ceph node-exporter.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:45.634509+0000 mgr.x (mgr.24778) 41 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:46.416303+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:46.426680+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:45.634509+0000 mgr.x (mgr.24778) 41 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:46.416303+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:46.426680+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:45.634509+0000 mgr.x (mgr.24778) 41 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:46.416303+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:47.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:46.426680+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:50:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:46.475348+0000 mgr.x (mgr.24778) 42 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:50:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:46.486100+0000 mgr.x (mgr.24778) 43 : cephadm [INF] Reconfiguring daemon grafana.a on smithi110 2023-10-09T09:50:48.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:46.475348+0000 mgr.x (mgr.24778) 42 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:50:48.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:46.486100+0000 mgr.x (mgr.24778) 43 : cephadm [INF] Reconfiguring daemon grafana.a on smithi110 2023-10-09T09:50:48.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:47 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:46.475348+0000 mgr.x (mgr.24778) 42 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:50:48.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:47 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:46.486100+0000 mgr.x (mgr.24778) 43 : cephadm [INF] Reconfiguring daemon grafana.a on smithi110 2023-10-09T09:50:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:47.634840+0000 mgr.x (mgr.24778) 44 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:47.969053+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.12:0/3945822496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:49.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:47.634840+0000 mgr.x (mgr.24778) 44 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:49.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:47.969053+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.12:0/3945822496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:47.634840+0000 mgr.x (mgr.24778) 44 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:47.969053+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.12:0/3945822496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:50.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:49.635155+0000 mgr.x (mgr.24778) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:50.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:49 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:49.635155+0000 mgr.x (mgr.24778) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:50.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:49.635155+0000 mgr.x (mgr.24778) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:50.373937+0000 mon.a (mon.0) 844 : audit [DBG] from='client.? 172.21.15.12:0/1682262704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:50.373937+0000 mon.a (mon.0) 844 : audit [DBG] from='client.? 172.21.15.12:0/1682262704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:51.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:50.373937+0000 mon.a (mon.0) 844 : audit [DBG] from='client.? 172.21.15.12:0/1682262704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:51.635598+0000 mgr.x (mgr.24778) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:51.635598+0000 mgr.x (mgr.24778) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:52.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:51.635598+0000 mgr.x (mgr.24778) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:54.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:52.776689+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.12:0/966542076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:54.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:52.776689+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.12:0/966542076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:54.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:52.776689+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.12:0/966542076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:54.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:50:53] "GET /metrics HTTP/1.1" 200 33859 "" "Prometheus/2.33.4" 2023-10-09T09:50:55.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:53.635931+0000 mgr.x (mgr.24778) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:55.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:53.635931+0000 mgr.x (mgr.24778) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:53.635931+0000 mgr.x (mgr.24778) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:55.186096+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.12:0/2201541754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:56.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:55.186096+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.12:0/2201541754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:56.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:55.186096+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.12:0/2201541754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:55.636396+0000 mgr.x (mgr.24778) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:57.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:55.636396+0000 mgr.x (mgr.24778) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:57.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:55.636396+0000 mgr.x (mgr.24778) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:57.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:57.591185+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.12:0/1041069583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:57.945 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:57 smithi110 systemd[1]: Stopping Ceph grafana.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:57.945 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:57 smithi110 bash[126060]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana.a" found: no such container 2023-10-09T09:50:58.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:57.591185+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.12:0/1041069583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:58.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:57.591185+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.12:0/1041069583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:50:58.259 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[118107]: t=2023-10-09T09:50:57+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T09:50:58.259 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 bash[126102]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T09:50:58.692 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T09:50:58.693 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 systemd[1]: Stopped Ceph grafana.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:58.693 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 systemd[1]: Starting Ceph grafana.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:50:58.693 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:50:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:50:58.989 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 podman[126225]: 2023-10-09T09:50:58.989 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:57.636752+0000 mgr.x (mgr.24778) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:59.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:57.636752+0000 mgr.x (mgr.24778) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:59.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:57.636752+0000 mgr.x (mgr.24778) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:50:59.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:59] ENGINE Bus STOPPING 2023-10-09T09:50:59.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:50:59.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:59] ENGINE Bus STOPPED 2023-10-09T09:50:59.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:59] ENGINE Bus STARTING 2023-10-09T09:50:59.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:59] ENGINE Serving on http://:::9283 2023-10-09T09:50:59.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:50:59] ENGINE Bus STARTED 2023-10-09T09:50:59.443 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T09:50:59.443 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T09:50:59.444 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:58+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.173935ms 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 bash[126225]: 3c5f62b02d381cebd2531449dd17e10c89d6e5df1ca945bbeef41a66653e9c26 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 systemd[1]: Started Ceph grafana.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T09:50:59.445 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T09:50:59.446 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-10-09T09:50:59.446 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T09:50:59.446 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-10-09T09:50:59.446 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T09:50:59.446 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T09:50:59.446 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T09:50:59+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T09:51:00.115 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:58.835012+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:00.115 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.050546+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.059904+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.072961+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.074740+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.083136+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.088041+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.095635+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.103434+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:58.835012+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.050546+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.116 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.059904+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.072961+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:51:00.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.074740+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:51:00.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.083136+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.088041+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:00.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.095635+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:51:00.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:50:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.103434+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:58.835012+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.050546+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.059904+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.072961+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:51:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.074740+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:51:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.083136+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.088041+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.095635+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:51:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:50:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.103434+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:00.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.073585+0000 mgr.x (mgr.24778) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:51:00.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.075163+0000 mgr.x (mgr.24778) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:51:00.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.088521+0000 mgr.x (mgr.24778) 52 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:00.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.096222+0000 mgr.x (mgr.24778) 53 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:51:00.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:59.105708+0000 mgr.x (mgr.24778) 54 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:51:00.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:50:59.106100+0000 mgr.x (mgr.24778) 55 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:51:00.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:50:59.637130+0000 mgr.x (mgr.24778) 56 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:00.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:50:59.989867+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.12:0/311601469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:00.479498+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:00.480472+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:00.480793+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:00.481818+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:00.482709+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.073585+0000 mgr.x (mgr.24778) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.075163+0000 mgr.x (mgr.24778) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.088521+0000 mgr.x (mgr.24778) 52 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:00.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.096222+0000 mgr.x (mgr.24778) 53 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:59.105708+0000 mgr.x (mgr.24778) 54 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:50:59.106100+0000 mgr.x (mgr.24778) 55 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:50:59.637130+0000 mgr.x (mgr.24778) 56 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:50:59.989867+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.12:0/311601469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:00.479498+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:00.480472+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:00.480793+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:00.481818+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:51:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:00.482709+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.073585+0000 mgr.x (mgr.24778) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T09:51:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.075163+0000 mgr.x (mgr.24778) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.088521+0000 mgr.x (mgr.24778) 52 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.096222+0000 mgr.x (mgr.24778) 53 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:59.105708+0000 mgr.x (mgr.24778) 54 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:50:59.106100+0000 mgr.x (mgr.24778) 55 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:50:59.637130+0000 mgr.x (mgr.24778) 56 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:50:59.989867+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.12:0/311601469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:00.479498+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:00.480472+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:00.480793+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:00.481818+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:51:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:00.482709+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:01.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:00.472689+0000 mgr.x (mgr.24778) 57 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T09:51:01.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:00.483749+0000 mgr.x (mgr.24778) 58 : cephadm [INF] Deploying daemon mgr.y on smithi012 2023-10-09T09:51:01.928 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:01 smithi012 systemd[1]: Stopping Ceph mgr.y for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:51:01.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:00.472689+0000 mgr.x (mgr.24778) 57 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T09:51:01.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:00.483749+0000 mgr.x (mgr.24778) 58 : cephadm [INF] Deploying daemon mgr.y on smithi012 2023-10-09T09:51:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:00.472689+0000 mgr.x (mgr.24778) 57 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T09:51:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:00.483749+0000 mgr.x (mgr.24778) 58 : cephadm [INF] Deploying daemon mgr.y on smithi012 2023-10-09T09:51:03.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:01.637941+0000 mgr.x (mgr.24778) 59 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:03.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:02.396269+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.12:0/3958004814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:03.025 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:02 smithi012 bash[142428]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T09:51:03.025 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:02 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:51:03.025 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:01.637941+0000 mgr.x (mgr.24778) 59 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:03.026 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:02.396269+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.12:0/3958004814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:01.637941+0000 mgr.x (mgr.24778) 59 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:02.396269+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.12:0/3958004814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:03.430 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T09:51:03.430 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 systemd[1]: Stopped Ceph mgr.y for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:51:03.430 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 systemd[1]: Starting Ceph mgr.y for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:51:03.727 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 podman[142605]: 2023-10-09T09:51:03.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:03.638631+0000 mgr.x (mgr.24778) 60 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:03.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:03.638631+0000 mgr.x (mgr.24778) 60 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:03.982 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 bash[142605]: 3a9297092bc214b3b218d290dc23b6be6ff321b9a818f1e4e6cb629a3d3d02b0 2023-10-09T09:51:03.982 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 ceph-mgr[142645]: -- 172.21.15.12:0/1113244297 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d97ef791e0 con 0x55d97ef4e800 2023-10-09T09:51:03.982 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 systemd[1]: Started Ceph mgr.y for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:51:03.982 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:03.882+0000 7f7b266ad200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:51:03.982 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:03.980+0000 7f7b266ad200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:04.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:03.638631+0000 mgr.x (mgr.24778) 60 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:04.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:51:03] "GET /metrics HTTP/1.1" 200 33863 "" "Prometheus/2.33.4" 2023-10-09T09:51:04.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:04.098+0000 7f7b266ad200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:51:04.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:04.216+0000 7f7b266ad200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:51:04.816 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:04.435+0000 7f7b266ad200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:51:05.106 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:04.943+0000 7f7b266ad200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:51:05.107 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:03.816995+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:05.107 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:03.821916+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:05.107 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:03.855246+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:05.107 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:03.816995+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:05.107 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:03.821916+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:05.107 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:03.855246+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:05.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:03.816995+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:05.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:03.821916+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:05.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:03.855246+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:05.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:05.105+0000 7f7b266ad200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:51:05.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:05.214+0000 7f7b266ad200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:51:05.427 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:05.336+0000 7f7b266ad200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:51:06.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:05.880+0000 7f7b266ad200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:51:06.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:04.816269+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.12:0/4062735566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:06.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:05.639163+0000 mgr.x (mgr.24778) 61 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:04.816269+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.12:0/4062735566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:05.639163+0000 mgr.x (mgr.24778) 61 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:06.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:04.816269+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.12:0/4062735566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:06.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:05.639163+0000 mgr.x (mgr.24778) 61 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:07.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:06.700+0000 7f7b266ad200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:51:07.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:06.814+0000 7f7b266ad200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:51:07.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:07.242523+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.12:0/1937355927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:07.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:07.242523+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.12:0/1937355927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:07.242523+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.12:0/1937355927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:07.867+0000 7f7b266ad200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:07.993+0000 7f7b266ad200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:51:08.605 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:08.222+0000 7f7b266ad200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:08.605 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:08.604+0000 7f7b266ad200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:51:08.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:07.639555+0000 mgr.x (mgr.24778) 62 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:08.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:08.385804+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:08.395476+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:08.537078+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.871 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:08.543248+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:07.639555+0000 mgr.x (mgr.24778) 62 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:08.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:08.385804+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:08.395476+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:08.537078+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:08.543248+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:07.639555+0000 mgr.x (mgr.24778) 62 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:08.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:08.385804+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:08.395476+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:08.537078+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:08.543248+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:08.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:51:09.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:08.868+0000 7f7b266ad200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:51:09.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:09.061+0000 7f7b266ad200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:51:09.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:09.415+0000 7f7b266ad200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:51:10.475 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:09.381464+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:10.476 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:09.391309+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:10.476 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:09.640189+0000 mgr.x (mgr.24778) 63 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:10.476 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:09.671576+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.12:0/3628495592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:10.476 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:10.303+0000 7f7b266ad200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:51:10.476 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:10.474+0000 7f7b266ad200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:51:10.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:09.381464+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:10.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:09.391309+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:10.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:09.640189+0000 mgr.x (mgr.24778) 63 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:10.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:09.671576+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.12:0/3628495592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:09.381464+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:09.391309+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:09.640189+0000 mgr.x (mgr.24778) 63 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:09.671576+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.12:0/3628495592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:11.118 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:10.732+0000 7f7b266ad200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:51:11.119 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:10.876+0000 7f7b266ad200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:11.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:11.169+0000 7f7b266ad200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:51:12.314 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.291+0000 7f7b266ad200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:51:12.314 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE Bus STARTING 2023-10-09T09:51:12.542 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:12.287+0000 7fa2c5115700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T09:51:12.542 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: ignoring --setuser ceph since I am not root 2023-10-09T09:51:12.542 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: ignoring --setgroup ceph since I am not root 2023-10-09T09:51:12.542 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:12.342+0000 7f953a0f9700 1 -- 172.21.15.110:0/200911061 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fb0e77d1e0 con 0x55fb0e752800 2023-10-09T09:51:12.542 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:12.457+0000 7f9543e29200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:51:12.542 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:12.541+0000 7f9543e29200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:12.602 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: CherryPy Checker: 2023-10-09T09:51:12.602 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: The Application mounted at '' has an empty config. 2023-10-09T09:51:12.602 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.602 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE Serving on http://:::9283 2023-10-09T09:51:12.602 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE Bus STARTED 2023-10-09T09:51:12.602 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE Bus STOPPING 2023-10-09T09:51:12.602 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:51:12.603 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE Bus STOPPED 2023-10-09T09:51:12.603 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE Bus STARTING 2023-10-09T09:51:12.908 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:12.638+0000 7f9543e29200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:51:12.908 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:12.735+0000 7f9543e29200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:51:12.909 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:12.907+0000 7f9543e29200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:51:12.909 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:11.640893+0000 mgr.x (mgr.24778) 64 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:12.909 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.086803+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.12:0/1707063297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:12.909 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.119903+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.909 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.127030+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.129734+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.130945+0000 mon.c (mon.2) 110 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.136662+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.276459+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.282472+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.282813+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:12.289655+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.289917+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:12.289992+0000 mon.a (mon.0) 869 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00672102s) 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:12.306616+0000 mon.a (mon.0) 870 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:12.306931+0000 mon.a (mon.0) 871 : cluster [INF] Activating manager daemon y 2023-10-09T09:51:12.910 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:12.312883+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:12.313326+0000 mon.a (mon.0) 873 : cluster [DBG] mgrmap e27: y(active, starting, since 0.00649551s) 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.314197+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.? 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.315699+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.315893+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.316080+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.316256+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.316600+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.316854+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.317267+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.317757+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:51:12.911 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.318162+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.318592+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.319022+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.319421+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.319842+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.320221+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.320645+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.321092+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.321497+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.322370+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.322695+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.324507+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:51:12.912 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.324924+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:12.485177+0000 mon.a (mon.0) 875 : cluster [INF] Manager daemon y is now available 2023-10-09T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.508936+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.512147+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.533354+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.540026+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.540363+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.564308+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:12.564795+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.603+0000 7f7a8331d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.603+0000 7f7a8331d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.603+0000 7f7a8331d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.603+0000 7f7a8331d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.603+0000 7f7a8331d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.604+0000 7f7a80317700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.604+0000 7f7a80317700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.604+0000 7f7a80317700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.604+0000 7f7a80317700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:12.604+0000 7f7a80317700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:12.928 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE Serving on http://:::9283 2023-10-09T09:51:12.928 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:12] ENGINE Bus STARTED 2023-10-09T09:51:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:11.640893+0000 mgr.x (mgr.24778) 64 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.086803+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.12:0/1707063297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.119903+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.127030+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.129734+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.130945+0000 mon.c (mon.2) 110 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.136662+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.276459+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.282472+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.282813+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:12.289655+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.289917+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:12.289992+0000 mon.a (mon.0) 869 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00672102s) 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:12.306616+0000 mon.a (mon.0) 870 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:12.306931+0000 mon.a (mon.0) 871 : cluster [INF] Activating manager daemon y 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:12.312883+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:12.313326+0000 mon.a (mon.0) 873 : cluster [DBG] mgrmap e27: y(active, starting, since 0.00649551s) 2023-10-09T09:51:12.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.314197+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.? 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:11.640893+0000 mgr.x (mgr.24778) 64 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.086803+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.12:0/1707063297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.119903+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.127030+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.129734+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.130945+0000 mon.c (mon.2) 110 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.136662+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24778 ' entity='mgr.x' 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.276459+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.282472+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.24778 172.21.15.110:0/1793897862' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.282813+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:12.289655+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.289917+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24778 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:12.289992+0000 mon.a (mon.0) 869 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00672102s) 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:12.306616+0000 mon.a (mon.0) 870 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:12.306931+0000 mon.a (mon.0) 871 : cluster [INF] Activating manager daemon y 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:12.312883+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:12.313326+0000 mon.a (mon.0) 873 : cluster [DBG] mgrmap e27: y(active, starting, since 0.00649551s) 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.314197+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.? 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:51:12.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.315699+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.315893+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.316080+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.316256+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.316600+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.316854+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.317267+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.317757+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.318162+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.318592+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.319022+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:51:12.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.319421+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.319842+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.320221+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.320645+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.315699+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.315893+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.316080+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.316256+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.316600+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:51:12.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.316854+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.317267+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.317757+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.318162+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.318592+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.319022+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.319421+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.319842+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.320221+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:51:12.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.320645+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.321092+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.321497+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.322370+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.322695+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.324507+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.324924+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:12.485177+0000 mon.a (mon.0) 875 : cluster [INF] Manager daemon y is now available 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.508936+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.512147+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.533354+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.540026+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:12.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.540363+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.564308+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:12.564795+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.321092+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.321497+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.322370+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.322695+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.324507+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.324924+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:12.936 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:12.485177+0000 mon.a (mon.0) 875 : cluster [INF] Manager daemon y is now available 2023-10-09T09:51:12.937 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.508936+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.937 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.512147+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:12.937 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.533354+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:12.937 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.540026+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:12.937 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.540363+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:12.937 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.564308+0000 mon.b (mon.1) 52 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:12.937 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:12.564795+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:13.619 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:13.288+0000 7f9543e29200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:51:13.619 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:13.426+0000 7f9543e29200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:51:13.619 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:13.516+0000 7f9543e29200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:51:13.619 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:13.617+0000 7f9543e29200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:51:14.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:14.068+0000 7f9543e29200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:51:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:13.084208+0000 mgr.y (mgr.15129) 1 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Bus STARTING 2023-10-09T09:51:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:13.187606+0000 mgr.y (mgr.15129) 2 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:51:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:13.313814+0000 mgr.y (mgr.15129) 3 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:51:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:13.313976+0000 mgr.y (mgr.15129) 4 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Bus STARTED 2023-10-09T09:51:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:13.316225+0000 mon.a (mon.0) 878 : cluster [DBG] mgrmap e28: y(active, since 1.00941s) 2023-10-09T09:51:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:13.320597+0000 mgr.y (mgr.15129) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:13.084208+0000 mgr.y (mgr.15129) 1 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Bus STARTING 2023-10-09T09:51:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:13.187606+0000 mgr.y (mgr.15129) 2 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:51:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:13.313814+0000 mgr.y (mgr.15129) 3 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:51:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:13.313976+0000 mgr.y (mgr.15129) 4 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Bus STARTED 2023-10-09T09:51:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:13.316225+0000 mon.a (mon.0) 878 : cluster [DBG] mgrmap e28: y(active, since 1.00941s) 2023-10-09T09:51:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:13.320597+0000 mgr.y (mgr.15129) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:13.084208+0000 mgr.y (mgr.15129) 1 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Bus STARTING 2023-10-09T09:51:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:13.187606+0000 mgr.y (mgr.15129) 2 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:51:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:13.313814+0000 mgr.y (mgr.15129) 3 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:51:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:13.313976+0000 mgr.y (mgr.15129) 4 : cephadm [INF] [09/Oct/2023:09:51:13] ENGINE Bus STARTED 2023-10-09T09:51:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:13.316225+0000 mon.a (mon.0) 878 : cluster [DBG] mgrmap e28: y(active, since 1.00941s) 2023-10-09T09:51:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:13.320597+0000 mgr.y (mgr.15129) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:15.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:14.762+0000 7f9543e29200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:51:15.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:14.850+0000 7f9543e29200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:51:15.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:14.318716+0000 mgr.y (mgr.15129) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:15.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:14.508093+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.12:0/3982652006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:15.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:14.318716+0000 mgr.y (mgr.15129) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:15.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:14.508093+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.12:0/3982652006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:15.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:14.318716+0000 mgr.y (mgr.15129) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:15.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:14.508093+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.12:0/3982652006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:16.000 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:15.692+0000 7f9543e29200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:16.001 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:15.791+0000 7f9543e29200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:51:16.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:15.323663+0000 mon.a (mon.0) 879 : cluster [DBG] mgrmap e29: y(active, since 3s) 2023-10-09T09:51:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:15.323663+0000 mon.a (mon.0) 879 : cluster [DBG] mgrmap e29: y(active, since 3s) 2023-10-09T09:51:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:15.323663+0000 mon.a (mon.0) 879 : cluster [DBG] mgrmap e29: y(active, since 3s) 2023-10-09T09:51:16.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:15.999+0000 7f9543e29200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:16.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:16.295+0000 7f9543e29200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:51:16.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:16.542+0000 7f9543e29200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:51:16.978 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:16.698+0000 7f9543e29200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:51:17.373 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:16.977+0000 7f9543e29200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:51:17.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:16.319057+0000 mgr.y (mgr.15129) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:16.912060+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.12:0/3990878656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:17.268520+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:17.273441+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:17.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:16.319057+0000 mgr.y (mgr.15129) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:17.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:16.912060+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.12:0/3990878656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:17.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:17.268520+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:17.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:17.273441+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:16.319057+0000 mgr.y (mgr.15129) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:16.912060+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.12:0/3990878656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:17.268520+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:17.273441+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.102 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:17.761+0000 7f9543e29200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:51:18.103 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:17 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:17.897+0000 7f9543e29200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:51:18.103 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:18.101+0000 7f9543e29200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:51:18.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:18.227+0000 7f9543e29200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:18.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:17.558292+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:17.563271+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:18.165013+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:18.169738+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:18.462199+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:18.467057+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:17.558292+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:17.563271+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:18.165013+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:18.169738+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:18.462199+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:18.467057+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:18.460+0000 7f9543e29200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:51:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:17.558292+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:17.563271+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:18.165013+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:18.169738+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:18.462199+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:18.467057+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:19.562 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:18.319524+0000 mgr.y (mgr.15129) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:19.562 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:19.320982+0000 mon.a (mon.0) 888 : audit [DBG] from='client.? 172.21.15.12:0/980974037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:19.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:19.340437+0000 mon.a (mon.0) 889 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:51:19.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:19.342395+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:51:19.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:19.342805+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:19.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:19.344565+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:51:19.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:19.345632+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:19.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: debug 2023-10-09T09:51:19.326+0000 7f9543e29200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:51:19.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:51:19] ENGINE Bus STARTING 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: CherryPy Checker: 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: The Application mounted at '' has an empty config. 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:51:19] ENGINE Serving on http://:::9283 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: [09/Oct/2023:09:51:19] ENGINE Bus STARTED 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:18.319524+0000 mgr.y (mgr.15129) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:19.320982+0000 mon.a (mon.0) 888 : audit [DBG] from='client.? 172.21.15.12:0/980974037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:19.340437+0000 mon.a (mon.0) 889 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:19.342395+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:51:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:19.342805+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:19.344565+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:51:19.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:19 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:19.345632+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:19.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:18.319524+0000 mgr.y (mgr.15129) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:19.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:19.320982+0000 mon.a (mon.0) 888 : audit [DBG] from='client.? 172.21.15.12:0/980974037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:19.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:19.340437+0000 mon.a (mon.0) 889 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:51:19.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:19.342395+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:51:19.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:19.342805+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:19.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:19.344565+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:51:19.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:19.345632+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.? 172.21.15.110:0/2580039176' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:20.180846+0000 mon.a (mon.0) 894 : cluster [DBG] mgrmap e30: y(active, since 7s), standbys: x 2023-10-09T09:51:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:20.181751+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:51:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:20.319850+0000 mgr.y (mgr.15129) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:20.745927+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:20.754409+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:20.756612+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:20.757502+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:21.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:20.180846+0000 mon.a (mon.0) 894 : cluster [DBG] mgrmap e30: y(active, since 7s), standbys: x 2023-10-09T09:51:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:20.181751+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:51:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:20.319850+0000 mgr.y (mgr.15129) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:20.745927+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:20.754409+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:20.756612+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:20.757502+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:20.180846+0000 mon.a (mon.0) 894 : cluster [DBG] mgrmap e30: y(active, since 7s), standbys: x 2023-10-09T09:51:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:20.181751+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:51:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:20.319850+0000 mgr.y (mgr.15129) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:20.745927+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:20.754409+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:20.756612+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:21 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:20.757502+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:22.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:20.758570+0000 mgr.y (mgr.15129) 10 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:20.758753+0000 mgr.y (mgr.15129) 11 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:21.022944+0000 mgr.y (mgr.15129) 12 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:21.023087+0000 mgr.y (mgr.15129) 13 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:21.287666+0000 mgr.y (mgr.15129) 14 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:21.287878+0000 mgr.y (mgr.15129) 15 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:21.551947+0000 mgr.y (mgr.15129) 16 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:21.552536+0000 mgr.y (mgr.15129) 17 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:21.728991+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.12:0/1293745124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:21.821409+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:21.824366+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:21.828584+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:21.832127+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:21.836670+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:20.758570+0000 mgr.y (mgr.15129) 10 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:20.758753+0000 mgr.y (mgr.15129) 11 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:21.022944+0000 mgr.y (mgr.15129) 12 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:21.023087+0000 mgr.y (mgr.15129) 13 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:21.287666+0000 mgr.y (mgr.15129) 14 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:21.287878+0000 mgr.y (mgr.15129) 15 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:21.551947+0000 mgr.y (mgr.15129) 16 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:21.552536+0000 mgr.y (mgr.15129) 17 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:51:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:21.728991+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.12:0/1293745124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:21.821409+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:21.824366+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:21.828584+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:21.832127+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:21.836670+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:20.758570+0000 mgr.y (mgr.15129) 10 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:20.758753+0000 mgr.y (mgr.15129) 11 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:21.022944+0000 mgr.y (mgr.15129) 12 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:51:22.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:21.023087+0000 mgr.y (mgr.15129) 13 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:21.287666+0000 mgr.y (mgr.15129) 14 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:21.287878+0000 mgr.y (mgr.15129) 15 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:21.551947+0000 mgr.y (mgr.15129) 16 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:21.552536+0000 mgr.y (mgr.15129) 17 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:21.728991+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.12:0/1293745124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:21.821409+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:21.824366+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:21.828584+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:21.832127+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:22.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:21.836670+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:23.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:21.937918+0000 mgr.y (mgr.15129) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:51:23.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:22.194450+0000 mgr.y (mgr.15129) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:51:23.894 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:22.320484+0000 mgr.y (mgr.15129) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:24.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:21.937918+0000 mgr.y (mgr.15129) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:51:24.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:22.194450+0000 mgr.y (mgr.15129) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:51:24.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:22.320484+0000 mgr.y (mgr.15129) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:24.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:21.937918+0000 mgr.y (mgr.15129) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:51:24.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:22.194450+0000 mgr.y (mgr.15129) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:51:24.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:22.320484+0000 mgr.y (mgr.15129) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:24.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[121335]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T09:51:24.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 systemd[1]: Stopping Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:51:24.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 bash[128905]: Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus.a" found: no such container 2023-10-09T09:51:24.647 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.389Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T09:51:24.647 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.389Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T09:51:24.647 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.389Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T09:51:24.647 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.389Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T09:51:24.647 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.389Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T09:51:24.647 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.389Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T09:51:24.648 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.390Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T09:51:24.648 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.390Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T09:51:24.648 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.390Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T09:51:24.648 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.390Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T09:51:24.648 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.390Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T09:51:24.648 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[125480]: ts=2023-10-09T09:51:24.390Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T09:51:24.925 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:24.138159+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.12:0/2252998261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:24.926 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 bash[128949]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T09:51:25.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:24.138159+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.12:0/2252998261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:25.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:24.138159+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.12:0/2252998261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:25.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T09:51:25.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 systemd[1]: Stopped Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:51:25.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:24 smithi110 systemd[1]: Starting Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:51:25.505 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 podman[129069]: 2023-10-09T09:51:25.505 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 bash[129069]: 82ffa853b6d7853334379f3a2c08ad9c4493564a6f4b47ed1a95cb73a59eee95 2023-10-09T09:51:25.505 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.435Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T09:51:25.505 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.435Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T09:51:25.505 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.435Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi110 (none))" 2023-10-09T09:51:25.505 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.435Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T09:51:25.505 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.435Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T09:51:25.506 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.437Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T09:51:25.506 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.437Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T09:51:25.506 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.438Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T09:51:25.506 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.440Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T09:51:25.506 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.440Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.791µs 2023-10-09T09:51:25.506 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.441Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T09:51:25.861 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 systemd[1]: Started Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:51:25.861 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.616Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T09:51:25.861 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:24.321297+0000 mgr.y (mgr.15129) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:25.861 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:25.533264+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:25.861 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:25.541872+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:25.861 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:25.571976+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:25.862 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:25.579767+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:26.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:24.321297+0000 mgr.y (mgr.15129) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:26.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:25.533264+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:26.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:25.541872+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:26.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:25.571976+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:26.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:25.579767+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:26.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:24.321297+0000 mgr.y (mgr.15129) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:26.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:25.533264+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:26.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:25.541872+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:26.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:25.571976+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:26.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:25 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:25.579767+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:26.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.861Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T09:51:26.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.908Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T09:51:26.192 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.908Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T09:51:26.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.908Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.841µs wal_replay_duration=467.760087ms total_replay_duration=467.810609ms 2023-10-09T09:51:26.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.912Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T09:51:26.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.912Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T09:51:26.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.912Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T09:51:26.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.928Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.474994ms db_storage=754ns remote_storage=1.936µs web_handler=357ns query_engine=696ns scrape=244.18µs scrape_sd=71.698µs notify=12.632µs notify_sd=9.755µs rules=14.80716ms 2023-10-09T09:51:26.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:25.928Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T09:51:27.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:25.572683+0000 mgr.y (mgr.15129) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:27.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:26.544297+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.12:0/2451550389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:27.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:25.572683+0000 mgr.y (mgr.15129) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:27.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:26.544297+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.12:0/2451550389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:27.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:25.572683+0000 mgr.y (mgr.15129) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T09:51:27.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:26.544297+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.12:0/2451550389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:28.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:26.321664+0000 mgr.y (mgr.15129) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:28.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:27.534128+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:28.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:26.321664+0000 mgr.y (mgr.15129) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:28.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:27.534128+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:28.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:26.321664+0000 mgr.y (mgr.15129) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:28.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:27.534128+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:30.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:28.322207+0000 mgr.y (mgr.15129) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:28.951622+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.12:0/2979480492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:28.322207+0000 mgr.y (mgr.15129) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:28.951622+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.12:0/2979480492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:28.322207+0000 mgr.y (mgr.15129) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:28.951622+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.12:0/2979480492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:31.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:30.461848+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:31.740 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:30.462474+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:31.854 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:30.463935+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:51:31.854 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:30.465015+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:31.854 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:30.461848+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:31.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:30.462474+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:31.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:30.463935+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:51:31.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:30.465015+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:31.861 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:30.461848+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:31.861 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:30.462474+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:51:31.861 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:30.463935+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:51:31.861 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:30.465015+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:32.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:30.322704+0000 mgr.y (mgr.15129) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:32.578 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:30.461247+0000 mgr.y (mgr.15129) 26 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:51:32.578 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:30.466055+0000 mgr.y (mgr.15129) 27 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:51:32.578 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:31.352951+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.12:0/11062732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:32.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:30.322704+0000 mgr.y (mgr.15129) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:32.581 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:30.461247+0000 mgr.y (mgr.15129) 26 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:51:32.581 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:30.466055+0000 mgr.y (mgr.15129) 27 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:51:32.581 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:31 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:31.352951+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.12:0/11062732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:32.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:30.322704+0000 mgr.y (mgr.15129) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:32.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:30.461247+0000 mgr.y (mgr.15129) 26 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T09:51:32.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:30.466055+0000 mgr.y (mgr.15129) 27 : cephadm [INF] Deploying daemon mgr.x on smithi110 2023-10-09T09:51:32.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:31.352951+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.12:0/11062732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:33.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:32 smithi110 systemd[1]: Stopping Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:51:33.683 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:33 smithi110 bash[129478]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T09:51:33.683 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:33 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T09:51:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:32.323237+0000 mgr.y (mgr.15129) 28 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:33.945 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:33 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T09:51:33.945 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:33 smithi110 systemd[1]: Stopped Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:51:33.945 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:33 smithi110 systemd[1]: Starting Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:51:34.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:32.323237+0000 mgr.y (mgr.15129) 28 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:34.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:32.323237+0000 mgr.y (mgr.15129) 28 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:34.394 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 podman[129626]: 2023-10-09T09:51:34.394 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 bash[129626]: 6e3769c16b69a27d97d0bf03343f8416e52845067d813ca16e9f359b744c5f7b 2023-10-09T09:51:34.394 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-mgr[129666]: -- 172.21.15.110:0/2552664936 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563e8d5d91e0 con 0x563e8d5ae800 2023-10-09T09:51:34.651 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 systemd[1]: Started Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:51:34.651 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:34.469+0000 7ff4b1a2c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:51:34.651 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:34.557+0000 7ff4b1a2c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:34.940 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:33.761348+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.12:0/2153525257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:34.940 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:34.457028+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:34.940 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:34.462885+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:34.940 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:34.489348+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:34.940 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:34.651+0000 7ff4b1a2c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:51:34.941 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:34.753+0000 7ff4b1a2c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:51:34.941 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:34.939+0000 7ff4b1a2c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:51:35.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:33.761348+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.12:0/2153525257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:35.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:34.457028+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:35.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:34.462885+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:35.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:34.489348+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:35.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:33.761348+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.12:0/2153525257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:35.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:34.457028+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:35.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:34.462885+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:35.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:34.489348+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:35.673 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:35.325+0000 7ff4b1a2c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:51:35.673 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:35.460+0000 7ff4b1a2c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:51:35.673 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:35.564+0000 7ff4b1a2c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:51:35.673 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:35.672+0000 7ff4b1a2c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:51:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:34.323890+0000 mgr.y (mgr.15129) 29 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:36.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:34.323890+0000 mgr.y (mgr.15129) 29 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:36.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:35 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:34.323890+0000 mgr.y (mgr.15129) 29 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:36.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:36.124+0000 7ff4b1a2c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:51:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:36.166926+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.12:0/1275809591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:36.324324+0000 mgr.y (mgr.15129) 30 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:37.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:36.166926+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.12:0/1275809591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:37.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:36.324324+0000 mgr.y (mgr.15129) 30 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:37.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:36.166926+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.12:0/1275809591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:37.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:36.324324+0000 mgr.y (mgr.15129) 30 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:37.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:36.825+0000 7ff4b1a2c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:51:37.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:36.914+0000 7ff4b1a2c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:51:38.037 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:37.748+0000 7ff4b1a2c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:38.037 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:37.842+0000 7ff4b1a2c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:51:38.037 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:38.036+0000 7ff4b1a2c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:38.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:51:37] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.33.4" 2023-10-09T09:51:38.334 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:38.334+0000 7ff4b1a2c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:51:38.660 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:38.571+0000 7ff4b1a2c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:51:38.660 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:51:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:38.580033+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.12:0/1558486732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:38.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:38.728+0000 7ff4b1a2c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:51:39.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:38.580033+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.12:0/1558486732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:38.580033+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.12:0/1558486732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:39.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:39.016+0000 7ff4b1a2c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:51:40.106 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:38.324790+0000 mgr.y (mgr.15129) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:40.106 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:39.275613+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:40.107 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:39.284553+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:40.107 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:39.759+0000 7ff4b1a2c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:51:40.107 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:39.899+0000 7ff4b1a2c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:51:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:38.324790+0000 mgr.y (mgr.15129) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:39.275613+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:40.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:39.284553+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:40.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:38.324790+0000 mgr.y (mgr.15129) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:40.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:39.275613+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:40.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:39.284553+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:40.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:40.106+0000 7ff4b1a2c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:51:40.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:40.232+0000 7ff4b1a2c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:40.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:40.466+0000 7ff4b1a2c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:51:41.288 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:40.171238+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:41.288 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:40.177338+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:41.288 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:40.325177+0000 mgr.y (mgr.15129) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:41.288 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:40.986385+0000 mon.a (mon.0) 914 : audit [DBG] from='client.? 172.21.15.12:0/910793218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:41.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:40.171238+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:41.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:40.177338+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:41.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:40.325177+0000 mgr.y (mgr.15129) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:41.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:40.986385+0000 mon.a (mon.0) 914 : audit [DBG] from='client.? 172.21.15.12:0/910793218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:41.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:40.171238+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:41.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:40.177338+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:41.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:40.325177+0000 mgr.y (mgr.15129) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:41.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:40.986385+0000 mon.a (mon.0) 914 : audit [DBG] from='client.? 172.21.15.12:0/910793218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:41.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:41.399+0000 7ff4b1a2c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:51:41.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: [09/Oct/2023:09:51:41] ENGINE Bus STARTING 2023-10-09T09:51:41.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: CherryPy Checker: 2023-10-09T09:51:41.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: The Application mounted at '' has an empty config. 2023-10-09T09:51:41.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:41.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: [09/Oct/2023:09:51:41] ENGINE Serving on http://:::9283 2023-10-09T09:51:41.693 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: [09/Oct/2023:09:51:41] ENGINE Bus STARTED 2023-10-09T09:51:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:41.402068+0000 mon.a (mon.0) 915 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:51:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:41.402170+0000 mon.a (mon.0) 916 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:51:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:41.404733+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:51:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:41.405249+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:41.407425+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:51:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:41.408562+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:41.402068+0000 mon.a (mon.0) 915 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:51:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:41.402170+0000 mon.a (mon.0) 916 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:51:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:41.404733+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:51:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:41.405249+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:41.407425+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:51:42.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:41.408562+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:42.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:41.402068+0000 mon.a (mon.0) 915 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:51:42.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:41.402170+0000 mon.a (mon.0) 916 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:51:42.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:41.404733+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:51:42.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:41.405249+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:42.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:41.407425+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:51:42.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:41.408562+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.? 172.21.15.110:0/3959464565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:43.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:42.194219+0000 mon.a (mon.0) 917 : cluster [DBG] mgrmap e31: y(active, since 29s), standbys: x 2023-10-09T09:51:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:42.325636+0000 mgr.y (mgr.15129) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.534234+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.655837+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.661983+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.663663+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.664758+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.670867+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.811274+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.815241+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.821942+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.823147+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.823508+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.827844+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.828987+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.829329+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.832680+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.834599+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:51:43.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:42.835692+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T09:51:43.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:42.194219+0000 mon.a (mon.0) 917 : cluster [DBG] mgrmap e31: y(active, since 29s), standbys: x 2023-10-09T09:51:43.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:42.325636+0000 mgr.y (mgr.15129) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:43.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.534234+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:43.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.655837+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.661983+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.663663+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.664758+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.670867+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.811274+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.815241+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.821942+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.823147+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.823508+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.827844+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.828987+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:51:43.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.829329+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:51:43.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.832680+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T09:51:43.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.834599+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:51:43.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:42.835692+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T09:51:43.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:42.194219+0000 mon.a (mon.0) 917 : cluster [DBG] mgrmap e31: y(active, since 29s), standbys: x 2023-10-09T09:51:43.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:42.325636+0000 mgr.y (mgr.15129) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:43.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.534234+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:43.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.655837+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.661983+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.663663+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:43.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.664758+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:51:43.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.670867+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.811274+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:43.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.815241+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:51:43.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.821942+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:43.556 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.823147+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:51:43.556 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.823508+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T09:51:43.556 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.827844+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T09:51:43.556 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.828987+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:51:43.556 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.829329+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T09:51:43.556 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.832680+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.15129 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T09:51:43.556 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.834599+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:51:43.556 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:42.835692+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T09:51:44.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:42.815959+0000 mgr.y (mgr.15129) 34 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T09:51:44.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:42.836297+0000 mgr.y (mgr.15129) 35 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T09:51:44.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:43.399518+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.12:0/29752512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:42.815959+0000 mgr.y (mgr.15129) 34 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T09:51:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:42.836297+0000 mgr.y (mgr.15129) 35 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T09:51:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:43.399518+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.12:0/29752512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:44.699 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:42.815959+0000 mgr.y (mgr.15129) 34 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T09:51:44.699 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:42.836297+0000 mgr.y (mgr.15129) 35 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T09:51:44.699 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:43.399518+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.12:0/29752512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:44.302625+0000 mgr.y (mgr.15129) 36 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:44.307907+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:44.309126+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:44.309959+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: audit 2023-10-09T09:51:44.310691+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cephadm 2023-10-09T09:51:44.311457+0000 mgr.y (mgr.15129) 37 : cephadm [INF] Deploying daemon mon.a on smithi012 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: cluster 2023-10-09T09:51:44.326210+0000 mgr.y (mgr.15129) 38 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:44.302625+0000 mgr.y (mgr.15129) 36 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T09:51:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:44.307907+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:45.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:44.309126+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:51:45.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:44.309959+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:51:45.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:44.310691+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:45.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:44.311457+0000 mgr.y (mgr.15129) 37 : cephadm [INF] Deploying daemon mon.a on smithi012 2023-10-09T09:51:45.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:44.326210+0000 mgr.y (mgr.15129) 38 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:45.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:44.302625+0000 mgr.y (mgr.15129) 36 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T09:51:45.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:44.307907+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15129 ' entity='mgr.y' 2023-10-09T09:51:45.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:44.309126+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:51:45.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:44.309959+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:51:45.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:44.310691+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.15129 172.21.15.12:0/1918989408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:51:45.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:44.311457+0000 mgr.y (mgr.15129) 37 : cephadm [INF] Deploying daemon mon.a on smithi012 2023-10-09T09:51:45.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:44.326210+0000 mgr.y (mgr.15129) 38 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:45.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 systemd[1]: Stopping Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:51:46.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:51:45.960+0000 7f7852af4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:51:46.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:45 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[96670]: debug 2023-10-09T09:51:45.960+0000 7f7852af4700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T09:51:46.819 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:46 smithi012 bash[147730]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T09:51:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:46 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T09:51:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:46 smithi012 systemd[1]: Stopped Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:51:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:46 smithi012 systemd[1]: Starting Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:51:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 podman[147886]: 2023-10-09T09:51:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:51:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T09:51:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: pidfile_write: ignore empty --pid-file 2023-10-09T09:51:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 bash[147886]: 9f2783659fac5af987a2eede2e36004ceabd1479bbcf1973ad6d68fbbc79ea4c 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: load: jerasure load: lrc 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: RocksDB version: 7.9.2 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Git sha 0 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: DB SUMMARY 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: DB Session ID: UA89ES007JQBDK6O0ME3 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: CURRENT file: CURRENT 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: MANIFEST file: MANIFEST-000008 size: 579 Bytes 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000017.sst 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000015.log size: 2463894 ; 2023-10-09T09:51:47.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.error_if_exists: 0 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.create_if_missing: 0 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.paranoid_checks: 1 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.env: 0x556d3dccc720 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.info_log: 0x556d40739420 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.statistics: (nil) 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.use_fsync: 0 2023-10-09T09:51:47.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_log_file_size: 0 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.allow_fallocate: 1 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.use_direct_reads: 0 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:51:47.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.db_log_dir: 2023-10-09T09:51:47.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.wal_dir: 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.write_buffer_manager: 0x556d3f9cac80 2023-10-09T09:51:47.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.unordered_write: 0 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:51:47.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.row_cache: None 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.wal_filter: None 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.two_write_queues: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.wal_compression: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.atomic_flush: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.log_readahead_size: 0 2023-10-09T09:51:47.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_background_jobs: 2 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_background_compactions: -1 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_subcompactions: 1 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:51:47.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_open_files: -1 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_background_flushes: -1 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Compression algorithms supported: 2023-10-09T09:51:47.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: kZSTD supported: 0 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: kXpressCompression supported: 0 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: kZlibCompression supported: 1 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: kSnappyCompression supported: 1 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: kLZ4Compression supported: 1 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: kBZip2Compression supported: 0 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:51:47.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.merge_operator: 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_filter: None 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:51:47.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556d40739520) 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: cache_index_and_filter_blocks: 1 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: pin_top_level_index_and_filter: 1 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_type: 0 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: data_block_index_type: 0 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_shortening: 1 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: checksum: 4 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: no_block_cache: 0 2023-10-09T09:51:47.689 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache: 0x556d3fa4b350 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_name: BinnedLRUCache 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_options: 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: capacity : 536870912 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: num_shard_bits : 4 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: strict_capacity_limit : 0 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: high_pri_pool_ratio: 0.000 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_compressed: (nil) 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: persistent_cache: (nil) 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_size: 4096 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_size_deviation: 10 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_restart_interval: 16 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_block_restart_interval: 1 2023-10-09T09:51:47.690 INFO:journalctl@ceph.mon.a.smithi012.stdout: metadata_block_size: 4096 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: partition_filters: 0 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: use_delta_encoding: 1 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: filter_policy: bloomfilter 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: whole_key_filtering: 1 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: verify_compression: 0 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: read_amp_bytes_per_bit: 0 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: format_version: 5 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: enable_index_compression: 1 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_align: 0 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: max_auto_readahead_size: 262144 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: prepopulate_block_cache: 0 2023-10-09T09:51:47.691 INFO:journalctl@ceph.mon.a.smithi012.stdout: initial_auto_readahead_size: 8192 2023-10-09T09:51:47.692 INFO:journalctl@ceph.mon.a.smithi012.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T09:51:47.692 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:51:47.692 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:51:47.692 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression: NoCompression 2023-10-09T09:51:47.692 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:51:47.692 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:51:47.694 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:51:47.694 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.num_levels: 7 2023-10-09T09:51:47.694 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:51:47.694 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:51:47.695 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:51:47.696 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:51:47.697 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:51:47.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:51:47.700 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:51:47.700 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:51:47.700 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.table_properties_collectors: 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.inplace_update_support: 0 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:51:47.701 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.bloom_locality: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.max_successive_merges: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.ttl: 2592000 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T09:51:47.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.enable_blob_files: false 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.min_blob_size: 0 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 17.sst 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T09:51:47.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 19, last_sequence is 8879, log_number is 15,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 15 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: cb1e7303-e8bd-40e3-a394-6b8a2a5111e3 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845107438096, "job": 1, "event": "recovery_started", "wal_files": [15]} 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #15 mode 2 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 systemd[1]: Started Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845107445807, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 20, "file_size": 2351405, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8880, "largest_seqno": 9447, "table_properties": {"data_size": 2347215, "index_size": 2660, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4210, "raw_average_key_size": 21, "raw_value_size": 2341412, "raw_average_value_size": 11765, "num_data_blocks": 120, "num_entries": 199, "num_filter_entries": 199, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696845107, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "cb1e7303-e8bd-40e3-a394-6b8a2a5111e3", "db_session_id": "UA89ES007JQBDK6O0ME3", "orig_file_number": 20, "seqno_to_time_mapping": "N/A"}} 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845107445925, "job": 1, "event": "recovery_finished"} 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/version_set.cc:5047] Creating manifest 22 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000015.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556d3fae8000 2023-10-09T09:51:47.704 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: DB pointer 0x556d3fad4000 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** DB Stats ** 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.705 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** Compaction Stats [default] ** 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: L0 1/0 2.24 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 499.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: L6 1/0 21.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: Sum 2/0 23.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 499.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 499.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** Compaction Stats [default] ** 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 499.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T09:51:47.706 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative compaction: 0.00 GB write, 187.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval compaction: 0.00 GB write, 187.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:51:47.707 INFO:journalctl@ceph.mon.a.smithi012.stdout: Block cache BinnedLRUCache@0x556d3fa4b350#7 capacity: 512.00 MB usage: 3.42 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: starting mon.a rank 0 at public addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] at bind addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: mon.a@-1(???) e3 preinit fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: mon.a@-1(???).mds e7 new map 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: mon.a@-1(???).mds e7 print_map 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: e7 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: legacy client fscid: 1 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.708 INFO:journalctl@ceph.mon.a.smithi012.stdout: Filesystem 'foo' (1) 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: fs_name foo 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: epoch 5 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: created 2023-10-09T09:48:25.508326+0000 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: modified 2023-10-09T09:48:40.668865+0000 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: tableserver 0 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: root 0 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: session_timeout 60 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: session_autoclose 300 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: max_file_size 1099511627776 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: required_client_features {} 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: last_failure 0 2023-10-09T09:51:47.709 INFO:journalctl@ceph.mon.a.smithi012.stdout: last_failure_osd_epoch 0 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: max_mds 1 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: in 0 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: up {0=14652} 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: failed 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: damaged 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: stopped 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: data_pools [4] 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: metadata_pool 3 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: inline_data disabled 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: balancer 2023-10-09T09:51:47.710 INFO:journalctl@ceph.mon.a.smithi012.stdout: bal_rank_mask -1 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout: standby_count_wanted 1 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout: [mds.foo.smithi012.mbnrgj{0:14652} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout: Standby daemons: 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout: [mds.foo.smithi110.sgosap{-1:24556} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.110:6832/704760757,v1:172.21.15.110:6833/704760757] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: mon.a@-1(???).osd e74 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: mon.a@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: mon.a@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: mon.a@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:51:47.711 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta expand map: {default=false} 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta from 'false' to 'false' 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta expanded map: {default=false} 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta expand map: {default=info} 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta from 'info' to 'info' 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta expanded map: {default=info} 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta expand map: {default=daemon} 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta expand map: {default=debug} 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T09:51:47.712 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-mon[147926]: expand_channel_meta expanded map: {default=debug} 2023-10-09T09:51:48.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:51:47] "GET /metrics HTTP/1.1" 200 33859 "" "Prometheus/2.33.4" 2023-10-09T09:51:48.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:51:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:51:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:51:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T09:51:50.798 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ignoring --setuser ceph since I am not root 2023-10-09T09:51:50.798 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ignoring --setgroup ceph since I am not root 2023-10-09T09:51:50.798 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-mgr[142645]: -- 172.21.15.12:0/723624734 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55df8baeb1e0 con 0x55df8bac0c00 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:50.692+0000 7f2cfb693200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:50.797+0000 7f2cfb693200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:46.326704+0000 mgr.y (mgr.15129) 39 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:48.231792+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.12:0/761051157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:48.327217+0000 mgr.y (mgr.15129) 40 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:50.463168+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:50.477782+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:50.482717+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:51:50.799 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:50.483192+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:51:50.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:50.483222+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T09:51:50.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:50.484106+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e31: y(active, since 38s), standbys: x 2023-10-09T09:51:50.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:50.495959+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e32: y(active, since 38s), standbys: x 2023-10-09T09:51:50.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:50.496005+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T09:51:50.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:50.496109+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15129 ' entity='' 2023-10-09T09:51:50.829 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: ignoring --setuser ceph since I am not root 2023-10-09T09:51:50.830 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: ignoring --setgroup ceph since I am not root 2023-10-09T09:51:50.830 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-mgr[129666]: -- 172.21.15.110:0/2709007481 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d78dfff1e0 con 0x55d78dfd4c00 2023-10-09T09:51:50.830 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:50.660+0000 7fb282a10200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T09:51:50.830 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:50.739+0000 7fb282a10200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:50.830 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:50.828+0000 7fb282a10200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:51:50.830 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:46.326704+0000 mgr.y (mgr.15129) 39 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:50.830 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:48.231792+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.12:0/761051157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:50.830 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:48.327217+0000 mgr.y (mgr.15129) 40 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:50.463168+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:50.477782+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:50.482717+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:50.483192+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:50.483222+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:50.484106+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e31: y(active, since 38s), standbys: x 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:50.495959+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e32: y(active, since 38s), standbys: x 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:50.496005+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T09:51:50.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:50.496109+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15129 ' entity='' 2023-10-09T09:51:51.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:51:51.048 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:50.907+0000 7f2cfb693200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T09:51:51.048 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:51.022+0000 7f2cfb693200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:51:51.091 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:50.921+0000 7fb282a10200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T09:51:51.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:51.238+0000 7f2cfb693200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:51:51.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:51.089+0000 7fb282a10200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T09:51:51.763 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:50.655192+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.12:0/2263639211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:51.764 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:51.451+0000 7fb282a10200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:51:51.764 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:51.581+0000 7fb282a10200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:51:51.764 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:51.668+0000 7fb282a10200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:51:51.764 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:51.762+0000 7fb282a10200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:51:51.885 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:51.724+0000 7f2cfb693200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T09:51:51.885 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:50.655192+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.12:0/2263639211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:51:52.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:51.885+0000 7f2cfb693200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T09:51:52.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:51.994+0000 7f2cfb693200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T09:51:52.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:52.115+0000 7f2cfb693200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T09:51:52.442 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:52.179+0000 7fb282a10200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:51:52.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:52.665+0000 7f2cfb693200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T09:51:53.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:52.830+0000 7fb282a10200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:51:53.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:52.913+0000 7fb282a10200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:51:53.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:53.449+0000 7f2cfb693200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T09:51:53.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:53.553+0000 7f2cfb693200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T09:51:53.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:53.689+0000 7fb282a10200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:53.942 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:53.782+0000 7fb282a10200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:51:54.239 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:53.956+0000 7fb282a10200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:54.615 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:54.238+0000 7fb282a10200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:51:54.615 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:54.464+0000 7fb282a10200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:51:54.875 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:54.536+0000 7f2cfb693200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:54.875 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:54.649+0000 7f2cfb693200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T09:51:54.882 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:54.614+0000 7fb282a10200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:51:55.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:54.874+0000 7f2cfb693200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T09:51:55.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:54.881+0000 7fb282a10200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:51:55.491 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:55.233+0000 7f2cfb693200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T09:51:55.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:55.491+0000 7f2cfb693200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T09:51:55.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:55 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:55.676+0000 7f2cfb693200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T09:51:55.929 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:55.586+0000 7fb282a10200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:51:55.929 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:55.724+0000 7fb282a10200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:51:55.929 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:55 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:55.928+0000 7fb282a10200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:51:56.192 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:56.048+0000 7fb282a10200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:56.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:56.017+0000 7f2cfb693200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T09:51:56.692 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:56.270+0000 7fb282a10200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:51:57.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:56.857+0000 7f2cfb693200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T09:51:57.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:57.019+0000 7f2cfb693200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T09:51:57.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:57.145954+0000 mon.a (mon.0) 10 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:51:57.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:57.146044+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:57.147682+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.? 172.21.15.110:0/2540403980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:57.148113+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.? 172.21.15.110:0/2540403980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:57.149908+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.? 172.21.15.110:0/2540403980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:57.150989+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.? 172.21.15.110:0/2540403980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:57.130+0000 7fb282a10200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: [09/Oct/2023:09:51:57] ENGINE Bus STARTING 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: CherryPy Checker: 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: The Application mounted at '' has an empty config. 2023-10-09T09:51:57.443 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: 2023-10-09T09:51:57.444 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: [09/Oct/2023:09:51:57] ENGINE Serving on http://:::9283 2023-10-09T09:51:57.444 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 09:51:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x[129642]: [09/Oct/2023:09:51:57] ENGINE Bus STARTED 2023-10-09T09:51:57.678 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:57.276+0000 7f2cfb693200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T09:51:57.678 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:57.416+0000 7f2cfb693200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T09:51:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:57.145954+0000 mon.a (mon.0) 10 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T09:51:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:57.146044+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2023-10-09T09:51:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:57.147682+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.? 172.21.15.110:0/2540403980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T09:51:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:57.148113+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.? 172.21.15.110:0/2540403980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T09:51:57.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:57.149908+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.? 172.21.15.110:0/2540403980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T09:51:57.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:57.150989+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.? 172.21.15.110:0/2540403980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T09:51:58.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:57.701+0000 7f2cfb693200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T09:51:58.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:57.201557+0000 mon.a (mon.0) 12 : cluster [DBG] mgrmap e33: y(active, since 44s), standbys: x 2023-10-09T09:51:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:57.201557+0000 mon.a (mon.0) 12 : cluster [DBG] mgrmap e33: y(active, since 44s), standbys: x 2023-10-09T09:51:59.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:51:58.781+0000 7f058f287700 -1 mon.c@2(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:51:59.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.774+0000 7f2cfb693200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T09:51:59.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:58] ENGINE Bus STARTING 2023-10-09T09:51:59.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c5c1fc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c5c1fc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c5c1fc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c5c1fc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c5c1fc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c559ef700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c559ef700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c559ef700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c559ef700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:58.932+0000 7f2c559ef700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: CherryPy Checker: 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: The Application mounted at '' has an empty config. 2023-10-09T09:51:59.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:59.179 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:59] ENGINE Serving on http://:::9283 2023-10-09T09:51:59.179 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:51:59] ENGINE Bus STARTED 2023-10-09T09:51:59.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:51:58.778+0000 7fd1944fb700 -1 mon.b@1(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T09:51:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.776407+0000 mon.a (mon.0) 13 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:51:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.776706+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2023-10-09T09:51:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.785148+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:51:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.785507+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e34: y(active, starting, since 0.00888089s), standbys: x 2023-10-09T09:51:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.787136+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.789486+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.792640+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.792871+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.792999+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.793790+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.794013+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.794385+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.794582+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.794824+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.795213+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:51:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.795568+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.795888+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.796210+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.796476+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.796758+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.797071+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.797373+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.797658+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.798169+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.806440+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.832175+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.834910+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.858468+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:59.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.871516+0000 mon.c (mon.2) 143 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:59.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.871924+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:59.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.900038+0000 mon.c (mon.2) 144 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:59.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:51:58.900474+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.776407+0000 mon.a (mon.0) 13 : cluster [INF] Active manager daemon y restarted 2023-10-09T09:51:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.776706+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.785148+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.785507+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e34: y(active, starting, since 0.00888089s), standbys: x 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.787136+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.789486+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.792640+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.792871+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.792999+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.793790+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.mbnrgj"}]: dispatch 2023-10-09T09:51:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.794013+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi110.sgosap"}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.794385+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.794582+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.794824+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.795213+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.795568+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.795888+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.796210+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.796476+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.796758+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.797071+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.797373+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T09:51:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.797658+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.798169+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.806440+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.832175+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.834910+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.858468+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.871516+0000 mon.c (mon.2) 143 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.871924+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.900038+0000 mon.c (mon.2) 144 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:51:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:51:58.900474+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T09:51:59.806 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (86s) - 4m 18.3M - 0.23.0 ba2b418f427c f37e3c8f7f66 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi110 *:3000 running (61s) - 3m 45.9M - 8.3.5 dad864ee21e9 3c5f62b02d38 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.mbnrgj smithi012 running (3m) - 3m 15.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5be3d4216704 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi110.sgosap smithi110 running (3m) - 3m 14.6M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f17cc81b8a5e 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi110 *:8443,9283,8765 running (25s) - 7m 58.4M - 18.2.0-598-gb4fa663f 3060897fdf97 6e3769c16b69 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (56s) - 9m 480M - 18.2.0-598-gb4fa663f 3060897fdf97 3a9297092bc2 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (9m) - 9m 115M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 3c03dd9903d9 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi110 running (7m) - 7m 83.2M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 99ff116de7cf 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (7m) - 7m 92.7M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8ea2dfe53e63 2023-10-09T09:51:59.807 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (93s) - 4m 14.4M - 1.3.1 1dbe0e931976 c780854e434e 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi110 *:9100 running (73s) - 4m 16.9M - 1.3.1 1dbe0e931976 c0d7506efa64 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (7m) - 7m 64.5M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 40f45dc46a61 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (6m) - 6m 62.4M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e2afac5c73f9 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (6m) - 6m 136M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 71decd1a84e3 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (6m) - 6m 58.0M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8587e6142c61 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi110 running (5m) - 5m 68.3M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 18029670387d 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi110 running (5m) - 5m 89.4M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c927f6e714d1 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi110 running (5m) - 5m 59.7M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b96408c9e315 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi110 running (4m) - 4m 96.5M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 ac326d7f40d1 2023-10-09T09:51:59.808 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi110 *:9095 running (34s) - 4m 48.1M - 2.33.4 514e6a882f6e 82ffa853b6d7 2023-10-09T09:52:00.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:51:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:51:59.791+0000 7f2cb4de3700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T09:52:00.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T09:52:00.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.783525+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-09T09:52:00.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:58.891088+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-09T09:52:00.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:51:59.788729+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e35: y(active, since 1.01211s), standbys: x 2023-10-09T09:52:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.783525+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-09T09:52:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:58.891088+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-09T09:52:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:51:59.788729+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e35: y(active, since 1.01211s), standbys: x 2023-10-09T09:52:01.213 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:59.859106+0000 mgr.y (mgr.25217) 3 : cephadm [INF] [09/Oct/2023:09:51:59] ENGINE Bus STARTING 2023-10-09T09:52:01.213 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:51:59.961098+0000 mgr.y (mgr.25217) 4 : cephadm [INF] [09/Oct/2023:09:51:59] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:52:01.213 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:00.078783+0000 mgr.y (mgr.25217) 5 : cephadm [INF] [09/Oct/2023:09:52:00] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:52:01.213 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:00.078998+0000 mgr.y (mgr.25217) 6 : cephadm [INF] [09/Oct/2023:09:52:00] ENGINE Bus STARTED 2023-10-09T09:52:01.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:59.859106+0000 mgr.y (mgr.25217) 3 : cephadm [INF] [09/Oct/2023:09:51:59] ENGINE Bus STARTING 2023-10-09T09:52:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:51:59.961098+0000 mgr.y (mgr.25217) 4 : cephadm [INF] [09/Oct/2023:09:51:59] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:52:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:00.078783+0000 mgr.y (mgr.25217) 5 : cephadm [INF] [09/Oct/2023:09:52:00] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:52:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:00.078998+0000 mgr.y (mgr.25217) 6 : cephadm [INF] [09/Oct/2023:09:52:00] ENGINE Bus STARTED 2023-10-09T09:52:02.407 INFO:teuthology.orchestra.run.smithi012.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:52:02.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:00.796161+0000 mgr.y (mgr.25217) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:02.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:02 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:01.791992+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e36: y(active, since 3s), standbys: x 2023-10-09T09:52:02.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:00.796161+0000 mgr.y (mgr.25217) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:02.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:01.791992+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e36: y(active, since 3s), standbys: x 2023-10-09T09:52:02.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T09:52:03.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:02.401855+0000 mgr.y (mgr.25217) 8 : audit [DBG] from='client.25262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:52:03.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:02.402332+0000 mgr.y (mgr.25217) 9 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:52:03.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:02.406670+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:03.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:02.401855+0000 mgr.y (mgr.25217) 8 : audit [DBG] from='client.25262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:52:03.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:02.402332+0000 mgr.y (mgr.25217) 9 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:52:03.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:03 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:02.406670+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:04.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:02.796505+0000 mgr.y (mgr.25217) 10 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:04.007353+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:04.015911+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:02.796505+0000 mgr.y (mgr.25217) 10 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:04.007353+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:04.015911+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:05.968 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:04.699143+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:05.968 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:04.707575+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:05.968 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:04.797004+0000 mgr.y (mgr.25217) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:05.968 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:04.904267+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:05.968 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:04.913208+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:05.968 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:05.603623+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:05.968 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:05 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:05.609530+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:06.176 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:04.699143+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:04.707575+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:04.797004+0000 mgr.y (mgr.25217) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:04.904267+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:04.913208+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:05.603623+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:06.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:05 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:05.609530+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:08.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:52:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:52:07] "GET /metrics HTTP/1.1" 200 31227 "" "Prometheus/2.33.4" 2023-10-09T09:52:08.659 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:06.797525+0000 mgr.y (mgr.25217) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:08.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:07.347809+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:08.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:07.355325+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:08.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:07.358004+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:52:08.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:07.359091+0000 mon.c (mon.2) 146 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:52:08.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:07.360598+0000 mgr.y (mgr.25217) 13 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:52:08.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:07.360872+0000 mgr.y (mgr.25217) 14 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:52:08.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:06.797525+0000 mgr.y (mgr.25217) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:07.347809+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:07.355325+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:07.358004+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:52:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:07.359091+0000 mon.c (mon.2) 146 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:52:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:07.360598+0000 mgr.y (mgr.25217) 13 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:52:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:07.360872+0000 mgr.y (mgr.25217) 14 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:52:08.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:52:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:52:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:52:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:07.625252+0000 mgr.y (mgr.25217) 15 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:52:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:07.625819+0000 mgr.y (mgr.25217) 16 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:52:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:07.889640+0000 mgr.y (mgr.25217) 17 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:52:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:07.890346+0000 mgr.y (mgr.25217) 18 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:52:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:08.154284+0000 mgr.y (mgr.25217) 19 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:52:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:52:08.154464+0000 mgr.y (mgr.25217) 20 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:52:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.423827+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.427132+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.434322+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.437669+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.444681+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.582850+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:52:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.584280+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:52:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.585020+0000 mon.c (mon.2) 149 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:52:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:08.592366+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:07.625252+0000 mgr.y (mgr.25217) 15 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:52:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:07.625819+0000 mgr.y (mgr.25217) 16 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:52:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:07.889640+0000 mgr.y (mgr.25217) 17 : cephadm [INF] Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:52:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:07.890346+0000 mgr.y (mgr.25217) 18 : cephadm [INF] Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:52:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:08.154284+0000 mgr.y (mgr.25217) 19 : cephadm [INF] Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:52:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:52:08.154464+0000 mgr.y (mgr.25217) 20 : cephadm [INF] Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.423827+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.427132+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.434322+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.437669+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.444681+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.582850+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.584280+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.585020+0000 mon.c (mon.2) 149 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:52:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:08.592366+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:52:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:08.798076+0000 mgr.y (mgr.25217) 21 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:10.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:08.798076+0000 mgr.y (mgr.25217) 21 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:10.798710+0000 mgr.y (mgr.25217) 22 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:12.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:10.798710+0000 mgr.y (mgr.25217) 22 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:12.799387+0000 mgr.y (mgr.25217) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:13.859166+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:12.799387+0000 mgr.y (mgr.25217) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:13.859166+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:16.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:14.800242+0000 mgr.y (mgr.25217) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:14.800242+0000 mgr.y (mgr.25217) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:52:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:52:17] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.33.4" 2023-10-09T09:52:18.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:16.800710+0000 mgr.y (mgr.25217) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:16.800710+0000 mgr.y (mgr.25217) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:18.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:52:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:52:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:52:20.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:20 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:18.801283+0000 mgr.y (mgr.25217) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:18.801283+0000 mgr.y (mgr.25217) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:22.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:22 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:20.801871+0000 mgr.y (mgr.25217) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:22.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:20.801871+0000 mgr.y (mgr.25217) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:24.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:22.802491+0000 mgr.y (mgr.25217) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:22.802491+0000 mgr.y (mgr.25217) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:26.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:24.803070+0000 mgr.y (mgr.25217) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:24.803070+0000 mgr.y (mgr.25217) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:28.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:52:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:52:27] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.33.4" 2023-10-09T09:52:28.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:26.803629+0000 mgr.y (mgr.25217) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:26.803629+0000 mgr.y (mgr.25217) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:28.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:52:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:52:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:52:29.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:29 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:28.859368+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:28.859368+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:30.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:30 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:28.804249+0000 mgr.y (mgr.25217) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:30.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:28.804249+0000 mgr.y (mgr.25217) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:32.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:32 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:30.804914+0000 mgr.y (mgr.25217) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:32.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:30.804914+0000 mgr.y (mgr.25217) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:34.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:32.805430+0000 mgr.y (mgr.25217) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:32.805430+0000 mgr.y (mgr.25217) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:36.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:36 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:34.806093+0000 mgr.y (mgr.25217) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:34.806093+0000 mgr.y (mgr.25217) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:52:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:52:37] "GET /metrics HTTP/1.1" 200 33856 "" "Prometheus/2.33.4" 2023-10-09T09:52:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:36.806568+0000 mgr.y (mgr.25217) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:36.806568+0000 mgr.y (mgr.25217) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:38.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:52:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:52:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:52:40.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:38.806984+0000 mgr.y (mgr.25217) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:38.806984+0000 mgr.y (mgr.25217) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:42.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:40.807675+0000 mgr.y (mgr.25217) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:40.807675+0000 mgr.y (mgr.25217) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:44.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:42.808282+0000 mgr.y (mgr.25217) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:44.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:44 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:43.859856+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:42.808282+0000 mgr.y (mgr.25217) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:43.859856+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:46.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:44.808902+0000 mgr.y (mgr.25217) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:44.808902+0000 mgr.y (mgr.25217) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:52:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:52:47] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.33.4" 2023-10-09T09:52:48.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:48 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:46.809286+0000 mgr.y (mgr.25217) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:46.809286+0000 mgr.y (mgr.25217) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:48.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:52:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:52:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:52:50.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:48.809684+0000 mgr.y (mgr.25217) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:48.809684+0000 mgr.y (mgr.25217) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:52.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:50.810523+0000 mgr.y (mgr.25217) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:50.810523+0000 mgr.y (mgr.25217) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:54 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:52.810970+0000 mgr.y (mgr.25217) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:52.810970+0000 mgr.y (mgr.25217) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:56.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:54.811578+0000 mgr.y (mgr.25217) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:54.811578+0000 mgr.y (mgr.25217) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:58.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:52:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:52:57] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.33.4" 2023-10-09T09:52:58.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:58 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:56.812112+0000 mgr.y (mgr.25217) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:56.812112+0000 mgr.y (mgr.25217) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:52:58.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:52:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:52:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:52:59.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:52:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:52:58.860490+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:52:59.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:52:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:52:58.860490+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:00.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:52:58.812572+0000 mgr.y (mgr.25217) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:00.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:52:58.812572+0000 mgr.y (mgr.25217) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:01.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:53:00.813359+0000 mgr.y (mgr.25217) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:01.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:01 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:00.813359+0000 mgr.y (mgr.25217) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:04.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:53:02.813835+0000 mgr.y (mgr.25217) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:02.813835+0000 mgr.y (mgr.25217) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:04.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T09:53:06.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:53:04.814547+0000 mgr.y (mgr.25217) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:06.700 INFO:teuthology.orchestra.run.smithi012.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:53:06.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:04.814547+0000 mgr.y (mgr.25217) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:07.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T09:53:08.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:53:07] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-10-09T09:53:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:06.696366+0000 mgr.y (mgr.25217) 50 : audit [DBG] from='client.25268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:53:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:53:06.696725+0000 mgr.y (mgr.25217) 51 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:53:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:06.700562+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:06.714321+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:53:08.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:53:06.814911+0000 mgr.y (mgr.25217) 52 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:08.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:06.696366+0000 mgr.y (mgr.25217) 50 : audit [DBG] from='client.25268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:53:08.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:53:06.696725+0000 mgr.y (mgr.25217) 51 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:53:08.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:06.700562+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:08.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:06.714321+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:53:08.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:06.814911+0000 mgr.y (mgr.25217) 52 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:08.900 INFO:teuthology.orchestra.run.smithi012.stdout:wait for majority of mons to upgrade 2023-10-09T09:53:08.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:53:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:53:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:53:09.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T09:53:10.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:09 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:53:08.815238+0000 mgr.y (mgr.25217) 53 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:10.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:08.815238+0000 mgr.y (mgr.25217) 53 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:12.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:53:10.816028+0000 mgr.y (mgr.25217) 54 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:11.350668+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:11.358595+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:11.385477+0000 mon.a (mon.0) 42 : audit [DBG] from='client.? 172.21.15.12:0/2158543482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:53:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.260134+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.266523+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:10.816028+0000 mgr.y (mgr.25217) 54 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:11.350668+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:11.358595+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:11.385477+0000 mon.a (mon.0) 42 : audit [DBG] from='client.? 172.21.15.12:0/2158543482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:53:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.260134+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.266523+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:13.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.786761+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:13.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.787514+0000 mon.c (mon.2) 156 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:53:13.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.793089+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:13.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.932323+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:53:13.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.935380+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:53:13.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.936546+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:53:13.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:12.937208+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T09:53:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.786761+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.787514+0000 mon.c (mon.2) 156 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:53:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.793089+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.932323+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:53:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.935380+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:53:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.936546+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:53:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:13 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:12.937208+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T09:53:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cluster 2023-10-09T09:53:12.816374+0000 mgr.y (mgr.25217) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:53:12.937704+0000 mgr.y (mgr.25217) 56 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T09:53:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:13.794507+0000 mon.a (mon.0) 46 : audit [DBG] from='client.? 172.21.15.12:0/3741105778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:53:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:13.860862+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:14.258699+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:14.259965+0000 mon.c (mon.2) 162 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:53:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:14.260873+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:53:14.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:14 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: audit 2023-10-09T09:53:14.261624+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:12.816374+0000 mgr.y (mgr.25217) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:53:12.937704+0000 mgr.y (mgr.25217) 56 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T09:53:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:13.794507+0000 mon.a (mon.0) 46 : audit [DBG] from='client.? 172.21.15.12:0/3741105778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:53:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:13.860862+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:14.258699+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:14.259965+0000 mon.c (mon.2) 162 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:53:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:14.260873+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:53:14.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:14.261624+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:15.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:53:14.253783+0000 mgr.y (mgr.25217) 57 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T09:53:15.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: cephadm 2023-10-09T09:53:14.262670+0000 mgr.y (mgr.25217) 58 : cephadm [INF] Deploying daemon mon.c on smithi012 2023-10-09T09:53:15.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:15 smithi012 systemd[1]: Stopping Ceph mon.c for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:53:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:53:14.253783+0000 mgr.y (mgr.25217) 57 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T09:53:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:15 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:53:14.262670+0000 mgr.y (mgr.25217) 58 : cephadm [INF] Deploying daemon mon.c on smithi012 2023-10-09T09:53:16.213 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:53:15.948+0000 7f0595a94700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:53:16.214 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[103023]: debug 2023-10-09T09:53:15.948+0000 7f0595a94700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T09:53:16.214 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:53:15] ENGINE Bus STOPPING 2023-10-09T09:53:16.214 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:53:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:53:16.214 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:53:15] ENGINE Bus STOPPED 2023-10-09T09:53:16.214 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:53:15] ENGINE Bus STARTING 2023-10-09T09:53:16.214 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:53:16] ENGINE Serving on http://:::9283 2023-10-09T09:53:16.214 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:53:16] ENGINE Bus STARTED 2023-10-09T09:53:16.483 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:16 smithi012 bash[152046]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T09:53:16.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:16 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T09:53:16.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:16 smithi012 systemd[1]: Stopped Ceph mon.c for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:53:16.897 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:16 smithi012 systemd[1]: Starting Ceph mon.c for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:53:17.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:16 smithi012 podman[152223]: 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 bash[152223]: 251f6625e45a310c2073aba8ac533b2d899524b69c4ce2fb812634c386d986d7 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: pidfile_write: ignore empty --pid-file 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: load: jerasure load: lrc 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: RocksDB version: 7.9.2 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Git sha 0 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: DB SUMMARY 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: DB Session ID: XHG6CY05L7KMIZTI452N 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: CURRENT file: CURRENT 2023-10-09T09:53:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: MANIFEST file: MANIFEST-000005 size: 690 Bytes 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000017.sst 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000015.log size: 991889 ; 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.error_if_exists: 0 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.create_if_missing: 0 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.paranoid_checks: 1 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.env: 0x55c959c10720 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T09:53:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.info_log: 0x55c95c0f1420 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.statistics: (nil) 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.use_fsync: 0 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_log_file_size: 0 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.allow_fallocate: 1 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:53:17.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.use_direct_reads: 0 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.db_log_dir: 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.wal_dir: 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:53:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.write_buffer_manager: 0x55c95b382c80 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.unordered_write: 0 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:53:17.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.row_cache: None 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.wal_filter: None 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.two_write_queues: 0 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.wal_compression: 0 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.atomic_flush: 0 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:53:17.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.log_readahead_size: 0 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_background_jobs: 2 2023-10-09T09:53:17.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_background_compactions: -1 2023-10-09T09:53:17.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_subcompactions: 1 2023-10-09T09:53:17.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:53:17.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:53:17.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:53:17.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:53:17.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:53:17.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:53:17.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_open_files: -1 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_background_flushes: -1 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Compression algorithms supported: 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: kZSTD supported: 0 2023-10-09T09:53:17.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: kXpressCompression supported: 0 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: kZlibCompression supported: 1 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: kSnappyCompression supported: 1 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: kLZ4Compression supported: 1 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: kBZip2Compression supported: 0 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.merge_operator: 2023-10-09T09:53:17.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_filter: None 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c95c0f1520) 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout: cache_index_and_filter_blocks: 1 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout: pin_top_level_index_and_filter: 1 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_type: 0 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout: data_block_index_type: 0 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_shortening: 1 2023-10-09T09:53:17.188 INFO:journalctl@ceph.mon.c.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: checksum: 4 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: no_block_cache: 0 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache: 0x55c95b4031f0 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_name: BinnedLRUCache 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_options: 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: capacity : 536870912 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: num_shard_bits : 4 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: strict_capacity_limit : 0 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: high_pri_pool_ratio: 0.000 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_compressed: (nil) 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: persistent_cache: (nil) 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_size: 4096 2023-10-09T09:53:17.189 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_size_deviation: 10 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_restart_interval: 16 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_block_restart_interval: 1 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: metadata_block_size: 4096 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: partition_filters: 0 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: use_delta_encoding: 1 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: filter_policy: bloomfilter 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: whole_key_filtering: 1 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: verify_compression: 0 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: read_amp_bytes_per_bit: 0 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: format_version: 5 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: enable_index_compression: 1 2023-10-09T09:53:17.190 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_align: 0 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout: max_auto_readahead_size: 262144 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout: prepopulate_block_cache: 0 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout: initial_auto_readahead_size: 8192 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression: NoCompression 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.num_levels: 7 2023-10-09T09:53:17.191 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:53:17.192 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:53:17.193 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:53:17.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:53:17.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T09:53:17.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T09:53:17.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:53:17.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:53:17.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:53:17.196 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.table_properties_collectors: 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.inplace_update_support: 0 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.bloom_locality: 0 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.max_successive_merges: 0 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:53:17.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.ttl: 2592000 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.enable_blob_files: false 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.min_blob_size: 0 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T09:53:17.198 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 17.sst 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 19, last_sequence is 11184, log_number is 15,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 15 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: cbe3e68b-523e-4b01-9ebf-af827eb80827 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845197087058, "job": 1, "event": "recovery_started", "wal_files": [15]} 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #15 mode 2 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845197090179, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 20, "file_size": 660175, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11179, "largest_seqno": 11392, "table_properties": {"data_size": 657967, "index_size": 940, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 1826, "raw_average_key_size": 23, "raw_value_size": 655654, "raw_average_value_size": 8299, "num_data_blocks": 40, "num_entries": 79, "num_filter_entries": 79, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696845197, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "cbe3e68b-523e-4b01-9ebf-af827eb80827", "db_session_id": "XHG6CY05L7KMIZTI452N", "orig_file_number": 20, "seqno_to_time_mapping": "N/A"}} 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845197090254, "job": 1, "event": "recovery_finished"} 2023-10-09T09:53:17.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/version_set.cc:5047] Creating manifest 22 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000015.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c95b4a0000 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: DB pointer 0x55c95b48c000 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** DB Stats ** 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:53:17.200 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** Compaction Stats [default] ** 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: L0 1/0 644.70 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 371.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: L6 1/0 26.49 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: Sum 2/0 27.12 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 371.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 371.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** Compaction Stats [default] ** 2023-10-09T09:53:17.201 INFO:journalctl@ceph.mon.c.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 371.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:53:17.202 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative compaction: 0.00 GB write, 116.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval compaction: 0.00 GB write, 116.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: Block cache BinnedLRUCache@0x55c95b4031f0#7 capacity: 512.00 MB usage: 1.39 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: starting mon.c rank 2 at public addrs [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] at bind addrs [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: mon.c@-1(???) e3 preinit fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: mon.c@-1(???).mds e7 new map 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: mon.c@-1(???).mds e7 print_map 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: e7 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:53:17.203 INFO:journalctl@ceph.mon.c.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: legacy client fscid: 1 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: Filesystem 'foo' (1) 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: fs_name foo 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: epoch 5 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: created 2023-10-09T09:48:25.508326+0000 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: modified 2023-10-09T09:48:40.668865+0000 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: tableserver 0 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: root 0 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: session_timeout 60 2023-10-09T09:53:17.204 INFO:journalctl@ceph.mon.c.smithi012.stdout: session_autoclose 300 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: max_file_size 1099511627776 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: required_client_features {} 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: last_failure 0 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: last_failure_osd_epoch 0 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: max_mds 1 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: in 0 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: up {0=14652} 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: failed 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: damaged 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: stopped 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: data_pools [4] 2023-10-09T09:53:17.205 INFO:journalctl@ceph.mon.c.smithi012.stdout: metadata_pool 3 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: inline_data disabled 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: balancer 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: bal_rank_mask -1 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: standby_count_wanted 1 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: [mds.foo.smithi012.mbnrgj{0:14652} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: Standby daemons: 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout: [mds.foo.smithi110.sgosap{-1:24556} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.110:6832/704760757,v1:172.21.15.110:6833/704760757] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:53:17.206 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: mon.c@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: mon.c@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: mon.c@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: mon.c@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta expand map: {default=false} 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta from 'false' to 'false' 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta expanded map: {default=false} 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta expand map: {default=info} 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta from 'info' to 'info' 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta expanded map: {default=info} 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta expand map: {default=daemon} 2023-10-09T09:53:17.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T09:53:17.208 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T09:53:17.208 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta expand map: {default=debug} 2023-10-09T09:53:17.208 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T09:53:17.208 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-mon[152263]: expand_channel_meta expanded map: {default=debug} 2023-10-09T09:53:17.208 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:53:17 smithi012 systemd[1]: Started Ceph mon.c for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:53:18.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:53:17] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-10-09T09:53:18.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:53:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:53:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:53:22.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:14.816880+0000 mgr.y (mgr.25217) 59 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:22.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:15.954637+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:53:22.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:15.956677+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:53:22.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:15.956920+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:53:22.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:16.209036+0000 mon.a (mon.0) 49 : audit [DBG] from='client.? 172.21.15.12:0/1784609491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:16.817446+0000 mgr.y (mgr.25217) 60 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:17.302075+0000 mon.b (mon.1) 86 : cluster [INF] mon.b calling monitor election 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:17.302676+0000 mon.a (mon.0) 50 : cluster [INF] mon.a calling monitor election 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:18.817957+0000 mgr.y (mgr.25217) 61 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:20.818822+0000 mgr.y (mgr.25217) 62 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.306885+0000 mon.a (mon.0) 51 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.312786+0000 mon.a (mon.0) 52 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.324976+0000 mon.a (mon.0) 53 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:53:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.325011+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.325459+0000 mon.a (mon.0) 55 : cluster [DBG] mgrmap e36: y(active, since 83s), standbys: x 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.325674+0000 mon.a (mon.0) 56 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.330941+0000 mon.a (mon.0) 57 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.330964+0000 mon.a (mon.0) 58 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.330972+0000 mon.a (mon.0) 59 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] is down (out of quorum) 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:22.333778+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:22.339875+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:22.346872+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:53:23.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:23.038651+0000 mon.a (mon.0) 62 : audit [DBG] from='client.? 172.21.15.12:0/3339767101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:17.301427+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:22.819150+0000 mgr.y (mgr.25217) 63 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.316705+0000 mon.a (mon.0) 63 : cluster [INF] mon.a calling monitor election 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.317521+0000 mon.b (mon.1) 88 : cluster [INF] mon.b calling monitor election 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.330925+0000 mon.a (mon.0) 64 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.333820+0000 mon.a (mon.0) 65 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.334267+0000 mon.a (mon.0) 66 : cluster [DBG] fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.334293+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.334599+0000 mon.a (mon.0) 68 : cluster [DBG] mgrmap e36: y(active, since 84s), standbys: x 2023-10-09T09:53:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.334735+0000 mon.a (mon.0) 69 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T09:53:24.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.334754+0000 mon.a (mon.0) 70 : cluster [INF] Cluster is now healthy 2023-10-09T09:53:24.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:23.346728+0000 mon.a (mon.0) 71 : cluster [INF] overall HEALTH_OK 2023-10-09T09:53:25.477 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T09:53:26.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:53:26.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:24.819608+0000 mgr.y (mgr.25217) 64 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:25.464096+0000 mon.a (mon.0) 72 : audit [DBG] from='client.? 172.21.15.12:0/4168504264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:53:27.426 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:53:27.098+0000 7f2cb4de3700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T09:53:28.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:53:27] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (2m) - 5m 25.1M - 0.23.0 ba2b418f427c f37e3c8f7f66 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi110 *:3000 running (2m) - 5m 54.0M - 8.3.5 dad864ee21e9 3c5f62b02d38 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.mbnrgj smithi012 running (4m) - 4m 16.9M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5be3d4216704 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi110.sgosap smithi110 running (4m) - 4m 15.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f17cc81b8a5e 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi110 *:8443,9283,8765 running (114s) - 8m 447M - 18.2.0-598-gb4fa663f 3060897fdf97 6e3769c16b69 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (2m) - 10m 499M - 18.2.0-598-gb4fa663f 3060897fdf97 3a9297092bc2 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (101s) - 10m 79.4M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 9f2783659fac 2023-10-09T09:53:28.232 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi110 running (9m) - 9m 93.6M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 99ff116de7cf 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (11s) - 9m 19.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 251f6625e45a 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (3m) - 5m 22.3M - 1.3.1 1dbe0e931976 c780854e434e 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi110 *:9100 running (2m) - 5m 19.2M - 1.3.1 1dbe0e931976 c0d7506efa64 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (8m) - 8m 69.3M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 40f45dc46a61 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (8m) - 8m 67.4M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e2afac5c73f9 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (7m) - 7m 140M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 71decd1a84e3 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (7m) - 7m 63.3M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8587e6142c61 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi110 running (7m) - 7m 72.9M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 18029670387d 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi110 running (7m) - 7m 92.4M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c927f6e714d1 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi110 running (6m) - 6m 63.1M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b96408c9e315 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi110 running (6m) - 6m 101M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 ac326d7f40d1 2023-10-09T09:53:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi110 *:9095 running (2m) - 5m 56.5M - 2.33.4 514e6a882f6e 82ffa853b6d7 2023-10-09T09:53:28.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:26.819940+0000 mgr.y (mgr.25217) 65 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:28.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:26.977736+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:28.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:26.984542+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:28.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T09:53:28.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:53:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:53:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:53:29.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:28.125765+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:29.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:28.132046+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:28.219726+0000 mgr.y (mgr.25217) 66 : audit [DBG] from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:53:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:28.861240+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:30.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:28.820271+0000 mgr.y (mgr.25217) 67 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:31.132 INFO:teuthology.orchestra.run.smithi012.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:53:31.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:30.142216+0000 mon.a (mon.0) 77 : cluster [DBG] mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T09:53:31.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:31.131037+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:31.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T09:53:32.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:30.820933+0000 mgr.y (mgr.25217) 68 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:53:32.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:31.126640+0000 mgr.y (mgr.25217) 69 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:53:32.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:53:31.126982+0000 mgr.y (mgr.25217) 70 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:53:32.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:31.910851+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:32.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:31.917893+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:32.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:31.919803+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:32.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:31.920835+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:53:32.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:31.926829+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:32.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:32.066299+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:53:32.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:32.068091+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:53:32.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:32.068955+0000 mon.b (mon.1) 94 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:53:32.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:32.074262+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:53:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:32.821261+0000 mgr.y (mgr.25217) 71 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:53:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:34.821793+0000 mgr.y (mgr.25217) 72 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:53:38.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:53:37] "GET /metrics HTTP/1.1" 200 33849 "" "Prometheus/2.33.4" 2023-10-09T09:53:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:36.822227+0000 mgr.y (mgr.25217) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:53:38.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:53:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:53:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:53:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:38.822908+0000 mgr.y (mgr.25217) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:53:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:40.823690+0000 mgr.y (mgr.25217) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:53:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:42.824159+0000 mgr.y (mgr.25217) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:43.861789+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:53:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:44.824689+0000 mgr.y (mgr.25217) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:48.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:53:47] "GET /metrics HTTP/1.1" 200 33846 "" "Prometheus/2.33.4" 2023-10-09T09:53:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:46.825142+0000 mgr.y (mgr.25217) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:48.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:53:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:53:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:53:50.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:48.825531+0000 mgr.y (mgr.25217) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:50.826141+0000 mgr.y (mgr.25217) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:54.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:52.826588+0000 mgr.y (mgr.25217) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:56 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:54.827249+0000 mgr.y (mgr.25217) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:58.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:53:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:53:57] "GET /metrics HTTP/1.1" 200 33846 "" "Prometheus/2.33.4" 2023-10-09T09:53:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:56.827613+0000 mgr.y (mgr.25217) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:53:58.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:53:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:53:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:53:59.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:53:59 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:53:58.867193+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:00.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:00 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:53:58.828017+0000 mgr.y (mgr.25217) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:02.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:02 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:00.828688+0000 mgr.y (mgr.25217) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:04 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:02.829218+0000 mgr.y (mgr.25217) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:06.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:06 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:04.829776+0000 mgr.y (mgr.25217) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:54:07] "GET /metrics HTTP/1.1" 200 33851 "" "Prometheus/2.33.4" 2023-10-09T09:54:08.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:06.830208+0000 mgr.y (mgr.25217) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:08.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:54:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:54:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:54:10.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:10 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:08.830758+0000 mgr.y (mgr.25217) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:12.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:12 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:10.831407+0000 mgr.y (mgr.25217) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:12.831876+0000 mgr.y (mgr.25217) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:14 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:13.867701+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:16 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:14.832615+0000 mgr.y (mgr.25217) 92 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:54:17] "GET /metrics HTTP/1.1" 200 33858 "" "Prometheus/2.33.4" 2023-10-09T09:54:18.948 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:16.833129+0000 mgr.y (mgr.25217) 93 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:18.948 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:54:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:54:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:54:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:20 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:18.833722+0000 mgr.y (mgr.25217) 94 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:22.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:20.834310+0000 mgr.y (mgr.25217) 95 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:22.834847+0000 mgr.y (mgr.25217) 96 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:24.835451+0000 mgr.y (mgr.25217) 97 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:28.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:54:27] "GET /metrics HTTP/1.1" 200 33858 "" "Prometheus/2.33.4" 2023-10-09T09:54:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:26.835875+0000 mgr.y (mgr.25217) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:28.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:54:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:54:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:54:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:28.867776+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:30.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:28.836344+0000 mgr.y (mgr.25217) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:30.836988+0000 mgr.y (mgr.25217) 100 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:32.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:32.212912+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:33.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T09:54:33.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:32.634726+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:33.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:32.638171+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:32.643691+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:32.647928+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:32.837325+0000 mgr.y (mgr.25217) 101 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:35.783 INFO:teuthology.orchestra.run.smithi012.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:36.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:34.837779+0000 mgr.y (mgr.25217) 102 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:36.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:35.782589+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:36.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T09:54:37.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:35.778385+0000 mgr.y (mgr.25217) 103 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:37.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:54:35.778747+0000 mgr.y (mgr.25217) 104 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:37.815 INFO:teuthology.orchestra.run.smithi012.stdout:wait for all mons to upgrade 2023-10-09T09:54:38.098 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:54:37] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.33.4" 2023-10-09T09:54:38.135 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T09:54:38.663 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:36.838110+0000 mgr.y (mgr.25217) 105 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:38.663 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:37.331016+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:38.663 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:37.340387+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:38.663 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:37.880619+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:38.663 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:37.886516+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:38.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:54:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:54:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:54:39.508 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:38.402956+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:39.508 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:38.404144+0000 mon.b (mon.1) 101 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:39.509 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:38.409131+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:39.509 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:38.547307+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:39.509 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:38.550588+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:54:39.509 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:38.552314+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:54:39.509 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:38.553259+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T09:54:39.509 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:54:38.553814+0000 mgr.y (mgr.25217) 106 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-09T09:54:40.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cluster 2023-10-09T09:54:38.838442+0000 mgr.y (mgr.25217) 107 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:40.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:39.875475+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:39.876421+0000 mon.b (mon.1) 106 : audit [INF] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:39.877199+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:39.877934+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: audit 2023-10-09T09:54:39.943626+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.12:0/2463681214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:41.418 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:41 smithi110 systemd[1]: Stopping Ceph mon.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:54:41.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:54:39.869963+0000 mgr.y (mgr.25217) 108 : cephadm [INF] Upgrade: Updating mon.b 2023-10-09T09:54:41.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: cephadm 2023-10-09T09:54:39.878848+0000 mgr.y (mgr.25217) 109 : cephadm [INF] Deploying daemon mon.b on smithi110 2023-10-09T09:54:41.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:54:41.439+0000 7fd19ad08700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T09:54:41.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[97076]: debug 2023-10-09T09:54:41.439+0000 7fd19ad08700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T09:54:41.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:54:41] ENGINE Bus STOPPING 2023-10-09T09:54:41.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:54:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T09:54:41.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:54:41] ENGINE Bus STOPPED 2023-10-09T09:54:41.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:54:41] ENGINE Bus STARTING 2023-10-09T09:54:41.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:54:41] ENGINE Serving on http://:::9283 2023-10-09T09:54:41.928 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:09:54:41] ENGINE Bus STARTED 2023-10-09T09:54:41.981 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:41 smithi110 bash[135843]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T09:54:42.403 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:41 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T09:54:42.403 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:41 smithi110 systemd[1]: Stopped Ceph mon.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:54:42.403 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 systemd[1]: Starting Ceph mon.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:54:42.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 podman[135992]: 2023-10-09T09:54:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T09:54:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T09:54:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 bash[135992]: 5fab451fcf370e802ca29fe2deefa30d40ce9bb5cb708971a4d2bcdb7b4b8484 2023-10-09T09:54:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: pidfile_write: ignore empty --pid-file 2023-10-09T09:54:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: load: jerasure load: lrc 2023-10-09T09:54:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: RocksDB version: 7.9.2 2023-10-09T09:54:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Git sha 0 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: DB SUMMARY 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: DB Session ID: 72PY85O0SFDZ7GYF3YBQ 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: CURRENT file: CURRENT 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: MANIFEST file: MANIFEST-000005 size: 1069 Bytes 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000023.sst 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000021.log size: 473209 ; 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.error_if_exists: 0 2023-10-09T09:54:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.create_if_missing: 0 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.paranoid_checks: 1 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.env: 0x5642409a9720 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.info_log: 0x564242eb9420 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T09:54:43.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.statistics: (nil) 2023-10-09T09:54:43.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.use_fsync: 0 2023-10-09T09:54:43.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_log_file_size: 0 2023-10-09T09:54:43.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T09:54:43.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T09:54:43.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T09:54:43.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T09:54:43.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.allow_fallocate: 1 2023-10-09T09:54:43.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T09:54:43.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T09:54:43.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.use_direct_reads: 0 2023-10-09T09:54:43.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T09:54:43.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T09:54:43.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.db_log_dir: 2023-10-09T09:54:43.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.wal_dir: 2023-10-09T09:54:43.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T09:54:43.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T09:54:43.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T09:54:43.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T09:54:43.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T09:54:43.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T09:54:43.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T09:54:43.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T09:54:43.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.write_buffer_manager: 0x56424214ac80 2023-10-09T09:54:43.199 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T09:54:43.200 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T09:54:43.200 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T09:54:43.200 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T09:54:43.200 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T09:54:43.200 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T09:54:43.200 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T09:54:43.201 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T09:54:43.201 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.unordered_write: 0 2023-10-09T09:54:43.201 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T09:54:43.201 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T09:54:43.201 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T09:54:43.201 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T09:54:43.201 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.row_cache: None 2023-10-09T09:54:43.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.wal_filter: None 2023-10-09T09:54:43.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T09:54:43.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T09:54:43.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.two_write_queues: 0 2023-10-09T09:54:43.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T09:54:43.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.wal_compression: 0 2023-10-09T09:54:43.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.atomic_flush: 0 2023-10-09T09:54:43.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T09:54:43.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T09:54:43.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T09:54:43.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.log_readahead_size: 0 2023-10-09T09:54:43.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T09:54:43.204 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T09:54:43.204 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T09:54:43.204 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T09:54:43.204 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T09:54:43.205 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T09:54:43.205 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T09:54:43.206 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_background_jobs: 2 2023-10-09T09:54:43.206 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_background_compactions: -1 2023-10-09T09:54:43.206 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_subcompactions: 1 2023-10-09T09:54:43.206 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T09:54:43.206 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T09:54:43.206 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T09:54:43.207 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T09:54:43.207 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T09:54:43.207 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T09:54:43.207 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T09:54:43.207 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T09:54:43.207 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_open_files: -1 2023-10-09T09:54:43.208 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T09:54:43.208 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T09:54:43.208 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T09:54:43.208 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T09:54:43.208 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_background_flushes: -1 2023-10-09T09:54:43.208 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Compression algorithms supported: 2023-10-09T09:54:43.208 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T09:54:43.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: kZSTD supported: 0 2023-10-09T09:54:43.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: kXpressCompression supported: 0 2023-10-09T09:54:43.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T09:54:43.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: kZlibCompression supported: 1 2023-10-09T09:54:43.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: kSnappyCompression supported: 1 2023-10-09T09:54:43.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: kLZ4Compression supported: 1 2023-10-09T09:54:43.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: kBZip2Compression supported: 0 2023-10-09T09:54:43.210 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T09:54:43.210 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T09:54:43.210 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-09T09:54:43.210 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T09:54:43.210 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T09:54:43.210 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.merge_operator: 2023-10-09T09:54:43.210 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_filter: None 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564242eb9520) 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout: cache_index_and_filter_blocks: 1 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout: pin_top_level_index_and_filter: 1 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout: index_type: 0 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout: data_block_index_type: 0 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout: index_shortening: 1 2023-10-09T09:54:43.211 INFO:journalctl@ceph.mon.b.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: checksum: 4 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: no_block_cache: 0 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: block_cache: 0x5642421cb350 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: block_cache_name: BinnedLRUCache 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: block_cache_options: 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: capacity : 536870912 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: num_shard_bits : 4 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: strict_capacity_limit : 0 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: high_pri_pool_ratio: 0.000 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: block_cache_compressed: (nil) 2023-10-09T09:54:43.212 INFO:journalctl@ceph.mon.b.smithi110.stdout: persistent_cache: (nil) 2023-10-09T09:54:43.213 INFO:journalctl@ceph.mon.b.smithi110.stdout: block_size: 4096 2023-10-09T09:54:43.213 INFO:journalctl@ceph.mon.b.smithi110.stdout: block_size_deviation: 10 2023-10-09T09:54:43.213 INFO:journalctl@ceph.mon.b.smithi110.stdout: block_restart_interval: 16 2023-10-09T09:54:43.213 INFO:journalctl@ceph.mon.b.smithi110.stdout: index_block_restart_interval: 1 2023-10-09T09:54:43.213 INFO:journalctl@ceph.mon.b.smithi110.stdout: metadata_block_size: 4096 2023-10-09T09:54:43.213 INFO:journalctl@ceph.mon.b.smithi110.stdout: partition_filters: 0 2023-10-09T09:54:43.213 INFO:journalctl@ceph.mon.b.smithi110.stdout: use_delta_encoding: 1 2023-10-09T09:54:43.214 INFO:journalctl@ceph.mon.b.smithi110.stdout: filter_policy: bloomfilter 2023-10-09T09:54:43.214 INFO:journalctl@ceph.mon.b.smithi110.stdout: whole_key_filtering: 1 2023-10-09T09:54:43.214 INFO:journalctl@ceph.mon.b.smithi110.stdout: verify_compression: 0 2023-10-09T09:54:43.214 INFO:journalctl@ceph.mon.b.smithi110.stdout: read_amp_bytes_per_bit: 0 2023-10-09T09:54:43.214 INFO:journalctl@ceph.mon.b.smithi110.stdout: format_version: 5 2023-10-09T09:54:43.214 INFO:journalctl@ceph.mon.b.smithi110.stdout: enable_index_compression: 1 2023-10-09T09:54:43.214 INFO:journalctl@ceph.mon.b.smithi110.stdout: block_align: 0 2023-10-09T09:54:43.215 INFO:journalctl@ceph.mon.b.smithi110.stdout: max_auto_readahead_size: 262144 2023-10-09T09:54:43.215 INFO:journalctl@ceph.mon.b.smithi110.stdout: prepopulate_block_cache: 0 2023-10-09T09:54:43.215 INFO:journalctl@ceph.mon.b.smithi110.stdout: initial_auto_readahead_size: 8192 2023-10-09T09:54:43.215 INFO:journalctl@ceph.mon.b.smithi110.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T09:54:43.215 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T09:54:43.215 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T09:54:43.215 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression: NoCompression 2023-10-09T09:54:43.215 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.num_levels: 7 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T09:54:43.216 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T09:54:43.217 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T09:54:43.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T09:54:43.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T09:54:43.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T09:54:43.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T09:54:43.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T09:54:43.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T09:54:43.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T09:54:43.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T09:54:43.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T09:54:43.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T09:54:43.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T09:54:43.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T09:54:43.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T09:54:43.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T09:54:43.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T09:54:43.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T09:54:43.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T09:54:43.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T09:54:43.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T09:54:43.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T09:54:43.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T09:54:43.222 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T09:54:43.222 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T09:54:43.222 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.table_properties_collectors: 2023-10-09T09:54:43.222 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.inplace_update_support: 0 2023-10-09T09:54:43.222 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T09:54:43.222 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T09:54:43.222 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T09:54:43.223 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T09:54:43.223 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.bloom_locality: 0 2023-10-09T09:54:43.223 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.max_successive_merges: 0 2023-10-09T09:54:43.223 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T09:54:43.223 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T09:54:43.223 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T09:54:43.223 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.ttl: 2592000 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.enable_blob_files: false 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.min_blob_size: 0 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T09:54:43.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 25, last_sequence is 13060, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-10-09T09:54:43.225 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3931cc18-e83e-4581-86b0-8a5573bec542 2023-10-09T09:54:43.226 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845282827468, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-10-09T09:54:43.226 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-10-09T09:54:43.226 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845282829760, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 316023, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13065, "largest_seqno": 13148, "table_properties": {"data_size": 314457, "index_size": 426, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 133, "raw_key_size": 1003, "raw_average_key_size": 25, "raw_value_size": 313239, "raw_average_value_size": 7830, "num_data_blocks": 18, "num_entries": 40, "num_filter_entries": 40, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696845282, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3931cc18-e83e-4581-86b0-8a5573bec542", "db_session_id": "72PY85O0SFDZ7GYF3YBQ", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-10-09T09:54:43.226 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696845282829896, "job": 1, "event": "recovery_finished"} 2023-10-09T09:54:43.226 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-10-09T09:54:43.226 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T09:54:43.226 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564242268000 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: DB pointer 0x564242254000 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout: ** DB Stats ** 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:54:43.227 INFO:journalctl@ceph.mon.b.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T09:54:43.228 INFO:journalctl@ceph.mon.b.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T09:54:43.228 INFO:journalctl@ceph.mon.b.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T09:54:43.228 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.228 INFO:journalctl@ceph.mon.b.smithi110.stdout: ** Compaction Stats [default] ** 2023-10-09T09:54:43.228 INFO:journalctl@ceph.mon.b.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:54:43.228 INFO:journalctl@ceph.mon.b.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T09:54:43.228 INFO:journalctl@ceph.mon.b.smithi110.stdout: L0 1/0 308.62 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 224.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: L6 1/0 25.70 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: Sum 2/0 26.00 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 224.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 224.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: ** Compaction Stats [default] ** 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 224.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T09:54:43.229 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T09:54:43.230 INFO:journalctl@ceph.mon.b.smithi110.stdout: Cumulative compaction: 0.00 GB write, 57.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout: Interval compaction: 0.00 GB write, 57.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout: Block cache BinnedLRUCache@0x5642421cb350#7 capacity: 512.00 MB usage: 0.69 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.17 KB,3.27826e-05%) IndexBlock(1,0.52 KB,9.83477e-05%) Misc(1,0.00 KB,0%) 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: starting mon.b rank 1 at public addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] at bind addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: mon.b@-1(???) e3 preinit fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: mon.b@-1(???).mds e7 new map 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: mon.b@-1(???).mds e7 print_map 2023-10-09T09:54:43.231 INFO:journalctl@ceph.mon.b.smithi110.stdout: e7 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: legacy client fscid: 1 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: Filesystem 'foo' (1) 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: fs_name foo 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: epoch 5 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: created 2023-10-09T09:48:25.508326+0000 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: modified 2023-10-09T09:48:40.668865+0000 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: tableserver 0 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: root 0 2023-10-09T09:54:43.232 INFO:journalctl@ceph.mon.b.smithi110.stdout: session_timeout 60 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: session_autoclose 300 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: max_file_size 1099511627776 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: required_client_features {} 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: last_failure 0 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: last_failure_osd_epoch 0 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: max_mds 1 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: in 0 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: up {0=14652} 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: failed 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: damaged 2023-10-09T09:54:43.233 INFO:journalctl@ceph.mon.b.smithi110.stdout: stopped 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: data_pools [4] 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: metadata_pool 3 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: inline_data disabled 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: balancer 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: bal_rank_mask -1 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: standby_count_wanted 1 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: [mds.foo.smithi012.mbnrgj{0:14652} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.12:6834/4171778208,v1:172.21.15.12:6835/4171778208] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.234 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout: Standby daemons: 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout: 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout: [mds.foo.smithi110.sgosap{-1:24556} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.110:6832/704760757,v1:172.21.15.110:6833/704760757] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: mon.b@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: mon.b@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: mon.b@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: mon.b@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta expand map: {default=false} 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta from 'false' to 'false' 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta expanded map: {default=false} 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta expand map: {default=info} 2023-10-09T09:54:43.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta from 'info' to 'info' 2023-10-09T09:54:43.236 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta expanded map: {default=info} 2023-10-09T09:54:43.236 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta expand map: {default=daemon} 2023-10-09T09:54:43.236 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T09:54:43.236 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T09:54:43.236 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta expand map: {default=debug} 2023-10-09T09:54:43.236 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T09:54:43.236 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 ceph-mon[136032]: expand_channel_meta expanded map: {default=debug} 2023-10-09T09:54:43.236 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:42 smithi110 systemd[1]: Started Ceph mon.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:54:43.982 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mkfs 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:54:43.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mkfs 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:54:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:54:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Reconfiguring daemon node-exporter.b on smithi110 2023-10-09T09:54:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:54:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Reconfiguring daemon grafana.a on smithi110 2023-10-09T09:54:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:54:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:54:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Updating mgr.y 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Deploying daemon mgr.y on smithi012 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: [09/Oct/2023:09:51:13] ENGINE Bus STARTING 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: [09/Oct/2023:09:51:13] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: [09/Oct/2023:09:51:13] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: [09/Oct/2023:09:51:13] ENGINE Bus STARTED 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Updating mgr.x 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Deploying daemon mgr.x on smithi110 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Setting container_image for all mgr 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: It appears safe to stop mon.a 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Updating mon.a 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Deploying daemon mon.a on smithi012 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: [09/Oct/2023:09:51:59] ENGINE Bus STARTING 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: [09/Oct/2023:09:51:59] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: [09/Oct/2023:09:52:00] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: [09/Oct/2023:09:52:00] ENGINE Bus STARTED 2023-10-09T09:54:44.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mkfs 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:54:44.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Reconfiguring daemon node-exporter.b on smithi110 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Reconfiguring daemon grafana.a on smithi110 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Updating mgr.y 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Deploying daemon mgr.y on smithi012 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: [09/Oct/2023:09:51:13] ENGINE Bus STARTING 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: [09/Oct/2023:09:51:13] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: [09/Oct/2023:09:51:13] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: [09/Oct/2023:09:51:13] ENGINE Bus STARTED 2023-10-09T09:54:44.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Updating mgr.x 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Deploying daemon mgr.x on smithi110 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Setting container_image for all mgr 2023-10-09T09:54:44.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: It appears safe to stop mon.a 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Updating mon.a 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Deploying daemon mon.a on smithi012 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: [09/Oct/2023:09:51:59] ENGINE Bus STARTING 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: [09/Oct/2023:09:51:59] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: [09/Oct/2023:09:52:00] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: [09/Oct/2023:09:52:00] ENGINE Bus STARTED 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: It appears safe to stop mon.c 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Updating mon.c 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Deploying daemon mon.c on smithi012 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3339767101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mon.a calling monitor election 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mon.b calling monitor election 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mgrmap e36: y(active, since 84s), standbys: x 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: overall HEALTH_OK 2023-10-09T09:54:44.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4168504264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.437 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: It appears safe to stop mon.c 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Updating mon.c 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Deploying daemon mon.c on smithi012 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3339767101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mon.a calling monitor election 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mon.b calling monitor election 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mgrmap e36: y(active, since 84s), standbys: x 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T09:54:44.438 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: overall HEALTH_OK 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4168504264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.439 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.440 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.441 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.443 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.444 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.445 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: It appears safe to stop mon.b 2023-10-09T09:54:44.448 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T09:54:44.451 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: It appears safe to stop mon.b 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Upgrade: Updating mon.b 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: Deploying daemon mon.b on smithi110 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2463681214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:54:44.452 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1917931511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: overall HEALTH_OK 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mon.b calling monitor election 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mon.a calling monitor election 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mon.c calling monitor election 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:54:44.453 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: monmap e4: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:54:44.454 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:54:44.454 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:54:44.454 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: mgrmap e37: y(active, since 2m), standbys: x 2023-10-09T09:54:44.454 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: overall HEALTH_OK 2023-10-09T09:54:44.454 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='' 2023-10-09T09:54:44.454 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.454 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.454 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Upgrade: Updating mon.b 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: Deploying daemon mon.b on smithi110 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2463681214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:54:44.455 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1917931511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: overall HEALTH_OK 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mon.b calling monitor election 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mon.a calling monitor election 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mon.c calling monitor election 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: monmap e4: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:54:44.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:54:44.457 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: mgrmap e37: y(active, since 2m), standbys: x 2023-10-09T09:54:44.457 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: overall HEALTH_OK 2023-10-09T09:54:44.457 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='' 2023-10-09T09:54:44.457 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.457 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.457 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T09:54:44.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Reconfiguring daemon node-exporter.b on smithi110 2023-10-09T09:54:44.458 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Reconfiguring daemon grafana.a on smithi110 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Updating mgr.y 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Deploying daemon mgr.y on smithi012 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: [09/Oct/2023:09:51:13] ENGINE Bus STARTING 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: [09/Oct/2023:09:51:13] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: [09/Oct/2023:09:51:13] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: [09/Oct/2023:09:51:13] ENGINE Bus STARTED 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:54:44.459 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Reconfiguring daemon prometheus.a on smithi110 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Updating mgr.x 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Deploying daemon mgr.x on smithi110 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Setting container_image for all mgr 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: It appears safe to stop mon.a 2023-10-09T09:54:44.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Updating mon.a 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Deploying daemon mon.a on smithi012 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: [09/Oct/2023:09:51:59] ENGINE Bus STARTING 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: [09/Oct/2023:09:51:59] ENGINE Serving on http://172.21.15.12:8765 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: [09/Oct/2023:09:52:00] ENGINE Serving on https://172.21.15.12:7150 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: [09/Oct/2023:09:52:00] ENGINE Bus STARTED 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi012:/etc/ceph/ceph.conf 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi110:/etc/ceph/ceph.conf 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.461 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Updating smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: It appears safe to stop mon.c 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Updating mon.c 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Deploying daemon mon.c on smithi012 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3339767101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mon.a calling monitor election 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mon.b calling monitor election 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:54:44.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mgrmap e36: y(active, since 84s), standbys: x 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: overall HEALTH_OK 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4168504264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T09:54:44.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.468 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:44.469 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: It appears safe to stop mon.b 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Upgrade: Updating mon.b 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:44.470 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: Deploying daemon mon.b on smithi110 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2463681214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1917931511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: overall HEALTH_OK 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T09:54:44.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mon.b calling monitor election 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mon.a calling monitor election 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mon.c calling monitor election 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: monmap e4: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: fsmap foo:1 {0=foo.smithi012.mbnrgj=up:active} 1 up:standby 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: mgrmap e37: y(active, since 2m), standbys: x 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: overall HEALTH_OK 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='' 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:54:44.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:44.771 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T09:54:45.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2979468477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:45.096 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2979468477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:45.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:54:45.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2979468477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T09:54:46.394 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:45 smithi012 ceph-mon[147926]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:46.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:45 smithi012 ceph-mon[152263]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:46.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:45 smithi110 ceph-mon[136032]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:46.806 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:54:46.806 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (4m) - 6m 25.1M - 0.23.0 ba2b418f427c f37e3c8f7f66 2023-10-09T09:54:46.806 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi110 *:3000 running (3m) - 6m 56.4M - 8.3.5 dad864ee21e9 3c5f62b02d38 2023-10-09T09:54:46.806 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.mbnrgj smithi012 running (6m) - 6m 17.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5be3d4216704 2023-10-09T09:54:46.806 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi110.sgosap smithi110 running (6m) - 6m 15.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f17cc81b8a5e 2023-10-09T09:54:46.806 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi110 *:8443,9283,8765 running (3m) - 10m 448M - 18.2.0-598-gb4fa663f 3060897fdf97 6e3769c16b69 2023-10-09T09:54:46.806 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (3m) - 11m 502M - 18.2.0-598-gb4fa663f 3060897fdf97 3a9297092bc2 2023-10-09T09:54:46.806 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (2m) - 11m 86.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 9f2783659fac 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi110 starting - - - 2048M 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (89s) - 10m 68.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 251f6625e45a 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (4m) - 7m 19.3M - 1.3.1 1dbe0e931976 c780854e434e 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi110 *:9100 running (4m) - 6m 19.2M - 1.3.1 1dbe0e931976 c0d7506efa64 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (9m) - 9m 71.7M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 40f45dc46a61 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (9m) - 9m 69.6M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e2afac5c73f9 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (9m) - 9m 141M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 71decd1a84e3 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (8m) - 8m 65.7M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8587e6142c61 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi110 running (8m) - 8m 76.0M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 18029670387d 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi110 running (8m) - 8m 94.7M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c927f6e714d1 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi110 running (8m) - 8m 65.5M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b96408c9e315 2023-10-09T09:54:46.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi110 running (7m) - 7m 103M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 ac326d7f40d1 2023-10-09T09:54:46.808 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi110 *:9095 running (3m) - 7m 68.5M - 2.33.4 514e6a882f6e 82ffa853b6d7 2023-10-09T09:54:47.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T09:54:48.095 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:54:47.841+0000 7f2cb4de3700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T09:54:48.095 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:54:47] "GET /metrics HTTP/1.1" 200 33787 "" "Prometheus/2.33.4" 2023-10-09T09:54:48.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:48 smithi012 ceph-mon[147926]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:48.956 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:48 smithi012 ceph-mon[147926]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:48.956 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:48.957 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:48.957 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:48 smithi012 ceph-mon[152263]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:48.957 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:48 smithi012 ceph-mon[152263]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:48.957 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:48.958 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:48.958 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:48 smithi110 ceph-mon[136032]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:48.959 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:48 smithi110 ceph-mon[136032]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:48.959 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:48.959 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:48.959 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:54:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:54:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:54:48.982 INFO:teuthology.orchestra.run.smithi012.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:49.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T09:54:49.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:49 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:49.832 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:49 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:49.832 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:49 smithi110 ceph-mon[136032]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:49.832 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:49 smithi110 ceph-mon[136032]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:49.832 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:49 smithi110 ceph-mon[136032]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:49.832 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:49 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:49.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:49.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:49.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[147926]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:49.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[147926]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:49.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[147926]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:49.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:49.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:49.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:49.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[152263]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:49.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[152263]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:54:49.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[152263]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:54:49.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:49 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:52.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:52.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: Reconfiguring mon.a (monmap changed)... 2023-10-09T09:54:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[147926]: Reconfiguring daemon mon.a on smithi012 2023-10-09T09:54:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: Reconfiguring mon.a (monmap changed)... 2023-10-09T09:54:52.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:52.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:52.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:52.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:52 smithi012 ceph-mon[152263]: Reconfiguring daemon mon.a on smithi012 2023-10-09T09:54:52.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:52.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:54:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: Reconfiguring mon.a (monmap changed)... 2023-10-09T09:54:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:52.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:52.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:52 smithi110 ceph-mon[136032]: Reconfiguring daemon mon.a on smithi012 2023-10-09T09:54:53.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:53.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[147926]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T09:54:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:54:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:54:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:54:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[147926]: Reconfiguring daemon mgr.y on smithi012 2023-10-09T09:54:53.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:53.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[152263]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T09:54:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:54:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:54:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:54:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:53 smithi012 ceph-mon[152263]: Reconfiguring daemon mgr.y on smithi012 2023-10-09T09:54:53.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:53 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:53.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:53 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:53.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:53 smithi110 ceph-mon[136032]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T09:54:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:53 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:54:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:53 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:54:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:53 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:54:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:53 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:53 smithi110 ceph-mon[136032]: Reconfiguring daemon mgr.y on smithi012 2023-10-09T09:54:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[152263]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:54.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:54.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:54.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:54.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:54.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[147926]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:54.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:54.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:54.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:54.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:54.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:54 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:54 smithi110 ceph-mon[136032]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:54 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:54 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:54 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:54:54.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:54 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:54:54.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:54 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:55.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[147926]: Reconfiguring mon.c (monmap changed)... 2023-10-09T09:54:55.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[147926]: Reconfiguring daemon mon.c on smithi012 2023-10-09T09:54:55.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[152263]: Reconfiguring mon.c (monmap changed)... 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[152263]: Reconfiguring daemon mon.c on smithi012 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:54:55.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:55 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:55.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:55 smithi110 ceph-mon[136032]: Reconfiguring mon.c (monmap changed)... 2023-10-09T09:54:55.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:55 smithi110 ceph-mon[136032]: Reconfiguring daemon mon.c on smithi012 2023-10-09T09:54:55.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:55 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:55.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:55 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:55.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:55 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:54:55.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:55 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:56.883 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[147926]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[147926]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[147926]: Reconfiguring daemon osd.0 on smithi012 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[152263]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[152263]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[152263]: Reconfiguring daemon osd.0 on smithi012 2023-10-09T09:54:56.884 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:56.885 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:56.885 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:54:56.885 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:56 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:56 smithi110 ceph-mon[136032]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:56 smithi110 ceph-mon[136032]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T09:54:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:56 smithi110 ceph-mon[136032]: Reconfiguring daemon osd.0 on smithi012 2023-10-09T09:54:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:56 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:56 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:56 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:54:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:56 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:57.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[147926]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T09:54:57.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[147926]: Reconfiguring daemon osd.1 on smithi012 2023-10-09T09:54:57.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:57.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:57.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:54:57.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:57.889 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[152263]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T09:54:57.889 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[152263]: Reconfiguring daemon osd.1 on smithi012 2023-10-09T09:54:57.889 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:57.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:57.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:54:57.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:57.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:57 smithi110 ceph-mon[136032]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T09:54:57.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:57 smithi110 ceph-mon[136032]: Reconfiguring daemon osd.1 on smithi012 2023-10-09T09:54:57.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:57 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:57.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:57 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:57.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:57 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:54:57.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:57 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:58.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:54:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:54:57] "GET /metrics HTTP/1.1" 200 33787 "" "Prometheus/2.33.4" 2023-10-09T09:54:58.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[147926]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:58.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[147926]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T09:54:58.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[147926]: Reconfiguring daemon osd.2 on smithi012 2023-10-09T09:54:58.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:58.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:58.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[152263]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:58.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[152263]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T09:54:58.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[152263]: Reconfiguring daemon osd.2 on smithi012 2023-10-09T09:54:58.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:58.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:58 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:58 smithi110 ceph-mon[136032]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:54:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:58 smithi110 ceph-mon[136032]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T09:54:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:58 smithi110 ceph-mon[136032]: Reconfiguring daemon osd.2 on smithi012 2023-10-09T09:54:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:58 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:58.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:58 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:54:58.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:54:58 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:54:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:54:59.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[147926]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[147926]: Reconfiguring daemon osd.3 on smithi012 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[152263]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[152263]: Reconfiguring daemon osd.3 on smithi012 2023-10-09T09:54:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:54:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:54:59.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:59 smithi110 ceph-mon[136032]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T09:54:59.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:54:59.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:54:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:59 smithi110 ceph-mon[136032]: Reconfiguring daemon osd.3 on smithi012 2023-10-09T09:54:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:54:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:00.563 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[147926]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:00.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:00.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:00.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:00.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:00.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:00.564 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[152263]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:00.564 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:00.564 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:00.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:00.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:00.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:00 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:00.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:00 smithi110 ceph-mon[136032]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:00.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:00 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.mbnrgj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:00 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:01.804 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:01 smithi110 ceph-mon[136032]: Reconfiguring mds.foo.smithi012.mbnrgj (monmap changed)... 2023-10-09T09:55:01.804 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:01 smithi110 ceph-mon[136032]: Reconfiguring daemon mds.foo.smithi012.mbnrgj on smithi012 2023-10-09T09:55:01.805 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:01.805 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:01.805 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:01 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:55:01.805 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:01 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:55:01.805 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:01 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:01.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[147926]: Reconfiguring mds.foo.smithi012.mbnrgj (monmap changed)... 2023-10-09T09:55:01.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[147926]: Reconfiguring daemon mds.foo.smithi012.mbnrgj on smithi012 2023-10-09T09:55:01.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:01.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:01.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:55:01.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:55:01.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:01.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[152263]: Reconfiguring mds.foo.smithi012.mbnrgj (monmap changed)... 2023-10-09T09:55:01.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[152263]: Reconfiguring daemon mds.foo.smithi012.mbnrgj on smithi012 2023-10-09T09:55:01.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:01.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:01.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T09:55:01.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T09:55:01.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:01 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:02.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:02.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: Reconfiguring mon.b (monmap changed)... 2023-10-09T09:55:02.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: Reconfiguring daemon mon.b on smithi110 2023-10-09T09:55:02.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:02.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:02.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:55:02.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:55:02.847 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:55:02.847 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:02.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:02.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: Reconfiguring mon.b (monmap changed)... 2023-10-09T09:55:02.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: Reconfiguring daemon mon.b on smithi110 2023-10-09T09:55:02.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:02.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:02.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:55:02.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: Reconfiguring mon.b (monmap changed)... 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: Reconfiguring daemon mon.b on smithi110 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T09:55:02.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:03.847 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:03 smithi110 ceph-mon[136032]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T09:55:03.847 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:03 smithi110 ceph-mon[136032]: Reconfiguring daemon mgr.x on smithi110 2023-10-09T09:55:03.848 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:03 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:03.848 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:03 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:03.848 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:03 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:55:03.848 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:03 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:03.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[152263]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T09:55:03.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[152263]: Reconfiguring daemon mgr.x on smithi110 2023-10-09T09:55:03.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:03.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:03.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:55:03.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:03.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[147926]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T09:55:03.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[147926]: Reconfiguring daemon mgr.x on smithi110 2023-10-09T09:55:03.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:03.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:03.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T09:55:03.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:03 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[152263]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[152263]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T09:55:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[152263]: Reconfiguring daemon osd.4 on smithi110 2023-10-09T09:55:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:04.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:55:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[147926]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[147926]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T09:55:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[147926]: Reconfiguring daemon osd.4 on smithi110 2023-10-09T09:55:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:55:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:04 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:04 smithi110 ceph-mon[136032]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:04 smithi110 ceph-mon[136032]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T09:55:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:04 smithi110 ceph-mon[136032]: Reconfiguring daemon osd.4 on smithi110 2023-10-09T09:55:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:04 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:04 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:04 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T09:55:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:04 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:05.679 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:05 smithi110 ceph-mon[136032]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T09:55:05.679 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:05 smithi110 ceph-mon[136032]: Reconfiguring daemon osd.5 on smithi110 2023-10-09T09:55:05.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:05 smithi012 ceph-mon[147926]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T09:55:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:05 smithi012 ceph-mon[147926]: Reconfiguring daemon osd.5 on smithi110 2023-10-09T09:55:05.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:05 smithi012 ceph-mon[152263]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T09:55:05.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:05 smithi012 ceph-mon[152263]: Reconfiguring daemon osd.5 on smithi110 2023-10-09T09:55:06.873 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:06 smithi110 ceph-mon[136032]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:06.873 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:06 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:06.873 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:06 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:06.873 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:06 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:55:06.873 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:06 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[147926]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:06.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:06.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:06.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:55:06.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:06.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[152263]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:06.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:06.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:06.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T09:55:06.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:06 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:07.810 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:07 smithi110 ceph-mon[136032]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T09:55:07.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:07 smithi110 ceph-mon[136032]: Reconfiguring daemon osd.6 on smithi110 2023-10-09T09:55:07.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:07 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:07.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:07 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:07.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:07 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:55:07.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:07 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:07.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[147926]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T09:55:07.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[147926]: Reconfiguring daemon osd.6 on smithi110 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[152263]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[152263]: Reconfiguring daemon osd.6 on smithi110 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T09:55:07.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:08.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:55:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:55:07] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.33.4" 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: Reconfiguring daemon osd.7 on smithi110 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: Reconfiguring mds.foo.smithi110.sgosap (monmap changed)... 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[147926]: Reconfiguring daemon mds.foo.smithi110.sgosap on smithi110 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: Reconfiguring daemon osd.7 on smithi110 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: Reconfiguring mds.foo.smithi110.sgosap (monmap changed)... 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:08 smithi012 ceph-mon[152263]: Reconfiguring daemon mds.foo.smithi110.sgosap on smithi110 2023-10-09T09:55:08.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:55:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:55:08.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: Reconfiguring daemon osd.7 on smithi110 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: Reconfiguring mds.foo.smithi110.sgosap (monmap changed)... 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi110.sgosap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:08 smithi110 ceph-mon[136032]: Reconfiguring daemon mds.foo.smithi110.sgosap on smithi110 2023-10-09T09:55:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[147926]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[152263]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:10 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:10 smithi110 ceph-mon[136032]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:10 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:10 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:10 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:10 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:10 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:10 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:12.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:12 smithi012 ceph-mon[147926]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:12 smithi012 ceph-mon[152263]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:12 smithi110 ceph-mon[136032]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:14.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:14 smithi012 ceph-mon[147926]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:14 smithi012 ceph-mon[152263]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:14 smithi110 ceph-mon[136032]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:16 smithi012 ceph-mon[147926]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:16.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:16 smithi012 ceph-mon[152263]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:16 smithi110 ceph-mon[136032]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:18.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:55:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:55:17] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-10-09T09:55:18.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:18 smithi012 ceph-mon[147926]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:18 smithi012 ceph-mon[152263]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:18 smithi110 ceph-mon[136032]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:18.943 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:55:18 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:55:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:55:20.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:20 smithi012 ceph-mon[147926]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:20.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:20 smithi012 ceph-mon[152263]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:20 smithi110 ceph-mon[136032]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:22.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:22 smithi012 ceph-mon[147926]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:22.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:22 smithi012 ceph-mon[152263]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:22.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:22 smithi110 ceph-mon[136032]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:24.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:24 smithi012 ceph-mon[147926]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:24.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:24 smithi012 ceph-mon[152263]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:24 smithi110 ceph-mon[136032]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:26.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:26 smithi012 ceph-mon[147926]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:26.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:26 smithi012 ceph-mon[152263]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:26 smithi110 ceph-mon[136032]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:28.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:55:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:55:27] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-10-09T09:55:28.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:28 smithi012 ceph-mon[147926]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:28 smithi012 ceph-mon[152263]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:28 smithi110 ceph-mon[136032]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:28.942 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 09:55:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T09:55:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi110.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T09:55:29.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:29.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:30 smithi012 ceph-mon[147926]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:30.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:30 smithi012 ceph-mon[152263]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:30.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:30 smithi110 ceph-mon[136032]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:32.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:32 smithi012 ceph-mon[147926]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:32.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:32 smithi012 ceph-mon[152263]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:32.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:32 smithi110 ceph-mon[136032]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:34.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:34 smithi012 ceph-mon[147926]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:34.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:34 smithi012 ceph-mon[152263]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:34 smithi110 ceph-mon[136032]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:36 smithi012 ceph-mon[147926]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:36.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:36 smithi012 ceph-mon[152263]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:36 smithi110 ceph-mon[136032]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:38.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:55:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:55:37] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-10-09T09:55:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:38 smithi012 ceph-mon[147926]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:39.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:38 smithi012 ceph-mon[152263]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:39.178 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:38 smithi110 ceph-mon[136032]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:40 smithi012 ceph-mon[147926]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:40.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:40 smithi012 ceph-mon[152263]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:40 smithi110 ceph-mon[136032]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:42 smithi012 ceph-mon[147926]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:42.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:42 smithi012 ceph-mon[152263]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:42 smithi110 ceph-mon[136032]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:44.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:44 smithi012 ceph-mon[147926]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:44.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:44 smithi012 ceph-mon[152263]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:44.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:44 smithi110 ceph-mon[136032]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:46.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:46 smithi012 ceph-mon[147926]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:46.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:46 smithi012 ceph-mon[152263]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:46 smithi110 ceph-mon[136032]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:48.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:55:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:55:47] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-10-09T09:55:48.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:48 smithi012 ceph-mon[147926]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:48.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:48 smithi012 ceph-mon[152263]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:48 smithi110 ceph-mon[136032]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:50.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:50 smithi012 ceph-mon[152263]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:50.852 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:50 smithi012 ceph-mon[147926]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:50 smithi110 ceph-mon[136032]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:51.219 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T09:55:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:52 smithi012 ceph-mon[147926]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:52.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:52 smithi012 ceph-mon[152263]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:52 smithi110 ceph-mon[136032]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:52.943 INFO:teuthology.orchestra.run.smithi012.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:55:53.379 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T09:55:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[147926]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[147926]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:55:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[147926]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:55:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:54.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[152263]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:54.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[152263]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:55:54.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[152263]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:55:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:53 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:54.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:53 smithi110 ceph-mon[136032]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:54.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:53 smithi110 ceph-mon[136032]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:55:54.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:53 smithi110 ceph-mon[136032]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:55:54.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:53 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:54.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:53 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:55.019 INFO:teuthology.orchestra.run.smithi012.stdout:wait for half of osds to upgrade 2023-10-09T09:55:55.546 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T09:55:56.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:55 smithi012 ceph-mon[147926]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:56.218 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:55 smithi012 ceph-mon[152263]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:56.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:55 smithi110 ceph-mon[136032]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:58.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:55:57] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-10-09T09:55:58.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-mon[147926]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:58.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/554900245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:55:58.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:58.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-mon[152263]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/554900245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:55:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:57 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:58.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:57 smithi110 ceph-mon[136032]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:55:58.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/554900245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:55:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:57 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:57 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T09:55:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:55:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T09:55:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:55:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:55:59.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T09:55:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:55:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:55:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:56:00.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[152263]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:00.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[152263]: Upgrade: Setting container_image for all mon 2023-10-09T09:56:00.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[152263]: Upgrade: Setting container_image for all crash 2023-10-09T09:56:00.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[152263]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:56:00.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[152263]: Upgrade: osd.0 is safe to restart 2023-10-09T09:56:00.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2134232539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:00.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[147926]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:00.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[147926]: Upgrade: Setting container_image for all mon 2023-10-09T09:56:00.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[147926]: Upgrade: Setting container_image for all crash 2023-10-09T09:56:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[147926]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:56:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[147926]: Upgrade: osd.0 is safe to restart 2023-10-09T09:56:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2134232539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:00 smithi110 ceph-mon[136032]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:00 smithi110 ceph-mon[136032]: Upgrade: Setting container_image for all mon 2023-10-09T09:56:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:00 smithi110 ceph-mon[136032]: Upgrade: Setting container_image for all crash 2023-10-09T09:56:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:00 smithi110 ceph-mon[136032]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T09:56:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:00 smithi110 ceph-mon[136032]: Upgrade: osd.0 is safe to restart 2023-10-09T09:56:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2134232539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[147926]: Upgrade: Updating osd.0 2023-10-09T09:56:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:56:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[147926]: Deploying daemon osd.0 on smithi012 2023-10-09T09:56:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[152263]: Upgrade: Updating osd.0 2023-10-09T09:56:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:56:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:01 smithi012 ceph-mon[152263]: Deploying daemon osd.0 on smithi012 2023-10-09T09:56:01.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:01 smithi110 ceph-mon[136032]: Upgrade: Updating osd.0 2023-10-09T09:56:01.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:01 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T09:56:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:01 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:01 smithi110 ceph-mon[136032]: Deploying daemon osd.0 on smithi012 2023-10-09T09:56:02.658 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:02 smithi012 ceph-mon[147926]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:02.658 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/313361122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:02.658 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:02 smithi012 ceph-mon[152263]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:02.658 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/313361122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:02.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:02 smithi110 ceph-mon[136032]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:02.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/313361122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:03.303 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:02 smithi012 systemd[1]: Stopping Ceph osd.0 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:56:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:03 smithi012 ceph-mon[147926]: osd.0 marked itself down and dead 2023-10-09T09:56:03.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:03 smithi012 ceph-mon[152263]: osd.0 marked itself down and dead 2023-10-09T09:56:03.677 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[106439]: debug 2023-10-09T09:56:03.302+0000 7f724d09f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:56:03.677 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[106439]: debug 2023-10-09T09:56:03.302+0000 7f724d09f700 -1 osd.0 75 *** Got signal Terminated *** 2023-10-09T09:56:03.677 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:03 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[106439]: debug 2023-10-09T09:56:03.302+0000 7f724d09f700 -1 osd.0 75 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:56:03.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:03 smithi110 ceph-mon[136032]: osd.0 marked itself down and dead 2023-10-09T09:56:04.176 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:03 smithi012 bash[159501]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T09:56:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[147926]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[147926]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:56:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[147926]: failed to encode map e76 with expected crc 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[147926]: failed to encode map e76 with expected crc 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[147926]: osdmap e76: 8 total, 7 up, 8 in 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/214196487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[152263]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[152263]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[152263]: failed to encode map e76 with expected crc 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[152263]: failed to encode map e76 with expected crc 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[152263]: osdmap e76: 8 total, 7 up, 8 in 2023-10-09T09:56:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/214196487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:04 smithi110 ceph-mon[136032]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:04 smithi110 ceph-mon[136032]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:56:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:04 smithi110 ceph-mon[136032]: failed to encode map e76 with expected crc 2023-10-09T09:56:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:04 smithi110 ceph-mon[136032]: failed to encode map e76 with expected crc 2023-10-09T09:56:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:04 smithi110 ceph-mon[136032]: osdmap e76: 8 total, 7 up, 8 in 2023-10-09T09:56:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/214196487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:05.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:05 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T09:56:05.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:05 smithi012 systemd[1]: Stopped Ceph osd.0 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:56:05.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:05 smithi012 systemd[1]: Starting Ceph osd.0 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:56:05.562 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.562 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.563 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: osdmap e77: 8 total, 7 up, 8 in 2023-10-09T09:56:05.563 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.563 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.563 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: osdmap e77: 8 total, 7 up, 8 in 2023-10-09T09:56:05.563 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.564 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.564 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[147926]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:05 smithi012 ceph-mon[152263]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: osdmap e77: 8 total, 7 up, 8 in 2023-10-09T09:56:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: failed to encode map e77 with expected crc 2023-10-09T09:56:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:05 smithi110 ceph-mon[136032]: failed to encode map e77 with expected crc 2023-10-09T09:56:06.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 bash[159802]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:56:06.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 bash[159802]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T09:56:06.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 bash[159802]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T09:56:06.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 bash[159802]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T09:56:06.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 bash[159802]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T09:56:06.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 bash[159802]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T09:56:06.562 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 bash[159802]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T09:56:06.563 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 bash[159802]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T09:56:06.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:06 smithi012 ceph-mon[147926]: pgmap v128: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:06.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:06 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:56:06.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:06 smithi012 ceph-mon[152263]: pgmap v128: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:06.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:06 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:56:06.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:06 smithi110 ceph-mon[136032]: pgmap v128: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:06.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:06 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T09:56:07.270 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:06 smithi012 podman[160079]: 2023-10-09T09:56:07.270 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-osd[160119]: -- 172.21.15.12:0/2362363724 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cf211ac4e0 con 0x55cf2117d000 2023-10-09T09:56:07.270 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:07 smithi012 bash[160079]: 51758888c5a84163355735756cf2134c58bfe2f851810c7541a0993f7fa4c17a 2023-10-09T09:56:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1771687670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:07.569 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:07.569 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:07 smithi012 systemd[1]: Started Ceph osd.0 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:56:07.913 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1771687670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:07.913 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:07.913 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:07.913 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:07.914 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[160095]: 2023-10-09T09:56:07.686+0000 7fe45dc08640 -1 Falling back to public interface 2023-10-09T09:56:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1771687670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:07 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:07 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:07 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:08.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:56:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:56:07] "GET /metrics HTTP/1.1" 200 33856 "" "Prometheus/2.33.4" 2023-10-09T09:56:08.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:08 smithi012 ceph-mon[147926]: pgmap v129: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:08.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:08 smithi012 ceph-mon[152263]: pgmap v129: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:08.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:08 smithi110 ceph-mon[136032]: pgmap v129: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:09.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4275255415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:09.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4275255415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:09.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4275255415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:10.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:10 smithi012 ceph-mon[147926]: pgmap v130: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:10.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:10 smithi012 ceph-mon[152263]: pgmap v130: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:10.927 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:10 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[160095]: 2023-10-09T09:56:10.648+0000 7fe45dc08640 -1 osd.0 75 log_to_monitors true 2023-10-09T09:56:10.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:10 smithi110 ceph-mon[136032]: pgmap v130: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:11.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:11 smithi012 ceph-mon[152263]: from='osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:56:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:11 smithi012 ceph-mon[152263]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:56:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2891056850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:11.927 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 09:56:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[160095]: 2023-10-09T09:56:11.602+0000 7fe44d070700 -1 osd.0 75 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:56:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:11 smithi012 ceph-mon[147926]: from='osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:56:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:11 smithi012 ceph-mon[147926]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:56:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2891056850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:11 smithi110 ceph-mon[136032]: from='osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:56:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:11 smithi110 ceph-mon[136032]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T09:56:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2891056850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: failed to encode map e78 with expected crc 2023-10-09T09:56:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: failed to encode map e78 with expected crc 2023-10-09T09:56:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:56:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: osdmap e78: 8 total, 7 up, 8 in 2023-10-09T09:56:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: from='osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: failed to encode map e78 with expected crc 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: failed to encode map e78 with expected crc 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: osdmap e78: 8 total, 7 up, 8 in 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: from='osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:12.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:12.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: failed to encode map e78 with expected crc 2023-10-09T09:56:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: failed to encode map e78 with expected crc 2023-10-09T09:56:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T09:56:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: osdmap e78: 8 total, 7 up, 8 in 2023-10-09T09:56:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: from='osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:13.800 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[147926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:56:13.800 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[147926]: failed to encode map e79 with expected crc 2023-10-09T09:56:13.800 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[147926]: failed to encode map e79 with expected crc 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[147926]: osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086] boot 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[147926]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[152263]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[152263]: failed to encode map e79 with expected crc 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[152263]: failed to encode map e79 with expected crc 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[152263]: osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086] boot 2023-10-09T09:56:13.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[152263]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T09:56:13.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:56:13.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:13.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:13 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:13 smithi110 ceph-mon[136032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:56:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:13 smithi110 ceph-mon[136032]: failed to encode map e79 with expected crc 2023-10-09T09:56:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:13 smithi110 ceph-mon[136032]: failed to encode map e79 with expected crc 2023-10-09T09:56:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:13 smithi110 ceph-mon[136032]: osd.0 [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086] boot 2023-10-09T09:56:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:13 smithi110 ceph-mon[136032]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T09:56:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:13 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T09:56:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:13 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:13 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:14.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: pgmap v134: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:14.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T09:56:14.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.868 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2528754025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: pgmap v134: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.869 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2528754025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:14.870 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: pgmap v134: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: failed to encode map e80 with expected crc 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2528754025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:14.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:15.926 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:56:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:56:15.490+0000 7f2cb4de3700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3495750743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:56:16.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:16 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3495750743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:56:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:16 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3495750743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:17 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T09:56:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:17 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T09:56:17.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:17 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T09:56:17.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:17 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T09:56:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:17 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T09:56:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:17 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T09:56:18.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:56:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:56:17] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.33.4" 2023-10-09T09:56:18.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:18 smithi012 ceph-mon[147926]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:18.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1623077680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:18 smithi012 ceph-mon[152263]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1623077680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:18 smithi110 ceph-mon[136032]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1623077680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:20 smithi012 ceph-mon[147926]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:20.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:20 smithi012 ceph-mon[152263]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:20 smithi110 ceph-mon[136032]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4032234728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:21.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4032234728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:21.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4032234728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:22.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:22 smithi012 ceph-mon[147926]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:22.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:22 smithi012 ceph-mon[152263]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:22.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:22 smithi110 ceph-mon[136032]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:23.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:23 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2013335238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:23.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:23 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2013335238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:23.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:23 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2013335238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:24.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:24 smithi012 ceph-mon[147926]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:24.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:24 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:24.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:24 smithi012 ceph-mon[152263]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:24.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:24 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:24 smithi110 ceph-mon[136032]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:24 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:25.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3597376497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3597376497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:25.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3597376497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:26.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:26 smithi012 ceph-mon[147926]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:26.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:26 smithi012 ceph-mon[152263]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:26 smithi110 ceph-mon[136032]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:28.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:56:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:56:27] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.33.4" 2023-10-09T09:56:28.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:28 smithi012 ceph-mon[147926]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:28.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/392217217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:28 smithi012 ceph-mon[152263]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:28.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/392217217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:28 smithi110 ceph-mon[136032]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/392217217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:30.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:29 smithi012 ceph-mon[147926]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:29 smithi012 ceph-mon[152263]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:29 smithi110 ceph-mon[136032]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/576620682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:30 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:30 smithi012 ceph-mon[147926]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:30 smithi012 ceph-mon[147926]: Upgrade: osd.1 is safe to restart 2023-10-09T09:56:31.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/576620682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:31.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:30 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:31.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:30 smithi012 ceph-mon[152263]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:31.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:30 smithi012 ceph-mon[152263]: Upgrade: osd.1 is safe to restart 2023-10-09T09:56:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/576620682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:30 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:30 smithi110 ceph-mon[136032]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T09:56:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:30 smithi110 ceph-mon[136032]: Upgrade: osd.1 is safe to restart 2023-10-09T09:56:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:31 smithi012 ceph-mon[147926]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:31.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:31 smithi012 ceph-mon[152263]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:32.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:31 smithi110 ceph-mon[136032]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:33.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[147926]: Upgrade: Updating osd.1 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[147926]: Deploying daemon osd.1 on smithi012 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/872497780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[152263]: Upgrade: Updating osd.1 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:56:33.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:33.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[152263]: Deploying daemon osd.1 on smithi012 2023-10-09T09:56:33.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/872497780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:33.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:33 smithi110 ceph-mon[136032]: Upgrade: Updating osd.1 2023-10-09T09:56:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:33 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:33 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T09:56:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:33 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:33 smithi110 ceph-mon[136032]: Deploying daemon osd.1 on smithi012 2023-10-09T09:56:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/872497780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:34.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:34 smithi012 ceph-mon[147926]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:34.831 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:34 smithi012 ceph-mon[152263]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:34.831 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:34 smithi012 systemd[1]: Stopping Ceph osd.1 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:56:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:34 smithi110 ceph-mon[136032]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:35.177 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[110447]: debug 2023-10-09T09:56:34.828+0000 7f9cd678e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:56:35.177 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[110447]: debug 2023-10-09T09:56:34.828+0000 7f9cd678e700 -1 osd.1 80 *** Got signal Terminated *** 2023-10-09T09:56:35.177 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[110447]: debug 2023-10-09T09:56:34.828+0000 7f9cd678e700 -1 osd.1 80 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:56:35.818 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3039653345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:35.819 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:35 smithi012 ceph-mon[147926]: osd.1 marked itself down and dead 2023-10-09T09:56:35.819 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3039653345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:35.819 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:35 smithi012 ceph-mon[152263]: osd.1 marked itself down and dead 2023-10-09T09:56:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3039653345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:35 smithi110 ceph-mon[136032]: osd.1 marked itself down and dead 2023-10-09T09:56:36.176 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:35 smithi012 bash[162832]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T09:56:36.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[147926]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:36.894 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[147926]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:56:36.894 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[147926]: failed to encode map e81 with expected crc 2023-10-09T09:56:36.894 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[147926]: failed to encode map e81 with expected crc 2023-10-09T09:56:36.894 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[147926]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T09:56:36.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[152263]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:36.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[152263]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:56:36.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[152263]: failed to encode map e81 with expected crc 2023-10-09T09:56:36.895 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[152263]: failed to encode map e81 with expected crc 2023-10-09T09:56:36.895 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:36 smithi012 ceph-mon[152263]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T09:56:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:36 smithi110 ceph-mon[136032]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:36 smithi110 ceph-mon[136032]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:56:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:36 smithi110 ceph-mon[136032]: failed to encode map e81 with expected crc 2023-10-09T09:56:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:36 smithi110 ceph-mon[136032]: failed to encode map e81 with expected crc 2023-10-09T09:56:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:36 smithi110 ceph-mon[136032]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T09:56:37.544 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:37 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T09:56:37.544 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:37 smithi012 systemd[1]: Stopped Ceph osd.1 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:56:37.544 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:37 smithi012 systemd[1]: Starting Ceph osd.1 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:56:37.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T09:56:37.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3529648754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.819 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.820 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.820 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.820 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.820 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.820 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3529648754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: failed to encode map e82 with expected crc 2023-10-09T09:56:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3529648754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:38.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:56:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:56:37] "GET /metrics HTTP/1.1" 200 33918 "" "Prometheus/2.33.4" 2023-10-09T09:56:38.544 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 bash[163123]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:56:38.545 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 bash[163123]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T09:56:38.545 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 bash[163123]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T09:56:38.545 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 bash[163123]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T09:56:38.545 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 bash[163123]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T09:56:38.545 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 bash[163123]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T09:56:38.545 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 bash[163123]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T09:56:38.545 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 bash[163123]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T09:56:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:38 smithi012 ceph-mon[147926]: pgmap v149: 73 pgs: 1 active+undersized+degraded, 8 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:38 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:56:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:38 smithi012 ceph-mon[152263]: pgmap v149: 73 pgs: 1 active+undersized+degraded, 8 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:38 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:56:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:38 smithi110 ceph-mon[136032]: pgmap v149: 73 pgs: 1 active+undersized+degraded, 8 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:38 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:56:39.427 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:38 smithi012 podman[163395]: 2023-10-09T09:56:39.532 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:39 smithi012 ceph-osd[163458]: -- 172.21.15.12:0/2961532329 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5573d204a4e0 con 0x5573d201ac00 2023-10-09T09:56:39.532 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:39 smithi012 bash[163395]: a82d721b3b8f2fb265683f32178f5af997e8f16152b14d57bcac225b7503c751 2023-10-09T09:56:39.532 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:39 smithi012 systemd[1]: Started Ceph osd.1 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:56:39.926 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[163414]: 2023-10-09T09:56:39.661+0000 7fa38128e640 -1 Falling back to public interface 2023-10-09T09:56:40.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[147926]: pgmap v150: 73 pgs: 1 active+undersized+degraded, 8 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:40.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:40.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:40.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:40.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/310885449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:40.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[152263]: pgmap v150: 73 pgs: 1 active+undersized+degraded, 8 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:40.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:40.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:40.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:40.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/310885449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:40.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:40 smithi110 ceph-mon[136032]: pgmap v150: 73 pgs: 1 active+undersized+degraded, 8 stale+active+clean, 2 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:40.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:40 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:40.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:40 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:40.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:40 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/310885449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:42.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:42 smithi012 ceph-mon[147926]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:42.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4140717757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:42.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:42 smithi012 ceph-mon[152263]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:42.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4140717757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:42.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:42 smithi110 ceph-mon[136032]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:42.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4140717757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:42.926 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:42 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[163414]: 2023-10-09T09:56:42.591+0000 7fa38128e640 -1 osd.1 80 log_to_monitors true 2023-10-09T09:56:43.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:43 smithi012 ceph-mon[147926]: from='osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:56:43.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:43 smithi012 ceph-mon[147926]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:56:43.927 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 09:56:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[163414]: 2023-10-09T09:56:43.557+0000 7fa36fef5700 -1 osd.1 80 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:56:43.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:43 smithi012 ceph-mon[152263]: from='osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:56:43.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:43 smithi012 ceph-mon[152263]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:56:43.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:43 smithi110 ceph-mon[136032]: from='osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:56:43.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:43 smithi110 ceph-mon[136032]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: pgmap v152: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: failed to encode map e83 with expected crc 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: failed to encode map e83 with expected crc 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: from='osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2702388273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: pgmap v152: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: failed to encode map e83 with expected crc 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: failed to encode map e83 with expected crc 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: from='osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2702388273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: pgmap v152: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T09:56:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: failed to encode map e83 with expected crc 2023-10-09T09:56:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: failed to encode map e83 with expected crc 2023-10-09T09:56:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T09:56:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T09:56:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: from='osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:56:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:44.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:56:44.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2702388273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:45.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[147926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:56:45.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[147926]: failed to encode map e84 with expected crc 2023-10-09T09:56:45.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[147926]: osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322] boot 2023-10-09T09:56:45.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[147926]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T09:56:45.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:56:45.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:45.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:45.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[147926]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T09:56:45.662 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[152263]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:56:45.662 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[152263]: failed to encode map e84 with expected crc 2023-10-09T09:56:45.662 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[152263]: osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322] boot 2023-10-09T09:56:45.662 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[152263]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T09:56:45.662 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:56:45.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:45.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:45.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:45 smithi012 ceph-mon[152263]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T09:56:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:45 smithi110 ceph-mon[136032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:56:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:45 smithi110 ceph-mon[136032]: failed to encode map e84 with expected crc 2023-10-09T09:56:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:45 smithi110 ceph-mon[136032]: osd.1 [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322] boot 2023-10-09T09:56:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:45 smithi110 ceph-mon[136032]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T09:56:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:45 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T09:56:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:45 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:45 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:45 smithi110 ceph-mon[136032]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T09:56:46.878 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: pgmap v155: 73 pgs: 21 peering, 1 active+undersized+degraded, 10 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:56:46.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3323419798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: pgmap v155: 73 pgs: 21 peering, 1 active+undersized+degraded, 10 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:56:46.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:46 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3323419798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: pgmap v155: 73 pgs: 21 peering, 1 active+undersized+degraded, 10 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: failed to encode map e85 with expected crc 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:56:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:46 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3323419798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:47.913 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:56:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:56:47.501+0000 7f2cb4de3700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T09:56:48.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:56:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:56:47] "GET /metrics HTTP/1.1" 200 33833 "" "Prometheus/2.33.4" 2023-10-09T09:56:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: pgmap v157: 73 pgs: 21 peering, 9 active+undersized, 43 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:56:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: pgmap v157: 73 pgs: 21 peering, 9 active+undersized, 43 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:56:48.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:56:48.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T09:56:48.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T09:56:48.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:48 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T09:56:48.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: pgmap v157: 73 pgs: 21 peering, 9 active+undersized, 43 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:48.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T09:56:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T09:56:48.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:48 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T09:56:49.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3250070624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:49.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:49 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:49.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3250070624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:49.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:49 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:49.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3250070624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:49.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:49 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:56:50.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:50 smithi012 ceph-mon[147926]: pgmap v158: 73 pgs: 21 peering, 52 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:50.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:50 smithi012 ceph-mon[152263]: pgmap v158: 73 pgs: 21 peering, 52 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:50.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:50 smithi110 ceph-mon[136032]: pgmap v158: 73 pgs: 21 peering, 52 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:51.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1226864275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:51.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1226864275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:51.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1226864275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:52.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:52 smithi012 ceph-mon[147926]: pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:52.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:52 smithi012 ceph-mon[152263]: pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:52.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:52 smithi110 ceph-mon[136032]: pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:53.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3923486214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:53.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3923486214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:53.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3923486214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:54.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:54 smithi012 ceph-mon[147926]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:54 smithi012 ceph-mon[152263]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:54 smithi110 ceph-mon[136032]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:56.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:56 smithi012 ceph-mon[147926]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:56.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1707778428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:56.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:56 smithi012 ceph-mon[152263]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:56.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1707778428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:56 smithi110 ceph-mon[136032]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1707778428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:58.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:56:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:56:57] "GET /metrics HTTP/1.1" 200 33833 "" "Prometheus/2.33.4" 2023-10-09T09:56:58.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:58 smithi012 ceph-mon[147926]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:58.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2854768568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:58.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:58 smithi012 ceph-mon[152263]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:58.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2854768568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:56:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:58 smithi110 ceph-mon[136032]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:56:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2854768568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:00.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:59 smithi012 ceph-mon[147926]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:56:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:00.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:59 smithi012 ceph-mon[152263]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:00.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:00.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:56:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:59 smithi110 ceph-mon[136032]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:56:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:01.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4238992160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4238992160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4238992160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:01 smithi012 ceph-mon[147926]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:02.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:01 smithi012 ceph-mon[152263]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:01 smithi110 ceph-mon[136032]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:03.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:57:03.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:02 smithi012 ceph-mon[147926]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:57:03.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:02 smithi012 ceph-mon[147926]: Upgrade: osd.2 is safe to restart 2023-10-09T09:57:03.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1177598575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:03.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:57:03.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:02 smithi012 ceph-mon[152263]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:57:03.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:02 smithi012 ceph-mon[152263]: Upgrade: osd.2 is safe to restart 2023-10-09T09:57:03.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1177598575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:57:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:02 smithi110 ceph-mon[136032]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T09:57:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:02 smithi110 ceph-mon[136032]: Upgrade: osd.2 is safe to restart 2023-10-09T09:57:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1177598575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:04.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:04 smithi012 ceph-mon[147926]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:04.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:04 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:04.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:04 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:57:04.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:04 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:04.780 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:04 smithi012 ceph-mon[152263]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:04.780 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:04 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:04.780 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:04 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:57:04.780 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:04 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:04 smithi110 ceph-mon[136032]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:04 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:04 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T09:57:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:04 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:05.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:05 smithi012 ceph-mon[147926]: Upgrade: Updating osd.2 2023-10-09T09:57:05.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:05 smithi012 ceph-mon[147926]: Deploying daemon osd.2 on smithi012 2023-10-09T09:57:05.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/161475368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:05.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:05 smithi012 ceph-mon[152263]: Upgrade: Updating osd.2 2023-10-09T09:57:05.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:05 smithi012 ceph-mon[152263]: Deploying daemon osd.2 on smithi012 2023-10-09T09:57:05.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/161475368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:05.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:05 smithi110 ceph-mon[136032]: Upgrade: Updating osd.2 2023-10-09T09:57:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:05 smithi110 ceph-mon[136032]: Deploying daemon osd.2 on smithi012 2023-10-09T09:57:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/161475368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:06.537 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:06 smithi012 systemd[1]: Stopping Ceph osd.2 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:57:06.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:06 smithi012 ceph-mon[147926]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:06.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:06 smithi012 ceph-mon[152263]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:06.927 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[114661]: debug 2023-10-09T09:57:06.587+0000 7f381b546700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:57:06.927 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[114661]: debug 2023-10-09T09:57:06.587+0000 7f381b546700 -1 osd.2 85 *** Got signal Terminated *** 2023-10-09T09:57:06.927 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[114661]: debug 2023-10-09T09:57:06.587+0000 7f381b546700 -1 osd.2 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:57:06.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:06 smithi110 ceph-mon[136032]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:07.838 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:07 smithi012 bash[166251]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T09:57:07.838 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:07 smithi012 ceph-mon[147926]: osd.2 marked itself down and dead 2023-10-09T09:57:07.839 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1113272455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:07.839 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:07 smithi012 ceph-mon[152263]: osd.2 marked itself down and dead 2023-10-09T09:57:07.839 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1113272455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:07 smithi110 ceph-mon[136032]: osd.2 marked itself down and dead 2023-10-09T09:57:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1113272455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:08.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:57:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:57:07] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.33.4" 2023-10-09T09:57:08.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[147926]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:08.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[147926]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:57:08.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[147926]: failed to encode map e86 with expected crc 2023-10-09T09:57:08.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[147926]: failed to encode map e86 with expected crc 2023-10-09T09:57:08.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[147926]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T09:57:08.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[152263]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:08.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[152263]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:57:08.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[152263]: failed to encode map e86 with expected crc 2023-10-09T09:57:08.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[152263]: failed to encode map e86 with expected crc 2023-10-09T09:57:08.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:08 smithi012 ceph-mon[152263]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T09:57:08.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:08 smithi110 ceph-mon[136032]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:08.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:08 smithi110 ceph-mon[136032]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:57:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:08 smithi110 ceph-mon[136032]: failed to encode map e86 with expected crc 2023-10-09T09:57:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:08 smithi110 ceph-mon[136032]: failed to encode map e86 with expected crc 2023-10-09T09:57:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:08 smithi110 ceph-mon[136032]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T09:57:09.553 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:09 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T09:57:09.553 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:09 smithi012 systemd[1]: Stopped Ceph osd.2 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:57:09.553 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:09 smithi012 systemd[1]: Starting Ceph osd.2 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:57:09.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[147926]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[147926]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[147926]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T09:57:09.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[147926]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[147926]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[147926]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.817 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[147926]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.817 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[147926]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.817 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[152263]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.817 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[152263]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.817 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[152263]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T09:57:09.817 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[152263]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.817 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[152263]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[152263]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[152263]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:09 smithi012 ceph-mon[152263]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:09 smithi110 ceph-mon[136032]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:09 smithi110 ceph-mon[136032]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:09 smithi110 ceph-mon[136032]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T09:57:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:09 smithi110 ceph-mon[136032]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:09 smithi110 ceph-mon[136032]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:09 smithi110 ceph-mon[136032]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:09 smithi110 ceph-mon[136032]: failed to encode map e87 with expected crc 2023-10-09T09:57:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:09 smithi110 ceph-mon[136032]: failed to encode map e87 with expected crc 2023-10-09T09:57:10.554 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 bash[166571]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:57:10.554 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 bash[166571]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T09:57:10.554 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 bash[166571]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T09:57:10.554 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 bash[166571]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T09:57:10.554 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 bash[166571]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T09:57:10.554 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 bash[166571]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T09:57:10.554 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 bash[166571]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T09:57:10.555 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 bash[166571]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T09:57:10.861 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:10 smithi012 ceph-mon[147926]: pgmap v170: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:10.861 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1120995025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:10.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:10 smithi012 ceph-mon[152263]: pgmap v170: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:10.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1120995025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:10.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:10 smithi110 ceph-mon[136032]: pgmap v170: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T09:57:10.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1120995025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:11.157 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:10 smithi012 podman[166816]: 2023-10-09T09:57:11.157 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:11 smithi012 ceph-osd[166856]: -- 172.21.15.12:0/3811008100 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f2649004e0 con 0x55f2648d0c00 2023-10-09T09:57:11.157 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:11 smithi012 bash[166816]: 60afe019512d9e54232f9d4abd749e344e1fd61aa33bd6423d3e06f749e5015c 2023-10-09T09:57:11.426 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:11 smithi012 systemd[1]: Started Ceph osd.2 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:57:11.927 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[166832]: 2023-10-09T09:57:11.572+0000 7f3c34d3c640 -1 Falling back to public interface 2023-10-09T09:57:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[147926]: pgmap v171: 73 pgs: 1 active+undersized+degraded, 5 peering, 11 active+undersized, 56 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4188679837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[152263]: pgmap v171: 73 pgs: 1 active+undersized+degraded, 5 peering, 11 active+undersized, 56 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4188679837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:12 smithi110 ceph-mon[136032]: pgmap v171: 73 pgs: 1 active+undersized+degraded, 5 peering, 11 active+undersized, 56 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:12 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:12 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4188679837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:14 smithi012 ceph-mon[147926]: pgmap v172: 73 pgs: 2 active+undersized+degraded, 5 peering, 12 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2381581486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:14 smithi012 ceph-mon[152263]: pgmap v172: 73 pgs: 2 active+undersized+degraded, 5 peering, 12 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2381581486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:14 smithi110 ceph-mon[136032]: pgmap v172: 73 pgs: 2 active+undersized+degraded, 5 peering, 12 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2381581486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:16.176 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:15 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[166832]: 2023-10-09T09:57:15.832+0000 7f3c34d3c640 -1 osd.2 85 log_to_monitors true 2023-10-09T09:57:16.836 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: from='osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:16.837 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.838 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.838 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.838 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.838 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: from='osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:57:16.838 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:57:16.838 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.838 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: from='osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:57:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T09:57:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:17.176 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 09:57:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[166832]: 2023-10-09T09:57:16.835+0000 7f3c259a7700 -1 osd.2 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:57:17.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1717124749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:17.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[147926]: failed to encode map e88 with expected crc 2023-10-09T09:57:17.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[147926]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[147926]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[147926]: from='osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[147926]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[147926]: failed to encode map e88 with expected crc 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[147926]: failed to encode map e88 with expected crc 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1717124749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[152263]: failed to encode map e88 with expected crc 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[152263]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[152263]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T09:57:17.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[152263]: from='osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:17.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[152263]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:17.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[152263]: failed to encode map e88 with expected crc 2023-10-09T09:57:17.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-mon[152263]: failed to encode map e88 with expected crc 2023-10-09T09:57:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:17 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1717124749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:17 smithi110 ceph-mon[136032]: failed to encode map e88 with expected crc 2023-10-09T09:57:17.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:17 smithi110 ceph-mon[136032]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T09:57:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:17 smithi110 ceph-mon[136032]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T09:57:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:17 smithi110 ceph-mon[136032]: from='osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:17 smithi110 ceph-mon[136032]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:17 smithi110 ceph-mon[136032]: failed to encode map e88 with expected crc 2023-10-09T09:57:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:17 smithi110 ceph-mon[136032]: failed to encode map e88 with expected crc 2023-10-09T09:57:18.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:57:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:57:17] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.33.4" 2023-10-09T09:57:18.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[152263]: pgmap v175: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:18.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:57:18.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[152263]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:57:18.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[152263]: failed to encode map e89 with expected crc 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[152263]: failed to encode map e89 with expected crc 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[152263]: osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385] boot 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[152263]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[147926]: pgmap v175: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[147926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[147926]: failed to encode map e89 with expected crc 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[147926]: failed to encode map e89 with expected crc 2023-10-09T09:57:18.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[147926]: osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385] boot 2023-10-09T09:57:18.804 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[147926]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T09:57:18.804 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:18 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:57:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:18 smithi110 ceph-mon[136032]: pgmap v175: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:57:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:18 smithi110 ceph-mon[136032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:57:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:18 smithi110 ceph-mon[136032]: failed to encode map e89 with expected crc 2023-10-09T09:57:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:18 smithi110 ceph-mon[136032]: failed to encode map e89 with expected crc 2023-10-09T09:57:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:18 smithi110 ceph-mon[136032]: osd.2 [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385] boot 2023-10-09T09:57:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:18 smithi110 ceph-mon[136032]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T09:57:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:18 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T09:57:19.676 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T09:57:19.248+0000 7f2cb4de3700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T09:57:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T09:57:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: pgmap v178: 73 pgs: 4 peering, 2 active+undersized+degraded, 13 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:20.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2123704962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:20.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:20.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:20.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:20.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:20.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:20.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[147926]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T09:57:20.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T09:57:20.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: pgmap v178: 73 pgs: 4 peering, 2 active+undersized+degraded, 13 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:20.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2123704962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:20.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:20.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:20.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:20.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:20.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:20.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:19 smithi012 ceph-mon[152263]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T09:57:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T09:57:20.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: failed to encode map e90 with expected crc 2023-10-09T09:57:20.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: pgmap v178: 73 pgs: 4 peering, 2 active+undersized+degraded, 13 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:20.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2123704962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:20.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:20.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:20.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:20.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:20.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:57:20.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:20.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:20.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:19 smithi110 ceph-mon[136032]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T09:57:22.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:21 smithi110 ceph-mon[136032]: pgmap v179: 73 pgs: 4 peering, 1 active+undersized+degraded, 4 active+undersized, 64 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:22.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2297176858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:22.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:21 smithi012 ceph-mon[147926]: pgmap v179: 73 pgs: 4 peering, 1 active+undersized+degraded, 4 active+undersized, 64 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:22.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2297176858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:22.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:21 smithi012 ceph-mon[152263]: pgmap v179: 73 pgs: 4 peering, 1 active+undersized+degraded, 4 active+undersized, 64 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:22.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2297176858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:23.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:22 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:23.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:22 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:23.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:22 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:24.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:23 smithi110 ceph-mon[136032]: pgmap v180: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:24.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:23 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T09:57:24.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:23 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T09:57:24.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:23 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1824925923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:24.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:23 smithi012 ceph-mon[147926]: pgmap v180: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:23 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T09:57:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:23 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T09:57:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:23 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1824925923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:24.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:23 smithi012 ceph-mon[152263]: pgmap v180: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:24.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:23 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T09:57:24.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:23 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T09:57:24.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:23 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1824925923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:26.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:25 smithi110 ceph-mon[136032]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2748513372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:26.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:25 smithi012 ceph-mon[147926]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:26.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2748513372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:26.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:25 smithi012 ceph-mon[152263]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:26.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2748513372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:28.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:57:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:57:27] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.33.4" 2023-10-09T09:57:28.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:28 smithi012 ceph-mon[147926]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:28.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3126594725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:28.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:28 smithi012 ceph-mon[152263]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:28.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3126594725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:28 smithi110 ceph-mon[136032]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3126594725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:29 smithi012 ceph-mon[147926]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:30.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:29 smithi012 ceph-mon[152263]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:30.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:29 smithi110 ceph-mon[136032]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3086313177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:31.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3086313177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3086313177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:32.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:31 smithi012 ceph-mon[147926]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:32.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:31 smithi012 ceph-mon[152263]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:32.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:31 smithi110 ceph-mon[136032]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:33.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1958728649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:33.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1958728649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:33.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1958728649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:34.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:34 smithi012 ceph-mon[147926]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:34.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:34 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:34.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:34 smithi012 ceph-mon[152263]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:34.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:34 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:34 smithi110 ceph-mon[136032]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:34 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:35.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:35 smithi012 ceph-mon[147926]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:35.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:35 smithi012 ceph-mon[147926]: Upgrade: osd.3 is safe to restart 2023-10-09T09:57:35.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1098137208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:35 smithi012 ceph-mon[152263]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:35 smithi012 ceph-mon[152263]: Upgrade: osd.3 is safe to restart 2023-10-09T09:57:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1098137208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:35 smithi110 ceph-mon[136032]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T09:57:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:35 smithi110 ceph-mon[136032]: Upgrade: osd.3 is safe to restart 2023-10-09T09:57:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1098137208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:36.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:36 smithi012 ceph-mon[147926]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:36.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:36 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:36.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:36 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:57:36.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:36 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:36.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:36 smithi012 ceph-mon[152263]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:36.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:36 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:36.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:36 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:57:36.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:36 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:36 smithi110 ceph-mon[136032]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:36 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:36 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T09:57:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:36 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:37 smithi012 ceph-mon[147926]: Upgrade: Updating osd.3 2023-10-09T09:57:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:37 smithi012 ceph-mon[147926]: Deploying daemon osd.3 on smithi012 2023-10-09T09:57:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2069548698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:37.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:37 smithi012 ceph-mon[152263]: Upgrade: Updating osd.3 2023-10-09T09:57:37.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:37 smithi012 ceph-mon[152263]: Deploying daemon osd.3 on smithi012 2023-10-09T09:57:37.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2069548698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:37 smithi110 ceph-mon[136032]: Upgrade: Updating osd.3 2023-10-09T09:57:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:37 smithi110 ceph-mon[136032]: Deploying daemon osd.3 on smithi012 2023-10-09T09:57:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2069548698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:37.970 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:57:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:57:37] "GET /metrics HTTP/1.1" 200 33969 "" "Prometheus/2.33.4" 2023-10-09T09:57:38.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:38 smithi012 ceph-mon[152263]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:38.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:38 smithi012 ceph-mon[147926]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:38.677 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:38 smithi012 systemd[1]: Stopping Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:57:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:38 smithi110 ceph-mon[136032]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:39.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[118796]: debug 2023-10-09T09:57:38.827+0000 7efdb2942700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T09:57:39.177 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[118796]: debug 2023-10-09T09:57:38.827+0000 7efdb2942700 -1 osd.3 90 *** Got signal Terminated *** 2023-10-09T09:57:39.177 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[118796]: debug 2023-10-09T09:57:38.827+0000 7efdb2942700 -1 osd.3 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T09:57:39.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:39 smithi012 ceph-mon[147926]: osd.3 marked itself down and dead 2023-10-09T09:57:39.837 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:39 smithi012 ceph-mon[152263]: osd.3 marked itself down and dead 2023-10-09T09:57:39.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:39 smithi110 ceph-mon[136032]: osd.3 marked itself down and dead 2023-10-09T09:57:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:39 smithi012 bash[169548]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T09:57:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[147926]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[147926]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:57:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[147926]: failed to encode map e91 with expected crc 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[147926]: failed to encode map e91 with expected crc 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[147926]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3844215761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[152263]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[152263]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[152263]: failed to encode map e91 with expected crc 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[152263]: failed to encode map e91 with expected crc 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[152263]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T09:57:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3844215761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:40 smithi110 ceph-mon[136032]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:40 smithi110 ceph-mon[136032]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T09:57:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:40 smithi110 ceph-mon[136032]: failed to encode map e91 with expected crc 2023-10-09T09:57:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:40 smithi110 ceph-mon[136032]: failed to encode map e91 with expected crc 2023-10-09T09:57:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:40 smithi110 ceph-mon[136032]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T09:57:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3844215761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:41.676 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:41 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T09:57:41.677 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:41 smithi012 systemd[1]: Stopped Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:57:41.677 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:41 smithi012 systemd[1]: Starting Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T09:57:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[147926]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[147926]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T09:57:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[147926]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[147926]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[147926]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[147926]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[147926]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[147926]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[152263]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[152263]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[152263]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[152263]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[152263]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[152263]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[152263]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:41 smithi012 ceph-mon[152263]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:41 smithi110 ceph-mon[136032]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:41 smithi110 ceph-mon[136032]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T09:57:41.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:41 smithi110 ceph-mon[136032]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:41 smithi110 ceph-mon[136032]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:41 smithi110 ceph-mon[136032]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:41 smithi110 ceph-mon[136032]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:41 smithi110 ceph-mon[136032]: failed to encode map e92 with expected crc 2023-10-09T09:57:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:41 smithi110 ceph-mon[136032]: failed to encode map e92 with expected crc 2023-10-09T09:57:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:42 smithi012 ceph-mon[147926]: pgmap v191: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:42 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T09:57:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1729161450' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:42.927 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:42 smithi012 bash[169860]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T09:57:42.928 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:42 smithi012 bash[169860]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T09:57:42.928 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:42 smithi012 bash[169860]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T09:57:42.928 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:42 smithi012 bash[169860]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T09:57:42.928 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:42 smithi012 bash[169860]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T09:57:42.928 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:42 smithi012 bash[169860]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T09:57:42.928 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:42 smithi012 bash[169860]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T09:57:42.928 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:42 smithi012 bash[169860]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T09:57:42.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:42 smithi012 ceph-mon[152263]: pgmap v191: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:42.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:42 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T09:57:42.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1729161450' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:42 smithi110 ceph-mon[136032]: pgmap v191: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:42 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T09:57:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1729161450' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:43.498 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:43 smithi012 podman[170156]: 2023-10-09T09:57:43.499 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:43 smithi012 ceph-osd[170196]: -- 172.21.15.12:0/3663991333 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5651eb35c4e0 con 0x5651eb32d400 2023-10-09T09:57:43.499 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:43 smithi012 bash[170156]: 06b62af57f5fb70d9a2caca003dd368368f24e5060c106a4769d30acdeb6c2db 2023-10-09T09:57:43.920 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:43 smithi012 systemd[1]: Started Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T09:57:44.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:43 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[170172]: 2023-10-09T09:57:43.918+0000 7fac26cdd640 -1 Falling back to public interface 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[147926]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 14 peering, 4 stale+active+clean, 9 active+undersized, 45 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4140127256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:44.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[152263]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 14 peering, 4 stale+active+clean, 9 active+undersized, 45 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:57:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:44.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4140127256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:44 smithi110 ceph-mon[136032]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 14 peering, 4 stale+active+clean, 9 active+undersized, 45 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T09:57:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:44 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T09:57:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:57:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4140127256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:45.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:45 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T09:57:45.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:45 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T09:57:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:45 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T09:57:46.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:46 smithi012 ceph-mon[147926]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:46.927 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[170172]: 2023-10-09T09:57:46.798+0000 7fac26cdd640 -1 osd.3 90 log_to_monitors true 2023-10-09T09:57:46.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:46 smithi012 ceph-mon[152263]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:46 smithi110 ceph-mon[136032]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:47.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:47 smithi012 ceph-mon[147926]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:57:47.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2526060237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:47.913 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:47 smithi012 ceph-mon[152263]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:57:47.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2526060237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:47.914 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 09:57:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[170172]: 2023-10-09T09:57:47.630+0000 7fac18149700 -1 osd.3 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T09:57:47.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:47 smithi110 ceph-mon[136032]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T09:57:47.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2526060237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:48.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:57:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:57:47] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.33.4" 2023-10-09T09:57:48.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[147926]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:48.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[147926]: failed to encode map e93 with expected crc 2023-10-09T09:57:48.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[147926]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:57:48.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[147926]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T09:57:48.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[147926]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:48.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:48.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:48.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[152263]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:48.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[152263]: failed to encode map e93 with expected crc 2023-10-09T09:57:48.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[152263]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:57:48.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[152263]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T09:57:48.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[152263]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:48.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:48.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:48 smithi110 ceph-mon[136032]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:48 smithi110 ceph-mon[136032]: failed to encode map e93 with expected crc 2023-10-09T09:57:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:48 smithi110 ceph-mon[136032]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T09:57:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:48 smithi110 ceph-mon[136032]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T09:57:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:48 smithi110 ceph-mon[136032]: from='osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T09:57:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:49.155 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T09:57:49.562 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T09:57:49.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:57:49.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: failed to encode map e93 with expected crc 2023-10-09T09:57:49.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:57:49.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893] boot 2023-10-09T09:57:49.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T09:57:49.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: pgmap v197: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:49.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:49.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:49.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3921771957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:49.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[147926]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T09:57:49.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:57:49.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: failed to encode map e93 with expected crc 2023-10-09T09:57:49.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:57:49.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893] boot 2023-10-09T09:57:49.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T09:57:49.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: pgmap v197: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:49.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:49.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:49.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3921771957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:49.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:49 smithi012 ceph-mon[152263]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T09:57:49.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T09:57:49.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: failed to encode map e93 with expected crc 2023-10-09T09:57:49.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T09:57:49.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: osd.3 [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893] boot 2023-10-09T09:57:49.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T09:57:49.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: pgmap v197: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T09:57:49.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:49.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:49.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3921771957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T09:57:49.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:49 smithi110 ceph-mon[136032]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T09:57:50.878 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[147926]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[147926]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[147926]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[147926]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[147926]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[147926]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[147926]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[147926]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[152263]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[152263]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.879 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[152263]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[152263]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[152263]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[152263]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[152263]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:50 smithi012 ceph-mon[152263]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:50 smithi110 ceph-mon[136032]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:50 smithi110 ceph-mon[136032]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:50 smithi110 ceph-mon[136032]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:50 smithi110 ceph-mon[136032]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:50 smithi110 ceph-mon[136032]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:50 smithi110 ceph-mon[136032]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:50 smithi110 ceph-mon[136032]: failed to encode map e95 with expected crc 2023-10-09T09:57:50.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:50 smithi110 ceph-mon[136032]: failed to encode map e95 with expected crc 2023-10-09T09:57:51.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:51 smithi012 ceph-mon[147926]: pgmap v199: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:51.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:51 smithi012 ceph-mon[152263]: pgmap v199: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:51.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:51 smithi110 ceph-mon[136032]: pgmap v199: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T09:57:52.017 INFO:teuthology.orchestra.run.smithi012.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:57:52.609 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:57:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:57:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:57:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T09:57:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:57:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:53.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:53 smithi012 ceph-mon[147926]: pgmap v200: 73 pgs: 3 active+undersized, 70 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:53.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:53 smithi012 ceph-mon[152263]: pgmap v200: 73 pgs: 3 active+undersized, 70 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:53.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:53 smithi110 ceph-mon[136032]: pgmap v200: 73 pgs: 3 active+undersized, 70 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (7m) - 9m 25.2M - 0.23.0 ba2b418f427c f37e3c8f7f66 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi110 *:3000 running (6m) - 9m 56.2M - 8.3.5 dad864ee21e9 3c5f62b02d38 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.mbnrgj smithi012 running (9m) - 9m 18.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5be3d4216704 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi110.sgosap smithi110 running (9m) - 9m 16.1M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f17cc81b8a5e 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi110 *:8443,9283,8765 running (6m) - 13m 450M - 18.2.0-598-gb4fa663f 3060897fdf97 6e3769c16b69 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (6m) - 14m 515M - 18.2.0-598-gb4fa663f 3060897fdf97 3a9297092bc2 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (6m) - 15m 111M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 9f2783659fac 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi110 running (3m) - 13m 45.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 5fab451fcf37 2023-10-09T09:57:54.466 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (4m) - 13m 89.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 251f6625e45a 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (7m) - 10m 20.0M - 1.3.1 1dbe0e931976 c780854e434e 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi110 *:9100 running (7m) - 10m 20.0M - 1.3.1 1dbe0e931976 c0d7506efa64 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (107s) - 13m 57.8M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 51758888c5a8 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (75s) - 12m 53.2M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 a82d721b3b8f 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (43s) - 12m 48.0M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 60afe019512d 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (11s) - 12m 13.9M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 06b62af57f5f 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi110 running (11m) - 11m 90.5M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 18029670387d 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi110 running (11m) - 11m 103M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c927f6e714d1 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi110 running (11m) - 11m 75.0M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b96408c9e315 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi110 running (10m) - 10m 115M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 ac326d7f40d1 2023-10-09T09:57:54.467 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi110 *:9095 running (6m) - 10m 68.8M - 2.33.4 514e6a882f6e 82ffa853b6d7 2023-10-09T09:57:54.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:54 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T09:57:54.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:54 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T09:57:54.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:54 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:54.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:54 smithi012 ceph-mon[147926]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:54.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:54 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T09:57:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:54 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T09:57:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:54 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:54.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:54 smithi012 ceph-mon[152263]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:54 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T09:57:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:54 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T09:57:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:54 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:57:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:54 smithi110 ceph-mon[136032]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:54.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T09:57:55.802 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:55 smithi012 ceph-mon[147926]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:55.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:55 smithi012 ceph-mon[152263]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:55.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:55 smithi110 ceph-mon[136032]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:56.738 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T09:57:56.738 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (7m) - 10m 25.2M - 0.23.0 ba2b418f427c f37e3c8f7f66 2023-10-09T09:57:56.738 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi110 *:3000 running (6m) - 9m 56.2M - 8.3.5 dad864ee21e9 3c5f62b02d38 2023-10-09T09:57:56.738 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.mbnrgj smithi012 running (9m) - 9m 18.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 5be3d4216704 2023-10-09T09:57:56.739 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi110.sgosap smithi110 running (9m) - 9m 16.1M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f17cc81b8a5e 2023-10-09T09:57:56.739 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi110 *:8443,9283,8765 running (6m) - 13m 450M - 18.2.0-598-gb4fa663f 3060897fdf97 6e3769c16b69 2023-10-09T09:57:56.739 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (6m) - 14m 515M - 18.2.0-598-gb4fa663f 3060897fdf97 3a9297092bc2 2023-10-09T09:57:56.739 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (6m) - 15m 111M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 9f2783659fac 2023-10-09T09:57:56.739 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi110 running (3m) - 13m 45.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 5fab451fcf37 2023-10-09T09:57:56.739 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (4m) - 13m 89.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 251f6625e45a 2023-10-09T09:57:56.739 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (7m) - 10m 20.0M - 1.3.1 1dbe0e931976 c780854e434e 2023-10-09T09:57:56.739 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi110 *:9100 running (7m) - 10m 20.0M - 1.3.1 1dbe0e931976 c0d7506efa64 2023-10-09T09:57:56.740 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (109s) - 13m 57.8M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 51758888c5a8 2023-10-09T09:57:56.740 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (77s) - 12m 53.2M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 a82d721b3b8f 2023-10-09T09:57:56.740 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (45s) - 12m 48.0M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 60afe019512d 2023-10-09T09:57:56.740 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (13s) - 12m 13.9M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 06b62af57f5f 2023-10-09T09:57:56.740 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi110 running (11m) - 11m 90.5M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 18029670387d 2023-10-09T09:57:56.740 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi110 running (11m) - 11m 103M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c927f6e714d1 2023-10-09T09:57:56.741 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi110 running (11m) - 11m 75.0M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b96408c9e315 2023-10-09T09:57:56.741 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi110 running (10m) - 10m 115M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 ac326d7f40d1 2023-10-09T09:57:56.741 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi110 *:9095 running (6m) - 10m 68.8M - 2.33.4 514e6a882f6e 82ffa853b6d7 2023-10-09T09:57:57.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T09:57:58.149 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:57:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:57:57] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.33.4" 2023-10-09T09:57:58.753 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:58 smithi012 ceph-mon[147926]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:58.753 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:58 smithi012 ceph-mon[147926]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:58.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:58 smithi012 ceph-mon[152263]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:58.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:58 smithi012 ceph-mon[152263]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:58 smithi110 ceph-mon[136032]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T09:57:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:58 smithi110 ceph-mon[136032]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:57:59.224 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:57:59.224 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2023-10-09T09:57:59.224 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T09:57:59.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:57:59.224 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2023-10-09T09:57:59.224 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T09:57:59.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:57:59.225 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2023-10-09T09:57:59.225 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 4, 2023-10-09T09:57:59.225 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T09:57:59.225 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:57:59.225 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2023-10-09T09:57:59.225 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 2 2023-10-09T09:57:59.225 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:57:59.226 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2023-10-09T09:57:59.226 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 6, 2023-10-09T09:57:59.226 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T09:57:59.226 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:57:59.226 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:57:59.683 DEBUG:teuthology.parallel:result is None 2023-10-09T09:57:59.683 INFO:teuthology.run_tasks:Running task print... 2023-10-09T09:57:59.695 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T09:57:59.695 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T09:57:59.742 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T09:57:59.742 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T09:57:59.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T09:58:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:59 smithi012 ceph-mon[147926]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:57:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4043926444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:58:00.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:00.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:59 smithi012 ceph-mon[152263]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:57:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4043926444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:58:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:59 smithi110 ceph-mon[136032]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:57:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4043926444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T09:58:00.356 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:01.914 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:58:02.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:01 smithi012 ceph-mon[147926]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:02.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2037948667' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:58:02.369 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:01 smithi012 ceph-mon[152263]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:02.370 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2037948667' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:58:02.380 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":37,"active_gid":25217,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":1503907773},{"type":"v1","addr":"172.21.15.12:6801","nonce":1503907773}]},"active_addr":"172.21.15.12:6801/1503907773","active_change":"2023-10-09T09:51:58.776612+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25129,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":75,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":3872680902}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2168106054}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2721409080}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":1359539837}]}]} 2023-10-09T09:58:02.385 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T09:58:02.385 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T09:58:02.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:58:02.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:01 smithi110 ceph-mon[136032]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:02.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2037948667' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T09:58:02.996 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:04.493 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:58:04.493 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":95,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T09:57:49.606098+0000","last_up_change":"2023-10-09T09:57:48.604458+0000","last_in_change":"2023-10-09T09:46:55.568500+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:47:41.614981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:48:24.361405+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:48:24.948128+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":75,"up_from":79,"up_thru":94,"down_at":76,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6803","nonce":3324439086}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6805","nonce":3324439086}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6809","nonce":3324439086}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6807","nonce":3324439086}]},"public_addr":"172.21.15.12:6803/3324439086","cluster_addr":"172.21.15.12:6805/3324439086","heartbeat_back_addr":"172.21.15.12:6809/3324439086","heartbeat_front_addr":"172.21.15.12:6807/3324439086","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":80,"up_from":84,"up_thru":94,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6811","nonce":715957322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6813","nonce":715957322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6817","nonce":715957322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6815","nonce":715957322}]},"public_addr":"172.21.15.12:6811/715957322","cluster_addr":"172.21.15.12:6813/715957322","heartbeat_back_addr":"172.21.15.12:6817/715957322","heartbeat_front_addr":"172.21.15.12:6815/715957322","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":85,"up_from":89,"up_thru":94,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4293129385},{"type":"v1","addr":"172.21.15.12:6819","nonce":4293129385}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4293129385},{"type":"v1","addr":"172.21.15.12:6821","nonce":4293129385}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4293129385},{"type":"v1","addr":"172.21.15.12:6825","nonce":4293129385}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4293129385},{"type":"v1","addr":"172.21.15.12:6823","nonce":4293129385}]},"public_addr":"172.21.15.12:6819/4293129385","cluster_addr":"172.21.15.12:6821/4293129385","heartbeat_back_addr":"172.21.15.12:6825/4293129385","heartbeat_front_addr":"172.21.15.12:6823/4293129385","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":90,"up_from":94,"up_thru":94,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":2914552893},{"type":"v1","addr":"172.21.15.12:6827","nonce":2914552893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":2914552893},{"type":"v1","addr":"172.21.15.12:6829","nonce":2914552893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":2914552893},{"type":"v1","addr":"172.21.15.12:6833","nonce":2914552893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":2914552893},{"type":"v1","addr":"172.21.15.12:6831","nonce":2914552893}]},"public_addr":"172.21.15.12:6827/2914552893","cluster_addr":"172.21.15.12:6829/2914552893","heartbeat_back_addr":"172.21.15.12:6833/2914552893","heartbeat_front_addr":"172.21.15.12:6831/2914552893","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6801","nonce":3628101753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6803","nonce":3628101753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6807","nonce":3628101753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6805","nonce":3628101753}]},"public_addr":"172.21.15.110:6801/3628101753","cluster_addr":"172.21.15.110:6803/3628101753","heartbeat_back_addr":"172.21.15.110:6807/3628101753","heartbeat_front_addr":"172.21.15.110:6805/3628101753","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":75},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":80},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":85},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":90},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/2293469660":"2023-10-10T09:50:12.280108+0000","172.21.15.12:0/2348702677":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/2990486696":"2023-10-10T09:51:58.776424+0000","172.21.15.110:0/1671269215":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/1146481933":"2023-10-10T09:51:58.776424+0000","172.21.15.110:0/3560901131":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/1925191424":"2023-10-10T09:50:12.280108+0000","172.21.15.12:6801/1925191424":"2023-10-10T09:50:12.280108+0000","172.21.15.110:0/4208451228":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/1146481933":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/1614086989":"2023-10-10T09:51:58.776424+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.110:0/1606345106":"2023-10-10T09:51:12.282860+0000","172.21.15.110:0/1952966694":"2023-10-10T09:51:12.282860+0000","172.21.15.12:0/3260703278":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/807677503":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/928078357":"2023-10-10T09:50:12.280108+0000","172.21.15.12:0/518580822":"2023-10-10T09:51:58.776424+0000","172.21.15.110:6835/4098121826":"2023-10-10T09:51:12.282860+0000","172.21.15.110:6834/4098121826":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.110:0/3728005719":"2023-10-10T09:51:12.282860+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:58:04.891 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:04 smithi012 ceph-mon[152263]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:04.891 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:04 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:04.891 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2682685846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:58:04.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:04 smithi012 ceph-mon[147926]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:04.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:04 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:04.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2682685846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:58:04.925 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T09:58:04.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T09:58:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:04 smithi110 ceph-mon[136032]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:04 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:04.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2682685846' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:58:05.532 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:06.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:06 smithi012 ceph-mon[147926]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:06.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:06 smithi012 ceph-mon[152263]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:06.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:06 smithi110 ceph-mon[136032]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:07.024 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:58:07.024 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":95,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T09:57:49.606098+0000","last_up_change":"2023-10-09T09:57:48.604458+0000","last_in_change":"2023-10-09T09:46:55.568500+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:47:41.614981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:48:24.361405+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:48:24.948128+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":75,"up_from":79,"up_thru":94,"down_at":76,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6803","nonce":3324439086}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6805","nonce":3324439086}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6809","nonce":3324439086}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6807","nonce":3324439086}]},"public_addr":"172.21.15.12:6803/3324439086","cluster_addr":"172.21.15.12:6805/3324439086","heartbeat_back_addr":"172.21.15.12:6809/3324439086","heartbeat_front_addr":"172.21.15.12:6807/3324439086","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":80,"up_from":84,"up_thru":94,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6811","nonce":715957322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6813","nonce":715957322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6817","nonce":715957322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6815","nonce":715957322}]},"public_addr":"172.21.15.12:6811/715957322","cluster_addr":"172.21.15.12:6813/715957322","heartbeat_back_addr":"172.21.15.12:6817/715957322","heartbeat_front_addr":"172.21.15.12:6815/715957322","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":85,"up_from":89,"up_thru":94,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4293129385},{"type":"v1","addr":"172.21.15.12:6819","nonce":4293129385}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4293129385},{"type":"v1","addr":"172.21.15.12:6821","nonce":4293129385}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4293129385},{"type":"v1","addr":"172.21.15.12:6825","nonce":4293129385}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4293129385},{"type":"v1","addr":"172.21.15.12:6823","nonce":4293129385}]},"public_addr":"172.21.15.12:6819/4293129385","cluster_addr":"172.21.15.12:6821/4293129385","heartbeat_back_addr":"172.21.15.12:6825/4293129385","heartbeat_front_addr":"172.21.15.12:6823/4293129385","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":90,"up_from":94,"up_thru":94,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":2914552893},{"type":"v1","addr":"172.21.15.12:6827","nonce":2914552893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":2914552893},{"type":"v1","addr":"172.21.15.12:6829","nonce":2914552893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":2914552893},{"type":"v1","addr":"172.21.15.12:6833","nonce":2914552893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":2914552893},{"type":"v1","addr":"172.21.15.12:6831","nonce":2914552893}]},"public_addr":"172.21.15.12:6827/2914552893","cluster_addr":"172.21.15.12:6829/2914552893","heartbeat_back_addr":"172.21.15.12:6833/2914552893","heartbeat_front_addr":"172.21.15.12:6831/2914552893","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6801","nonce":3628101753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6803","nonce":3628101753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6807","nonce":3628101753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3628101753},{"type":"v1","addr":"172.21.15.110:6805","nonce":3628101753}]},"public_addr":"172.21.15.110:6801/3628101753","cluster_addr":"172.21.15.110:6803/3628101753","heartbeat_back_addr":"172.21.15.110:6807/3628101753","heartbeat_front_addr":"172.21.15.110:6805/3628101753","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":75},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":80},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":85},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":90},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/2293469660":"2023-10-10T09:50:12.280108+0000","172.21.15.12:0/2348702677":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/2990486696":"2023-10-10T09:51:58.776424+0000","172.21.15.110:0/1671269215":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/1146481933":"2023-10-10T09:51:58.776424+0000","172.21.15.110:0/3560901131":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/1925191424":"2023-10-10T09:50:12.280108+0000","172.21.15.12:6801/1925191424":"2023-10-10T09:50:12.280108+0000","172.21.15.110:0/4208451228":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/1146481933":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/1614086989":"2023-10-10T09:51:58.776424+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.110:0/1606345106":"2023-10-10T09:51:12.282860+0000","172.21.15.110:0/1952966694":"2023-10-10T09:51:12.282860+0000","172.21.15.12:0/3260703278":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/807677503":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/928078357":"2023-10-10T09:50:12.280108+0000","172.21.15.12:0/518580822":"2023-10-10T09:51:58.776424+0000","172.21.15.110:6835/4098121826":"2023-10-10T09:51:12.282860+0000","172.21.15.110:6834/4098121826":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.110:0/3728005719":"2023-10-10T09:51:12.282860+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T09:58:07.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T09:58:07.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T09:58:07.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T09:58:07.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T09:58:07.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T09:58:07.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T09:58:07.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T09:58:07.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T09:58:07.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2932173481' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:58:07.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2932173481' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:58:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2932173481' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T09:58:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:58:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:58:07] "GET /metrics HTTP/1.1" 200 34041 "" "Prometheus/2.33.4" 2023-10-09T09:58:08.282 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:08.614 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:08.617 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:08.617 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:08.621 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:08.800 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:08.801 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:08.881 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:08 smithi012 ceph-mon[147926]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:08.882 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:08 smithi012 ceph-mon[152263]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:08.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:08 smithi110 ceph-mon[136032]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:09.204 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:10.920 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:10 smithi012 ceph-mon[147926]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:10.921 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:10 smithi012 ceph-mon[152263]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:10.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:10 smithi110 ceph-mon[136032]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:12.894 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:12 smithi012 ceph-mon[147926]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:12.895 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:12 smithi012 ceph-mon[152263]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:12.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:12 smithi110 ceph-mon[136032]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:14.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:14 smithi012 ceph-mon[152263]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:14.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:14.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:14 smithi012 ceph-mon[147926]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:14.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:14.778 INFO:teuthology.orchestra.run.smithi012.stdout:154618822798 2023-10-09T09:58:14.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T09:58:14.838 INFO:teuthology.orchestra.run.smithi012.stdout:403726925831 2023-10-09T09:58:14.838 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T09:58:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:14 smithi110 ceph-mon[136032]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:15.386 INFO:teuthology.orchestra.run.smithi012.stdout:360777252884 2023-10-09T09:58:15.387 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T09:58:16.017 INFO:teuthology.orchestra.run.smithi012.stdout:382252089357 2023-10-09T09:58:16.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T09:58:16.069 INFO:teuthology.orchestra.run.smithi012.stdout:128849019027 2023-10-09T09:58:16.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T09:58:16.322 INFO:teuthology.orchestra.run.smithi012.stdout:206158430342 2023-10-09T09:58:16.323 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T09:58:16.395 INFO:teuthology.orchestra.run.smithi012.stdout:339302416410 2023-10-09T09:58:16.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T09:58:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:16 smithi012 ceph-mon[147926]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:16.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:16 smithi012 ceph-mon[152263]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:16 smithi110 ceph-mon[136032]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:16.946 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:17.170 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:17.782 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:17.998 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:58:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:58:17] "GET /metrics HTTP/1.1" 200 34039 "" "Prometheus/2.33.4" 2023-10-09T09:58:18.323 INFO:teuthology.orchestra.run.smithi012.stdout:180388626571 2023-10-09T09:58:18.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T09:58:18.539 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:18.593 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:18.698 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:18.750 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:18.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:18 smithi012 ceph-mon[147926]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:18.795 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:18 smithi012 ceph-mon[152263]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:18 smithi110 ceph-mon[136032]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:19.213 INFO:teuthology.orchestra.run.smithi012.stdout:403726925831 2023-10-09T09:58:19.861 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:19 smithi012 ceph-mon[152263]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:19.861 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3942905724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:58:19.861 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:19 smithi012 ceph-mon[147926]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:19.861 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3942905724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:58:19.934 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925831 got 403726925831 for osd.3 2023-10-09T09:58:19.934 DEBUG:teuthology.parallel:result is None 2023-10-09T09:58:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:19 smithi110 ceph-mon[136032]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3942905724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T09:58:21.354 INFO:teuthology.orchestra.run.smithi012.stdout:360777252885 2023-10-09T09:58:21.833 INFO:tasks.cephadm.ceph_manager.ceph:need seq 360777252884 got 360777252885 for osd.1 2023-10-09T09:58:21.833 DEBUG:teuthology.parallel:result is None 2023-10-09T09:58:22.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:21 smithi012 ceph-mon[147926]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:22.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1153683098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:58:22.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:21 smithi012 ceph-mon[152263]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:22.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1153683098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:58:22.385 INFO:teuthology.orchestra.run.smithi012.stdout:154618822800 2023-10-09T09:58:22.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:21 smithi110 ceph-mon[136032]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:22.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1153683098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T09:58:22.946 INFO:teuthology.orchestra.run.smithi012.stdout:128849019028 2023-10-09T09:58:23.010 INFO:teuthology.orchestra.run.smithi012.stdout:382252089358 2023-10-09T09:58:23.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3293888934' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:58:23.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1287116502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:58:23.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2816026338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:58:23.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3293888934' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:58:23.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1287116502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:58:23.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2816026338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:58:23.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3293888934' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T09:58:23.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1287116502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T09:58:23.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2816026338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T09:58:23.844 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089357 got 382252089358 for osd.2 2023-10-09T09:58:23.844 DEBUG:teuthology.parallel:result is None 2023-10-09T09:58:23.846 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:23.937 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019027 got 128849019028 for osd.4 2023-10-09T09:58:23.937 DEBUG:teuthology.parallel:result is None 2023-10-09T09:58:24.271 INFO:teuthology.orchestra.run.smithi012.stdout:339302416412 2023-10-09T09:58:24.382 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822798 got 154618822800 for osd.5 2023-10-09T09:58:24.382 DEBUG:teuthology.parallel:result is None 2023-10-09T09:58:24.707 INFO:tasks.cephadm.ceph_manager.ceph:need seq 339302416410 got 339302416412 for osd.0 2023-10-09T09:58:24.896 DEBUG:teuthology.parallel:result is None 2023-10-09T09:58:24.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:24 smithi012 ceph-mon[147926]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:24.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2907328290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:58:24.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:24 smithi012 ceph-mon[152263]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:24.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2907328290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:58:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:24 smithi110 ceph-mon[136032]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2907328290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T09:58:25.671 INFO:teuthology.orchestra.run.smithi012.stdout:206158430345 2023-10-09T09:58:26.286 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430342 got 206158430345 for osd.7 2023-10-09T09:58:26.287 DEBUG:teuthology.parallel:result is None 2023-10-09T09:58:26.414 INFO:teuthology.orchestra.run.smithi012.stdout:180388626573 2023-10-09T09:58:26.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:26 smithi012 ceph-mon[147926]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:26.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2861834080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:58:26.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1491536215' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:58:26.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:26 smithi012 ceph-mon[152263]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:26.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2861834080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:58:26.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1491536215' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:58:26.892 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626571 got 180388626573 for osd.6 2023-10-09T09:58:26.892 DEBUG:teuthology.parallel:result is None 2023-10-09T09:58:26.893 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T09:58:26.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:58:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:26 smithi110 ceph-mon[136032]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2861834080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T09:58:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1491536215' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T09:58:27.502 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:28.053 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:58:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:58:27] "GET /metrics HTTP/1.1" 200 34039 "" "Prometheus/2.33.4" 2023-10-09T09:58:28.901 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:58:28.903 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:58:28.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:28 smithi012 ceph-mon[147926]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:28.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:28 smithi012 ceph-mon[152263]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:28 smithi110 ceph-mon[136032]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:29.333 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":217,"stamp":"2023-10-09T09:58:26.904927+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1348276,"kb_used_data":9396,"kb_used_omap":126,"kb_used_meta":1338689,"kb_avail":748612940,"statfs":{"total":767960285184,"available":766579650560,"internally_reserved":0,"allocated":9621504,"data_stored":5943134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1370818171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003161"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.616419+0000","last_change":"2023-10-09T09:56:45.556456+0000","last_active":"2023-10-09T09:57:48.616419+0000","last_peered":"2023-10-09T09:57:48.616419+0000","last_clean":"2023-10-09T09:57:48.616419+0000","last_became_active":"2023-10-09T09:56:45.555878+0000","last_became_peered":"2023-10-09T09:56:45.555878+0000","last_unstale":"2023-10-09T09:57:48.616419+0000","last_undegraded":"2023-10-09T09:57:48.616419+0000","last_fullsized":"2023-10-09T09:57:48.616419+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612158+0000","last_change":"2023-10-09T09:57:18.842453+0000","last_active":"2023-10-09T09:57:49.612158+0000","last_peered":"2023-10-09T09:57:49.612158+0000","last_clean":"2023-10-09T09:57:49.612158+0000","last_became_active":"2023-10-09T09:57:18.842133+0000","last_became_peered":"2023-10-09T09:57:18.842133+0000","last_unstale":"2023-10-09T09:57:49.612158+0000","last_undegraded":"2023-10-09T09:57:49.612158+0000","last_fullsized":"2023-10-09T09:57:49.612158+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:07.459692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620527+0000","last_change":"2023-10-09T09:57:49.620527+0000","last_active":"2023-10-09T09:57:49.620527+0000","last_peered":"2023-10-09T09:57:49.620527+0000","last_clean":"2023-10-09T09:57:49.620527+0000","last_became_active":"2023-10-09T09:57:49.620109+0000","last_became_peered":"2023-10-09T09:57:49.620109+0000","last_unstale":"2023-10-09T09:57:49.620527+0000","last_undegraded":"2023-10-09T09:57:49.620527+0000","last_fullsized":"2023-10-09T09:57:49.620527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:09.284902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":68,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623102+0000","last_change":"2023-10-09T09:57:49.623102+0000","last_active":"2023-10-09T09:57:49.623102+0000","last_peered":"2023-10-09T09:57:49.623102+0000","last_clean":"2023-10-09T09:57:49.623102+0000","last_became_active":"2023-10-09T09:57:49.620638+0000","last_became_peered":"2023-10-09T09:57:49.620638+0000","last_unstale":"2023-10-09T09:57:49.623102+0000","last_undegraded":"2023-10-09T09:57:49.623102+0000","last_fullsized":"2023-10-09T09:57:49.623102+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:11.322675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620967+0000","last_change":"2023-10-09T09:57:49.620967+0000","last_active":"2023-10-09T09:57:49.620967+0000","last_peered":"2023-10-09T09:57:49.620967+0000","last_clean":"2023-10-09T09:57:49.620967+0000","last_became_active":"2023-10-09T09:57:49.620567+0000","last_became_peered":"2023-10-09T09:57:49.620567+0000","last_unstale":"2023-10-09T09:57:49.620967+0000","last_undegraded":"2023-10-09T09:57:49.620967+0000","last_fullsized":"2023-10-09T09:57:49.620967+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620814+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T09:57:48.620814+0000","last_peered":"2023-10-09T09:57:48.620814+0000","last_clean":"2023-10-09T09:57:48.620814+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T09:57:48.620814+0000","last_undegraded":"2023-10-09T09:57:48.620814+0000","last_fullsized":"2023-10-09T09:57:48.620814+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622634+0000","last_change":"2023-10-09T09:57:49.622634+0000","last_active":"2023-10-09T09:57:49.622634+0000","last_peered":"2023-10-09T09:57:49.622634+0000","last_clean":"2023-10-09T09:57:49.622634+0000","last_became_active":"2023-10-09T09:57:49.622247+0000","last_became_peered":"2023-10-09T09:57:49.622247+0000","last_unstale":"2023-10-09T09:57:49.622634+0000","last_undegraded":"2023-10-09T09:57:49.622634+0000","last_fullsized":"2023-10-09T09:57:49.622634+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621538+0000","last_change":"2023-10-09T09:56:45.554958+0000","last_active":"2023-10-09T09:57:48.621538+0000","last_peered":"2023-10-09T09:57:48.621538+0000","last_clean":"2023-10-09T09:57:48.621538+0000","last_became_active":"2023-10-09T09:56:45.554597+0000","last_became_peered":"2023-10-09T09:56:45.554597+0000","last_unstale":"2023-10-09T09:57:48.621538+0000","last_undegraded":"2023-10-09T09:57:48.621538+0000","last_fullsized":"2023-10-09T09:57:48.621538+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623531+0000","last_change":"2023-10-09T09:57:49.623531+0000","last_active":"2023-10-09T09:57:49.623531+0000","last_peered":"2023-10-09T09:57:49.623531+0000","last_clean":"2023-10-09T09:57:49.623531+0000","last_became_active":"2023-10-09T09:57:49.623045+0000","last_became_peered":"2023-10-09T09:57:49.623045+0000","last_unstale":"2023-10-09T09:57:49.623531+0000","last_undegraded":"2023-10-09T09:57:49.623531+0000","last_fullsized":"2023-10-09T09:57:49.623531+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:49.655085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622370+0000","last_change":"2023-10-09T09:56:13.614416+0000","last_active":"2023-10-09T09:57:48.622370+0000","last_peered":"2023-10-09T09:57:48.622370+0000","last_clean":"2023-10-09T09:57:48.622370+0000","last_became_active":"2023-10-09T09:56:13.614050+0000","last_became_peered":"2023-10-09T09:56:13.614050+0000","last_unstale":"2023-10-09T09:57:48.622370+0000","last_undegraded":"2023-10-09T09:57:48.622370+0000","last_fullsized":"2023-10-09T09:57:48.622370+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:17.068217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617691+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T09:57:48.617691+0000","last_peered":"2023-10-09T09:57:48.617691+0000","last_clean":"2023-10-09T09:57:48.617691+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T09:57:48.617691+0000","last_undegraded":"2023-10-09T09:57:48.617691+0000","last_fullsized":"2023-10-09T09:57:48.617691+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612468+0000","last_change":"2023-10-09T09:56:45.554759+0000","last_active":"2023-10-09T09:57:49.612468+0000","last_peered":"2023-10-09T09:57:49.612468+0000","last_clean":"2023-10-09T09:57:49.612468+0000","last_became_active":"2023-10-09T09:56:45.554390+0000","last_became_peered":"2023-10-09T09:56:45.554390+0000","last_unstale":"2023-10-09T09:57:49.612468+0000","last_undegraded":"2023-10-09T09:57:49.612468+0000","last_fullsized":"2023-10-09T09:57:49.612468+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:43.005737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622066+0000","last_change":"2023-10-09T09:57:18.844262+0000","last_active":"2023-10-09T09:57:48.622066+0000","last_peered":"2023-10-09T09:57:48.622066+0000","last_clean":"2023-10-09T09:57:48.622066+0000","last_became_active":"2023-10-09T09:57:18.843935+0000","last_became_peered":"2023-10-09T09:57:18.843935+0000","last_unstale":"2023-10-09T09:57:48.622066+0000","last_undegraded":"2023-10-09T09:57:48.622066+0000","last_fullsized":"2023-10-09T09:57:48.622066+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624797+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T09:57:49.624797+0000","last_peered":"2023-10-09T09:57:49.624797+0000","last_clean":"2023-10-09T09:57:49.624797+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T09:57:49.624797+0000","last_undegraded":"2023-10-09T09:57:49.624797+0000","last_fullsized":"2023-10-09T09:57:49.624797+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622011+0000","last_change":"2023-10-09T09:56:45.549837+0000","last_active":"2023-10-09T09:57:48.622011+0000","last_peered":"2023-10-09T09:57:48.622011+0000","last_clean":"2023-10-09T09:57:48.622011+0000","last_became_active":"2023-10-09T09:56:45.549546+0000","last_became_peered":"2023-10-09T09:56:45.549546+0000","last_unstale":"2023-10-09T09:57:48.622011+0000","last_undegraded":"2023-10-09T09:57:48.622011+0000","last_fullsized":"2023-10-09T09:57:48.622011+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:26.779071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624047+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T09:57:49.624047+0000","last_peered":"2023-10-09T09:57:49.624047+0000","last_clean":"2023-10-09T09:57:49.624047+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T09:57:49.624047+0000","last_undegraded":"2023-10-09T09:57:49.624047+0000","last_fullsized":"2023-10-09T09:57:49.624047+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.611604+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T09:57:49.611604+0000","last_peered":"2023-10-09T09:57:49.611604+0000","last_clean":"2023-10-09T09:57:49.611604+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T09:57:49.611604+0000","last_undegraded":"2023-10-09T09:57:49.611604+0000","last_fullsized":"2023-10-09T09:57:49.611604+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626860+0000","last_change":"2023-10-09T09:57:18.844843+0000","last_active":"2023-10-09T09:57:48.626860+0000","last_peered":"2023-10-09T09:57:48.626860+0000","last_clean":"2023-10-09T09:57:48.626860+0000","last_became_active":"2023-10-09T09:57:18.844530+0000","last_became_peered":"2023-10-09T09:57:18.844530+0000","last_unstale":"2023-10-09T09:57:48.626860+0000","last_undegraded":"2023-10-09T09:57:48.626860+0000","last_fullsized":"2023-10-09T09:57:48.626860+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:33.761788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625926+0000","last_change":"2023-10-09T09:56:13.614282+0000","last_active":"2023-10-09T09:57:48.625926+0000","last_peered":"2023-10-09T09:57:48.625926+0000","last_clean":"2023-10-09T09:57:48.625926+0000","last_became_active":"2023-10-09T09:56:13.613888+0000","last_became_peered":"2023-10-09T09:56:13.613888+0000","last_unstale":"2023-10-09T09:57:48.625926+0000","last_undegraded":"2023-10-09T09:57:48.625926+0000","last_fullsized":"2023-10-09T09:57:48.625926+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":71,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612551+0000","last_change":"2023-10-09T09:57:18.840413+0000","last_active":"2023-10-09T09:57:49.612551+0000","last_peered":"2023-10-09T09:57:49.612551+0000","last_clean":"2023-10-09T09:57:49.612551+0000","last_became_active":"2023-10-09T09:57:18.840093+0000","last_became_peered":"2023-10-09T09:57:18.840093+0000","last_unstale":"2023-10-09T09:57:49.612551+0000","last_undegraded":"2023-10-09T09:57:49.612551+0000","last_fullsized":"2023-10-09T09:57:49.612551+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:58.170995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626616+0000","last_change":"2023-10-09T09:57:49.624279+0000","last_active":"2023-10-09T09:57:49.626616+0000","last_peered":"2023-10-09T09:57:49.626616+0000","last_clean":"2023-10-09T09:57:49.626616+0000","last_became_active":"2023-10-09T09:57:49.624010+0000","last_became_peered":"2023-10-09T09:57:49.624010+0000","last_unstale":"2023-10-09T09:57:49.626616+0000","last_undegraded":"2023-10-09T09:57:49.626616+0000","last_fullsized":"2023-10-09T09:57:49.626616+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:47.717690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626997+0000","last_change":"2023-10-09T09:56:45.555907+0000","last_active":"2023-10-09T09:57:48.626997+0000","last_peered":"2023-10-09T09:57:48.626997+0000","last_clean":"2023-10-09T09:57:48.626997+0000","last_became_active":"2023-10-09T09:56:45.555548+0000","last_became_peered":"2023-10-09T09:56:45.555548+0000","last_unstale":"2023-10-09T09:57:48.626997+0000","last_undegraded":"2023-10-09T09:57:48.626997+0000","last_fullsized":"2023-10-09T09:57:48.626997+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622200+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T09:57:48.622200+0000","last_peered":"2023-10-09T09:57:48.622200+0000","last_clean":"2023-10-09T09:57:48.622200+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T09:57:48.622200+0000","last_undegraded":"2023-10-09T09:57:48.622200+0000","last_fullsized":"2023-10-09T09:57:48.622200+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622586+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T09:57:49.622586+0000","last_peered":"2023-10-09T09:57:49.622586+0000","last_clean":"2023-10-09T09:57:49.622586+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T09:57:49.622586+0000","last_undegraded":"2023-10-09T09:57:49.622586+0000","last_fullsized":"2023-10-09T09:57:49.622586+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":296,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626889+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T09:57:48.626889+0000","last_peered":"2023-10-09T09:57:48.626889+0000","last_clean":"2023-10-09T09:57:48.626889+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T09:57:48.626889+0000","last_undegraded":"2023-10-09T09:57:48.626889+0000","last_fullsized":"2023-10-09T09:57:48.626889+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619019+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:57:48.619019+0000","last_peered":"2023-10-09T09:57:48.619019+0000","last_clean":"2023-10-09T09:57:48.619019+0000","last_became_active":"2023-10-09T09:57:18.841472+0000","last_became_peered":"2023-10-09T09:57:18.841472+0000","last_unstale":"2023-10-09T09:57:48.619019+0000","last_undegraded":"2023-10-09T09:57:48.619019+0000","last_fullsized":"2023-10-09T09:57:48.619019+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622632+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T09:57:48.622632+0000","last_peered":"2023-10-09T09:57:48.622632+0000","last_clean":"2023-10-09T09:57:48.622632+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T09:57:48.622632+0000","last_undegraded":"2023-10-09T09:57:48.622632+0000","last_fullsized":"2023-10-09T09:57:48.622632+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.627470+0000","last_change":"2023-10-09T09:57:18.837334+0000","last_active":"2023-10-09T09:57:48.627470+0000","last_peered":"2023-10-09T09:57:48.627470+0000","last_clean":"2023-10-09T09:57:48.627470+0000","last_became_active":"2023-10-09T09:57:18.837037+0000","last_became_peered":"2023-10-09T09:57:18.837037+0000","last_unstale":"2023-10-09T09:57:48.627470+0000","last_undegraded":"2023-10-09T09:57:48.627470+0000","last_fullsized":"2023-10-09T09:57:48.627470+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:11.650333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":83,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621174+0000","last_change":"2023-10-09T09:57:49.621174+0000","last_active":"2023-10-09T09:57:49.621174+0000","last_peered":"2023-10-09T09:57:49.621174+0000","last_clean":"2023-10-09T09:57:49.621174+0000","last_became_active":"2023-10-09T09:57:49.620780+0000","last_became_peered":"2023-10-09T09:57:49.620780+0000","last_unstale":"2023-10-09T09:57:49.621174+0000","last_undegraded":"2023-10-09T09:57:49.621174+0000","last_fullsized":"2023-10-09T09:57:49.621174+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:01.389768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621680+0000","last_change":"2023-10-09T09:57:49.621680+0000","last_active":"2023-10-09T09:57:49.621680+0000","last_peered":"2023-10-09T09:57:49.621680+0000","last_clean":"2023-10-09T09:57:49.621680+0000","last_became_active":"2023-10-09T09:57:49.621345+0000","last_became_peered":"2023-10-09T09:57:49.621345+0000","last_unstale":"2023-10-09T09:57:49.621680+0000","last_undegraded":"2023-10-09T09:57:49.621680+0000","last_fullsized":"2023-10-09T09:57:49.621680+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:58.483689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617369+0000","last_change":"2023-10-09T09:56:13.613717+0000","last_active":"2023-10-09T09:57:48.617369+0000","last_peered":"2023-10-09T09:57:48.617369+0000","last_clean":"2023-10-09T09:57:48.617369+0000","last_became_active":"2023-10-09T09:56:13.613329+0000","last_became_peered":"2023-10-09T09:56:13.613329+0000","last_unstale":"2023-10-09T09:57:48.617369+0000","last_undegraded":"2023-10-09T09:57:48.617369+0000","last_fullsized":"2023-10-09T09:57:48.617369+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619103+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T09:57:48.619103+0000","last_peered":"2023-10-09T09:57:48.619103+0000","last_clean":"2023-10-09T09:57:48.619103+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T09:57:48.619103+0000","last_undegraded":"2023-10-09T09:57:48.619103+0000","last_fullsized":"2023-10-09T09:57:48.619103+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.618192+0000","last_change":"2023-10-09T09:57:18.842325+0000","last_active":"2023-10-09T09:57:48.618192+0000","last_peered":"2023-10-09T09:57:48.618192+0000","last_clean":"2023-10-09T09:57:48.618192+0000","last_became_active":"2023-10-09T09:57:18.841987+0000","last_became_peered":"2023-10-09T09:57:18.841987+0000","last_unstale":"2023-10-09T09:57:48.618192+0000","last_undegraded":"2023-10-09T09:57:48.618192+0000","last_fullsized":"2023-10-09T09:57:48.618192+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:28.615689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.628678+0000","last_change":"2023-10-09T09:57:49.628678+0000","last_active":"2023-10-09T09:57:49.628678+0000","last_peered":"2023-10-09T09:57:49.628678+0000","last_clean":"2023-10-09T09:57:49.628678+0000","last_became_active":"2023-10-09T09:57:49.628332+0000","last_became_peered":"2023-10-09T09:57:49.628332+0000","last_unstale":"2023-10-09T09:57:49.628678+0000","last_undegraded":"2023-10-09T09:57:49.628678+0000","last_fullsized":"2023-10-09T09:57:49.628678+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626775+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T09:57:48.626775+0000","last_peered":"2023-10-09T09:57:48.626775+0000","last_clean":"2023-10-09T09:57:48.626775+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T09:57:48.626775+0000","last_undegraded":"2023-10-09T09:57:48.626775+0000","last_fullsized":"2023-10-09T09:57:48.626775+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622055+0000","last_change":"2023-10-09T09:57:49.622055+0000","last_active":"2023-10-09T09:57:49.622055+0000","last_peered":"2023-10-09T09:57:49.622055+0000","last_clean":"2023-10-09T09:57:49.622055+0000","last_became_active":"2023-10-09T09:57:49.621784+0000","last_became_peered":"2023-10-09T09:57:49.621784+0000","last_unstale":"2023-10-09T09:57:49.622055+0000","last_undegraded":"2023-10-09T09:57:49.622055+0000","last_fullsized":"2023-10-09T09:57:49.622055+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:50.025391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624469+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T09:57:49.624469+0000","last_peered":"2023-10-09T09:57:49.624469+0000","last_clean":"2023-10-09T09:57:49.624469+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T09:57:49.624469+0000","last_undegraded":"2023-10-09T09:57:49.624469+0000","last_fullsized":"2023-10-09T09:57:49.624469+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619190+0000","last_change":"2023-10-09T09:56:45.555100+0000","last_active":"2023-10-09T09:57:48.619190+0000","last_peered":"2023-10-09T09:57:48.619190+0000","last_clean":"2023-10-09T09:57:48.619190+0000","last_became_active":"2023-10-09T09:56:45.554704+0000","last_became_peered":"2023-10-09T09:56:45.554704+0000","last_unstale":"2023-10-09T09:57:48.619190+0000","last_undegraded":"2023-10-09T09:57:48.619190+0000","last_fullsized":"2023-10-09T09:57:48.619190+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.616656+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:57:48.616656+0000","last_peered":"2023-10-09T09:57:48.616656+0000","last_clean":"2023-10-09T09:57:48.616656+0000","last_became_active":"2023-10-09T09:57:18.841522+0000","last_became_peered":"2023-10-09T09:57:18.841522+0000","last_unstale":"2023-10-09T09:57:48.616656+0000","last_undegraded":"2023-10-09T09:57:48.616656+0000","last_fullsized":"2023-10-09T09:57:48.616656+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.618427+0000","last_change":"2023-10-09T09:57:18.845748+0000","last_active":"2023-10-09T09:57:48.618427+0000","last_peered":"2023-10-09T09:57:48.618427+0000","last_clean":"2023-10-09T09:57:48.618427+0000","last_became_active":"2023-10-09T09:57:18.845397+0000","last_became_peered":"2023-10-09T09:57:18.845397+0000","last_unstale":"2023-10-09T09:57:48.618427+0000","last_undegraded":"2023-10-09T09:57:48.618427+0000","last_fullsized":"2023-10-09T09:57:48.618427+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622270+0000","last_change":"2023-10-09T09:56:45.556627+0000","last_active":"2023-10-09T09:57:48.622270+0000","last_peered":"2023-10-09T09:57:48.622270+0000","last_clean":"2023-10-09T09:57:48.622270+0000","last_became_active":"2023-10-09T09:56:45.556265+0000","last_became_peered":"2023-10-09T09:56:45.556265+0000","last_unstale":"2023-10-09T09:57:48.622270+0000","last_undegraded":"2023-10-09T09:57:48.622270+0000","last_fullsized":"2023-10-09T09:57:48.622270+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621587+0000","last_change":"2023-10-09T09:57:49.621587+0000","last_active":"2023-10-09T09:57:49.621587+0000","last_peered":"2023-10-09T09:57:49.621587+0000","last_clean":"2023-10-09T09:57:49.621587+0000","last_became_active":"2023-10-09T09:57:49.621245+0000","last_became_peered":"2023-10-09T09:57:49.621245+0000","last_unstale":"2023-10-09T09:57:49.621587+0000","last_undegraded":"2023-10-09T09:57:49.621587+0000","last_fullsized":"2023-10-09T09:57:49.621587+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620199+0000","last_change":"2023-10-09T09:57:49.620199+0000","last_active":"2023-10-09T09:57:49.620199+0000","last_peered":"2023-10-09T09:57:49.620199+0000","last_clean":"2023-10-09T09:57:49.620199+0000","last_became_active":"2023-10-09T09:57:49.619695+0000","last_became_peered":"2023-10-09T09:57:49.619695+0000","last_unstale":"2023-10-09T09:57:49.620199+0000","last_undegraded":"2023-10-09T09:57:49.620199+0000","last_fullsized":"2023-10-09T09:57:49.620199+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:48.818881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.611800+0000","last_change":"2023-10-09T09:56:45.553024+0000","last_active":"2023-10-09T09:57:49.611800+0000","last_peered":"2023-10-09T09:57:49.611800+0000","last_clean":"2023-10-09T09:57:49.611800+0000","last_became_active":"2023-10-09T09:56:45.552670+0000","last_became_peered":"2023-10-09T09:56:45.552670+0000","last_unstale":"2023-10-09T09:57:49.611800+0000","last_undegraded":"2023-10-09T09:57:49.611800+0000","last_fullsized":"2023-10-09T09:57:49.611800+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:58.132008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627539+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T09:57:49.627539+0000","last_peered":"2023-10-09T09:57:49.627539+0000","last_clean":"2023-10-09T09:57:49.627539+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T09:57:49.627539+0000","last_undegraded":"2023-10-09T09:57:49.627539+0000","last_fullsized":"2023-10-09T09:57:49.627539+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621676+0000","last_change":"2023-10-09T09:56:45.556365+0000","last_active":"2023-10-09T09:57:48.621676+0000","last_peered":"2023-10-09T09:57:48.621676+0000","last_clean":"2023-10-09T09:57:48.621676+0000","last_became_active":"2023-10-09T09:56:45.555832+0000","last_became_peered":"2023-10-09T09:56:45.555832+0000","last_unstale":"2023-10-09T09:57:48.621676+0000","last_undegraded":"2023-10-09T09:57:48.621676+0000","last_fullsized":"2023-10-09T09:57:48.621676+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626073+0000","last_change":"2023-10-09T09:48:25.514756+0000","last_active":"2023-10-09T09:57:48.626073+0000","last_peered":"2023-10-09T09:57:48.626073+0000","last_clean":"2023-10-09T09:57:48.626073+0000","last_became_active":"2023-10-09T09:48:25.514387+0000","last_became_peered":"2023-10-09T09:48:25.514387+0000","last_unstale":"2023-10-09T09:57:48.626073+0000","last_undegraded":"2023-10-09T09:57:48.626073+0000","last_fullsized":"2023-10-09T09:57:48.626073+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627073+0000","last_change":"2023-10-09T09:57:49.627073+0000","last_active":"2023-10-09T09:57:49.627073+0000","last_peered":"2023-10-09T09:57:49.627073+0000","last_clean":"2023-10-09T09:57:49.627073+0000","last_became_active":"2023-10-09T09:57:49.626706+0000","last_became_peered":"2023-10-09T09:57:49.626706+0000","last_unstale":"2023-10-09T09:57:49.627073+0000","last_undegraded":"2023-10-09T09:57:49.627073+0000","last_fullsized":"2023-10-09T09:57:49.627073+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625250+0000","last_change":"2023-10-09T09:56:45.554326+0000","last_active":"2023-10-09T09:57:48.625250+0000","last_peered":"2023-10-09T09:57:48.625250+0000","last_clean":"2023-10-09T09:57:48.625250+0000","last_became_active":"2023-10-09T09:56:45.553876+0000","last_became_peered":"2023-10-09T09:56:45.553876+0000","last_unstale":"2023-10-09T09:57:48.625250+0000","last_undegraded":"2023-10-09T09:57:48.625250+0000","last_fullsized":"2023-10-09T09:57:48.625250+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619582+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T09:57:48.619582+0000","last_peered":"2023-10-09T09:57:48.619582+0000","last_clean":"2023-10-09T09:57:48.619582+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T09:57:48.619582+0000","last_undegraded":"2023-10-09T09:57:48.619582+0000","last_fullsized":"2023-10-09T09:57:48.619582+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.624744+0000","last_change":"2023-10-09T09:56:13.614485+0000","last_active":"2023-10-09T09:57:48.624744+0000","last_peered":"2023-10-09T09:57:48.624744+0000","last_clean":"2023-10-09T09:57:48.624744+0000","last_became_active":"2023-10-09T09:56:13.614109+0000","last_became_peered":"2023-10-09T09:56:13.614109+0000","last_unstale":"2023-10-09T09:57:48.624744+0000","last_undegraded":"2023-10-09T09:57:48.624744+0000","last_fullsized":"2023-10-09T09:57:48.624744+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623806+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T09:57:49.623806+0000","last_peered":"2023-10-09T09:57:49.623806+0000","last_clean":"2023-10-09T09:57:49.623806+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T09:57:49.623806+0000","last_undegraded":"2023-10-09T09:57:49.623806+0000","last_fullsized":"2023-10-09T09:57:49.623806+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620753+0000","last_change":"2023-10-09T09:56:13.623204+0000","last_active":"2023-10-09T09:57:48.620753+0000","last_peered":"2023-10-09T09:57:48.620753+0000","last_clean":"2023-10-09T09:57:48.620753+0000","last_became_active":"2023-10-09T09:56:13.622853+0000","last_became_peered":"2023-10-09T09:56:13.622853+0000","last_unstale":"2023-10-09T09:57:48.620753+0000","last_undegraded":"2023-10-09T09:57:48.620753+0000","last_fullsized":"2023-10-09T09:57:48.620753+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622372+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T09:57:49.622372+0000","last_peered":"2023-10-09T09:57:49.622372+0000","last_clean":"2023-10-09T09:57:49.622372+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T09:57:49.622372+0000","last_undegraded":"2023-10-09T09:57:49.622372+0000","last_fullsized":"2023-10-09T09:57:49.622372+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625768+0000","last_change":"2023-10-09T09:48:25.523230+0000","last_active":"2023-10-09T09:57:48.625768+0000","last_peered":"2023-10-09T09:57:48.625768+0000","last_clean":"2023-10-09T09:57:48.625768+0000","last_became_active":"2023-10-09T09:48:25.522608+0000","last_became_peered":"2023-10-09T09:48:25.522608+0000","last_unstale":"2023-10-09T09:57:48.625768+0000","last_undegraded":"2023-10-09T09:57:48.625768+0000","last_fullsized":"2023-10-09T09:57:48.625768+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627770+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T09:57:49.627770+0000","last_peered":"2023-10-09T09:57:49.627770+0000","last_clean":"2023-10-09T09:57:49.627770+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T09:57:49.627770+0000","last_undegraded":"2023-10-09T09:57:49.627770+0000","last_fullsized":"2023-10-09T09:57:49.627770+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624030+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T09:57:49.624030+0000","last_peered":"2023-10-09T09:57:49.624030+0000","last_clean":"2023-10-09T09:57:49.624030+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T09:57:49.624030+0000","last_undegraded":"2023-10-09T09:57:49.624030+0000","last_fullsized":"2023-10-09T09:57:49.624030+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622141+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T09:57:49.622141+0000","last_peered":"2023-10-09T09:57:49.622141+0000","last_clean":"2023-10-09T09:57:49.622141+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T09:57:49.622141+0000","last_undegraded":"2023-10-09T09:57:49.622141+0000","last_fullsized":"2023-10-09T09:57:49.622141+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.624235+0000","last_change":"2023-10-09T09:57:18.843731+0000","last_active":"2023-10-09T09:57:48.624235+0000","last_peered":"2023-10-09T09:57:48.624235+0000","last_clean":"2023-10-09T09:57:48.624235+0000","last_became_active":"2023-10-09T09:57:18.843280+0000","last_became_peered":"2023-10-09T09:57:18.843280+0000","last_unstale":"2023-10-09T09:57:48.624235+0000","last_undegraded":"2023-10-09T09:57:48.624235+0000","last_fullsized":"2023-10-09T09:57:48.624235+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617918+0000","last_change":"2023-10-09T09:56:45.551325+0000","last_active":"2023-10-09T09:57:48.617918+0000","last_peered":"2023-10-09T09:57:48.617918+0000","last_clean":"2023-10-09T09:57:48.617918+0000","last_became_active":"2023-10-09T09:56:45.550946+0000","last_became_peered":"2023-10-09T09:56:45.550946+0000","last_unstale":"2023-10-09T09:57:48.617918+0000","last_undegraded":"2023-10-09T09:57:48.617918+0000","last_fullsized":"2023-10-09T09:57:48.617918+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":69,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617387+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T09:57:48.617387+0000","last_peered":"2023-10-09T09:57:48.617387+0000","last_clean":"2023-10-09T09:57:48.617387+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T09:57:48.617387+0000","last_undegraded":"2023-10-09T09:57:48.617387+0000","last_fullsized":"2023-10-09T09:57:48.617387+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621810+0000","last_change":"2023-10-09T09:57:18.840471+0000","last_active":"2023-10-09T09:57:48.621810+0000","last_peered":"2023-10-09T09:57:48.621810+0000","last_clean":"2023-10-09T09:57:48.621810+0000","last_became_active":"2023-10-09T09:57:18.840125+0000","last_became_peered":"2023-10-09T09:57:18.840125+0000","last_unstale":"2023-10-09T09:57:48.621810+0000","last_undegraded":"2023-10-09T09:57:48.621810+0000","last_fullsized":"2023-10-09T09:57:48.621810+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:07.418455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626281+0000","last_change":"2023-10-09T09:57:49.626281+0000","last_active":"2023-10-09T09:57:49.626281+0000","last_peered":"2023-10-09T09:57:49.626281+0000","last_clean":"2023-10-09T09:57:49.626281+0000","last_became_active":"2023-10-09T09:57:49.625726+0000","last_became_peered":"2023-10-09T09:57:49.625726+0000","last_unstale":"2023-10-09T09:57:49.626281+0000","last_undegraded":"2023-10-09T09:57:49.626281+0000","last_fullsized":"2023-10-09T09:57:49.626281+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625511+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T09:57:49.625511+0000","last_peered":"2023-10-09T09:57:49.625511+0000","last_clean":"2023-10-09T09:57:49.625511+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T09:57:49.625511+0000","last_undegraded":"2023-10-09T09:57:49.625511+0000","last_fullsized":"2023-10-09T09:57:49.625511+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620020+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T09:57:48.620020+0000","last_peered":"2023-10-09T09:57:48.620020+0000","last_clean":"2023-10-09T09:57:48.620020+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T09:57:48.620020+0000","last_undegraded":"2023-10-09T09:57:48.620020+0000","last_fullsized":"2023-10-09T09:57:48.620020+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624665+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T09:57:49.624665+0000","last_peered":"2023-10-09T09:57:49.624665+0000","last_clean":"2023-10-09T09:57:49.624665+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T09:57:49.624665+0000","last_undegraded":"2023-10-09T09:57:49.624665+0000","last_fullsized":"2023-10-09T09:57:49.624665+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626421+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T09:57:49.626421+0000","last_peered":"2023-10-09T09:57:49.626421+0000","last_clean":"2023-10-09T09:57:49.626421+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T09:57:49.626421+0000","last_undegraded":"2023-10-09T09:57:49.626421+0000","last_fullsized":"2023-10-09T09:57:49.626421+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622159+0000","last_change":"2023-10-09T09:57:18.838985+0000","last_active":"2023-10-09T09:57:48.622159+0000","last_peered":"2023-10-09T09:57:48.622159+0000","last_clean":"2023-10-09T09:57:48.622159+0000","last_became_active":"2023-10-09T09:57:18.838625+0000","last_became_peered":"2023-10-09T09:57:18.838625+0000","last_unstale":"2023-10-09T09:57:48.622159+0000","last_undegraded":"2023-10-09T09:57:48.622159+0000","last_fullsized":"2023-10-09T09:57:48.622159+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624475+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T09:57:49.624475+0000","last_peered":"2023-10-09T09:57:49.624475+0000","last_clean":"2023-10-09T09:57:49.624475+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T09:57:49.624475+0000","last_undegraded":"2023-10-09T09:57:49.624475+0000","last_fullsized":"2023-10-09T09:57:49.624475+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625707+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T09:57:49.625707+0000","last_peered":"2023-10-09T09:57:49.625707+0000","last_clean":"2023-10-09T09:57:49.625707+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T09:57:49.625707+0000","last_undegraded":"2023-10-09T09:57:49.625707+0000","last_fullsized":"2023-10-09T09:57:49.625707+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621495+0000","last_change":"2023-10-09T09:57:49.621495+0000","last_active":"2023-10-09T09:57:49.621495+0000","last_peered":"2023-10-09T09:57:49.621495+0000","last_clean":"2023-10-09T09:57:49.621495+0000","last_became_active":"2023-10-09T09:57:49.621144+0000","last_became_peered":"2023-10-09T09:57:49.621144+0000","last_unstale":"2023-10-09T09:57:49.621495+0000","last_undegraded":"2023-10-09T09:57:49.621495+0000","last_fullsized":"2023-10-09T09:57:49.621495+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:11.852424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622753+0000","last_change":"2023-10-09T09:57:49.622753+0000","last_active":"2023-10-09T09:57:49.622753+0000","last_peered":"2023-10-09T09:57:49.622753+0000","last_clean":"2023-10-09T09:57:49.622753+0000","last_became_active":"2023-10-09T09:57:49.622463+0000","last_became_peered":"2023-10-09T09:57:49.622463+0000","last_unstale":"2023-10-09T09:57:49.622753+0000","last_undegraded":"2023-10-09T09:57:49.622753+0000","last_fullsized":"2023-10-09T09:57:49.622753+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:51.020605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":78,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617029+0000","last_change":"2023-10-09T09:57:18.841362+0000","last_active":"2023-10-09T09:57:48.617029+0000","last_peered":"2023-10-09T09:57:48.617029+0000","last_clean":"2023-10-09T09:57:48.617029+0000","last_became_active":"2023-10-09T09:57:18.840951+0000","last_became_peered":"2023-10-09T09:57:18.840951+0000","last_unstale":"2023-10-09T09:57:48.617029+0000","last_undegraded":"2023-10-09T09:57:48.617029+0000","last_fullsized":"2023-10-09T09:57:48.617029+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430345,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310920,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434232,"statfs":{"total":95995035648,"available":95676653568,"internally_reserved":0,"allocated":1572864,"data_stored":1111479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58399999999999996,"15min":0.58999999999999997},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.443},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.41799999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73599999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.496}]},{"osd":1,"last update":"Mon Oct 9 09:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.57999999999999996,"15min":0.53700000000000003},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.36099999999999999},"max":{"1min":0.81399999999999995,"5min":0.89700000000000002,"15min":0.91600000000000004},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.56499999999999995},"min":{"1min":0.39700000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.76800000000000002,"5min":0.78100000000000003,"15min":0.83799999999999997},"last":0.47899999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.46000000000000002},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.71099999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.51100000000000001,"15min":0.47299999999999998},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":0.68100000000000005,"5min":0.87,"15min":0.87},"last":0.56999999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52000000000000002,"15min":0.498},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66600000000000004,"5min":0.79600000000000004,"15min":0.83599999999999997},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.36599999999999999,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":0.78600000000000003,"5min":0.81200000000000006,"15min":0.85399999999999998},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57599999999999996,"15min":0.54300000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.32900000000000001},"max":{"1min":0.78300000000000003,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58399999999999996,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.40300000000000002,"15min":0.33100000000000002},"max":{"1min":0.75,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.60699999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626573,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1556480,"data_stored":1111018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71999999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.51900000000000002,"15min":0.51000000000000001},"min":{"1min":0.42299999999999999,"5min":0.308,"15min":0.308},"max":{"1min":0.749,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.55700000000000005}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.61799999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.68300000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:57:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.57999999999999996,"15min":0.55000000000000004},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.33100000000000002},"max":{"1min":0.79300000000000004,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.60099999999999998,"15min":0.56299999999999994},"min":{"1min":0.39300000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.0169999999999999,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.627}]},{"osd":4,"last update":"Mon Oct 9 09:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46700000000000003,"15min":0.45000000000000001},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.23300000000000001},"max":{"1min":0.64500000000000002,"5min":0.75800000000000001,"15min":0.77600000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.496,"15min":0.48899999999999999},"min":{"1min":0.36299999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.66800000000000004,"5min":0.753,"15min":0.75600000000000001},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.50600000000000001,"15min":0.503},"min":{"1min":0.36599999999999999,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":0.63600000000000001,"5min":0.76100000000000001,"15min":0.80600000000000005},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51700000000000002,"15min":0.505},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68500000000000005,"5min":0.76800000000000002,"15min":1.0600000000000001},"last":0.47099999999999997}]},{"osd":7,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.54400000000000004,"15min":0.52600000000000002},"min":{"1min":0.35099999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.68000000000000005,"5min":0.79400000000000004,"15min":0.81399999999999995},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.627,"5min":0.61499999999999999,"15min":0.58699999999999997},"min":{"1min":0.52600000000000002,"5min":0.34899999999999998,"15min":0.34399999999999997},"max":{"1min":0.754,"5min":0.82999999999999996,"15min":0.84099999999999997},"last":0.66300000000000003}]}]},{"osd":1,"up_from":84,"seq":360777252886,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28308,"kb_used_data":972,"kb_used_omap":9,"kb_used_meta":27318,"kb_avail":93716844,"statfs":{"total":95995035648,"available":95966048256,"internally_reserved":0,"allocated":995328,"data_stored":524748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":27974633},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.31},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.252}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.33600000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.51800000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.53100000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.47199999999999998}]}]},{"osd":0,"up_from":79,"seq":339302416413,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33056,"kb_used_data":1560,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712096,"statfs":{"total":95995035648,"available":95961186304,"internally_reserved":0,"allocated":1597440,"data_stored":1114630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.39000000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:57:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.66900000000000004,"15min":0.66100000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68200000000000005,"15min":0.67800000000000005},"min":{"1min":0.44600000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.028,"5min":1.101,"15min":1.101},"last":0.68500000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64600000000000002,"15min":0.64400000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85799999999999998,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.94499999999999995,"5min":1.1180000000000001,"15min":1.1180000000000001},"last":0.63300000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.68300000000000005,"15min":0.68700000000000006},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89700000000000002,"5min":1.161,"15min":1.161},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.68899999999999995,"15min":0.69099999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.99199999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.59999999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.66500000000000004,"15min":0.66600000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.89600000000000002,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.68400000000000005,"15min":0.68300000000000005},"min":{"1min":0.441,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.98099999999999998,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.67600000000000005}]}]},{"osd":2,"up_from":89,"seq":382252089360,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36624,"kb_used_data":968,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708528,"statfs":{"total":95995035648,"available":95957532672,"internally_reserved":0,"allocated":991232,"data_stored":522739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.61499999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.68400000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.86799999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.85999999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.90200000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.83099999999999996}]}]},{"osd":3,"up_from":94,"seq":403726925833,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28288,"kb_used_data":952,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716864,"statfs":{"total":95995035648,"available":95966068736,"internally_reserved":0,"allocated":974848,"data_stored":520694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.311}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51400000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019029,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302144,"kb_used_data":952,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443008,"statfs":{"total":95995035648,"available":95685640192,"internally_reserved":0,"allocated":974848,"data_stored":521041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.66800000000000004,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56699999999999995,"15min":0.56999999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64300000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.56699999999999995}]},{"osd":2,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.437},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.59999999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:56:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.28199999999999997},"max":{"1min":0.753,"5min":0.877,"15min":2.2109999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.47899999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.75,"5min":0.81399999999999995,"15min":0.997},"last":0.48099999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.45600000000000002,"15min":0.43099999999999999},"min":{"1min":0.34399999999999997,"5min":0.28399999999999997,"15min":0.27100000000000002},"max":{"1min":0.77200000000000002,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.48999999999999999,"15min":0.46600000000000003},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70999999999999996,"5min":0.72099999999999997,"15min":0.83999999999999997},"last":0.39300000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.499,"15min":0.46800000000000003},"min":{"1min":0.41199999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78600000000000003,"5min":0.79300000000000004,"15min":0.86599999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53200000000000003,"15min":0.497},"min":{"1min":0.379,"5min":0.318,"15min":0.312},"max":{"1min":0.80700000000000005,"5min":0.81499999999999995,"15min":0.89900000000000002},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55800000000000005,"15min":0.52500000000000002},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.69899999999999995,"5min":0.76600000000000001,"15min":0.84999999999999998},"last":0.496},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59699999999999998,"15min":0.55800000000000005},"min":{"1min":0.45800000000000002,"5min":0.33700000000000002,"15min":0.316},"max":{"1min":0.76000000000000001,"5min":0.82899999999999996,"15min":0.85199999999999998},"last":0.53000000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822801,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310320,"kb_used_data":936,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434832,"statfs":{"total":95995035648,"available":95677267968,"internally_reserved":0,"allocated":958464,"data_stored":516785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.75,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.55000000000000004,"15min":0.54100000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.4289999999999998,"5min":2.4289999999999998,"15min":2.4289999999999998},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.495},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.47699999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":2.5950000000000002,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.50900000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.56899999999999995,"15min":0.54700000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":0.70699999999999996,"5min":0.84999999999999998,"15min":1.1479999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.60699999999999998,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":2.637},"last":0.436}]},{"osd":4,"last update":"Mon Oct 9 09:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47699999999999998,"15min":0.44400000000000001},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":0.74299999999999999,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.51000000000000001,"15min":0.46800000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64900000000000002,"5min":0.79600000000000004,"15min":0.88200000000000001},"last":0.41699999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.53000000000000003,"15min":0.503},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":0.76200000000000001,"5min":0.85099999999999998,"15min":0.92800000000000005},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.52100000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81000000000000005,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.376}]},{"osd":7,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58099999999999996,"15min":0.55500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":0.71299999999999997,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55000000000000004,"15min":0.53600000000000003},"min":{"1min":0.37,"5min":0.318,"15min":0.317},"max":{"1min":0.68200000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.54200000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:58:29.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:58:29.563 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:29.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:29.980 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:30.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:30 smithi012 ceph-mon[147926]: from='client.44612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:30.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:30 smithi012 ceph-mon[147926]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:30.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:30 smithi012 ceph-mon[152263]: from='client.44612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:30.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:30 smithi012 ceph-mon[152263]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:30.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:30 smithi110 ceph-mon[136032]: from='client.44612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:30.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:30 smithi110 ceph-mon[136032]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:31.404 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:58:31.405 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:58:31.812 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":219,"stamp":"2023-10-09T09:58:30.905806+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1348276,"kb_used_data":9396,"kb_used_omap":126,"kb_used_meta":1338689,"kb_avail":748612940,"statfs":{"total":767960285184,"available":766579650560,"internally_reserved":0,"allocated":9621504,"data_stored":5943134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1370818171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002869"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.616419+0000","last_change":"2023-10-09T09:56:45.556456+0000","last_active":"2023-10-09T09:57:48.616419+0000","last_peered":"2023-10-09T09:57:48.616419+0000","last_clean":"2023-10-09T09:57:48.616419+0000","last_became_active":"2023-10-09T09:56:45.555878+0000","last_became_peered":"2023-10-09T09:56:45.555878+0000","last_unstale":"2023-10-09T09:57:48.616419+0000","last_undegraded":"2023-10-09T09:57:48.616419+0000","last_fullsized":"2023-10-09T09:57:48.616419+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612158+0000","last_change":"2023-10-09T09:57:18.842453+0000","last_active":"2023-10-09T09:57:49.612158+0000","last_peered":"2023-10-09T09:57:49.612158+0000","last_clean":"2023-10-09T09:57:49.612158+0000","last_became_active":"2023-10-09T09:57:18.842133+0000","last_became_peered":"2023-10-09T09:57:18.842133+0000","last_unstale":"2023-10-09T09:57:49.612158+0000","last_undegraded":"2023-10-09T09:57:49.612158+0000","last_fullsized":"2023-10-09T09:57:49.612158+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:07.459692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620527+0000","last_change":"2023-10-09T09:57:49.620527+0000","last_active":"2023-10-09T09:57:49.620527+0000","last_peered":"2023-10-09T09:57:49.620527+0000","last_clean":"2023-10-09T09:57:49.620527+0000","last_became_active":"2023-10-09T09:57:49.620109+0000","last_became_peered":"2023-10-09T09:57:49.620109+0000","last_unstale":"2023-10-09T09:57:49.620527+0000","last_undegraded":"2023-10-09T09:57:49.620527+0000","last_fullsized":"2023-10-09T09:57:49.620527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:09.284902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":68,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623102+0000","last_change":"2023-10-09T09:57:49.623102+0000","last_active":"2023-10-09T09:57:49.623102+0000","last_peered":"2023-10-09T09:57:49.623102+0000","last_clean":"2023-10-09T09:57:49.623102+0000","last_became_active":"2023-10-09T09:57:49.620638+0000","last_became_peered":"2023-10-09T09:57:49.620638+0000","last_unstale":"2023-10-09T09:57:49.623102+0000","last_undegraded":"2023-10-09T09:57:49.623102+0000","last_fullsized":"2023-10-09T09:57:49.623102+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:11.322675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620967+0000","last_change":"2023-10-09T09:57:49.620967+0000","last_active":"2023-10-09T09:57:49.620967+0000","last_peered":"2023-10-09T09:57:49.620967+0000","last_clean":"2023-10-09T09:57:49.620967+0000","last_became_active":"2023-10-09T09:57:49.620567+0000","last_became_peered":"2023-10-09T09:57:49.620567+0000","last_unstale":"2023-10-09T09:57:49.620967+0000","last_undegraded":"2023-10-09T09:57:49.620967+0000","last_fullsized":"2023-10-09T09:57:49.620967+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620814+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T09:57:48.620814+0000","last_peered":"2023-10-09T09:57:48.620814+0000","last_clean":"2023-10-09T09:57:48.620814+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T09:57:48.620814+0000","last_undegraded":"2023-10-09T09:57:48.620814+0000","last_fullsized":"2023-10-09T09:57:48.620814+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622634+0000","last_change":"2023-10-09T09:57:49.622634+0000","last_active":"2023-10-09T09:57:49.622634+0000","last_peered":"2023-10-09T09:57:49.622634+0000","last_clean":"2023-10-09T09:57:49.622634+0000","last_became_active":"2023-10-09T09:57:49.622247+0000","last_became_peered":"2023-10-09T09:57:49.622247+0000","last_unstale":"2023-10-09T09:57:49.622634+0000","last_undegraded":"2023-10-09T09:57:49.622634+0000","last_fullsized":"2023-10-09T09:57:49.622634+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621538+0000","last_change":"2023-10-09T09:56:45.554958+0000","last_active":"2023-10-09T09:57:48.621538+0000","last_peered":"2023-10-09T09:57:48.621538+0000","last_clean":"2023-10-09T09:57:48.621538+0000","last_became_active":"2023-10-09T09:56:45.554597+0000","last_became_peered":"2023-10-09T09:56:45.554597+0000","last_unstale":"2023-10-09T09:57:48.621538+0000","last_undegraded":"2023-10-09T09:57:48.621538+0000","last_fullsized":"2023-10-09T09:57:48.621538+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623531+0000","last_change":"2023-10-09T09:57:49.623531+0000","last_active":"2023-10-09T09:57:49.623531+0000","last_peered":"2023-10-09T09:57:49.623531+0000","last_clean":"2023-10-09T09:57:49.623531+0000","last_became_active":"2023-10-09T09:57:49.623045+0000","last_became_peered":"2023-10-09T09:57:49.623045+0000","last_unstale":"2023-10-09T09:57:49.623531+0000","last_undegraded":"2023-10-09T09:57:49.623531+0000","last_fullsized":"2023-10-09T09:57:49.623531+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:49.655085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622370+0000","last_change":"2023-10-09T09:56:13.614416+0000","last_active":"2023-10-09T09:57:48.622370+0000","last_peered":"2023-10-09T09:57:48.622370+0000","last_clean":"2023-10-09T09:57:48.622370+0000","last_became_active":"2023-10-09T09:56:13.614050+0000","last_became_peered":"2023-10-09T09:56:13.614050+0000","last_unstale":"2023-10-09T09:57:48.622370+0000","last_undegraded":"2023-10-09T09:57:48.622370+0000","last_fullsized":"2023-10-09T09:57:48.622370+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:17.068217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617691+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T09:57:48.617691+0000","last_peered":"2023-10-09T09:57:48.617691+0000","last_clean":"2023-10-09T09:57:48.617691+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T09:57:48.617691+0000","last_undegraded":"2023-10-09T09:57:48.617691+0000","last_fullsized":"2023-10-09T09:57:48.617691+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612468+0000","last_change":"2023-10-09T09:56:45.554759+0000","last_active":"2023-10-09T09:57:49.612468+0000","last_peered":"2023-10-09T09:57:49.612468+0000","last_clean":"2023-10-09T09:57:49.612468+0000","last_became_active":"2023-10-09T09:56:45.554390+0000","last_became_peered":"2023-10-09T09:56:45.554390+0000","last_unstale":"2023-10-09T09:57:49.612468+0000","last_undegraded":"2023-10-09T09:57:49.612468+0000","last_fullsized":"2023-10-09T09:57:49.612468+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:43.005737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622066+0000","last_change":"2023-10-09T09:57:18.844262+0000","last_active":"2023-10-09T09:57:48.622066+0000","last_peered":"2023-10-09T09:57:48.622066+0000","last_clean":"2023-10-09T09:57:48.622066+0000","last_became_active":"2023-10-09T09:57:18.843935+0000","last_became_peered":"2023-10-09T09:57:18.843935+0000","last_unstale":"2023-10-09T09:57:48.622066+0000","last_undegraded":"2023-10-09T09:57:48.622066+0000","last_fullsized":"2023-10-09T09:57:48.622066+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624797+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T09:57:49.624797+0000","last_peered":"2023-10-09T09:57:49.624797+0000","last_clean":"2023-10-09T09:57:49.624797+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T09:57:49.624797+0000","last_undegraded":"2023-10-09T09:57:49.624797+0000","last_fullsized":"2023-10-09T09:57:49.624797+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622011+0000","last_change":"2023-10-09T09:56:45.549837+0000","last_active":"2023-10-09T09:57:48.622011+0000","last_peered":"2023-10-09T09:57:48.622011+0000","last_clean":"2023-10-09T09:57:48.622011+0000","last_became_active":"2023-10-09T09:56:45.549546+0000","last_became_peered":"2023-10-09T09:56:45.549546+0000","last_unstale":"2023-10-09T09:57:48.622011+0000","last_undegraded":"2023-10-09T09:57:48.622011+0000","last_fullsized":"2023-10-09T09:57:48.622011+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:26.779071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624047+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T09:57:49.624047+0000","last_peered":"2023-10-09T09:57:49.624047+0000","last_clean":"2023-10-09T09:57:49.624047+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T09:57:49.624047+0000","last_undegraded":"2023-10-09T09:57:49.624047+0000","last_fullsized":"2023-10-09T09:57:49.624047+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.611604+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T09:57:49.611604+0000","last_peered":"2023-10-09T09:57:49.611604+0000","last_clean":"2023-10-09T09:57:49.611604+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T09:57:49.611604+0000","last_undegraded":"2023-10-09T09:57:49.611604+0000","last_fullsized":"2023-10-09T09:57:49.611604+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626860+0000","last_change":"2023-10-09T09:57:18.844843+0000","last_active":"2023-10-09T09:57:48.626860+0000","last_peered":"2023-10-09T09:57:48.626860+0000","last_clean":"2023-10-09T09:57:48.626860+0000","last_became_active":"2023-10-09T09:57:18.844530+0000","last_became_peered":"2023-10-09T09:57:18.844530+0000","last_unstale":"2023-10-09T09:57:48.626860+0000","last_undegraded":"2023-10-09T09:57:48.626860+0000","last_fullsized":"2023-10-09T09:57:48.626860+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:33.761788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625926+0000","last_change":"2023-10-09T09:56:13.614282+0000","last_active":"2023-10-09T09:57:48.625926+0000","last_peered":"2023-10-09T09:57:48.625926+0000","last_clean":"2023-10-09T09:57:48.625926+0000","last_became_active":"2023-10-09T09:56:13.613888+0000","last_became_peered":"2023-10-09T09:56:13.613888+0000","last_unstale":"2023-10-09T09:57:48.625926+0000","last_undegraded":"2023-10-09T09:57:48.625926+0000","last_fullsized":"2023-10-09T09:57:48.625926+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":71,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612551+0000","last_change":"2023-10-09T09:57:18.840413+0000","last_active":"2023-10-09T09:57:49.612551+0000","last_peered":"2023-10-09T09:57:49.612551+0000","last_clean":"2023-10-09T09:57:49.612551+0000","last_became_active":"2023-10-09T09:57:18.840093+0000","last_became_peered":"2023-10-09T09:57:18.840093+0000","last_unstale":"2023-10-09T09:57:49.612551+0000","last_undegraded":"2023-10-09T09:57:49.612551+0000","last_fullsized":"2023-10-09T09:57:49.612551+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:58.170995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626616+0000","last_change":"2023-10-09T09:57:49.624279+0000","last_active":"2023-10-09T09:57:49.626616+0000","last_peered":"2023-10-09T09:57:49.626616+0000","last_clean":"2023-10-09T09:57:49.626616+0000","last_became_active":"2023-10-09T09:57:49.624010+0000","last_became_peered":"2023-10-09T09:57:49.624010+0000","last_unstale":"2023-10-09T09:57:49.626616+0000","last_undegraded":"2023-10-09T09:57:49.626616+0000","last_fullsized":"2023-10-09T09:57:49.626616+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:47.717690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626997+0000","last_change":"2023-10-09T09:56:45.555907+0000","last_active":"2023-10-09T09:57:48.626997+0000","last_peered":"2023-10-09T09:57:48.626997+0000","last_clean":"2023-10-09T09:57:48.626997+0000","last_became_active":"2023-10-09T09:56:45.555548+0000","last_became_peered":"2023-10-09T09:56:45.555548+0000","last_unstale":"2023-10-09T09:57:48.626997+0000","last_undegraded":"2023-10-09T09:57:48.626997+0000","last_fullsized":"2023-10-09T09:57:48.626997+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622200+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T09:57:48.622200+0000","last_peered":"2023-10-09T09:57:48.622200+0000","last_clean":"2023-10-09T09:57:48.622200+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T09:57:48.622200+0000","last_undegraded":"2023-10-09T09:57:48.622200+0000","last_fullsized":"2023-10-09T09:57:48.622200+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622586+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T09:57:49.622586+0000","last_peered":"2023-10-09T09:57:49.622586+0000","last_clean":"2023-10-09T09:57:49.622586+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T09:57:49.622586+0000","last_undegraded":"2023-10-09T09:57:49.622586+0000","last_fullsized":"2023-10-09T09:57:49.622586+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":296,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626889+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T09:57:48.626889+0000","last_peered":"2023-10-09T09:57:48.626889+0000","last_clean":"2023-10-09T09:57:48.626889+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T09:57:48.626889+0000","last_undegraded":"2023-10-09T09:57:48.626889+0000","last_fullsized":"2023-10-09T09:57:48.626889+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619019+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:57:48.619019+0000","last_peered":"2023-10-09T09:57:48.619019+0000","last_clean":"2023-10-09T09:57:48.619019+0000","last_became_active":"2023-10-09T09:57:18.841472+0000","last_became_peered":"2023-10-09T09:57:18.841472+0000","last_unstale":"2023-10-09T09:57:48.619019+0000","last_undegraded":"2023-10-09T09:57:48.619019+0000","last_fullsized":"2023-10-09T09:57:48.619019+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622632+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T09:57:48.622632+0000","last_peered":"2023-10-09T09:57:48.622632+0000","last_clean":"2023-10-09T09:57:48.622632+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T09:57:48.622632+0000","last_undegraded":"2023-10-09T09:57:48.622632+0000","last_fullsized":"2023-10-09T09:57:48.622632+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.627470+0000","last_change":"2023-10-09T09:57:18.837334+0000","last_active":"2023-10-09T09:57:48.627470+0000","last_peered":"2023-10-09T09:57:48.627470+0000","last_clean":"2023-10-09T09:57:48.627470+0000","last_became_active":"2023-10-09T09:57:18.837037+0000","last_became_peered":"2023-10-09T09:57:18.837037+0000","last_unstale":"2023-10-09T09:57:48.627470+0000","last_undegraded":"2023-10-09T09:57:48.627470+0000","last_fullsized":"2023-10-09T09:57:48.627470+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:11.650333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":83,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621174+0000","last_change":"2023-10-09T09:57:49.621174+0000","last_active":"2023-10-09T09:57:49.621174+0000","last_peered":"2023-10-09T09:57:49.621174+0000","last_clean":"2023-10-09T09:57:49.621174+0000","last_became_active":"2023-10-09T09:57:49.620780+0000","last_became_peered":"2023-10-09T09:57:49.620780+0000","last_unstale":"2023-10-09T09:57:49.621174+0000","last_undegraded":"2023-10-09T09:57:49.621174+0000","last_fullsized":"2023-10-09T09:57:49.621174+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:01.389768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621680+0000","last_change":"2023-10-09T09:57:49.621680+0000","last_active":"2023-10-09T09:57:49.621680+0000","last_peered":"2023-10-09T09:57:49.621680+0000","last_clean":"2023-10-09T09:57:49.621680+0000","last_became_active":"2023-10-09T09:57:49.621345+0000","last_became_peered":"2023-10-09T09:57:49.621345+0000","last_unstale":"2023-10-09T09:57:49.621680+0000","last_undegraded":"2023-10-09T09:57:49.621680+0000","last_fullsized":"2023-10-09T09:57:49.621680+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:58.483689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617369+0000","last_change":"2023-10-09T09:56:13.613717+0000","last_active":"2023-10-09T09:57:48.617369+0000","last_peered":"2023-10-09T09:57:48.617369+0000","last_clean":"2023-10-09T09:57:48.617369+0000","last_became_active":"2023-10-09T09:56:13.613329+0000","last_became_peered":"2023-10-09T09:56:13.613329+0000","last_unstale":"2023-10-09T09:57:48.617369+0000","last_undegraded":"2023-10-09T09:57:48.617369+0000","last_fullsized":"2023-10-09T09:57:48.617369+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619103+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T09:57:48.619103+0000","last_peered":"2023-10-09T09:57:48.619103+0000","last_clean":"2023-10-09T09:57:48.619103+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T09:57:48.619103+0000","last_undegraded":"2023-10-09T09:57:48.619103+0000","last_fullsized":"2023-10-09T09:57:48.619103+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.618192+0000","last_change":"2023-10-09T09:57:18.842325+0000","last_active":"2023-10-09T09:57:48.618192+0000","last_peered":"2023-10-09T09:57:48.618192+0000","last_clean":"2023-10-09T09:57:48.618192+0000","last_became_active":"2023-10-09T09:57:18.841987+0000","last_became_peered":"2023-10-09T09:57:18.841987+0000","last_unstale":"2023-10-09T09:57:48.618192+0000","last_undegraded":"2023-10-09T09:57:48.618192+0000","last_fullsized":"2023-10-09T09:57:48.618192+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:28.615689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.628678+0000","last_change":"2023-10-09T09:57:49.628678+0000","last_active":"2023-10-09T09:57:49.628678+0000","last_peered":"2023-10-09T09:57:49.628678+0000","last_clean":"2023-10-09T09:57:49.628678+0000","last_became_active":"2023-10-09T09:57:49.628332+0000","last_became_peered":"2023-10-09T09:57:49.628332+0000","last_unstale":"2023-10-09T09:57:49.628678+0000","last_undegraded":"2023-10-09T09:57:49.628678+0000","last_fullsized":"2023-10-09T09:57:49.628678+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626775+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T09:57:48.626775+0000","last_peered":"2023-10-09T09:57:48.626775+0000","last_clean":"2023-10-09T09:57:48.626775+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T09:57:48.626775+0000","last_undegraded":"2023-10-09T09:57:48.626775+0000","last_fullsized":"2023-10-09T09:57:48.626775+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622055+0000","last_change":"2023-10-09T09:57:49.622055+0000","last_active":"2023-10-09T09:57:49.622055+0000","last_peered":"2023-10-09T09:57:49.622055+0000","last_clean":"2023-10-09T09:57:49.622055+0000","last_became_active":"2023-10-09T09:57:49.621784+0000","last_became_peered":"2023-10-09T09:57:49.621784+0000","last_unstale":"2023-10-09T09:57:49.622055+0000","last_undegraded":"2023-10-09T09:57:49.622055+0000","last_fullsized":"2023-10-09T09:57:49.622055+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:50.025391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624469+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T09:57:49.624469+0000","last_peered":"2023-10-09T09:57:49.624469+0000","last_clean":"2023-10-09T09:57:49.624469+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T09:57:49.624469+0000","last_undegraded":"2023-10-09T09:57:49.624469+0000","last_fullsized":"2023-10-09T09:57:49.624469+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619190+0000","last_change":"2023-10-09T09:56:45.555100+0000","last_active":"2023-10-09T09:57:48.619190+0000","last_peered":"2023-10-09T09:57:48.619190+0000","last_clean":"2023-10-09T09:57:48.619190+0000","last_became_active":"2023-10-09T09:56:45.554704+0000","last_became_peered":"2023-10-09T09:56:45.554704+0000","last_unstale":"2023-10-09T09:57:48.619190+0000","last_undegraded":"2023-10-09T09:57:48.619190+0000","last_fullsized":"2023-10-09T09:57:48.619190+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.616656+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:57:48.616656+0000","last_peered":"2023-10-09T09:57:48.616656+0000","last_clean":"2023-10-09T09:57:48.616656+0000","last_became_active":"2023-10-09T09:57:18.841522+0000","last_became_peered":"2023-10-09T09:57:18.841522+0000","last_unstale":"2023-10-09T09:57:48.616656+0000","last_undegraded":"2023-10-09T09:57:48.616656+0000","last_fullsized":"2023-10-09T09:57:48.616656+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.618427+0000","last_change":"2023-10-09T09:57:18.845748+0000","last_active":"2023-10-09T09:57:48.618427+0000","last_peered":"2023-10-09T09:57:48.618427+0000","last_clean":"2023-10-09T09:57:48.618427+0000","last_became_active":"2023-10-09T09:57:18.845397+0000","last_became_peered":"2023-10-09T09:57:18.845397+0000","last_unstale":"2023-10-09T09:57:48.618427+0000","last_undegraded":"2023-10-09T09:57:48.618427+0000","last_fullsized":"2023-10-09T09:57:48.618427+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622270+0000","last_change":"2023-10-09T09:56:45.556627+0000","last_active":"2023-10-09T09:57:48.622270+0000","last_peered":"2023-10-09T09:57:48.622270+0000","last_clean":"2023-10-09T09:57:48.622270+0000","last_became_active":"2023-10-09T09:56:45.556265+0000","last_became_peered":"2023-10-09T09:56:45.556265+0000","last_unstale":"2023-10-09T09:57:48.622270+0000","last_undegraded":"2023-10-09T09:57:48.622270+0000","last_fullsized":"2023-10-09T09:57:48.622270+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621587+0000","last_change":"2023-10-09T09:57:49.621587+0000","last_active":"2023-10-09T09:57:49.621587+0000","last_peered":"2023-10-09T09:57:49.621587+0000","last_clean":"2023-10-09T09:57:49.621587+0000","last_became_active":"2023-10-09T09:57:49.621245+0000","last_became_peered":"2023-10-09T09:57:49.621245+0000","last_unstale":"2023-10-09T09:57:49.621587+0000","last_undegraded":"2023-10-09T09:57:49.621587+0000","last_fullsized":"2023-10-09T09:57:49.621587+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620199+0000","last_change":"2023-10-09T09:57:49.620199+0000","last_active":"2023-10-09T09:57:49.620199+0000","last_peered":"2023-10-09T09:57:49.620199+0000","last_clean":"2023-10-09T09:57:49.620199+0000","last_became_active":"2023-10-09T09:57:49.619695+0000","last_became_peered":"2023-10-09T09:57:49.619695+0000","last_unstale":"2023-10-09T09:57:49.620199+0000","last_undegraded":"2023-10-09T09:57:49.620199+0000","last_fullsized":"2023-10-09T09:57:49.620199+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:48.818881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.611800+0000","last_change":"2023-10-09T09:56:45.553024+0000","last_active":"2023-10-09T09:57:49.611800+0000","last_peered":"2023-10-09T09:57:49.611800+0000","last_clean":"2023-10-09T09:57:49.611800+0000","last_became_active":"2023-10-09T09:56:45.552670+0000","last_became_peered":"2023-10-09T09:56:45.552670+0000","last_unstale":"2023-10-09T09:57:49.611800+0000","last_undegraded":"2023-10-09T09:57:49.611800+0000","last_fullsized":"2023-10-09T09:57:49.611800+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:58.132008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627539+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T09:57:49.627539+0000","last_peered":"2023-10-09T09:57:49.627539+0000","last_clean":"2023-10-09T09:57:49.627539+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T09:57:49.627539+0000","last_undegraded":"2023-10-09T09:57:49.627539+0000","last_fullsized":"2023-10-09T09:57:49.627539+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621676+0000","last_change":"2023-10-09T09:56:45.556365+0000","last_active":"2023-10-09T09:57:48.621676+0000","last_peered":"2023-10-09T09:57:48.621676+0000","last_clean":"2023-10-09T09:57:48.621676+0000","last_became_active":"2023-10-09T09:56:45.555832+0000","last_became_peered":"2023-10-09T09:56:45.555832+0000","last_unstale":"2023-10-09T09:57:48.621676+0000","last_undegraded":"2023-10-09T09:57:48.621676+0000","last_fullsized":"2023-10-09T09:57:48.621676+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626073+0000","last_change":"2023-10-09T09:48:25.514756+0000","last_active":"2023-10-09T09:57:48.626073+0000","last_peered":"2023-10-09T09:57:48.626073+0000","last_clean":"2023-10-09T09:57:48.626073+0000","last_became_active":"2023-10-09T09:48:25.514387+0000","last_became_peered":"2023-10-09T09:48:25.514387+0000","last_unstale":"2023-10-09T09:57:48.626073+0000","last_undegraded":"2023-10-09T09:57:48.626073+0000","last_fullsized":"2023-10-09T09:57:48.626073+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627073+0000","last_change":"2023-10-09T09:57:49.627073+0000","last_active":"2023-10-09T09:57:49.627073+0000","last_peered":"2023-10-09T09:57:49.627073+0000","last_clean":"2023-10-09T09:57:49.627073+0000","last_became_active":"2023-10-09T09:57:49.626706+0000","last_became_peered":"2023-10-09T09:57:49.626706+0000","last_unstale":"2023-10-09T09:57:49.627073+0000","last_undegraded":"2023-10-09T09:57:49.627073+0000","last_fullsized":"2023-10-09T09:57:49.627073+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625250+0000","last_change":"2023-10-09T09:56:45.554326+0000","last_active":"2023-10-09T09:57:48.625250+0000","last_peered":"2023-10-09T09:57:48.625250+0000","last_clean":"2023-10-09T09:57:48.625250+0000","last_became_active":"2023-10-09T09:56:45.553876+0000","last_became_peered":"2023-10-09T09:56:45.553876+0000","last_unstale":"2023-10-09T09:57:48.625250+0000","last_undegraded":"2023-10-09T09:57:48.625250+0000","last_fullsized":"2023-10-09T09:57:48.625250+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619582+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T09:57:48.619582+0000","last_peered":"2023-10-09T09:57:48.619582+0000","last_clean":"2023-10-09T09:57:48.619582+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T09:57:48.619582+0000","last_undegraded":"2023-10-09T09:57:48.619582+0000","last_fullsized":"2023-10-09T09:57:48.619582+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.624744+0000","last_change":"2023-10-09T09:56:13.614485+0000","last_active":"2023-10-09T09:57:48.624744+0000","last_peered":"2023-10-09T09:57:48.624744+0000","last_clean":"2023-10-09T09:57:48.624744+0000","last_became_active":"2023-10-09T09:56:13.614109+0000","last_became_peered":"2023-10-09T09:56:13.614109+0000","last_unstale":"2023-10-09T09:57:48.624744+0000","last_undegraded":"2023-10-09T09:57:48.624744+0000","last_fullsized":"2023-10-09T09:57:48.624744+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623806+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T09:57:49.623806+0000","last_peered":"2023-10-09T09:57:49.623806+0000","last_clean":"2023-10-09T09:57:49.623806+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T09:57:49.623806+0000","last_undegraded":"2023-10-09T09:57:49.623806+0000","last_fullsized":"2023-10-09T09:57:49.623806+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620753+0000","last_change":"2023-10-09T09:56:13.623204+0000","last_active":"2023-10-09T09:57:48.620753+0000","last_peered":"2023-10-09T09:57:48.620753+0000","last_clean":"2023-10-09T09:57:48.620753+0000","last_became_active":"2023-10-09T09:56:13.622853+0000","last_became_peered":"2023-10-09T09:56:13.622853+0000","last_unstale":"2023-10-09T09:57:48.620753+0000","last_undegraded":"2023-10-09T09:57:48.620753+0000","last_fullsized":"2023-10-09T09:57:48.620753+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622372+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T09:57:49.622372+0000","last_peered":"2023-10-09T09:57:49.622372+0000","last_clean":"2023-10-09T09:57:49.622372+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T09:57:49.622372+0000","last_undegraded":"2023-10-09T09:57:49.622372+0000","last_fullsized":"2023-10-09T09:57:49.622372+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625768+0000","last_change":"2023-10-09T09:48:25.523230+0000","last_active":"2023-10-09T09:57:48.625768+0000","last_peered":"2023-10-09T09:57:48.625768+0000","last_clean":"2023-10-09T09:57:48.625768+0000","last_became_active":"2023-10-09T09:48:25.522608+0000","last_became_peered":"2023-10-09T09:48:25.522608+0000","last_unstale":"2023-10-09T09:57:48.625768+0000","last_undegraded":"2023-10-09T09:57:48.625768+0000","last_fullsized":"2023-10-09T09:57:48.625768+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627770+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T09:57:49.627770+0000","last_peered":"2023-10-09T09:57:49.627770+0000","last_clean":"2023-10-09T09:57:49.627770+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T09:57:49.627770+0000","last_undegraded":"2023-10-09T09:57:49.627770+0000","last_fullsized":"2023-10-09T09:57:49.627770+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624030+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T09:57:49.624030+0000","last_peered":"2023-10-09T09:57:49.624030+0000","last_clean":"2023-10-09T09:57:49.624030+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T09:57:49.624030+0000","last_undegraded":"2023-10-09T09:57:49.624030+0000","last_fullsized":"2023-10-09T09:57:49.624030+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622141+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T09:57:49.622141+0000","last_peered":"2023-10-09T09:57:49.622141+0000","last_clean":"2023-10-09T09:57:49.622141+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T09:57:49.622141+0000","last_undegraded":"2023-10-09T09:57:49.622141+0000","last_fullsized":"2023-10-09T09:57:49.622141+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.624235+0000","last_change":"2023-10-09T09:57:18.843731+0000","last_active":"2023-10-09T09:57:48.624235+0000","last_peered":"2023-10-09T09:57:48.624235+0000","last_clean":"2023-10-09T09:57:48.624235+0000","last_became_active":"2023-10-09T09:57:18.843280+0000","last_became_peered":"2023-10-09T09:57:18.843280+0000","last_unstale":"2023-10-09T09:57:48.624235+0000","last_undegraded":"2023-10-09T09:57:48.624235+0000","last_fullsized":"2023-10-09T09:57:48.624235+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617918+0000","last_change":"2023-10-09T09:56:45.551325+0000","last_active":"2023-10-09T09:57:48.617918+0000","last_peered":"2023-10-09T09:57:48.617918+0000","last_clean":"2023-10-09T09:57:48.617918+0000","last_became_active":"2023-10-09T09:56:45.550946+0000","last_became_peered":"2023-10-09T09:56:45.550946+0000","last_unstale":"2023-10-09T09:57:48.617918+0000","last_undegraded":"2023-10-09T09:57:48.617918+0000","last_fullsized":"2023-10-09T09:57:48.617918+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":69,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617387+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T09:57:48.617387+0000","last_peered":"2023-10-09T09:57:48.617387+0000","last_clean":"2023-10-09T09:57:48.617387+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T09:57:48.617387+0000","last_undegraded":"2023-10-09T09:57:48.617387+0000","last_fullsized":"2023-10-09T09:57:48.617387+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621810+0000","last_change":"2023-10-09T09:57:18.840471+0000","last_active":"2023-10-09T09:57:48.621810+0000","last_peered":"2023-10-09T09:57:48.621810+0000","last_clean":"2023-10-09T09:57:48.621810+0000","last_became_active":"2023-10-09T09:57:18.840125+0000","last_became_peered":"2023-10-09T09:57:18.840125+0000","last_unstale":"2023-10-09T09:57:48.621810+0000","last_undegraded":"2023-10-09T09:57:48.621810+0000","last_fullsized":"2023-10-09T09:57:48.621810+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:07.418455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626281+0000","last_change":"2023-10-09T09:57:49.626281+0000","last_active":"2023-10-09T09:57:49.626281+0000","last_peered":"2023-10-09T09:57:49.626281+0000","last_clean":"2023-10-09T09:57:49.626281+0000","last_became_active":"2023-10-09T09:57:49.625726+0000","last_became_peered":"2023-10-09T09:57:49.625726+0000","last_unstale":"2023-10-09T09:57:49.626281+0000","last_undegraded":"2023-10-09T09:57:49.626281+0000","last_fullsized":"2023-10-09T09:57:49.626281+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625511+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T09:57:49.625511+0000","last_peered":"2023-10-09T09:57:49.625511+0000","last_clean":"2023-10-09T09:57:49.625511+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T09:57:49.625511+0000","last_undegraded":"2023-10-09T09:57:49.625511+0000","last_fullsized":"2023-10-09T09:57:49.625511+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620020+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T09:57:48.620020+0000","last_peered":"2023-10-09T09:57:48.620020+0000","last_clean":"2023-10-09T09:57:48.620020+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T09:57:48.620020+0000","last_undegraded":"2023-10-09T09:57:48.620020+0000","last_fullsized":"2023-10-09T09:57:48.620020+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624665+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T09:57:49.624665+0000","last_peered":"2023-10-09T09:57:49.624665+0000","last_clean":"2023-10-09T09:57:49.624665+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T09:57:49.624665+0000","last_undegraded":"2023-10-09T09:57:49.624665+0000","last_fullsized":"2023-10-09T09:57:49.624665+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626421+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T09:57:49.626421+0000","last_peered":"2023-10-09T09:57:49.626421+0000","last_clean":"2023-10-09T09:57:49.626421+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T09:57:49.626421+0000","last_undegraded":"2023-10-09T09:57:49.626421+0000","last_fullsized":"2023-10-09T09:57:49.626421+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622159+0000","last_change":"2023-10-09T09:57:18.838985+0000","last_active":"2023-10-09T09:57:48.622159+0000","last_peered":"2023-10-09T09:57:48.622159+0000","last_clean":"2023-10-09T09:57:48.622159+0000","last_became_active":"2023-10-09T09:57:18.838625+0000","last_became_peered":"2023-10-09T09:57:18.838625+0000","last_unstale":"2023-10-09T09:57:48.622159+0000","last_undegraded":"2023-10-09T09:57:48.622159+0000","last_fullsized":"2023-10-09T09:57:48.622159+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624475+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T09:57:49.624475+0000","last_peered":"2023-10-09T09:57:49.624475+0000","last_clean":"2023-10-09T09:57:49.624475+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T09:57:49.624475+0000","last_undegraded":"2023-10-09T09:57:49.624475+0000","last_fullsized":"2023-10-09T09:57:49.624475+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625707+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T09:57:49.625707+0000","last_peered":"2023-10-09T09:57:49.625707+0000","last_clean":"2023-10-09T09:57:49.625707+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T09:57:49.625707+0000","last_undegraded":"2023-10-09T09:57:49.625707+0000","last_fullsized":"2023-10-09T09:57:49.625707+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621495+0000","last_change":"2023-10-09T09:57:49.621495+0000","last_active":"2023-10-09T09:57:49.621495+0000","last_peered":"2023-10-09T09:57:49.621495+0000","last_clean":"2023-10-09T09:57:49.621495+0000","last_became_active":"2023-10-09T09:57:49.621144+0000","last_became_peered":"2023-10-09T09:57:49.621144+0000","last_unstale":"2023-10-09T09:57:49.621495+0000","last_undegraded":"2023-10-09T09:57:49.621495+0000","last_fullsized":"2023-10-09T09:57:49.621495+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:11.852424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622753+0000","last_change":"2023-10-09T09:57:49.622753+0000","last_active":"2023-10-09T09:57:49.622753+0000","last_peered":"2023-10-09T09:57:49.622753+0000","last_clean":"2023-10-09T09:57:49.622753+0000","last_became_active":"2023-10-09T09:57:49.622463+0000","last_became_peered":"2023-10-09T09:57:49.622463+0000","last_unstale":"2023-10-09T09:57:49.622753+0000","last_undegraded":"2023-10-09T09:57:49.622753+0000","last_fullsized":"2023-10-09T09:57:49.622753+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:51.020605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":78,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617029+0000","last_change":"2023-10-09T09:57:18.841362+0000","last_active":"2023-10-09T09:57:48.617029+0000","last_peered":"2023-10-09T09:57:48.617029+0000","last_clean":"2023-10-09T09:57:48.617029+0000","last_became_active":"2023-10-09T09:57:18.840951+0000","last_became_peered":"2023-10-09T09:57:18.840951+0000","last_unstale":"2023-10-09T09:57:48.617029+0000","last_undegraded":"2023-10-09T09:57:48.617029+0000","last_fullsized":"2023-10-09T09:57:48.617029+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430346,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310920,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434232,"statfs":{"total":95995035648,"available":95676653568,"internally_reserved":0,"allocated":1572864,"data_stored":1111479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58399999999999996,"15min":0.58999999999999997},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.41799999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73599999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.73599999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.53400000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.57999999999999996,"15min":0.53700000000000003},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.36099999999999999},"max":{"1min":0.81399999999999995,"5min":0.89700000000000002,"15min":0.91600000000000004},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.56499999999999995},"min":{"1min":0.39700000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.76800000000000002,"5min":0.78100000000000003,"15min":0.83799999999999997},"last":0.755}]},{"osd":4,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.46000000000000002},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.71099999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.51100000000000001,"15min":0.47299999999999998},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":0.68100000000000005,"5min":0.87,"15min":0.87},"last":0.622}]},{"osd":5,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52000000000000002,"15min":0.498},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66600000000000004,"5min":0.79600000000000004,"15min":0.83599999999999997},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.36599999999999999,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":0.78600000000000003,"5min":0.81200000000000006,"15min":0.85399999999999998},"last":0.80500000000000005}]},{"osd":6,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57599999999999996,"15min":0.54300000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.32900000000000001},"max":{"1min":0.78300000000000003,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58399999999999996,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.40300000000000002,"15min":0.33100000000000002},"max":{"1min":0.75,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.72099999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626574,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1556480,"data_stored":1111018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71999999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.51900000000000002,"15min":0.51000000000000001},"min":{"1min":0.42299999999999999,"5min":0.308,"15min":0.308},"max":{"1min":0.749,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.56399999999999995}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.53900000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.58499999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:57:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.57999999999999996,"15min":0.55000000000000004},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.33100000000000002},"max":{"1min":0.79300000000000004,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.60099999999999998,"15min":0.56299999999999994},"min":{"1min":0.39300000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.0169999999999999,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.64500000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46700000000000003,"15min":0.45000000000000001},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.23300000000000001},"max":{"1min":0.64500000000000002,"5min":0.75800000000000001,"15min":0.77600000000000002},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.503,"5min":0.496,"15min":0.48899999999999999},"min":{"1min":0.36299999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.66800000000000004,"5min":0.753,"15min":0.75600000000000001},"last":0.499}]},{"osd":5,"last update":"Mon Oct 9 09:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.50600000000000001,"15min":0.503},"min":{"1min":0.36599999999999999,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":0.63600000000000001,"5min":0.76100000000000001,"15min":0.80600000000000005},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51700000000000002,"15min":0.505},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68500000000000005,"5min":0.76800000000000002,"15min":1.0600000000000001},"last":0.437}]},{"osd":7,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.54400000000000004,"15min":0.52600000000000002},"min":{"1min":0.35099999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.68000000000000005,"5min":0.79400000000000004,"15min":0.81399999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.627,"5min":0.61499999999999999,"15min":0.58699999999999997},"min":{"1min":0.52600000000000002,"5min":0.34899999999999998,"15min":0.34399999999999997},"max":{"1min":0.754,"5min":0.82999999999999996,"15min":0.84099999999999997},"last":0.51400000000000001}]}]},{"osd":1,"up_from":84,"seq":360777252887,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28308,"kb_used_data":972,"kb_used_omap":9,"kb_used_meta":27318,"kb_avail":93716844,"statfs":{"total":95995035648,"available":95966048256,"internally_reserved":0,"allocated":995328,"data_stored":524748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":27974633},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.377}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.314},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.40100000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42799999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.625},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.71199999999999997}]},{"osd":5,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.66200000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.63500000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.76200000000000001}]}]},{"osd":0,"up_from":79,"seq":339302416413,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33056,"kb_used_data":1560,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712096,"statfs":{"total":95995035648,"available":95961186304,"internally_reserved":0,"allocated":1597440,"data_stored":1114630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.39000000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:57:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.66900000000000004,"15min":0.66100000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68200000000000005,"15min":0.67800000000000005},"min":{"1min":0.44600000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.028,"5min":1.101,"15min":1.101},"last":0.68500000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64600000000000002,"15min":0.64400000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85799999999999998,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.94499999999999995,"5min":1.1180000000000001,"15min":1.1180000000000001},"last":0.63300000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.68300000000000005,"15min":0.68700000000000006},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89700000000000002,"5min":1.161,"15min":1.161},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.68899999999999995,"15min":0.69099999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.99199999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.59999999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.66500000000000004,"15min":0.66600000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.89600000000000002,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.68400000000000005,"15min":0.68300000000000005},"min":{"1min":0.441,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.98099999999999998,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.67600000000000005}]}]},{"osd":2,"up_from":89,"seq":382252089360,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36624,"kb_used_data":968,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708528,"statfs":{"total":95995035648,"available":95957532672,"internally_reserved":0,"allocated":991232,"data_stored":522739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.61499999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.68400000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.86799999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.85999999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.90200000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.83099999999999996}]}]},{"osd":3,"up_from":94,"seq":403726925834,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28288,"kb_used_data":952,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716864,"statfs":{"total":95995035648,"available":95966068736,"internally_reserved":0,"allocated":974848,"data_stored":520694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.378}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.315}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26800000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.622}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66300000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019030,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302144,"kb_used_data":952,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443008,"statfs":{"total":95995035648,"available":95685640192,"internally_reserved":0,"allocated":974848,"data_stored":521041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.66800000000000004,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56699999999999995,"15min":0.56999999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.76200000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.66100000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.621}]},{"osd":3,"last update":"Mon Oct 9 09:56:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.28199999999999997},"max":{"1min":0.753,"5min":0.877,"15min":2.2109999999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.47899999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.75,"5min":0.81399999999999995,"15min":0.997},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.45600000000000002,"15min":0.43099999999999999},"min":{"1min":0.34399999999999997,"5min":0.28399999999999997,"15min":0.27100000000000002},"max":{"1min":0.77200000000000002,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.48999999999999999,"15min":0.46600000000000003},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70999999999999996,"5min":0.72099999999999997,"15min":0.83999999999999997},"last":0.51200000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.499,"15min":0.46800000000000003},"min":{"1min":0.41199999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78600000000000003,"5min":0.79300000000000004,"15min":0.86599999999999999},"last":0.501},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53200000000000003,"15min":0.497},"min":{"1min":0.379,"5min":0.318,"15min":0.312},"max":{"1min":0.80700000000000005,"5min":0.81499999999999995,"15min":0.89900000000000002},"last":0.53600000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55800000000000005,"15min":0.52500000000000002},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.69899999999999995,"5min":0.76600000000000001,"15min":0.84999999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59699999999999998,"15min":0.55800000000000005},"min":{"1min":0.45800000000000002,"5min":0.33700000000000002,"15min":0.316},"max":{"1min":0.76000000000000001,"5min":0.82899999999999996,"15min":0.85199999999999998},"last":0.57699999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822802,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310320,"kb_used_data":936,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434832,"statfs":{"total":95995035648,"available":95677267968,"internally_reserved":0,"allocated":958464,"data_stored":516785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.75,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.55000000000000004,"15min":0.54100000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.4289999999999998,"5min":2.4289999999999998,"15min":2.4289999999999998},"last":0.46800000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.58699999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":2.5950000000000002,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.68300000000000005}]},{"osd":3,"last update":"Mon Oct 9 09:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.56899999999999995,"15min":0.54700000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":0.70699999999999996,"5min":0.84999999999999998,"15min":1.1479999999999999},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.60699999999999998,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":2.637},"last":0.72399999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47699999999999998,"15min":0.44400000000000001},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":0.74299999999999999,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.752},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.51000000000000001,"15min":0.46800000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64900000000000002,"5min":0.79600000000000004,"15min":0.88200000000000001},"last":0.69499999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.53000000000000003,"15min":0.503},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":0.76200000000000001,"5min":0.85099999999999998,"15min":0.92800000000000005},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.52100000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81000000000000005,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.77200000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58099999999999996,"15min":0.55500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":0.71299999999999997,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55000000000000004,"15min":0.53600000000000003},"min":{"1min":0.37,"5min":0.318,"15min":0.317},"max":{"1min":0.68200000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.78700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:58:31.817 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T09:58:31.817 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T09:58:31.817 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T09:58:31.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T09:58:32.424 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:32.822 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:32 smithi012 ceph-mon[147926]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:32.823 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:32 smithi012 ceph-mon[147926]: from='client.34707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:32.823 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:32 smithi012 ceph-mon[152263]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:32.823 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:32 smithi012 ceph-mon[152263]: from='client.34707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:32.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:32 smithi110 ceph-mon[136032]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:32.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:32 smithi110 ceph-mon[136032]: from='client.34707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:33.886 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:58:33.886 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T09:58:34.344 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T09:58:34.344 INFO:teuthology.run_tasks:Running task print... 2023-10-09T09:58:34.357 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T09:58:34.357 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T09:58:34.387 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T09:58:34.387 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T09:58:34.387 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T09:58:34.397 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T09:58:34.397 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T09:58:34.397 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T09:58:34.397 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T09:58:34.397 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T09:58:34.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:58:34.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:34 smithi012 ceph-mon[147926]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:34.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:34 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1641249505' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:58:34.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:34 smithi012 ceph-mon[152263]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:34.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:34 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1641249505' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:58:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:34 smithi110 ceph-mon[136032]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:34 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1641249505' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T09:58:35.006 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:36.527 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:58:36.528 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:58:36.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:36 smithi012 ceph-mon[147926]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:36.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:36 smithi012 ceph-mon[152263]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:36 smithi110 ceph-mon[136032]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:37.023 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":221,"stamp":"2023-10-09T09:58:34.906916+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1348276,"kb_used_data":9396,"kb_used_omap":126,"kb_used_meta":1338689,"kb_avail":748612940,"statfs":{"total":767960285184,"available":766579650560,"internally_reserved":0,"allocated":9621504,"data_stored":5943134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1370818171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002977"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.616419+0000","last_change":"2023-10-09T09:56:45.556456+0000","last_active":"2023-10-09T09:57:48.616419+0000","last_peered":"2023-10-09T09:57:48.616419+0000","last_clean":"2023-10-09T09:57:48.616419+0000","last_became_active":"2023-10-09T09:56:45.555878+0000","last_became_peered":"2023-10-09T09:56:45.555878+0000","last_unstale":"2023-10-09T09:57:48.616419+0000","last_undegraded":"2023-10-09T09:57:48.616419+0000","last_fullsized":"2023-10-09T09:57:48.616419+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612158+0000","last_change":"2023-10-09T09:57:18.842453+0000","last_active":"2023-10-09T09:57:49.612158+0000","last_peered":"2023-10-09T09:57:49.612158+0000","last_clean":"2023-10-09T09:57:49.612158+0000","last_became_active":"2023-10-09T09:57:18.842133+0000","last_became_peered":"2023-10-09T09:57:18.842133+0000","last_unstale":"2023-10-09T09:57:49.612158+0000","last_undegraded":"2023-10-09T09:57:49.612158+0000","last_fullsized":"2023-10-09T09:57:49.612158+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:07.459692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620527+0000","last_change":"2023-10-09T09:57:49.620527+0000","last_active":"2023-10-09T09:57:49.620527+0000","last_peered":"2023-10-09T09:57:49.620527+0000","last_clean":"2023-10-09T09:57:49.620527+0000","last_became_active":"2023-10-09T09:57:49.620109+0000","last_became_peered":"2023-10-09T09:57:49.620109+0000","last_unstale":"2023-10-09T09:57:49.620527+0000","last_undegraded":"2023-10-09T09:57:49.620527+0000","last_fullsized":"2023-10-09T09:57:49.620527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:09.284902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":68,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623102+0000","last_change":"2023-10-09T09:57:49.623102+0000","last_active":"2023-10-09T09:57:49.623102+0000","last_peered":"2023-10-09T09:57:49.623102+0000","last_clean":"2023-10-09T09:57:49.623102+0000","last_became_active":"2023-10-09T09:57:49.620638+0000","last_became_peered":"2023-10-09T09:57:49.620638+0000","last_unstale":"2023-10-09T09:57:49.623102+0000","last_undegraded":"2023-10-09T09:57:49.623102+0000","last_fullsized":"2023-10-09T09:57:49.623102+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:11.322675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620967+0000","last_change":"2023-10-09T09:57:49.620967+0000","last_active":"2023-10-09T09:57:49.620967+0000","last_peered":"2023-10-09T09:57:49.620967+0000","last_clean":"2023-10-09T09:57:49.620967+0000","last_became_active":"2023-10-09T09:57:49.620567+0000","last_became_peered":"2023-10-09T09:57:49.620567+0000","last_unstale":"2023-10-09T09:57:49.620967+0000","last_undegraded":"2023-10-09T09:57:49.620967+0000","last_fullsized":"2023-10-09T09:57:49.620967+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620814+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T09:57:48.620814+0000","last_peered":"2023-10-09T09:57:48.620814+0000","last_clean":"2023-10-09T09:57:48.620814+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T09:57:48.620814+0000","last_undegraded":"2023-10-09T09:57:48.620814+0000","last_fullsized":"2023-10-09T09:57:48.620814+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622634+0000","last_change":"2023-10-09T09:57:49.622634+0000","last_active":"2023-10-09T09:57:49.622634+0000","last_peered":"2023-10-09T09:57:49.622634+0000","last_clean":"2023-10-09T09:57:49.622634+0000","last_became_active":"2023-10-09T09:57:49.622247+0000","last_became_peered":"2023-10-09T09:57:49.622247+0000","last_unstale":"2023-10-09T09:57:49.622634+0000","last_undegraded":"2023-10-09T09:57:49.622634+0000","last_fullsized":"2023-10-09T09:57:49.622634+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621538+0000","last_change":"2023-10-09T09:56:45.554958+0000","last_active":"2023-10-09T09:57:48.621538+0000","last_peered":"2023-10-09T09:57:48.621538+0000","last_clean":"2023-10-09T09:57:48.621538+0000","last_became_active":"2023-10-09T09:56:45.554597+0000","last_became_peered":"2023-10-09T09:56:45.554597+0000","last_unstale":"2023-10-09T09:57:48.621538+0000","last_undegraded":"2023-10-09T09:57:48.621538+0000","last_fullsized":"2023-10-09T09:57:48.621538+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623531+0000","last_change":"2023-10-09T09:57:49.623531+0000","last_active":"2023-10-09T09:57:49.623531+0000","last_peered":"2023-10-09T09:57:49.623531+0000","last_clean":"2023-10-09T09:57:49.623531+0000","last_became_active":"2023-10-09T09:57:49.623045+0000","last_became_peered":"2023-10-09T09:57:49.623045+0000","last_unstale":"2023-10-09T09:57:49.623531+0000","last_undegraded":"2023-10-09T09:57:49.623531+0000","last_fullsized":"2023-10-09T09:57:49.623531+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:49.655085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622370+0000","last_change":"2023-10-09T09:56:13.614416+0000","last_active":"2023-10-09T09:57:48.622370+0000","last_peered":"2023-10-09T09:57:48.622370+0000","last_clean":"2023-10-09T09:57:48.622370+0000","last_became_active":"2023-10-09T09:56:13.614050+0000","last_became_peered":"2023-10-09T09:56:13.614050+0000","last_unstale":"2023-10-09T09:57:48.622370+0000","last_undegraded":"2023-10-09T09:57:48.622370+0000","last_fullsized":"2023-10-09T09:57:48.622370+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:17.068217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617691+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T09:57:48.617691+0000","last_peered":"2023-10-09T09:57:48.617691+0000","last_clean":"2023-10-09T09:57:48.617691+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T09:57:48.617691+0000","last_undegraded":"2023-10-09T09:57:48.617691+0000","last_fullsized":"2023-10-09T09:57:48.617691+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612468+0000","last_change":"2023-10-09T09:56:45.554759+0000","last_active":"2023-10-09T09:57:49.612468+0000","last_peered":"2023-10-09T09:57:49.612468+0000","last_clean":"2023-10-09T09:57:49.612468+0000","last_became_active":"2023-10-09T09:56:45.554390+0000","last_became_peered":"2023-10-09T09:56:45.554390+0000","last_unstale":"2023-10-09T09:57:49.612468+0000","last_undegraded":"2023-10-09T09:57:49.612468+0000","last_fullsized":"2023-10-09T09:57:49.612468+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:43.005737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622066+0000","last_change":"2023-10-09T09:57:18.844262+0000","last_active":"2023-10-09T09:57:48.622066+0000","last_peered":"2023-10-09T09:57:48.622066+0000","last_clean":"2023-10-09T09:57:48.622066+0000","last_became_active":"2023-10-09T09:57:18.843935+0000","last_became_peered":"2023-10-09T09:57:18.843935+0000","last_unstale":"2023-10-09T09:57:48.622066+0000","last_undegraded":"2023-10-09T09:57:48.622066+0000","last_fullsized":"2023-10-09T09:57:48.622066+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624797+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T09:57:49.624797+0000","last_peered":"2023-10-09T09:57:49.624797+0000","last_clean":"2023-10-09T09:57:49.624797+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T09:57:49.624797+0000","last_undegraded":"2023-10-09T09:57:49.624797+0000","last_fullsized":"2023-10-09T09:57:49.624797+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622011+0000","last_change":"2023-10-09T09:56:45.549837+0000","last_active":"2023-10-09T09:57:48.622011+0000","last_peered":"2023-10-09T09:57:48.622011+0000","last_clean":"2023-10-09T09:57:48.622011+0000","last_became_active":"2023-10-09T09:56:45.549546+0000","last_became_peered":"2023-10-09T09:56:45.549546+0000","last_unstale":"2023-10-09T09:57:48.622011+0000","last_undegraded":"2023-10-09T09:57:48.622011+0000","last_fullsized":"2023-10-09T09:57:48.622011+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:26.779071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624047+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T09:57:49.624047+0000","last_peered":"2023-10-09T09:57:49.624047+0000","last_clean":"2023-10-09T09:57:49.624047+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T09:57:49.624047+0000","last_undegraded":"2023-10-09T09:57:49.624047+0000","last_fullsized":"2023-10-09T09:57:49.624047+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.611604+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T09:57:49.611604+0000","last_peered":"2023-10-09T09:57:49.611604+0000","last_clean":"2023-10-09T09:57:49.611604+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T09:57:49.611604+0000","last_undegraded":"2023-10-09T09:57:49.611604+0000","last_fullsized":"2023-10-09T09:57:49.611604+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626860+0000","last_change":"2023-10-09T09:57:18.844843+0000","last_active":"2023-10-09T09:57:48.626860+0000","last_peered":"2023-10-09T09:57:48.626860+0000","last_clean":"2023-10-09T09:57:48.626860+0000","last_became_active":"2023-10-09T09:57:18.844530+0000","last_became_peered":"2023-10-09T09:57:18.844530+0000","last_unstale":"2023-10-09T09:57:48.626860+0000","last_undegraded":"2023-10-09T09:57:48.626860+0000","last_fullsized":"2023-10-09T09:57:48.626860+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:33.761788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625926+0000","last_change":"2023-10-09T09:56:13.614282+0000","last_active":"2023-10-09T09:57:48.625926+0000","last_peered":"2023-10-09T09:57:48.625926+0000","last_clean":"2023-10-09T09:57:48.625926+0000","last_became_active":"2023-10-09T09:56:13.613888+0000","last_became_peered":"2023-10-09T09:56:13.613888+0000","last_unstale":"2023-10-09T09:57:48.625926+0000","last_undegraded":"2023-10-09T09:57:48.625926+0000","last_fullsized":"2023-10-09T09:57:48.625926+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":71,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612551+0000","last_change":"2023-10-09T09:57:18.840413+0000","last_active":"2023-10-09T09:57:49.612551+0000","last_peered":"2023-10-09T09:57:49.612551+0000","last_clean":"2023-10-09T09:57:49.612551+0000","last_became_active":"2023-10-09T09:57:18.840093+0000","last_became_peered":"2023-10-09T09:57:18.840093+0000","last_unstale":"2023-10-09T09:57:49.612551+0000","last_undegraded":"2023-10-09T09:57:49.612551+0000","last_fullsized":"2023-10-09T09:57:49.612551+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:58.170995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626616+0000","last_change":"2023-10-09T09:57:49.624279+0000","last_active":"2023-10-09T09:57:49.626616+0000","last_peered":"2023-10-09T09:57:49.626616+0000","last_clean":"2023-10-09T09:57:49.626616+0000","last_became_active":"2023-10-09T09:57:49.624010+0000","last_became_peered":"2023-10-09T09:57:49.624010+0000","last_unstale":"2023-10-09T09:57:49.626616+0000","last_undegraded":"2023-10-09T09:57:49.626616+0000","last_fullsized":"2023-10-09T09:57:49.626616+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:47.717690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626997+0000","last_change":"2023-10-09T09:56:45.555907+0000","last_active":"2023-10-09T09:57:48.626997+0000","last_peered":"2023-10-09T09:57:48.626997+0000","last_clean":"2023-10-09T09:57:48.626997+0000","last_became_active":"2023-10-09T09:56:45.555548+0000","last_became_peered":"2023-10-09T09:56:45.555548+0000","last_unstale":"2023-10-09T09:57:48.626997+0000","last_undegraded":"2023-10-09T09:57:48.626997+0000","last_fullsized":"2023-10-09T09:57:48.626997+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622200+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T09:57:48.622200+0000","last_peered":"2023-10-09T09:57:48.622200+0000","last_clean":"2023-10-09T09:57:48.622200+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T09:57:48.622200+0000","last_undegraded":"2023-10-09T09:57:48.622200+0000","last_fullsized":"2023-10-09T09:57:48.622200+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622586+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T09:57:49.622586+0000","last_peered":"2023-10-09T09:57:49.622586+0000","last_clean":"2023-10-09T09:57:49.622586+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T09:57:49.622586+0000","last_undegraded":"2023-10-09T09:57:49.622586+0000","last_fullsized":"2023-10-09T09:57:49.622586+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":296,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626889+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T09:57:48.626889+0000","last_peered":"2023-10-09T09:57:48.626889+0000","last_clean":"2023-10-09T09:57:48.626889+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T09:57:48.626889+0000","last_undegraded":"2023-10-09T09:57:48.626889+0000","last_fullsized":"2023-10-09T09:57:48.626889+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619019+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:57:48.619019+0000","last_peered":"2023-10-09T09:57:48.619019+0000","last_clean":"2023-10-09T09:57:48.619019+0000","last_became_active":"2023-10-09T09:57:18.841472+0000","last_became_peered":"2023-10-09T09:57:18.841472+0000","last_unstale":"2023-10-09T09:57:48.619019+0000","last_undegraded":"2023-10-09T09:57:48.619019+0000","last_fullsized":"2023-10-09T09:57:48.619019+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622632+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T09:57:48.622632+0000","last_peered":"2023-10-09T09:57:48.622632+0000","last_clean":"2023-10-09T09:57:48.622632+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T09:57:48.622632+0000","last_undegraded":"2023-10-09T09:57:48.622632+0000","last_fullsized":"2023-10-09T09:57:48.622632+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.627470+0000","last_change":"2023-10-09T09:57:18.837334+0000","last_active":"2023-10-09T09:57:48.627470+0000","last_peered":"2023-10-09T09:57:48.627470+0000","last_clean":"2023-10-09T09:57:48.627470+0000","last_became_active":"2023-10-09T09:57:18.837037+0000","last_became_peered":"2023-10-09T09:57:18.837037+0000","last_unstale":"2023-10-09T09:57:48.627470+0000","last_undegraded":"2023-10-09T09:57:48.627470+0000","last_fullsized":"2023-10-09T09:57:48.627470+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:11.650333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":83,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621174+0000","last_change":"2023-10-09T09:57:49.621174+0000","last_active":"2023-10-09T09:57:49.621174+0000","last_peered":"2023-10-09T09:57:49.621174+0000","last_clean":"2023-10-09T09:57:49.621174+0000","last_became_active":"2023-10-09T09:57:49.620780+0000","last_became_peered":"2023-10-09T09:57:49.620780+0000","last_unstale":"2023-10-09T09:57:49.621174+0000","last_undegraded":"2023-10-09T09:57:49.621174+0000","last_fullsized":"2023-10-09T09:57:49.621174+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:01.389768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621680+0000","last_change":"2023-10-09T09:57:49.621680+0000","last_active":"2023-10-09T09:57:49.621680+0000","last_peered":"2023-10-09T09:57:49.621680+0000","last_clean":"2023-10-09T09:57:49.621680+0000","last_became_active":"2023-10-09T09:57:49.621345+0000","last_became_peered":"2023-10-09T09:57:49.621345+0000","last_unstale":"2023-10-09T09:57:49.621680+0000","last_undegraded":"2023-10-09T09:57:49.621680+0000","last_fullsized":"2023-10-09T09:57:49.621680+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:58.483689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617369+0000","last_change":"2023-10-09T09:56:13.613717+0000","last_active":"2023-10-09T09:57:48.617369+0000","last_peered":"2023-10-09T09:57:48.617369+0000","last_clean":"2023-10-09T09:57:48.617369+0000","last_became_active":"2023-10-09T09:56:13.613329+0000","last_became_peered":"2023-10-09T09:56:13.613329+0000","last_unstale":"2023-10-09T09:57:48.617369+0000","last_undegraded":"2023-10-09T09:57:48.617369+0000","last_fullsized":"2023-10-09T09:57:48.617369+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619103+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T09:57:48.619103+0000","last_peered":"2023-10-09T09:57:48.619103+0000","last_clean":"2023-10-09T09:57:48.619103+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T09:57:48.619103+0000","last_undegraded":"2023-10-09T09:57:48.619103+0000","last_fullsized":"2023-10-09T09:57:48.619103+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.618192+0000","last_change":"2023-10-09T09:57:18.842325+0000","last_active":"2023-10-09T09:57:48.618192+0000","last_peered":"2023-10-09T09:57:48.618192+0000","last_clean":"2023-10-09T09:57:48.618192+0000","last_became_active":"2023-10-09T09:57:18.841987+0000","last_became_peered":"2023-10-09T09:57:18.841987+0000","last_unstale":"2023-10-09T09:57:48.618192+0000","last_undegraded":"2023-10-09T09:57:48.618192+0000","last_fullsized":"2023-10-09T09:57:48.618192+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:28.615689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.628678+0000","last_change":"2023-10-09T09:57:49.628678+0000","last_active":"2023-10-09T09:57:49.628678+0000","last_peered":"2023-10-09T09:57:49.628678+0000","last_clean":"2023-10-09T09:57:49.628678+0000","last_became_active":"2023-10-09T09:57:49.628332+0000","last_became_peered":"2023-10-09T09:57:49.628332+0000","last_unstale":"2023-10-09T09:57:49.628678+0000","last_undegraded":"2023-10-09T09:57:49.628678+0000","last_fullsized":"2023-10-09T09:57:49.628678+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626775+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T09:57:48.626775+0000","last_peered":"2023-10-09T09:57:48.626775+0000","last_clean":"2023-10-09T09:57:48.626775+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T09:57:48.626775+0000","last_undegraded":"2023-10-09T09:57:48.626775+0000","last_fullsized":"2023-10-09T09:57:48.626775+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622055+0000","last_change":"2023-10-09T09:57:49.622055+0000","last_active":"2023-10-09T09:57:49.622055+0000","last_peered":"2023-10-09T09:57:49.622055+0000","last_clean":"2023-10-09T09:57:49.622055+0000","last_became_active":"2023-10-09T09:57:49.621784+0000","last_became_peered":"2023-10-09T09:57:49.621784+0000","last_unstale":"2023-10-09T09:57:49.622055+0000","last_undegraded":"2023-10-09T09:57:49.622055+0000","last_fullsized":"2023-10-09T09:57:49.622055+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:50.025391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624469+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T09:57:49.624469+0000","last_peered":"2023-10-09T09:57:49.624469+0000","last_clean":"2023-10-09T09:57:49.624469+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T09:57:49.624469+0000","last_undegraded":"2023-10-09T09:57:49.624469+0000","last_fullsized":"2023-10-09T09:57:49.624469+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619190+0000","last_change":"2023-10-09T09:56:45.555100+0000","last_active":"2023-10-09T09:57:48.619190+0000","last_peered":"2023-10-09T09:57:48.619190+0000","last_clean":"2023-10-09T09:57:48.619190+0000","last_became_active":"2023-10-09T09:56:45.554704+0000","last_became_peered":"2023-10-09T09:56:45.554704+0000","last_unstale":"2023-10-09T09:57:48.619190+0000","last_undegraded":"2023-10-09T09:57:48.619190+0000","last_fullsized":"2023-10-09T09:57:48.619190+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.616656+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:57:48.616656+0000","last_peered":"2023-10-09T09:57:48.616656+0000","last_clean":"2023-10-09T09:57:48.616656+0000","last_became_active":"2023-10-09T09:57:18.841522+0000","last_became_peered":"2023-10-09T09:57:18.841522+0000","last_unstale":"2023-10-09T09:57:48.616656+0000","last_undegraded":"2023-10-09T09:57:48.616656+0000","last_fullsized":"2023-10-09T09:57:48.616656+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.618427+0000","last_change":"2023-10-09T09:57:18.845748+0000","last_active":"2023-10-09T09:57:48.618427+0000","last_peered":"2023-10-09T09:57:48.618427+0000","last_clean":"2023-10-09T09:57:48.618427+0000","last_became_active":"2023-10-09T09:57:18.845397+0000","last_became_peered":"2023-10-09T09:57:18.845397+0000","last_unstale":"2023-10-09T09:57:48.618427+0000","last_undegraded":"2023-10-09T09:57:48.618427+0000","last_fullsized":"2023-10-09T09:57:48.618427+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622270+0000","last_change":"2023-10-09T09:56:45.556627+0000","last_active":"2023-10-09T09:57:48.622270+0000","last_peered":"2023-10-09T09:57:48.622270+0000","last_clean":"2023-10-09T09:57:48.622270+0000","last_became_active":"2023-10-09T09:56:45.556265+0000","last_became_peered":"2023-10-09T09:56:45.556265+0000","last_unstale":"2023-10-09T09:57:48.622270+0000","last_undegraded":"2023-10-09T09:57:48.622270+0000","last_fullsized":"2023-10-09T09:57:48.622270+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621587+0000","last_change":"2023-10-09T09:57:49.621587+0000","last_active":"2023-10-09T09:57:49.621587+0000","last_peered":"2023-10-09T09:57:49.621587+0000","last_clean":"2023-10-09T09:57:49.621587+0000","last_became_active":"2023-10-09T09:57:49.621245+0000","last_became_peered":"2023-10-09T09:57:49.621245+0000","last_unstale":"2023-10-09T09:57:49.621587+0000","last_undegraded":"2023-10-09T09:57:49.621587+0000","last_fullsized":"2023-10-09T09:57:49.621587+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620199+0000","last_change":"2023-10-09T09:57:49.620199+0000","last_active":"2023-10-09T09:57:49.620199+0000","last_peered":"2023-10-09T09:57:49.620199+0000","last_clean":"2023-10-09T09:57:49.620199+0000","last_became_active":"2023-10-09T09:57:49.619695+0000","last_became_peered":"2023-10-09T09:57:49.619695+0000","last_unstale":"2023-10-09T09:57:49.620199+0000","last_undegraded":"2023-10-09T09:57:49.620199+0000","last_fullsized":"2023-10-09T09:57:49.620199+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:48.818881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.611800+0000","last_change":"2023-10-09T09:56:45.553024+0000","last_active":"2023-10-09T09:57:49.611800+0000","last_peered":"2023-10-09T09:57:49.611800+0000","last_clean":"2023-10-09T09:57:49.611800+0000","last_became_active":"2023-10-09T09:56:45.552670+0000","last_became_peered":"2023-10-09T09:56:45.552670+0000","last_unstale":"2023-10-09T09:57:49.611800+0000","last_undegraded":"2023-10-09T09:57:49.611800+0000","last_fullsized":"2023-10-09T09:57:49.611800+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:58.132008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627539+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T09:57:49.627539+0000","last_peered":"2023-10-09T09:57:49.627539+0000","last_clean":"2023-10-09T09:57:49.627539+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T09:57:49.627539+0000","last_undegraded":"2023-10-09T09:57:49.627539+0000","last_fullsized":"2023-10-09T09:57:49.627539+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621676+0000","last_change":"2023-10-09T09:56:45.556365+0000","last_active":"2023-10-09T09:57:48.621676+0000","last_peered":"2023-10-09T09:57:48.621676+0000","last_clean":"2023-10-09T09:57:48.621676+0000","last_became_active":"2023-10-09T09:56:45.555832+0000","last_became_peered":"2023-10-09T09:56:45.555832+0000","last_unstale":"2023-10-09T09:57:48.621676+0000","last_undegraded":"2023-10-09T09:57:48.621676+0000","last_fullsized":"2023-10-09T09:57:48.621676+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626073+0000","last_change":"2023-10-09T09:48:25.514756+0000","last_active":"2023-10-09T09:57:48.626073+0000","last_peered":"2023-10-09T09:57:48.626073+0000","last_clean":"2023-10-09T09:57:48.626073+0000","last_became_active":"2023-10-09T09:48:25.514387+0000","last_became_peered":"2023-10-09T09:48:25.514387+0000","last_unstale":"2023-10-09T09:57:48.626073+0000","last_undegraded":"2023-10-09T09:57:48.626073+0000","last_fullsized":"2023-10-09T09:57:48.626073+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627073+0000","last_change":"2023-10-09T09:57:49.627073+0000","last_active":"2023-10-09T09:57:49.627073+0000","last_peered":"2023-10-09T09:57:49.627073+0000","last_clean":"2023-10-09T09:57:49.627073+0000","last_became_active":"2023-10-09T09:57:49.626706+0000","last_became_peered":"2023-10-09T09:57:49.626706+0000","last_unstale":"2023-10-09T09:57:49.627073+0000","last_undegraded":"2023-10-09T09:57:49.627073+0000","last_fullsized":"2023-10-09T09:57:49.627073+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625250+0000","last_change":"2023-10-09T09:56:45.554326+0000","last_active":"2023-10-09T09:57:48.625250+0000","last_peered":"2023-10-09T09:57:48.625250+0000","last_clean":"2023-10-09T09:57:48.625250+0000","last_became_active":"2023-10-09T09:56:45.553876+0000","last_became_peered":"2023-10-09T09:56:45.553876+0000","last_unstale":"2023-10-09T09:57:48.625250+0000","last_undegraded":"2023-10-09T09:57:48.625250+0000","last_fullsized":"2023-10-09T09:57:48.625250+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619582+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T09:57:48.619582+0000","last_peered":"2023-10-09T09:57:48.619582+0000","last_clean":"2023-10-09T09:57:48.619582+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T09:57:48.619582+0000","last_undegraded":"2023-10-09T09:57:48.619582+0000","last_fullsized":"2023-10-09T09:57:48.619582+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.624744+0000","last_change":"2023-10-09T09:56:13.614485+0000","last_active":"2023-10-09T09:57:48.624744+0000","last_peered":"2023-10-09T09:57:48.624744+0000","last_clean":"2023-10-09T09:57:48.624744+0000","last_became_active":"2023-10-09T09:56:13.614109+0000","last_became_peered":"2023-10-09T09:56:13.614109+0000","last_unstale":"2023-10-09T09:57:48.624744+0000","last_undegraded":"2023-10-09T09:57:48.624744+0000","last_fullsized":"2023-10-09T09:57:48.624744+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623806+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T09:57:49.623806+0000","last_peered":"2023-10-09T09:57:49.623806+0000","last_clean":"2023-10-09T09:57:49.623806+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T09:57:49.623806+0000","last_undegraded":"2023-10-09T09:57:49.623806+0000","last_fullsized":"2023-10-09T09:57:49.623806+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620753+0000","last_change":"2023-10-09T09:56:13.623204+0000","last_active":"2023-10-09T09:57:48.620753+0000","last_peered":"2023-10-09T09:57:48.620753+0000","last_clean":"2023-10-09T09:57:48.620753+0000","last_became_active":"2023-10-09T09:56:13.622853+0000","last_became_peered":"2023-10-09T09:56:13.622853+0000","last_unstale":"2023-10-09T09:57:48.620753+0000","last_undegraded":"2023-10-09T09:57:48.620753+0000","last_fullsized":"2023-10-09T09:57:48.620753+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622372+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T09:57:49.622372+0000","last_peered":"2023-10-09T09:57:49.622372+0000","last_clean":"2023-10-09T09:57:49.622372+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T09:57:49.622372+0000","last_undegraded":"2023-10-09T09:57:49.622372+0000","last_fullsized":"2023-10-09T09:57:49.622372+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625768+0000","last_change":"2023-10-09T09:48:25.523230+0000","last_active":"2023-10-09T09:57:48.625768+0000","last_peered":"2023-10-09T09:57:48.625768+0000","last_clean":"2023-10-09T09:57:48.625768+0000","last_became_active":"2023-10-09T09:48:25.522608+0000","last_became_peered":"2023-10-09T09:48:25.522608+0000","last_unstale":"2023-10-09T09:57:48.625768+0000","last_undegraded":"2023-10-09T09:57:48.625768+0000","last_fullsized":"2023-10-09T09:57:48.625768+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627770+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T09:57:49.627770+0000","last_peered":"2023-10-09T09:57:49.627770+0000","last_clean":"2023-10-09T09:57:49.627770+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T09:57:49.627770+0000","last_undegraded":"2023-10-09T09:57:49.627770+0000","last_fullsized":"2023-10-09T09:57:49.627770+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626723+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T09:57:49.626723+0000","last_peered":"2023-10-09T09:57:49.626723+0000","last_clean":"2023-10-09T09:57:49.626723+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T09:57:49.626723+0000","last_undegraded":"2023-10-09T09:57:49.626723+0000","last_fullsized":"2023-10-09T09:57:49.626723+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622141+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T09:57:49.622141+0000","last_peered":"2023-10-09T09:57:49.622141+0000","last_clean":"2023-10-09T09:57:49.622141+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T09:57:49.622141+0000","last_undegraded":"2023-10-09T09:57:49.622141+0000","last_fullsized":"2023-10-09T09:57:49.622141+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.624235+0000","last_change":"2023-10-09T09:57:18.843731+0000","last_active":"2023-10-09T09:57:48.624235+0000","last_peered":"2023-10-09T09:57:48.624235+0000","last_clean":"2023-10-09T09:57:48.624235+0000","last_became_active":"2023-10-09T09:57:18.843280+0000","last_became_peered":"2023-10-09T09:57:18.843280+0000","last_unstale":"2023-10-09T09:57:48.624235+0000","last_undegraded":"2023-10-09T09:57:48.624235+0000","last_fullsized":"2023-10-09T09:57:48.624235+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617918+0000","last_change":"2023-10-09T09:56:45.551325+0000","last_active":"2023-10-09T09:57:48.617918+0000","last_peered":"2023-10-09T09:57:48.617918+0000","last_clean":"2023-10-09T09:57:48.617918+0000","last_became_active":"2023-10-09T09:56:45.550946+0000","last_became_peered":"2023-10-09T09:56:45.550946+0000","last_unstale":"2023-10-09T09:57:48.617918+0000","last_undegraded":"2023-10-09T09:57:48.617918+0000","last_fullsized":"2023-10-09T09:57:48.617918+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":69,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617387+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T09:57:48.617387+0000","last_peered":"2023-10-09T09:57:48.617387+0000","last_clean":"2023-10-09T09:57:48.617387+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T09:57:48.617387+0000","last_undegraded":"2023-10-09T09:57:48.617387+0000","last_fullsized":"2023-10-09T09:57:48.617387+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621810+0000","last_change":"2023-10-09T09:57:18.840471+0000","last_active":"2023-10-09T09:57:48.621810+0000","last_peered":"2023-10-09T09:57:48.621810+0000","last_clean":"2023-10-09T09:57:48.621810+0000","last_became_active":"2023-10-09T09:57:18.840125+0000","last_became_peered":"2023-10-09T09:57:18.840125+0000","last_unstale":"2023-10-09T09:57:48.621810+0000","last_undegraded":"2023-10-09T09:57:48.621810+0000","last_fullsized":"2023-10-09T09:57:48.621810+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:07.418455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626281+0000","last_change":"2023-10-09T09:57:49.626281+0000","last_active":"2023-10-09T09:57:49.626281+0000","last_peered":"2023-10-09T09:57:49.626281+0000","last_clean":"2023-10-09T09:57:49.626281+0000","last_became_active":"2023-10-09T09:57:49.625726+0000","last_became_peered":"2023-10-09T09:57:49.625726+0000","last_unstale":"2023-10-09T09:57:49.626281+0000","last_undegraded":"2023-10-09T09:57:49.626281+0000","last_fullsized":"2023-10-09T09:57:49.626281+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625511+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T09:57:49.625511+0000","last_peered":"2023-10-09T09:57:49.625511+0000","last_clean":"2023-10-09T09:57:49.625511+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T09:57:49.625511+0000","last_undegraded":"2023-10-09T09:57:49.625511+0000","last_fullsized":"2023-10-09T09:57:49.625511+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620020+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T09:57:48.620020+0000","last_peered":"2023-10-09T09:57:48.620020+0000","last_clean":"2023-10-09T09:57:48.620020+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T09:57:48.620020+0000","last_undegraded":"2023-10-09T09:57:48.620020+0000","last_fullsized":"2023-10-09T09:57:48.620020+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624665+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T09:57:49.624665+0000","last_peered":"2023-10-09T09:57:49.624665+0000","last_clean":"2023-10-09T09:57:49.624665+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T09:57:49.624665+0000","last_undegraded":"2023-10-09T09:57:49.624665+0000","last_fullsized":"2023-10-09T09:57:49.624665+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626421+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T09:57:49.626421+0000","last_peered":"2023-10-09T09:57:49.626421+0000","last_clean":"2023-10-09T09:57:49.626421+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T09:57:49.626421+0000","last_undegraded":"2023-10-09T09:57:49.626421+0000","last_fullsized":"2023-10-09T09:57:49.626421+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622159+0000","last_change":"2023-10-09T09:57:18.838985+0000","last_active":"2023-10-09T09:57:48.622159+0000","last_peered":"2023-10-09T09:57:48.622159+0000","last_clean":"2023-10-09T09:57:48.622159+0000","last_became_active":"2023-10-09T09:57:18.838625+0000","last_became_peered":"2023-10-09T09:57:18.838625+0000","last_unstale":"2023-10-09T09:57:48.622159+0000","last_undegraded":"2023-10-09T09:57:48.622159+0000","last_fullsized":"2023-10-09T09:57:48.622159+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624475+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T09:57:49.624475+0000","last_peered":"2023-10-09T09:57:49.624475+0000","last_clean":"2023-10-09T09:57:49.624475+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T09:57:49.624475+0000","last_undegraded":"2023-10-09T09:57:49.624475+0000","last_fullsized":"2023-10-09T09:57:49.624475+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625707+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T09:57:49.625707+0000","last_peered":"2023-10-09T09:57:49.625707+0000","last_clean":"2023-10-09T09:57:49.625707+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T09:57:49.625707+0000","last_undegraded":"2023-10-09T09:57:49.625707+0000","last_fullsized":"2023-10-09T09:57:49.625707+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621495+0000","last_change":"2023-10-09T09:57:49.621495+0000","last_active":"2023-10-09T09:57:49.621495+0000","last_peered":"2023-10-09T09:57:49.621495+0000","last_clean":"2023-10-09T09:57:49.621495+0000","last_became_active":"2023-10-09T09:57:49.621144+0000","last_became_peered":"2023-10-09T09:57:49.621144+0000","last_unstale":"2023-10-09T09:57:49.621495+0000","last_undegraded":"2023-10-09T09:57:49.621495+0000","last_fullsized":"2023-10-09T09:57:49.621495+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:11.852424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622753+0000","last_change":"2023-10-09T09:57:49.622753+0000","last_active":"2023-10-09T09:57:49.622753+0000","last_peered":"2023-10-09T09:57:49.622753+0000","last_clean":"2023-10-09T09:57:49.622753+0000","last_became_active":"2023-10-09T09:57:49.622463+0000","last_became_peered":"2023-10-09T09:57:49.622463+0000","last_unstale":"2023-10-09T09:57:49.622753+0000","last_undegraded":"2023-10-09T09:57:49.622753+0000","last_fullsized":"2023-10-09T09:57:49.622753+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:51.020605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":78,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617029+0000","last_change":"2023-10-09T09:57:18.841362+0000","last_active":"2023-10-09T09:57:48.617029+0000","last_peered":"2023-10-09T09:57:48.617029+0000","last_clean":"2023-10-09T09:57:48.617029+0000","last_became_active":"2023-10-09T09:57:18.840951+0000","last_became_peered":"2023-10-09T09:57:18.840951+0000","last_unstale":"2023-10-09T09:57:48.617029+0000","last_undegraded":"2023-10-09T09:57:48.617029+0000","last_fullsized":"2023-10-09T09:57:48.617029+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430347,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310920,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434232,"statfs":{"total":95995035648,"available":95676653568,"internally_reserved":0,"allocated":1572864,"data_stored":1111479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58399999999999996,"15min":0.58999999999999997},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.41799999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73599999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.58299999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.73299999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.57999999999999996,"15min":0.53700000000000003},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.36099999999999999},"max":{"1min":0.81399999999999995,"5min":0.89700000000000002,"15min":0.91600000000000004},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.56499999999999995},"min":{"1min":0.39700000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.76800000000000002,"5min":0.78100000000000003,"15min":0.83799999999999997},"last":0.60499999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.49299999999999999,"15min":0.46800000000000003},"min":{"1min":0.39200000000000002,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.51700000000000002,"15min":0.47999999999999998},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.87},"last":0.70499999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.52400000000000002,"15min":0.503},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.83599999999999997},"last":0.628},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.498,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.85399999999999998},"last":0.79700000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.59099999999999997,"15min":0.55200000000000005},"min":{"1min":0.52900000000000003,"5min":0.36799999999999999,"15min":0.32900000000000001},"max":{"1min":0.84599999999999997,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.58899999999999997,"15min":0.54400000000000004},"min":{"1min":0.42799999999999999,"5min":0.40300000000000002,"15min":0.33100000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.96599999999999997},"last":0.74399999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626575,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1556480,"data_stored":1111018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71999999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.51900000000000002,"15min":0.51000000000000001},"min":{"1min":0.42299999999999999,"5min":0.308,"15min":0.308},"max":{"1min":0.749,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.63500000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.69999999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57999999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:57:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.57999999999999996,"15min":0.55000000000000004},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.33100000000000002},"max":{"1min":0.79300000000000004,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.60099999999999998,"15min":0.56299999999999994},"min":{"1min":0.39300000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.0169999999999999,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.76700000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46700000000000003,"15min":0.45000000000000001},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.23300000000000001},"max":{"1min":0.64500000000000002,"5min":0.75800000000000001,"15min":0.77600000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.496,"15min":0.48899999999999999},"min":{"1min":0.36299999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.66800000000000004,"5min":0.753,"15min":0.75600000000000001},"last":0.48299999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.50600000000000001,"15min":0.503},"min":{"1min":0.36599999999999999,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":0.63600000000000001,"5min":0.76100000000000001,"15min":0.80600000000000005},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51700000000000002,"15min":0.505},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68500000000000005,"5min":0.76800000000000002,"15min":1.0600000000000001},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.54400000000000004,"15min":0.52600000000000002},"min":{"1min":0.35099999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.68000000000000005,"5min":0.79400000000000004,"15min":0.81399999999999995},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.627,"5min":0.61499999999999999,"15min":0.58699999999999997},"min":{"1min":0.52600000000000002,"5min":0.34899999999999998,"15min":0.34399999999999997},"max":{"1min":0.754,"5min":0.82999999999999996,"15min":0.84099999999999997},"last":0.71599999999999997}]}]},{"osd":1,"up_from":84,"seq":360777252888,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28308,"kb_used_data":972,"kb_used_omap":9,"kb_used_meta":27318,"kb_avail":93716844,"statfs":{"total":95995035648,"available":95966048256,"internally_reserved":0,"allocated":995328,"data_stored":524748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":27974633},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.27900000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.34399999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.67900000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.627}]}]},{"osd":0,"up_from":79,"seq":339302416414,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33056,"kb_used_data":1560,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712096,"statfs":{"total":95995035648,"available":95961186304,"internally_reserved":0,"allocated":1597440,"data_stored":1114630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.442},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.47099999999999997}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:57:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.53800000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.66900000000000004,"15min":0.66100000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68200000000000005,"15min":0.67800000000000005},"min":{"1min":0.44600000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.028,"5min":1.101,"15min":1.101},"last":0.93000000000000005}]},{"osd":5,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64600000000000002,"15min":0.64400000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85799999999999998,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.94499999999999995,"5min":1.1180000000000001,"15min":1.1180000000000001},"last":0.94799999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.68300000000000005,"15min":0.68700000000000006},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89700000000000002,"5min":1.161,"15min":1.161},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.68899999999999995,"15min":0.69099999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.99199999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.66500000000000004,"15min":0.66600000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.89600000000000002,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.68400000000000005,"15min":0.68300000000000005},"min":{"1min":0.441,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.98099999999999998,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.91700000000000004}]}]},{"osd":2,"up_from":89,"seq":382252089361,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36624,"kb_used_data":968,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708528,"statfs":{"total":95995035648,"available":95957532672,"internally_reserved":0,"allocated":991232,"data_stored":522739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.35399999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.40100000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36899999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.437},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.56999999999999995}]},{"osd":5,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.51500000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.48099999999999998}]}]},{"osd":3,"up_from":94,"seq":403726925835,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28288,"kb_used_data":952,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716864,"statfs":{"total":95995035648,"available":95966068736,"internally_reserved":0,"allocated":974848,"data_stored":520694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32400000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019031,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302144,"kb_used_data":952,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443008,"statfs":{"total":95995035648,"available":95685640192,"internally_reserved":0,"allocated":974848,"data_stored":521041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.66800000000000004,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56699999999999995,"15min":0.56999999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.47999999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.54400000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:56:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.28199999999999997},"max":{"1min":0.753,"5min":0.877,"15min":2.2109999999999999},"last":0.442},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.47899999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.75,"5min":0.81399999999999995,"15min":0.997},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.45600000000000002,"15min":0.43099999999999999},"min":{"1min":0.34399999999999997,"5min":0.28399999999999997,"15min":0.27100000000000002},"max":{"1min":0.77200000000000002,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.48999999999999999,"15min":0.46600000000000003},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70999999999999996,"5min":0.72099999999999997,"15min":0.83999999999999997},"last":0.40500000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.499,"15min":0.46800000000000003},"min":{"1min":0.41199999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78600000000000003,"5min":0.79300000000000004,"15min":0.86599999999999999},"last":0.374},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53200000000000003,"15min":0.497},"min":{"1min":0.379,"5min":0.318,"15min":0.312},"max":{"1min":0.80700000000000005,"5min":0.81499999999999995,"15min":0.89900000000000002},"last":0.51900000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55800000000000005,"15min":0.52500000000000002},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.69899999999999995,"5min":0.76600000000000001,"15min":0.84999999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59699999999999998,"15min":0.55800000000000005},"min":{"1min":0.45800000000000002,"5min":0.33700000000000002,"15min":0.316},"max":{"1min":0.76000000000000001,"5min":0.82899999999999996,"15min":0.85199999999999998},"last":0.53600000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822803,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310320,"kb_used_data":936,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434832,"statfs":{"total":95995035648,"available":95677267968,"internally_reserved":0,"allocated":958464,"data_stored":516785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.75,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.55000000000000004,"15min":0.54100000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.4289999999999998,"5min":2.4289999999999998,"15min":2.4289999999999998},"last":0.41099999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.503}]},{"osd":2,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":2.5950000000000002,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.62}]},{"osd":3,"last update":"Mon Oct 9 09:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.56899999999999995,"15min":0.54700000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":0.70699999999999996,"5min":0.84999999999999998,"15min":1.1479999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.60699999999999998,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":2.637},"last":0.67100000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47699999999999998,"15min":0.44400000000000001},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":0.74299999999999999,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.51000000000000001,"15min":0.46800000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64900000000000002,"5min":0.79600000000000004,"15min":0.88200000000000001},"last":0.53700000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.53000000000000003,"15min":0.503},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":0.76200000000000001,"5min":0.85099999999999998,"15min":0.92800000000000005},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.52100000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81000000000000005,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.66400000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58099999999999996,"15min":0.55500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":0.71299999999999997,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55000000000000004,"15min":0.53600000000000003},"min":{"1min":0.37,"5min":0.318,"15min":0.317},"max":{"1min":0.68200000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.72799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:58:37.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:58:37.639 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:37.857 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:37 smithi012 ceph-mon[147926]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:37.857 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:37 smithi012 ceph-mon[152263]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:37 smithi110 ceph-mon[136032]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:38.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:58:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:58:37] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.33.4" 2023-10-09T09:58:38.579 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:38 smithi012 ceph-mon[147926]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:38.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:38 smithi012 ceph-mon[152263]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:38 smithi110 ceph-mon[136032]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:58:39.140 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:58:39.561 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":223,"stamp":"2023-10-09T09:58:38.907827+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1348276,"kb_used_data":9396,"kb_used_omap":126,"kb_used_meta":1338689,"kb_avail":748612940,"statfs":{"total":767960285184,"available":766579650560,"internally_reserved":0,"allocated":9621504,"data_stored":5943134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1370818171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002900"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.616419+0000","last_change":"2023-10-09T09:56:45.556456+0000","last_active":"2023-10-09T09:57:48.616419+0000","last_peered":"2023-10-09T09:57:48.616419+0000","last_clean":"2023-10-09T09:57:48.616419+0000","last_became_active":"2023-10-09T09:56:45.555878+0000","last_became_peered":"2023-10-09T09:56:45.555878+0000","last_unstale":"2023-10-09T09:57:48.616419+0000","last_undegraded":"2023-10-09T09:57:48.616419+0000","last_fullsized":"2023-10-09T09:57:48.616419+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612158+0000","last_change":"2023-10-09T09:57:18.842453+0000","last_active":"2023-10-09T09:57:49.612158+0000","last_peered":"2023-10-09T09:57:49.612158+0000","last_clean":"2023-10-09T09:57:49.612158+0000","last_became_active":"2023-10-09T09:57:18.842133+0000","last_became_peered":"2023-10-09T09:57:18.842133+0000","last_unstale":"2023-10-09T09:57:49.612158+0000","last_undegraded":"2023-10-09T09:57:49.612158+0000","last_fullsized":"2023-10-09T09:57:49.612158+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:07.459692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620527+0000","last_change":"2023-10-09T09:57:49.620527+0000","last_active":"2023-10-09T09:57:49.620527+0000","last_peered":"2023-10-09T09:57:49.620527+0000","last_clean":"2023-10-09T09:57:49.620527+0000","last_became_active":"2023-10-09T09:57:49.620109+0000","last_became_peered":"2023-10-09T09:57:49.620109+0000","last_unstale":"2023-10-09T09:57:49.620527+0000","last_undegraded":"2023-10-09T09:57:49.620527+0000","last_fullsized":"2023-10-09T09:57:49.620527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:09.284902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":68,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623102+0000","last_change":"2023-10-09T09:57:49.623102+0000","last_active":"2023-10-09T09:57:49.623102+0000","last_peered":"2023-10-09T09:57:49.623102+0000","last_clean":"2023-10-09T09:57:49.623102+0000","last_became_active":"2023-10-09T09:57:49.620638+0000","last_became_peered":"2023-10-09T09:57:49.620638+0000","last_unstale":"2023-10-09T09:57:49.623102+0000","last_undegraded":"2023-10-09T09:57:49.623102+0000","last_fullsized":"2023-10-09T09:57:49.623102+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:11.322675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620967+0000","last_change":"2023-10-09T09:57:49.620967+0000","last_active":"2023-10-09T09:57:49.620967+0000","last_peered":"2023-10-09T09:57:49.620967+0000","last_clean":"2023-10-09T09:57:49.620967+0000","last_became_active":"2023-10-09T09:57:49.620567+0000","last_became_peered":"2023-10-09T09:57:49.620567+0000","last_unstale":"2023-10-09T09:57:49.620967+0000","last_undegraded":"2023-10-09T09:57:49.620967+0000","last_fullsized":"2023-10-09T09:57:49.620967+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620814+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T09:57:48.620814+0000","last_peered":"2023-10-09T09:57:48.620814+0000","last_clean":"2023-10-09T09:57:48.620814+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T09:57:48.620814+0000","last_undegraded":"2023-10-09T09:57:48.620814+0000","last_fullsized":"2023-10-09T09:57:48.620814+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622634+0000","last_change":"2023-10-09T09:57:49.622634+0000","last_active":"2023-10-09T09:57:49.622634+0000","last_peered":"2023-10-09T09:57:49.622634+0000","last_clean":"2023-10-09T09:57:49.622634+0000","last_became_active":"2023-10-09T09:57:49.622247+0000","last_became_peered":"2023-10-09T09:57:49.622247+0000","last_unstale":"2023-10-09T09:57:49.622634+0000","last_undegraded":"2023-10-09T09:57:49.622634+0000","last_fullsized":"2023-10-09T09:57:49.622634+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621538+0000","last_change":"2023-10-09T09:56:45.554958+0000","last_active":"2023-10-09T09:57:48.621538+0000","last_peered":"2023-10-09T09:57:48.621538+0000","last_clean":"2023-10-09T09:57:48.621538+0000","last_became_active":"2023-10-09T09:56:45.554597+0000","last_became_peered":"2023-10-09T09:56:45.554597+0000","last_unstale":"2023-10-09T09:57:48.621538+0000","last_undegraded":"2023-10-09T09:57:48.621538+0000","last_fullsized":"2023-10-09T09:57:48.621538+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623531+0000","last_change":"2023-10-09T09:57:49.623531+0000","last_active":"2023-10-09T09:57:49.623531+0000","last_peered":"2023-10-09T09:57:49.623531+0000","last_clean":"2023-10-09T09:57:49.623531+0000","last_became_active":"2023-10-09T09:57:49.623045+0000","last_became_peered":"2023-10-09T09:57:49.623045+0000","last_unstale":"2023-10-09T09:57:49.623531+0000","last_undegraded":"2023-10-09T09:57:49.623531+0000","last_fullsized":"2023-10-09T09:57:49.623531+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:49.655085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":40,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622370+0000","last_change":"2023-10-09T09:56:13.614416+0000","last_active":"2023-10-09T09:57:48.622370+0000","last_peered":"2023-10-09T09:57:48.622370+0000","last_clean":"2023-10-09T09:57:48.622370+0000","last_became_active":"2023-10-09T09:56:13.614050+0000","last_became_peered":"2023-10-09T09:56:13.614050+0000","last_unstale":"2023-10-09T09:57:48.622370+0000","last_undegraded":"2023-10-09T09:57:48.622370+0000","last_fullsized":"2023-10-09T09:57:48.622370+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:17.068217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617691+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T09:57:48.617691+0000","last_peered":"2023-10-09T09:57:48.617691+0000","last_clean":"2023-10-09T09:57:48.617691+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T09:57:48.617691+0000","last_undegraded":"2023-10-09T09:57:48.617691+0000","last_fullsized":"2023-10-09T09:57:48.617691+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612468+0000","last_change":"2023-10-09T09:56:45.554759+0000","last_active":"2023-10-09T09:57:49.612468+0000","last_peered":"2023-10-09T09:57:49.612468+0000","last_clean":"2023-10-09T09:57:49.612468+0000","last_became_active":"2023-10-09T09:56:45.554390+0000","last_became_peered":"2023-10-09T09:56:45.554390+0000","last_unstale":"2023-10-09T09:57:49.612468+0000","last_undegraded":"2023-10-09T09:57:49.612468+0000","last_fullsized":"2023-10-09T09:57:49.612468+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:43.005737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622066+0000","last_change":"2023-10-09T09:57:18.844262+0000","last_active":"2023-10-09T09:57:48.622066+0000","last_peered":"2023-10-09T09:57:48.622066+0000","last_clean":"2023-10-09T09:57:48.622066+0000","last_became_active":"2023-10-09T09:57:18.843935+0000","last_became_peered":"2023-10-09T09:57:18.843935+0000","last_unstale":"2023-10-09T09:57:48.622066+0000","last_undegraded":"2023-10-09T09:57:48.622066+0000","last_fullsized":"2023-10-09T09:57:48.622066+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624797+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T09:57:49.624797+0000","last_peered":"2023-10-09T09:57:49.624797+0000","last_clean":"2023-10-09T09:57:49.624797+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T09:57:49.624797+0000","last_undegraded":"2023-10-09T09:57:49.624797+0000","last_fullsized":"2023-10-09T09:57:49.624797+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622011+0000","last_change":"2023-10-09T09:56:45.549837+0000","last_active":"2023-10-09T09:57:48.622011+0000","last_peered":"2023-10-09T09:57:48.622011+0000","last_clean":"2023-10-09T09:57:48.622011+0000","last_became_active":"2023-10-09T09:56:45.549546+0000","last_became_peered":"2023-10-09T09:56:45.549546+0000","last_unstale":"2023-10-09T09:57:48.622011+0000","last_undegraded":"2023-10-09T09:57:48.622011+0000","last_fullsized":"2023-10-09T09:57:48.622011+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:26.779071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624047+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T09:57:49.624047+0000","last_peered":"2023-10-09T09:57:49.624047+0000","last_clean":"2023-10-09T09:57:49.624047+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T09:57:49.624047+0000","last_undegraded":"2023-10-09T09:57:49.624047+0000","last_fullsized":"2023-10-09T09:57:49.624047+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.611604+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T09:57:49.611604+0000","last_peered":"2023-10-09T09:57:49.611604+0000","last_clean":"2023-10-09T09:57:49.611604+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T09:57:49.611604+0000","last_undegraded":"2023-10-09T09:57:49.611604+0000","last_fullsized":"2023-10-09T09:57:49.611604+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626860+0000","last_change":"2023-10-09T09:57:18.844843+0000","last_active":"2023-10-09T09:57:48.626860+0000","last_peered":"2023-10-09T09:57:48.626860+0000","last_clean":"2023-10-09T09:57:48.626860+0000","last_became_active":"2023-10-09T09:57:18.844530+0000","last_became_peered":"2023-10-09T09:57:18.844530+0000","last_unstale":"2023-10-09T09:57:48.626860+0000","last_undegraded":"2023-10-09T09:57:48.626860+0000","last_fullsized":"2023-10-09T09:57:48.626860+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:33.761788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625926+0000","last_change":"2023-10-09T09:56:13.614282+0000","last_active":"2023-10-09T09:57:48.625926+0000","last_peered":"2023-10-09T09:57:48.625926+0000","last_clean":"2023-10-09T09:57:48.625926+0000","last_became_active":"2023-10-09T09:56:13.613888+0000","last_became_peered":"2023-10-09T09:56:13.613888+0000","last_unstale":"2023-10-09T09:57:48.625926+0000","last_undegraded":"2023-10-09T09:57:48.625926+0000","last_fullsized":"2023-10-09T09:57:48.625926+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":71,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.612551+0000","last_change":"2023-10-09T09:57:18.840413+0000","last_active":"2023-10-09T09:57:49.612551+0000","last_peered":"2023-10-09T09:57:49.612551+0000","last_clean":"2023-10-09T09:57:49.612551+0000","last_became_active":"2023-10-09T09:57:18.840093+0000","last_became_peered":"2023-10-09T09:57:18.840093+0000","last_unstale":"2023-10-09T09:57:49.612551+0000","last_undegraded":"2023-10-09T09:57:49.612551+0000","last_fullsized":"2023-10-09T09:57:49.612551+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:58.170995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626616+0000","last_change":"2023-10-09T09:57:49.624279+0000","last_active":"2023-10-09T09:57:49.626616+0000","last_peered":"2023-10-09T09:57:49.626616+0000","last_clean":"2023-10-09T09:57:49.626616+0000","last_became_active":"2023-10-09T09:57:49.624010+0000","last_became_peered":"2023-10-09T09:57:49.624010+0000","last_unstale":"2023-10-09T09:57:49.626616+0000","last_undegraded":"2023-10-09T09:57:49.626616+0000","last_fullsized":"2023-10-09T09:57:49.626616+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:47.717690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626997+0000","last_change":"2023-10-09T09:56:45.555907+0000","last_active":"2023-10-09T09:57:48.626997+0000","last_peered":"2023-10-09T09:57:48.626997+0000","last_clean":"2023-10-09T09:57:48.626997+0000","last_became_active":"2023-10-09T09:56:45.555548+0000","last_became_peered":"2023-10-09T09:56:45.555548+0000","last_unstale":"2023-10-09T09:57:48.626997+0000","last_undegraded":"2023-10-09T09:57:48.626997+0000","last_fullsized":"2023-10-09T09:57:48.626997+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622200+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T09:57:48.622200+0000","last_peered":"2023-10-09T09:57:48.622200+0000","last_clean":"2023-10-09T09:57:48.622200+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T09:57:48.622200+0000","last_undegraded":"2023-10-09T09:57:48.622200+0000","last_fullsized":"2023-10-09T09:57:48.622200+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622586+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T09:57:49.622586+0000","last_peered":"2023-10-09T09:57:49.622586+0000","last_clean":"2023-10-09T09:57:49.622586+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T09:57:49.622586+0000","last_undegraded":"2023-10-09T09:57:49.622586+0000","last_fullsized":"2023-10-09T09:57:49.622586+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":296,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626889+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T09:57:48.626889+0000","last_peered":"2023-10-09T09:57:48.626889+0000","last_clean":"2023-10-09T09:57:48.626889+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T09:57:48.626889+0000","last_undegraded":"2023-10-09T09:57:48.626889+0000","last_fullsized":"2023-10-09T09:57:48.626889+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619019+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:57:48.619019+0000","last_peered":"2023-10-09T09:57:48.619019+0000","last_clean":"2023-10-09T09:57:48.619019+0000","last_became_active":"2023-10-09T09:57:18.841472+0000","last_became_peered":"2023-10-09T09:57:18.841472+0000","last_unstale":"2023-10-09T09:57:48.619019+0000","last_undegraded":"2023-10-09T09:57:48.619019+0000","last_fullsized":"2023-10-09T09:57:48.619019+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622632+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T09:57:48.622632+0000","last_peered":"2023-10-09T09:57:48.622632+0000","last_clean":"2023-10-09T09:57:48.622632+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T09:57:48.622632+0000","last_undegraded":"2023-10-09T09:57:48.622632+0000","last_fullsized":"2023-10-09T09:57:48.622632+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.627470+0000","last_change":"2023-10-09T09:57:18.837334+0000","last_active":"2023-10-09T09:57:48.627470+0000","last_peered":"2023-10-09T09:57:48.627470+0000","last_clean":"2023-10-09T09:57:48.627470+0000","last_became_active":"2023-10-09T09:57:18.837037+0000","last_became_peered":"2023-10-09T09:57:18.837037+0000","last_unstale":"2023-10-09T09:57:48.627470+0000","last_undegraded":"2023-10-09T09:57:48.627470+0000","last_fullsized":"2023-10-09T09:57:48.627470+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:11.650333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":83,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621174+0000","last_change":"2023-10-09T09:57:49.621174+0000","last_active":"2023-10-09T09:57:49.621174+0000","last_peered":"2023-10-09T09:57:49.621174+0000","last_clean":"2023-10-09T09:57:49.621174+0000","last_became_active":"2023-10-09T09:57:49.620780+0000","last_became_peered":"2023-10-09T09:57:49.620780+0000","last_unstale":"2023-10-09T09:57:49.621174+0000","last_undegraded":"2023-10-09T09:57:49.621174+0000","last_fullsized":"2023-10-09T09:57:49.621174+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:01.389768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621680+0000","last_change":"2023-10-09T09:57:49.621680+0000","last_active":"2023-10-09T09:57:49.621680+0000","last_peered":"2023-10-09T09:57:49.621680+0000","last_clean":"2023-10-09T09:57:49.621680+0000","last_became_active":"2023-10-09T09:57:49.621345+0000","last_became_peered":"2023-10-09T09:57:49.621345+0000","last_unstale":"2023-10-09T09:57:49.621680+0000","last_undegraded":"2023-10-09T09:57:49.621680+0000","last_fullsized":"2023-10-09T09:57:49.621680+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:58.483689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617369+0000","last_change":"2023-10-09T09:56:13.613717+0000","last_active":"2023-10-09T09:57:48.617369+0000","last_peered":"2023-10-09T09:57:48.617369+0000","last_clean":"2023-10-09T09:57:48.617369+0000","last_became_active":"2023-10-09T09:56:13.613329+0000","last_became_peered":"2023-10-09T09:56:13.613329+0000","last_unstale":"2023-10-09T09:57:48.617369+0000","last_undegraded":"2023-10-09T09:57:48.617369+0000","last_fullsized":"2023-10-09T09:57:48.617369+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619103+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T09:57:48.619103+0000","last_peered":"2023-10-09T09:57:48.619103+0000","last_clean":"2023-10-09T09:57:48.619103+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T09:57:48.619103+0000","last_undegraded":"2023-10-09T09:57:48.619103+0000","last_fullsized":"2023-10-09T09:57:48.619103+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.618192+0000","last_change":"2023-10-09T09:57:18.842325+0000","last_active":"2023-10-09T09:57:48.618192+0000","last_peered":"2023-10-09T09:57:48.618192+0000","last_clean":"2023-10-09T09:57:48.618192+0000","last_became_active":"2023-10-09T09:57:18.841987+0000","last_became_peered":"2023-10-09T09:57:18.841987+0000","last_unstale":"2023-10-09T09:57:48.618192+0000","last_undegraded":"2023-10-09T09:57:48.618192+0000","last_fullsized":"2023-10-09T09:57:48.618192+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:28.615689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.628678+0000","last_change":"2023-10-09T09:57:49.628678+0000","last_active":"2023-10-09T09:57:49.628678+0000","last_peered":"2023-10-09T09:57:49.628678+0000","last_clean":"2023-10-09T09:57:49.628678+0000","last_became_active":"2023-10-09T09:57:49.628332+0000","last_became_peered":"2023-10-09T09:57:49.628332+0000","last_unstale":"2023-10-09T09:57:49.628678+0000","last_undegraded":"2023-10-09T09:57:49.628678+0000","last_fullsized":"2023-10-09T09:57:49.628678+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626775+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T09:57:48.626775+0000","last_peered":"2023-10-09T09:57:48.626775+0000","last_clean":"2023-10-09T09:57:48.626775+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T09:57:48.626775+0000","last_undegraded":"2023-10-09T09:57:48.626775+0000","last_fullsized":"2023-10-09T09:57:48.626775+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622055+0000","last_change":"2023-10-09T09:57:49.622055+0000","last_active":"2023-10-09T09:57:49.622055+0000","last_peered":"2023-10-09T09:57:49.622055+0000","last_clean":"2023-10-09T09:57:49.622055+0000","last_became_active":"2023-10-09T09:57:49.621784+0000","last_became_peered":"2023-10-09T09:57:49.621784+0000","last_unstale":"2023-10-09T09:57:49.622055+0000","last_undegraded":"2023-10-09T09:57:49.622055+0000","last_fullsized":"2023-10-09T09:57:49.622055+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:50.025391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624469+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T09:57:49.624469+0000","last_peered":"2023-10-09T09:57:49.624469+0000","last_clean":"2023-10-09T09:57:49.624469+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T09:57:49.624469+0000","last_undegraded":"2023-10-09T09:57:49.624469+0000","last_fullsized":"2023-10-09T09:57:49.624469+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619190+0000","last_change":"2023-10-09T09:56:45.555100+0000","last_active":"2023-10-09T09:57:48.619190+0000","last_peered":"2023-10-09T09:57:48.619190+0000","last_clean":"2023-10-09T09:57:48.619190+0000","last_became_active":"2023-10-09T09:56:45.554704+0000","last_became_peered":"2023-10-09T09:56:45.554704+0000","last_unstale":"2023-10-09T09:57:48.619190+0000","last_undegraded":"2023-10-09T09:57:48.619190+0000","last_fullsized":"2023-10-09T09:57:48.619190+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.616656+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:57:48.616656+0000","last_peered":"2023-10-09T09:57:48.616656+0000","last_clean":"2023-10-09T09:57:48.616656+0000","last_became_active":"2023-10-09T09:57:18.841522+0000","last_became_peered":"2023-10-09T09:57:18.841522+0000","last_unstale":"2023-10-09T09:57:48.616656+0000","last_undegraded":"2023-10-09T09:57:48.616656+0000","last_fullsized":"2023-10-09T09:57:48.616656+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.618427+0000","last_change":"2023-10-09T09:57:18.845748+0000","last_active":"2023-10-09T09:57:48.618427+0000","last_peered":"2023-10-09T09:57:48.618427+0000","last_clean":"2023-10-09T09:57:48.618427+0000","last_became_active":"2023-10-09T09:57:18.845397+0000","last_became_peered":"2023-10-09T09:57:18.845397+0000","last_unstale":"2023-10-09T09:57:48.618427+0000","last_undegraded":"2023-10-09T09:57:48.618427+0000","last_fullsized":"2023-10-09T09:57:48.618427+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622270+0000","last_change":"2023-10-09T09:56:45.556627+0000","last_active":"2023-10-09T09:57:48.622270+0000","last_peered":"2023-10-09T09:57:48.622270+0000","last_clean":"2023-10-09T09:57:48.622270+0000","last_became_active":"2023-10-09T09:56:45.556265+0000","last_became_peered":"2023-10-09T09:56:45.556265+0000","last_unstale":"2023-10-09T09:57:48.622270+0000","last_undegraded":"2023-10-09T09:57:48.622270+0000","last_fullsized":"2023-10-09T09:57:48.622270+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621587+0000","last_change":"2023-10-09T09:57:49.621587+0000","last_active":"2023-10-09T09:57:49.621587+0000","last_peered":"2023-10-09T09:57:49.621587+0000","last_clean":"2023-10-09T09:57:49.621587+0000","last_became_active":"2023-10-09T09:57:49.621245+0000","last_became_peered":"2023-10-09T09:57:49.621245+0000","last_unstale":"2023-10-09T09:57:49.621587+0000","last_undegraded":"2023-10-09T09:57:49.621587+0000","last_fullsized":"2023-10-09T09:57:49.621587+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.620199+0000","last_change":"2023-10-09T09:57:49.620199+0000","last_active":"2023-10-09T09:57:49.620199+0000","last_peered":"2023-10-09T09:57:49.620199+0000","last_clean":"2023-10-09T09:57:49.620199+0000","last_became_active":"2023-10-09T09:57:49.619695+0000","last_became_peered":"2023-10-09T09:57:49.619695+0000","last_unstale":"2023-10-09T09:57:49.620199+0000","last_undegraded":"2023-10-09T09:57:49.620199+0000","last_fullsized":"2023-10-09T09:57:49.620199+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:48.818881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.611800+0000","last_change":"2023-10-09T09:56:45.553024+0000","last_active":"2023-10-09T09:57:49.611800+0000","last_peered":"2023-10-09T09:57:49.611800+0000","last_clean":"2023-10-09T09:57:49.611800+0000","last_became_active":"2023-10-09T09:56:45.552670+0000","last_became_peered":"2023-10-09T09:56:45.552670+0000","last_unstale":"2023-10-09T09:57:49.611800+0000","last_undegraded":"2023-10-09T09:57:49.611800+0000","last_fullsized":"2023-10-09T09:57:49.611800+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:58.132008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627539+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T09:57:49.627539+0000","last_peered":"2023-10-09T09:57:49.627539+0000","last_clean":"2023-10-09T09:57:49.627539+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T09:57:49.627539+0000","last_undegraded":"2023-10-09T09:57:49.627539+0000","last_fullsized":"2023-10-09T09:57:49.627539+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621676+0000","last_change":"2023-10-09T09:56:45.556365+0000","last_active":"2023-10-09T09:57:48.621676+0000","last_peered":"2023-10-09T09:57:48.621676+0000","last_clean":"2023-10-09T09:57:48.621676+0000","last_became_active":"2023-10-09T09:56:45.555832+0000","last_became_peered":"2023-10-09T09:56:45.555832+0000","last_unstale":"2023-10-09T09:57:48.621676+0000","last_undegraded":"2023-10-09T09:57:48.621676+0000","last_fullsized":"2023-10-09T09:57:48.621676+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.626073+0000","last_change":"2023-10-09T09:48:25.514756+0000","last_active":"2023-10-09T09:57:48.626073+0000","last_peered":"2023-10-09T09:57:48.626073+0000","last_clean":"2023-10-09T09:57:48.626073+0000","last_became_active":"2023-10-09T09:48:25.514387+0000","last_became_peered":"2023-10-09T09:48:25.514387+0000","last_unstale":"2023-10-09T09:57:48.626073+0000","last_undegraded":"2023-10-09T09:57:48.626073+0000","last_fullsized":"2023-10-09T09:57:48.626073+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627073+0000","last_change":"2023-10-09T09:57:49.627073+0000","last_active":"2023-10-09T09:57:49.627073+0000","last_peered":"2023-10-09T09:57:49.627073+0000","last_clean":"2023-10-09T09:57:49.627073+0000","last_became_active":"2023-10-09T09:57:49.626706+0000","last_became_peered":"2023-10-09T09:57:49.626706+0000","last_unstale":"2023-10-09T09:57:49.627073+0000","last_undegraded":"2023-10-09T09:57:49.627073+0000","last_fullsized":"2023-10-09T09:57:49.627073+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625250+0000","last_change":"2023-10-09T09:56:45.554326+0000","last_active":"2023-10-09T09:57:48.625250+0000","last_peered":"2023-10-09T09:57:48.625250+0000","last_clean":"2023-10-09T09:57:48.625250+0000","last_became_active":"2023-10-09T09:56:45.553876+0000","last_became_peered":"2023-10-09T09:56:45.553876+0000","last_unstale":"2023-10-09T09:57:48.625250+0000","last_undegraded":"2023-10-09T09:57:48.625250+0000","last_fullsized":"2023-10-09T09:57:48.625250+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.619582+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T09:57:48.619582+0000","last_peered":"2023-10-09T09:57:48.619582+0000","last_clean":"2023-10-09T09:57:48.619582+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T09:57:48.619582+0000","last_undegraded":"2023-10-09T09:57:48.619582+0000","last_fullsized":"2023-10-09T09:57:48.619582+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.624744+0000","last_change":"2023-10-09T09:56:13.614485+0000","last_active":"2023-10-09T09:57:48.624744+0000","last_peered":"2023-10-09T09:57:48.624744+0000","last_clean":"2023-10-09T09:57:48.624744+0000","last_became_active":"2023-10-09T09:56:13.614109+0000","last_became_peered":"2023-10-09T09:56:13.614109+0000","last_unstale":"2023-10-09T09:57:48.624744+0000","last_undegraded":"2023-10-09T09:57:48.624744+0000","last_fullsized":"2023-10-09T09:57:48.624744+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.623806+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T09:57:49.623806+0000","last_peered":"2023-10-09T09:57:49.623806+0000","last_clean":"2023-10-09T09:57:49.623806+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T09:57:49.623806+0000","last_undegraded":"2023-10-09T09:57:49.623806+0000","last_fullsized":"2023-10-09T09:57:49.623806+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620753+0000","last_change":"2023-10-09T09:56:13.623204+0000","last_active":"2023-10-09T09:57:48.620753+0000","last_peered":"2023-10-09T09:57:48.620753+0000","last_clean":"2023-10-09T09:57:48.620753+0000","last_became_active":"2023-10-09T09:56:13.622853+0000","last_became_peered":"2023-10-09T09:56:13.622853+0000","last_unstale":"2023-10-09T09:57:48.620753+0000","last_undegraded":"2023-10-09T09:57:48.620753+0000","last_fullsized":"2023-10-09T09:57:48.620753+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625687+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T09:57:49.625687+0000","last_peered":"2023-10-09T09:57:49.625687+0000","last_clean":"2023-10-09T09:57:49.625687+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T09:57:49.625687+0000","last_undegraded":"2023-10-09T09:57:49.625687+0000","last_fullsized":"2023-10-09T09:57:49.625687+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.625768+0000","last_change":"2023-10-09T09:48:25.523230+0000","last_active":"2023-10-09T09:57:48.625768+0000","last_peered":"2023-10-09T09:57:48.625768+0000","last_clean":"2023-10-09T09:57:48.625768+0000","last_became_active":"2023-10-09T09:48:25.522608+0000","last_became_peered":"2023-10-09T09:48:25.522608+0000","last_unstale":"2023-10-09T09:57:48.625768+0000","last_undegraded":"2023-10-09T09:57:48.625768+0000","last_fullsized":"2023-10-09T09:57:48.625768+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.627770+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T09:57:49.627770+0000","last_peered":"2023-10-09T09:57:49.627770+0000","last_clean":"2023-10-09T09:57:49.627770+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T09:57:49.627770+0000","last_undegraded":"2023-10-09T09:57:49.627770+0000","last_fullsized":"2023-10-09T09:57:49.627770+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626723+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T09:57:49.626723+0000","last_peered":"2023-10-09T09:57:49.626723+0000","last_clean":"2023-10-09T09:57:49.626723+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T09:57:49.626723+0000","last_undegraded":"2023-10-09T09:57:49.626723+0000","last_fullsized":"2023-10-09T09:57:49.626723+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622141+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T09:57:49.622141+0000","last_peered":"2023-10-09T09:57:49.622141+0000","last_clean":"2023-10-09T09:57:49.622141+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T09:57:49.622141+0000","last_undegraded":"2023-10-09T09:57:49.622141+0000","last_fullsized":"2023-10-09T09:57:49.622141+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.624235+0000","last_change":"2023-10-09T09:57:18.843731+0000","last_active":"2023-10-09T09:57:48.624235+0000","last_peered":"2023-10-09T09:57:48.624235+0000","last_clean":"2023-10-09T09:57:48.624235+0000","last_became_active":"2023-10-09T09:57:18.843280+0000","last_became_peered":"2023-10-09T09:57:18.843280+0000","last_unstale":"2023-10-09T09:57:48.624235+0000","last_undegraded":"2023-10-09T09:57:48.624235+0000","last_fullsized":"2023-10-09T09:57:48.624235+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617918+0000","last_change":"2023-10-09T09:56:45.551325+0000","last_active":"2023-10-09T09:57:48.617918+0000","last_peered":"2023-10-09T09:57:48.617918+0000","last_clean":"2023-10-09T09:57:48.617918+0000","last_became_active":"2023-10-09T09:56:45.550946+0000","last_became_peered":"2023-10-09T09:56:45.550946+0000","last_unstale":"2023-10-09T09:57:48.617918+0000","last_undegraded":"2023-10-09T09:57:48.617918+0000","last_fullsized":"2023-10-09T09:57:48.617918+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":69,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617387+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T09:57:48.617387+0000","last_peered":"2023-10-09T09:57:48.617387+0000","last_clean":"2023-10-09T09:57:48.617387+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T09:57:48.617387+0000","last_undegraded":"2023-10-09T09:57:48.617387+0000","last_fullsized":"2023-10-09T09:57:48.617387+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.621810+0000","last_change":"2023-10-09T09:57:18.840471+0000","last_active":"2023-10-09T09:57:48.621810+0000","last_peered":"2023-10-09T09:57:48.621810+0000","last_clean":"2023-10-09T09:57:48.621810+0000","last_became_active":"2023-10-09T09:57:18.840125+0000","last_became_peered":"2023-10-09T09:57:18.840125+0000","last_unstale":"2023-10-09T09:57:48.621810+0000","last_undegraded":"2023-10-09T09:57:48.621810+0000","last_fullsized":"2023-10-09T09:57:48.621810+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:07.418455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626281+0000","last_change":"2023-10-09T09:57:49.626281+0000","last_active":"2023-10-09T09:57:49.626281+0000","last_peered":"2023-10-09T09:57:49.626281+0000","last_clean":"2023-10-09T09:57:49.626281+0000","last_became_active":"2023-10-09T09:57:49.625726+0000","last_became_peered":"2023-10-09T09:57:49.625726+0000","last_unstale":"2023-10-09T09:57:49.626281+0000","last_undegraded":"2023-10-09T09:57:49.626281+0000","last_fullsized":"2023-10-09T09:57:49.626281+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625511+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T09:57:49.625511+0000","last_peered":"2023-10-09T09:57:49.625511+0000","last_clean":"2023-10-09T09:57:49.625511+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T09:57:49.625511+0000","last_undegraded":"2023-10-09T09:57:49.625511+0000","last_fullsized":"2023-10-09T09:57:49.625511+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.620020+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T09:57:48.620020+0000","last_peered":"2023-10-09T09:57:48.620020+0000","last_clean":"2023-10-09T09:57:48.620020+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T09:57:48.620020+0000","last_undegraded":"2023-10-09T09:57:48.620020+0000","last_fullsized":"2023-10-09T09:57:48.620020+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624665+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T09:57:49.624665+0000","last_peered":"2023-10-09T09:57:49.624665+0000","last_clean":"2023-10-09T09:57:49.624665+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T09:57:49.624665+0000","last_undegraded":"2023-10-09T09:57:49.624665+0000","last_fullsized":"2023-10-09T09:57:49.624665+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.626966+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T09:57:49.626966+0000","last_peered":"2023-10-09T09:57:49.626966+0000","last_clean":"2023-10-09T09:57:49.626966+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T09:57:49.626966+0000","last_undegraded":"2023-10-09T09:57:49.626966+0000","last_fullsized":"2023-10-09T09:57:49.626966+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.622159+0000","last_change":"2023-10-09T09:57:18.838985+0000","last_active":"2023-10-09T09:57:48.622159+0000","last_peered":"2023-10-09T09:57:48.622159+0000","last_clean":"2023-10-09T09:57:48.622159+0000","last_became_active":"2023-10-09T09:57:18.838625+0000","last_became_peered":"2023-10-09T09:57:18.838625+0000","last_unstale":"2023-10-09T09:57:48.622159+0000","last_undegraded":"2023-10-09T09:57:48.622159+0000","last_fullsized":"2023-10-09T09:57:48.622159+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.624475+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T09:57:49.624475+0000","last_peered":"2023-10-09T09:57:49.624475+0000","last_clean":"2023-10-09T09:57:49.624475+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T09:57:49.624475+0000","last_undegraded":"2023-10-09T09:57:49.624475+0000","last_fullsized":"2023-10-09T09:57:49.624475+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.625707+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T09:57:49.625707+0000","last_peered":"2023-10-09T09:57:49.625707+0000","last_clean":"2023-10-09T09:57:49.625707+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T09:57:49.625707+0000","last_undegraded":"2023-10-09T09:57:49.625707+0000","last_fullsized":"2023-10-09T09:57:49.625707+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.621495+0000","last_change":"2023-10-09T09:57:49.621495+0000","last_active":"2023-10-09T09:57:49.621495+0000","last_peered":"2023-10-09T09:57:49.621495+0000","last_clean":"2023-10-09T09:57:49.621495+0000","last_became_active":"2023-10-09T09:57:49.621144+0000","last_became_peered":"2023-10-09T09:57:49.621144+0000","last_unstale":"2023-10-09T09:57:49.621495+0000","last_undegraded":"2023-10-09T09:57:49.621495+0000","last_fullsized":"2023-10-09T09:57:49.621495+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:11.852424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T09:57:49.622753+0000","last_change":"2023-10-09T09:57:49.622753+0000","last_active":"2023-10-09T09:57:49.622753+0000","last_peered":"2023-10-09T09:57:49.622753+0000","last_clean":"2023-10-09T09:57:49.622753+0000","last_became_active":"2023-10-09T09:57:49.622463+0000","last_became_peered":"2023-10-09T09:57:49.622463+0000","last_unstale":"2023-10-09T09:57:49.622753+0000","last_undegraded":"2023-10-09T09:57:49.622753+0000","last_fullsized":"2023-10-09T09:57:49.622753+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:51.020605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":78,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-10-09T09:57:48.617029+0000","last_change":"2023-10-09T09:57:18.841362+0000","last_active":"2023-10-09T09:57:48.617029+0000","last_peered":"2023-10-09T09:57:48.617029+0000","last_clean":"2023-10-09T09:57:48.617029+0000","last_became_active":"2023-10-09T09:57:18.840951+0000","last_became_peered":"2023-10-09T09:57:18.840951+0000","last_unstale":"2023-10-09T09:57:48.617029+0000","last_undegraded":"2023-10-09T09:57:48.617029+0000","last_fullsized":"2023-10-09T09:57:48.617029+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430347,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310920,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434232,"statfs":{"total":95995035648,"available":95676653568,"internally_reserved":0,"allocated":1572864,"data_stored":1111479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58399999999999996,"15min":0.58999999999999997},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.41799999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73599999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.58299999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.73299999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.57999999999999996,"15min":0.53700000000000003},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.36099999999999999},"max":{"1min":0.81399999999999995,"5min":0.89700000000000002,"15min":0.91600000000000004},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.56499999999999995},"min":{"1min":0.39700000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.76800000000000002,"5min":0.78100000000000003,"15min":0.83799999999999997},"last":0.60499999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.49299999999999999,"15min":0.46800000000000003},"min":{"1min":0.39200000000000002,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.51700000000000002,"15min":0.47999999999999998},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.87},"last":0.70499999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.52400000000000002,"15min":0.503},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.83599999999999997},"last":0.628},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.498,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.85399999999999998},"last":0.79700000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.59099999999999997,"15min":0.55200000000000005},"min":{"1min":0.52900000000000003,"5min":0.36799999999999999,"15min":0.32900000000000001},"max":{"1min":0.84599999999999997,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.58899999999999997,"15min":0.54400000000000004},"min":{"1min":0.42799999999999999,"5min":0.40300000000000002,"15min":0.33100000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.96599999999999997},"last":0.74399999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626575,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1556480,"data_stored":1111018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71999999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.51900000000000002,"15min":0.51000000000000001},"min":{"1min":0.42299999999999999,"5min":0.308,"15min":0.308},"max":{"1min":0.749,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.63500000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.69999999999999996}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57999999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:57:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.57999999999999996,"15min":0.55000000000000004},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.33100000000000002},"max":{"1min":0.79300000000000004,"5min":1.8260000000000001,"15min":1.8260000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.60099999999999998,"15min":0.56299999999999994},"min":{"1min":0.39300000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.0169999999999999,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.76700000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46700000000000003,"15min":0.45000000000000001},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.23300000000000001},"max":{"1min":0.64500000000000002,"5min":0.75800000000000001,"15min":0.77600000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.496,"15min":0.48899999999999999},"min":{"1min":0.36299999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.66800000000000004,"5min":0.753,"15min":0.75600000000000001},"last":0.48299999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.50600000000000001,"15min":0.503},"min":{"1min":0.36599999999999999,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":0.63600000000000001,"5min":0.76100000000000001,"15min":0.80600000000000005},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51700000000000002,"15min":0.505},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68500000000000005,"5min":0.76800000000000002,"15min":1.0600000000000001},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.54400000000000004,"15min":0.52600000000000002},"min":{"1min":0.35099999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.68000000000000005,"5min":0.79400000000000004,"15min":0.81399999999999995},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.627,"5min":0.61499999999999999,"15min":0.58699999999999997},"min":{"1min":0.52600000000000002,"5min":0.34899999999999998,"15min":0.34399999999999997},"max":{"1min":0.754,"5min":0.82999999999999996,"15min":0.84099999999999997},"last":0.71599999999999997}]}]},{"osd":1,"up_from":84,"seq":360777252889,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28308,"kb_used_data":972,"kb_used_omap":9,"kb_used_meta":27318,"kb_avail":93716844,"statfs":{"total":95995035648,"available":95966048256,"internally_reserved":0,"allocated":995328,"data_stored":524748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":27974633},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.30399999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.38400000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.52900000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.54700000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:57:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.57499999999999996}]}]},{"osd":0,"up_from":79,"seq":339302416415,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33056,"kb_used_data":1560,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712096,"statfs":{"total":95995035648,"available":95961186304,"internally_reserved":0,"allocated":1597440,"data_stored":1114630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.45100000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:57:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.41399999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.66900000000000004,"15min":0.66100000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68200000000000005,"15min":0.67800000000000005},"min":{"1min":0.44600000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.028,"5min":1.101,"15min":1.101},"last":0.64900000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64600000000000002,"15min":0.64400000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85799999999999998,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.5},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.94499999999999995,"5min":1.1180000000000001,"15min":1.1180000000000001},"last":0.57799999999999996}]},{"osd":6,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.68300000000000005,"15min":0.68700000000000006},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89700000000000002,"5min":1.161,"15min":1.161},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.68899999999999995,"15min":0.69099999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.99199999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.60299999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.66500000000000004,"15min":0.66600000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.89600000000000002,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.68400000000000005,"15min":0.68300000000000005},"min":{"1min":0.441,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.98099999999999998,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.627}]}]},{"osd":2,"up_from":89,"seq":382252089362,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36624,"kb_used_data":968,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708528,"statfs":{"total":95995035648,"available":95957532672,"internally_reserved":0,"allocated":991232,"data_stored":522739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.432}]},{"osd":1,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.50900000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.60699999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.73799999999999999}]},{"osd":6,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.66300000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.63},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.69799999999999995}]}]},{"osd":3,"up_from":94,"seq":403726925836,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28288,"kb_used_data":952,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716864,"statfs":{"total":95995035648,"available":95966068736,"internally_reserved":0,"allocated":974848,"data_stored":520694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34599999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35899999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51000000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019031,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302144,"kb_used_data":952,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443008,"statfs":{"total":95995035648,"available":95685640192,"internally_reserved":0,"allocated":974848,"data_stored":521041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.66800000000000004,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56699999999999995,"15min":0.56999999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.47999999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.54400000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:56:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.28199999999999997},"max":{"1min":0.753,"5min":0.877,"15min":2.2109999999999999},"last":0.442},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.47899999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.75,"5min":0.81399999999999995,"15min":0.997},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.45600000000000002,"15min":0.43099999999999999},"min":{"1min":0.34399999999999997,"5min":0.28399999999999997,"15min":0.27100000000000002},"max":{"1min":0.77200000000000002,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.48999999999999999,"15min":0.46600000000000003},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70999999999999996,"5min":0.72099999999999997,"15min":0.83999999999999997},"last":0.40500000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.499,"15min":0.46800000000000003},"min":{"1min":0.41199999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78600000000000003,"5min":0.79300000000000004,"15min":0.86599999999999999},"last":0.374},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53200000000000003,"15min":0.497},"min":{"1min":0.379,"5min":0.318,"15min":0.312},"max":{"1min":0.80700000000000005,"5min":0.81499999999999995,"15min":0.89900000000000002},"last":0.51900000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55800000000000005,"15min":0.52500000000000002},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.69899999999999995,"5min":0.76600000000000001,"15min":0.84999999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59699999999999998,"15min":0.55800000000000005},"min":{"1min":0.45800000000000002,"5min":0.33700000000000002,"15min":0.316},"max":{"1min":0.76000000000000001,"5min":0.82899999999999996,"15min":0.85199999999999998},"last":0.53600000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822803,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310320,"kb_used_data":936,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434832,"statfs":{"total":95995035648,"available":95677267968,"internally_reserved":0,"allocated":958464,"data_stored":516785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.75,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.55000000000000004,"15min":0.54100000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.4289999999999998,"5min":2.4289999999999998,"15min":2.4289999999999998},"last":0.41099999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:57:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.503}]},{"osd":2,"last update":"Mon Oct 9 09:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":2.5950000000000002,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.62}]},{"osd":3,"last update":"Mon Oct 9 09:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.56899999999999995,"15min":0.54700000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":0.70699999999999996,"5min":0.84999999999999998,"15min":1.1479999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.60699999999999998,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":2.637},"last":0.67100000000000004}]},{"osd":4,"last update":"Mon Oct 9 09:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47699999999999998,"15min":0.44400000000000001},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":0.74299999999999999,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.51000000000000001,"15min":0.46800000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64900000000000002,"5min":0.79600000000000004,"15min":0.88200000000000001},"last":0.53700000000000003}]},{"osd":6,"last update":"Mon Oct 9 09:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.53000000000000003,"15min":0.503},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":0.76200000000000001,"5min":0.85099999999999998,"15min":0.92800000000000005},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.52100000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81000000000000005,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.66400000000000003}]},{"osd":7,"last update":"Mon Oct 9 09:57:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58099999999999996,"15min":0.55500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":0.71299999999999997,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55000000000000004,"15min":0.53600000000000003},"min":{"1min":0.37,"5min":0.318,"15min":0.317},"max":{"1min":0.68200000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.72799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:58:39.567 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T09:58:39.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T09:58:40.174 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:40.742 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:40 smithi012 ceph-mon[152263]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:40.742 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:40 smithi012 ceph-mon[152263]: from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:40.742 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:40 smithi012 ceph-mon[147926]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:40.742 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:40 smithi012 ceph-mon[147926]: from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:40 smithi110 ceph-mon[136032]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:40 smithi110 ceph-mon[136032]: from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:58:41.667 INFO:teuthology.orchestra.run.smithi012.stdout:epoch 95 2023-10-09T09:58:41.667 INFO:teuthology.orchestra.run.smithi012.stdout:fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 2023-10-09T09:58:41.667 INFO:teuthology.orchestra.run.smithi012.stdout:created 2023-10-09T09:42:53.020199+0000 2023-10-09T09:58:41.667 INFO:teuthology.orchestra.run.smithi012.stdout:modified 2023-10-09T09:57:49.606098+0000 2023-10-09T09:58:41.667 INFO:teuthology.orchestra.run.smithi012.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T09:58:41.667 INFO:teuthology.orchestra.run.smithi012.stdout:crush_version 22 2023-10-09T09:58:41.667 INFO:teuthology.orchestra.run.smithi012.stdout:full_ratio 0.95 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:backfillfull_ratio 0.9 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:nearfull_ratio 0.85 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:require_min_compat_client luminous 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:min_compat_client luminous 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:require_osd_release pacific 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:stretch_mode_enabled false 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 72 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 55 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T09:58:41.668 INFO:teuthology.orchestra.run.smithi012.stdout:max_osd 8 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 up in weight 1 up_from 79 up_thru 94 down_at 76 last_clean_interval [8,75) [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086] [v2:172.21.15.12:6804/3324439086,v1:172.21.15.12:6805/3324439086] exists,up 23895d43-58c0-4692-9ca5-d05fc064a71e 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 up in weight 1 up_from 84 up_thru 94 down_at 81 last_clean_interval [13,80) [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322] [v2:172.21.15.12:6812/715957322,v1:172.21.15.12:6813/715957322] exists,up 8657aa57-580d-4318-a093-2cc78c0f72a9 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 up in weight 1 up_from 89 up_thru 94 down_at 86 last_clean_interval [18,85) [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385] [v2:172.21.15.12:6820/4293129385,v1:172.21.15.12:6821/4293129385] exists,up 4bab5bc5-77e6-4b60-9cd1-511c5d18ff03 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 up in weight 1 up_from 94 up_thru 94 down_at 91 last_clean_interval [25,90) [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893] [v2:172.21.15.12:6828/2914552893,v1:172.21.15.12:6829/2914552893] exists,up bc089815-f2a8-41ef-9b69-505b7b51963e 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 up in weight 1 up_from 30 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753] [v2:172.21.15.110:6802/3628101753,v1:172.21.15.110:6803/3628101753] exists,up ad0b6f60-bf01-465b-acee-565d4d4a3df5 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 up in weight 1 up_from 36 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369] [v2:172.21.15.110:6810/3210978369,v1:172.21.15.110:6811/3210978369] exists,up 63ba8578-42c2-4852-acac-e616de4ccd47 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 up in weight 1 up_from 42 up_thru 91 down_at 0 last_clean_interval [0,0) [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135] [v2:172.21.15.110:6818/3541801135,v1:172.21.15.110:6819/3541801135] exists,up 00113d3e-cc98-49fe-ba99-7172c2c8c80b 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 up in weight 1 up_from 48 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104] [v2:172.21.15.110:6826/4270614104,v1:172.21.15.110:6827/4270614104] exists,up a34ee707-c52a-48c8-be23-cce0e1cc0801 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:pg_upmap_items 4.2 [1,2] 2023-10-09T09:58:41.669 INFO:teuthology.orchestra.run.smithi012.stdout:pg_upmap_items 4.6 [1,5] 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:pg_upmap_items 4.1f [1,2] 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/1937850108 expires 2023-10-10T09:43:46.740224+0000 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/847521526 expires 2023-10-10T09:43:25.068166+0000 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/497354987 expires 2023-10-10T09:47:27.376359+0000 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/2293469660 expires 2023-10-10T09:50:12.280108+0000 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/2348702677 expires 2023-10-10T09:51:58.776424+0000 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/3530794450 expires 2023-10-10T09:47:27.376359+0000 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/826788601 expires 2023-10-10T09:43:46.740224+0000 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/1874684344 expires 2023-10-10T09:43:25.068166+0000 2023-10-09T09:58:41.670 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6800/819667578 expires 2023-10-10T09:43:25.068166+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/2990486696 expires 2023-10-10T09:51:58.776424+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.110:0/1671269215 expires 2023-10-10T09:51:12.282860+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6800/1146481933 expires 2023-10-10T09:51:58.776424+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.110:0/3560901131 expires 2023-10-10T09:51:12.282860+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6801/106778378 expires 2023-10-10T09:47:27.376359+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6800/1925191424 expires 2023-10-10T09:50:12.280108+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6801/1925191424 expires 2023-10-10T09:50:12.280108+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.110:0/4208451228 expires 2023-10-10T09:51:12.282860+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6800/3616444435 expires 2023-10-10T09:43:46.740224+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6801/819667578 expires 2023-10-10T09:43:25.068166+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6801/1146481933 expires 2023-10-10T09:51:58.776424+0000 2023-10-09T09:58:41.671 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/1614086989 expires 2023-10-10T09:51:58.776424+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6801/3616444435 expires 2023-10-10T09:43:46.740224+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.110:0/1606345106 expires 2023-10-10T09:51:12.282860+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.110:0/1952966694 expires 2023-10-10T09:51:12.282860+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/3260703278 expires 2023-10-10T09:51:58.776424+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/807677503 expires 2023-10-10T09:51:58.776424+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/928078357 expires 2023-10-10T09:50:12.280108+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:0/518580822 expires 2023-10-10T09:51:58.776424+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.110:6835/4098121826 expires 2023-10-10T09:51:12.282860+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.110:6834/4098121826 expires 2023-10-10T09:51:12.282860+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.12:6800/106778378 expires 2023-10-10T09:47:27.376359+0000 2023-10-09T09:58:41.672 INFO:teuthology.orchestra.run.smithi012.stdout:blocklist 172.21.15.110:0/3728005719 expires 2023-10-10T09:51:12.282860+0000 2023-10-09T09:58:42.196 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 79 up_thru 94 down_at 76 last_clean_interval [8,75) [v2:172.21.15.12:6802/3324439086,v1:172.21.15.12:6803/3324439086] [v2:172.21.15.12:6804/3324439086,v1:172.21.15.12:6805/3324439086] exists,up 23895d43-58c0-4692-9ca5-d05fc064a71e', 'osd.1 up in weight 1 up_from 84 up_thru 94 down_at 81 last_clean_interval [13,80) [v2:172.21.15.12:6810/715957322,v1:172.21.15.12:6811/715957322] [v2:172.21.15.12:6812/715957322,v1:172.21.15.12:6813/715957322] exists,up 8657aa57-580d-4318-a093-2cc78c0f72a9', 'osd.2 up in weight 1 up_from 89 up_thru 94 down_at 86 last_clean_interval [18,85) [v2:172.21.15.12:6818/4293129385,v1:172.21.15.12:6819/4293129385] [v2:172.21.15.12:6820/4293129385,v1:172.21.15.12:6821/4293129385] exists,up 4bab5bc5-77e6-4b60-9cd1-511c5d18ff03', 'osd.3 up in weight 1 up_from 94 up_thru 94 down_at 91 last_clean_interval [25,90) [v2:172.21.15.12:6826/2914552893,v1:172.21.15.12:6827/2914552893] [v2:172.21.15.12:6828/2914552893,v1:172.21.15.12:6829/2914552893] exists,up bc089815-f2a8-41ef-9b69-505b7b51963e', 'osd.4 up in weight 1 up_from 30 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.110:6800/3628101753,v1:172.21.15.110:6801/3628101753] [v2:172.21.15.110:6802/3628101753,v1:172.21.15.110:6803/3628101753] exists,up ad0b6f60-bf01-465b-acee-565d4d4a3df5', 'osd.5 up in weight 1 up_from 36 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.110:6808/3210978369,v1:172.21.15.110:6809/3210978369] [v2:172.21.15.110:6810/3210978369,v1:172.21.15.110:6811/3210978369] exists,up 63ba8578-42c2-4852-acac-e616de4ccd47', 'osd.6 up in weight 1 up_from 42 up_thru 91 down_at 0 last_clean_interval [0,0) [v2:172.21.15.110:6816/3541801135,v1:172.21.15.110:6817/3541801135] [v2:172.21.15.110:6818/3541801135,v1:172.21.15.110:6819/3541801135] exists,up 00113d3e-cc98-49fe-ba99-7172c2c8c80b', 'osd.7 up in weight 1 up_from 48 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.110:6824/4270614104,v1:172.21.15.110:6825/4270614104] [v2:172.21.15.110:6826/4270614104,v1:172.21.15.110:6827/4270614104] exists,up a34ee707-c52a-48c8-be23-cce0e1cc0801'] 2023-10-09T09:58:42.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T09:58:42.801 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:42.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:42 smithi012 ceph-mon[147926]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2219875437' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:58:42.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:42 smithi012 ceph-mon[152263]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:42.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2219875437' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:58:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:42 smithi110 ceph-mon[136032]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2219875437' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T09:58:44.009 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "name": "mon.a", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "cluster": "ceph", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "admin_socket_mode": "", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_client_required": "cephx, none", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_cluster_required": "cephx", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_debug": "true", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T09:58:44.010 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_service_required": "cephx", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "auth_supported": "", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_aio": "true", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_aio_reap_max": "16", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_async_discard": "false", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_block_size": "4096", 2023-10-09T09:58:44.011 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_debug_aio": "false", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_enable_discard": "false", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_flock_retry": "3", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_inject_crash": "0", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_ioring": "false", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_ioring_hipri": "false", 2023-10-09T09:58:44.012 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bdev_type": "", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_allocator": "hybrid", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_buffered_io": "true", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T09:58:44.013 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_replay_recovery": "false", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T09:58:44.014 INFO:teuthology.orchestra.run.smithi012.stdout: "bluefs_sync_write": "false", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_allocator": "hybrid", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T09:58:44.015 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_create": "true", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_db_create": "false", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_db_path": "", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_db_size": "0", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_path": "", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_size": "107374182400", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_wal_create": "false", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_wal_path": "", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T09:58:44.016 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_bluefs": "true", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_autotune": "true", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T09:58:44.017 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_size": "0", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cache_type": "2q", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_clone_cow": "true", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T09:58:44.018 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_mode": "none", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_freelist": "false", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T09:58:44.019 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T09:58:44.020 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fail_eio": "false", 2023-10-09T09:58:44.021 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T09:58:44.022 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T09:58:44.023 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_max_blob_size": "0", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T09:58:44.024 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_spdk_mem": "512", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T09:58:44.025 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_tracing": "false", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T09:58:44.026 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T09:58:44.027 INFO:teuthology.orchestra.run.smithi012.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephx_cluster_require_version": "2", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephx_require_signatures": "false", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephx_require_version": "2", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephx_service_require_signatures": "false", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephx_service_require_version": "2", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "cephx_sign_messages": "true", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "chdir": "", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client_acl_type": "", 2023-10-09T09:58:44.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client_asio_thread_count": "2", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_cache_mid": "0.750000", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_cache_size": "16384", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_caps_release_delay": "5", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_check_pool_perm": "true", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_debug_force_sync_read": "false", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_debug_getattr_caps": "false", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_die_on_failed_remount": "false", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_dirsize_rbytes": "true", 2023-10-09T09:58:44.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_force_lazyio": "false", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_fs": "", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_inject_release_failure": "false", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_max_inline_size": "4096", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_mds_namespace": "", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_metadata": "", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_mount_gid": "-1", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_mount_timeout": "300", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_mount_uid": "-1", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_mountpoint": "/", 2023-10-09T09:58:44.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_notify_timeout": "10", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_oc": "true", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_oc_max_objects": "1000", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_oc_size": "209715200", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_permissions": "true", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_quota": "true", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_quota_df": "true", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_readahead_max_bytes": "0", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_readahead_max_periods": "4", 2023-10-09T09:58:44.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_readahead_min": "131072", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_reconnect_stale": "false", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_shutdown_timeout": "30", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_snapdir": ".snap", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_tick_interval": "1", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_trace": "", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_use_faked_inos": "false", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_use_random_mds": "false", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "clog_to_graylog": "false", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "clog_to_graylog_port": "12201", 2023-10-09T09:58:44.032 INFO:teuthology.orchestra.run.smithi012.stdout: "clog_to_monitors": "default=true", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "clog_to_syslog": "false", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "clog_to_syslog_level": "info", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "cluster_addr": "-", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "cluster_network": "", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "cluster_network_interface": "", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "compressor_zlib_isal": "false", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "compressor_zlib_level": "5", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "compressor_zstd_level": "1", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T09:58:44.033 INFO:teuthology.orchestra.run.smithi012.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "crimson_seastar_smp": "1", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "crush_location": "", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "crush_location_hook": "", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "crush_location_hook_timeout": "10", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "daemonize": "false", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "daos_pool": "tank", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T09:58:44.034 INFO:teuthology.orchestra.run.smithi012.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_alienstore": "0/5", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_asok": "1/5", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_asok_assert_abort": "false", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_auth": "1/5", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_bdev": "1/3", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_bluefs": "1/5", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_bluestore": "1/5", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_buffer": "0/1", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T09:58:44.035 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_cephsqlite": "0/5", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_client": "0/5", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_compressor": "1/5", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_context": "0/1", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_crush": "1/1", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_crypto": "1/5", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_cyanstore": "0/5", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_dpdk": "1/5", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_eventtrace": "1/5", 2023-10-09T09:58:44.036 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_filer": "0/1", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_filestore": "1/3", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_finisher": "1/1", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_fuse": "1/5", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_javaclient": "1/5", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_journal": "1/3", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_journaler": "0/5", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_kstore": "1/5", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_leveldb": "4/5", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_lockdep": "0/1", 2023-10-09T09:58:44.037 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mclock": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mds": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mds_balancer": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mds_locker": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mds_log": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mds_migrator": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_memstore": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mgr": "2/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mgrc": "1/5", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_mon": "20/20", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_monc": "0/10", 2023-10-09T09:58:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_ms": "1/1", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_none": "0/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_objclass": "0/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_objectcacher": "0/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_objecter": "0/1", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_optracker": "0/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_osd": "1/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_paxos": "20/20", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_perfcounter": "1/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_prioritycache": "1/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rados": "0/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rbd": "0/5", 2023-10-09T09:58:44.039 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rbd_replay": "0/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_refs": "0/0", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_reserver": "1/1", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rgw": "1/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rgw_access": "1/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rgw_flight": "1/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rgw_sync": "1/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_rocksdb": "4/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore": "0/5", 2023-10-09T09:58:44.040 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_backref": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_cache": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_device": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_epm": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_journal": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_lba": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_odata": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_omap": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_onode": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_t": "0/5", 2023-10-09T09:58:44.041 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_seastore_tm": "0/5", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_striper": "0/1", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_test": "0/5", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_throttle": "1/1", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_timer": "0/1", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "debug_tp": "0/5", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "device_failure_prediction_mode": "none", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "err_to_graylog": "false", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "err_to_journald": "false", 2023-10-09T09:58:44.042 INFO:teuthology.orchestra.run.smithi012.stdout: "err_to_stderr": "true", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "err_to_syslog": "false", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event_tracing": "false", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "exporter_http_port": "9926", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "exporter_prio_limit": "5", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "exporter_sort_metrics": "true", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "exporter_stats_period": "5", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "fake_statfs_for_testing": "0", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "fatal_signal_handlers": "true", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "filer_max_purge_ops": "10", 2023-10-09T09:58:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: "filer_max_truncate_ops": "128", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_blackhole": "false", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_btrfs_snap": "true", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_caller_concurrency": "10", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_debug_omap_check": "false", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_debug_verify_split": "false", 2023-10-09T09:58:44.044 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_dump_file": "", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_fadvise": "true", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_fail_eio": "true", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_fd_cache_size": "128", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_fiemap": "false", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_inject_stall": "0", 2023-10-09T09:58:44.045 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_journal_parallel": "false", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_journal_trailing": "false", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_journal_writeahead": "false", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_kill_at": "0", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T09:58:44.046 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_merge_threshold": "-10", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_odsync_write": "false", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_omap_backend_path": "", 2023-10-09T09:58:44.047 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_op_threads": "2", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_punch_hole": "false", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T09:58:44.048 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_queue_max_ops": "50", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_seek_data_hole": "false", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_sloppy_crc": "false", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_splice": "false", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_split_multiple": "2", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_split_rand_factor": "20", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_update_to": "1000", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T09:58:44.049 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T09:58:44.050 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_xfs_extsize": "false", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "filestore_zfs_snap": "false", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fio_dir": "/tmp/fio", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fsid": "1881d6a6-6688-11ee-8db5-212e2dc638e7", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_allow_other": "true", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_big_writes": "true", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_debug": "false", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_default_permissions": "false", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_disable_pagecache": "false", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_max_write": "0", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_multithreaded": "true", 2023-10-09T09:58:44.051 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_require_active_mds": "true", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_set_user_groups": "true", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_splice_move": "true", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_splice_read": "true", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_splice_write": "true", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "gss_target_name": "ceph", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "heartbeat_file": "", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "heartbeat_inject_failure": "0", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "heartbeat_interval": "5", 2023-10-09T09:58:44.052 INFO:teuthology.orchestra.run.smithi012.stdout: "host": "smithi012", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T09:58:44.053 INFO:teuthology.orchestra.run.smithi012.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "inject_early_sigterm": "false", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "jaeger_agent_port": "6799", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "jaeger_tracing_enable": "false", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_aio": "true", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_align_min_size": "65536", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_block_align": "true", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_block_size": "4096", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_dio": "true", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_discard": "false", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_force_aio": "false", 2023-10-09T09:58:44.054 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_ignore_corruption": "false", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_max_write_entries": "100", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_replay_from": "0", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_write_header_frequency": "0", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journal_zero_on_create": "false", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journaler_prefetch_periods": "10", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journaler_prezero_periods": "5", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "journaler_write_head_interval": "15", 2023-10-09T09:58:44.055 INFO:teuthology.orchestra.run.smithi012.stdout: "key": "", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "keyfile": "", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_backend": "rocksdb", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_max_bytes": "67108864", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_max_ops": "512", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_onode_map_size": "1024", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T09:58:44.056 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "kstore_sync_transaction": "false", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_block_size": "65536", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_bloom_size": "0", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_cache_size": "536870912", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_compression": "false", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_log": "", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_max_open_files": "0", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_paranoid": "false", 2023-10-09T09:58:44.057 INFO:teuthology.orchestra.run.smithi012.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "librados_thread_count": "2", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "lockdep": "false", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "lockdep_force_backtrace": "false", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_coarse_timestamps": "true", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_flush_on_exit": "false", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_graylog_port": "12201", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_max_new": "1000", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_max_recent": "10000", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_stderr_prefix": "", 2023-10-09T09:58:44.058 INFO:teuthology.orchestra.run.smithi012.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "log_to_file": "true", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "log_to_graylog": "false", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "log_to_journald": "false", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "log_to_stderr": "false", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "log_to_syslog": "false", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_action_on_write_error": "1", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_alternate_name_max": "8192", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_asio_thread_count": "2", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_export_pin": "true", 2023-10-09T09:58:44.059 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_interval": "10", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_max": "-1", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_max_until": "-1", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_merge_size": "50", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T09:58:44.060 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_mode": "0", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_split_bits": "3", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_split_size": "10000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T09:58:44.061 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cache_mid": "0.700000", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cache_trim_interval": "1", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T09:58:44.062 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_debug_auth_pins": "false", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_debug_frag": "false", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_debug_scatterstat": "false", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_debug_subtrees": "false", 2023-10-09T09:58:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_default_dir_hash": "2", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_defer_session_stale": "true", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dir_max_entries": "0", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dir_prefetch": "true", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T09:58:44.064 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_early_reply": "true", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_enable_op_tracker": "true", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_enforce_unique_name": "true", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T09:58:44.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_inject_health_dummy": "false", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T09:58:44.066 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_join_fs": "", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_journal_format": "1", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_create_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_export_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_import_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_journal_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_link_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_openc_at": "0", 2023-10-09T09:58:44.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_rename_at": "0", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_log_max_events": "-1", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_log_max_segments": "128", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_log_pause": "false", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_log_segment_size": "0", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_completed_requests": "100000", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_export_size": "20971520", 2023-10-09T09:58:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_file_recover": "32", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_purge_files": "64", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_purge_ops": "8192", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_metrics_update_interval": "2", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_min_caps_per_client": "100", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T09:58:44.069 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_numa_node": "-1", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_op_history_duration": "600", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_op_history_size": "20", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_op_log_threshold": "5", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_ping_grace": "15", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_ping_interval": "5", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_recall_max_caps": "30000", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T09:58:44.070 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_replay_interval": "1.000000", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_root_ino_gid": "0", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_root_ino_uid": "0", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T09:58:44.071 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_shutdown_check": "0", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_skip_ino": "0", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_snap_min_uid": "0", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_snap_rstat": "false", 2023-10-09T09:58:44.072 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_symlink_recovery": "true", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_thrash_exports": "0", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_thrash_fragments": "0", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_tick_interval": "5.000000", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_valgrind_exit": "false", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_verify_backtrace": "1", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_verify_scatter": "false", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mds_wipe_sessions": "false", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "mempool_debug": "false", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T09:58:44.073 INFO:teuthology.orchestra.run.smithi012.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "memstore_page_set": "false", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "memstore_page_size": "65536", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_client_bytes": "134217728", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_client_messages": "512", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_disabled_modules": "", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T09:58:44.074 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_mds_messages": "128", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_mon_messages": "128", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_osd_messages": "8192", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_pool": "true", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_standby_modules": "true", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_stats_period": "5", 2023-10-09T09:58:44.075 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_stats_threshold": "5", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_tick_period": "2", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_allow_pool_delete": "true", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_bytes": "104857600", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T09:58:44.076 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T09:58:44.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_journald": "true", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_stderr": "false", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_compact_on_start": "false", 2023-10-09T09:58:44.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_compact_on_trim": "true", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_cpu_threads": "4", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_data_avail_crit": "5", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_data_avail_warn": "5", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T09:58:44.079 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_dump_json": "false", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_extra_checks": "false", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T09:58:44.080 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_election_default_strategy": "1", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_election_timeout": "5.000000", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_enable_op_tracker": "true", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_fake_pool_delete": "false", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_force_quorum_join": "false", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_health_log_update_period": "5", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_health_max_detail": "50", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_health_to_clog": "true", 2023-10-09T09:58:44.081 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_host": "[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_host_override": "", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_initial_members": "", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_lease": "5.000000", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T09:58:44.082 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_log_full_interval": "50", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_log_max": "10000", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_log_max_summary": "50", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_max_log_epochs": "500", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_max_osd": "10000", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T09:58:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_memory_autotune": "true", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_memory_target": "2147483648", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mgr_digest_period": "5", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T09:58:44.084 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_op_complaint_time": "30", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_op_history_duration": "600", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_op_history_size": "20", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_op_log_threshold": "5", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T09:58:44.085 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_cache_size": "500", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T09:58:44.086 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_report_timeout": "900", 2023-10-09T09:58:44.087 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T09:58:44.088 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_reweight_max_osds": "4", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T09:58:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_scrub_interval": "86400", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_scrub_max_keys": "100", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_scrub_timeout": "300", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_session_timeout": "300", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_smart_report_timeout": "5", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_stretch_pool_size": "4", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_sync_debug": "false", 2023-10-09T09:58:44.090 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_tick_interval": "5", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T09:58:44.091 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_older_version": "true", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T09:58:44.092 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "monmap": "", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T09:58:44.093 INFO:teuthology.orchestra.run.smithi012.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "motr_tracing_enabled": "false", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_op_threads": "3", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_cm": "false", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_device_name": "", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T09:58:44.094 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_sl": "3", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_rdma_type": "ib", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_async_reap_threshold": "5", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_before_connect": "false", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_ipv4": "true", 2023-10-09T09:58:44.095 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_ipv6": "false", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_msgr1": "true", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_msgr2": "true", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_port_max": "7568", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_port_min": "6800", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_retry_count": "3", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_bind_retry_delay": "5", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_blackhole_client": "false", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_blackhole_mds": "false", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_blackhole_mgr": "false", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_blackhole_mon": "false", 2023-10-09T09:58:44.096 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_blackhole_osd": "false", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_client_mode": "crc secure", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_cluster_type": "", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_compress_secure": "false", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_crc_data": "true", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_crc_header": "true", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_die_on_bug": "true", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_die_on_old_message": "true", 2023-10-09T09:58:44.097 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_hugepages": "", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T09:58:44.098 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_lro": "true", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_pmd": "", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_port_id": "0", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_dump_on_send": "false", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_inject_delay_type": "", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T09:58:44.099 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_inject_network_congestion": "0", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_inject_socket_failures": "0", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_max_accept_failures": "4", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_max_backoff": "15.000000", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_osd_compress_mode": "none", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T09:58:44.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_pq_min_cost": "65536", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_public_type": "", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_service_mode": "crc secure", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_tcp_nodelay": "true", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "ms_type": "async+posix", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "no_config_file": "false", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T09:58:44.101 INFO:teuthology.orchestra.run.smithi012.stdout: "objecter_inflight_ops": "1024", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "objecter_timeout": "10.000000", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "objectstore_blackhole": "false", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "openssl_engine_opts": "", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_agent_max_ops": "4", 2023-10-09T09:58:44.102 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_agent_slop": "0.020000", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_asio_thread_count": "2", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_backfill_scan_max": "512", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_backfill_scan_min": "64", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T09:58:44.103 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_backoff_on_peering": "false", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_beacon_report_interval": "300", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_bench_duration": "30", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_blkin_trace_all": "false", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T09:58:44.104 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_class_update_on_start": "true", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_client_message_cap": "256", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_client_op_priority": "63", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_client_watch_timeout": "30", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_command_max_records": "256", 2023-10-09T09:58:44.105 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_compact_on_start": "false", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_crush_update_on_start": "true", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T09:58:44.106 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_op_order": "false", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_shutdown": "false", 2023-10-09T09:58:44.107 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T09:58:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_default_notify_timeout": "30", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_enable_op_tracker": "true", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T09:58:44.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_fast_info": "true", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_fast_shutdown": "true", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_function_tracing": "false", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_heartbeat_grace": "20", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_heartbeat_interval": "6", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T09:58:44.110 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_heartbeat_stale": "600", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T09:58:44.111 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_journal_size": "5120", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_kill_backfill_at": "0", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_map_cache_size": "50", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_map_dedup": "true", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_map_message_max": "40", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_attr_name_len": "100", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_attr_size": "0", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_backfills": "1", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_markdown_count": "5", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_markdown_period": "600", 2023-10-09T09:58:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_object_name_len": "2048", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_object_size": "134217728", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_pgls": "1024", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_push_cost": "8388608", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_push_objects": "10", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_scrubs": "1", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T09:58:44.113 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_max_write_size": "90", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_profile": "balanced", 2023-10-09T09:58:44.114 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_memory_base": "805306368", 2023-10-09T09:58:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_memory_target": "4294967296", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_memory_target_autotune": "false", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_min_recovery_priority": "0", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mon_report_interval": "5", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T09:58:44.116 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_num_cache_shards": "32", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_numa_node": "-1", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_objecter_finishers": "1", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_objectstore": "bluestore", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_objectstore_fuse": "false", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_objectstore_tracing": "false", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_history_duration": "600", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_history_size": "20", 2023-10-09T09:58:44.117 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_log_threshold": "5", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_num_shards": "0", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T09:58:44.118 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_op_thread_timeout": "15", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_open_classes_on_start": "true", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_os_flags": "0", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_peering_op_priority": "255", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_delete_priority": "5", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T09:58:44.119 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_crimson": "false", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T09:58:44.120 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_flags": "0", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_min_size": "0", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T09:58:44.121 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_size": "3", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_default_type": "replicated", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_cost": "20971520", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_max_active": "0", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T09:58:44.122 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_op_priority": "3", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_priority": "5", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T09:58:44.123 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_repair_during_recovery": "false", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T09:58:44.124 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_cost": "52428800", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_end_hour": "0", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T09:58:44.125 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_priority": "5", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_skip_data_digest": "false", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_smart_report_timeout": "5", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_snap_trim_priority": "5", 2023-10-09T09:58:44.126 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_target_transaction_size": "30", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T09:58:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_tracing": "false", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_use_stale_snap": "false", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_kill_at": "0", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_max_join_drift": "10", 2023-10-09T09:58:44.128 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_min": "500", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_min_wait": "0.050000", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_service_trim_max": "500", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_service_trim_min": "250", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_stash_full_interval": "25", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_trim_max": "500", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "paxos_trim_min": "250", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "perf": "true", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "pid_file": "", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T09:58:44.129 INFO:teuthology.orchestra.run.smithi012.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "public_addr": "-", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "public_addrv": "", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "public_bind_addr": "-", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "public_network": "172.21.0.0/20", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "public_network_interface": "", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "qat_compressor_enabled": "false", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "rados_mon_op_timeout": "0", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "rados_osd_op_timeout": "0", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "rados_tracing": "false", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_atime_update_interval": "60", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T09:58:44.130 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache": "true", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache_policy": "writearound", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache_size": "33554432", 2023-10-09T09:58:44.131 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_compression_hint": "none", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_clone_format": "auto", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_data_pool": "", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_features": "61", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_format": "2", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_map_options": "", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_order": "22", 2023-10-09T09:58:44.132 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_pool": "rbd", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_stripe_count": "0", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_io_scheduler": "simple", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T09:58:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_order": "24", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_pool": "", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_journal_splay_width": "4", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T09:58:44.134 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T09:58:44.135 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T09:58:44.136 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_op_threads": "1", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T09:58:44.137 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_plugins": "", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T09:58:44.138 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T09:58:44.139 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_tracing": "false", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_validate_names": "true", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rbd_validate_pool": "true", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "restapi_base_url": "", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "restapi_log_level": "", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T09:58:44.140 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_admin_entry": "admin", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_backend_store": "rados", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_barbican_url": "", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_beast_enable_async": "true", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T09:58:44.141 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_cache_enabled": "true", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_config_store": "rados", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_content_length_compat": "false", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_cross_domain_policy": "", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T09:58:44.142 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T09:58:44.143 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T09:58:44.144 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T09:58:44.145 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T09:58:44.146 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_data_log_window": "30", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T09:58:44.147 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T09:58:44.148 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dns_name": "", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dns_s3website_name": "", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_enable_ops_log": "false", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_enable_static_website": "false", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_enable_usage_log": "false", 2023-10-09T09:58:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_expose_bucket": "false", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_extended_http_attrs": "", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_filter": "none", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_max_objs": "32", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T09:58:44.150 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_init_timeout": "300", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T09:58:44.151 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_admin_password": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_admin_project": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_admin_token": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_admin_user": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_api_version": "2", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T09:58:44.152 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_url": "", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_lc_max_objs": "32", 2023-10-09T09:58:44.153 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_lc_max_worker": "3", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T09:58:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_log_http_headers": "", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_attr_size": "0", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T09:58:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_listing_results": "1000", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_notify_retries": "10", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T09:58:44.156 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T09:58:44.157 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_num_control_oids": "8", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_numa_node": "-1", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T09:58:44.158 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_op_tracing": "false", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_opa_token": "", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_opa_url": "", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ops_log_rados": "false", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T09:58:44.159 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_print_continue": "true", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_rados_tracing": "false", 2023-10-09T09:58:44.160 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_realm": "", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_realm_id": "", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_region": "", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_request_uri": "", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T09:58:44.161 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_resolve_cname": "false", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_run_sync_thread": "true", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T09:58:44.162 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_script_uri": "", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_service_provider_name": "", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sts_client_id": "", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sts_client_secret": "", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sts_entry": "sts", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sts_key": "sts", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T09:58:44.163 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_auth_url": "", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_custom_header": "", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_need_stats": "true", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_tenant_name": "", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_url": "", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_lease_period": "120", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T09:58:44.164 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_thread_pool_size": "512", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_torrent_comment": "", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_torrent_createby": "", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_torrent_encoding": "", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_torrent_flag": "false", 2023-10-09T09:58:44.165 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_torrent_origin": "", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_torrent_tracker": "", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_usage_max_shards": "32", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_use_opa_authz": "false", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T09:58:44.166 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_user_unique_email": "true", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_verify_ssl": "true", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_zone": "", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_zone_id": "", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_zonegroup": "", 2023-10-09T09:58:44.167 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_zonegroup_id": "", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_block_size": "4096", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T09:58:44.168 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_partition_filters": "false", 2023-10-09T09:58:44.169 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_perf": "false", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "run_dir": "/var/run/ceph", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_block_create": "true", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_device_size": "53687091200", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T09:58:44.170 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_main_device_type": "SSD", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "seastore_segment_size": "67108864", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "set_keepcaps": "false", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "setgroup": "ceph", 2023-10-09T09:58:44.171 INFO:teuthology.orchestra.run.smithi012.stdout: "setuser": "ceph", 2023-10-09T09:58:44.172 INFO:teuthology.orchestra.run.smithi012.stdout: "setuser_match_path": "", 2023-10-09T09:58:44.172 INFO:teuthology.orchestra.run.smithi012.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T09:58:44.172 INFO:teuthology.orchestra.run.smithi012.stdout: "thp": "false", 2023-10-09T09:58:44.172 INFO:teuthology.orchestra.run.smithi012.stdout: "threadpool_default_timeout": "60", 2023-10-09T09:58:44.172 INFO:teuthology.orchestra.run.smithi012.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T09:58:44.172 INFO:teuthology.orchestra.run.smithi012.stdout: "throttler_perf_counter": "true" 2023-10-09T09:58:44.172 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:58:44.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T09:58:44.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:44 smithi012 ceph-mon[147926]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:44.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:44 smithi012 ceph-mon[152263]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:44.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:44 smithi110 ceph-mon[136032]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:45.130 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:46.357 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a: {} 2023-10-09T09:58:46.367 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b: {} 2023-10-09T09:58:46.376 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c: {} 2023-10-09T09:58:46.377 INFO:teuthology.orchestra.run.smithi012.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T09:58:46.377 INFO:teuthology.orchestra.run.smithi012.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T09:58:46.377 INFO:teuthology.orchestra.run.smithi012.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T09:58:46.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:46 smithi012 ceph-mon[147926]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:46.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:46 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1886900957' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:58:46.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:46 smithi012 ceph-mon[152263]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:46.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:46 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1886900957' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:58:46.746 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T09:58:46.746 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T09:58:46.758 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T09:58:46.772 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T09:58:46.772 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T09:58:46.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T09:58:46.773 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T09:58:46.773 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:58:46.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T09:58:46.774 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T09:58:46.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:46.774 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T09:58:46.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:58:46.775 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T09:58:46.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T09:58:46.775 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T09:58:46.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:58:46.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:46.896 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:46.898 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:46 smithi110 ceph-mon[136032]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:46.898 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:46 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1886900957' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T09:58:46.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:46.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:47.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:58:47.083 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:47.120 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:58:47.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:47.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:58:47.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:58:47.253 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:47.293 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:58:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:47.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:58:47.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:47.416 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:47.431 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:47.467 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:47.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:47.495 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:47.571 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:58:47.598 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:47.604 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:47.606 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:47.607 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:47.693 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:58:47.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:47.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:58:47.797 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:47.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:47.861 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:47.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:47.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:47.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:58:47.994 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:48.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:58:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:58:47] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-10-09T09:58:48.285 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:58:48.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:48.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:58:48.389 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:48.421 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:48.456 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:48.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:48.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:58:48.603 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:48 smithi110 ceph-mon[136032]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:48.790 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:58:48.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:48.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:58:48.862 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:48 smithi012 ceph-mon[147926]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:48.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:48 smithi012 ceph-mon[152263]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:48.897 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:48.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:48.972 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:48.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:49.074 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:49.104 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:49.138 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:49.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:49.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:49.243 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:58:49.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:49.303 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:58:49.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:49.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:58:49.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:58:49.438 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:49.652 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:58:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:49.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:58:49.755 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:49.782 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:49.815 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:49.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:49.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:49.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:58:49.949 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:50.037 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:58:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:50.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:58:50.142 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:50.169 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:50.200 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:50.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:50.303 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:58:50.329 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:50.360 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:58:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:50.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:58:50.463 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:50.490 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:50.521 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:50.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:50.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:50.603 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T09:58:50.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:50 smithi012 ceph-mon[147926]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:50.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/798942483' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:58:50.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:50 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:58:50.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:50 smithi012 ceph-mon[152263]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:50.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/798942483' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:58:50.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:50 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:58:50.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:58:50.657 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:50 smithi110 ceph-mon[136032]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/798942483' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:58:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:50 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T09:58:51.195 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:58:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:51.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:58:51.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:58:51.334 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:51.546 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:58:51.546 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T09:58:51.546 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T09:58:51.546 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:58:51.561 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:58:51.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:51.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:58:51.627 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T09:58:51.628 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_0' created 2023-10-09T09:58:51.657 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T09:58:51.666 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:51.697 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:51.732 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:51.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:51.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:58:51.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T09:58:51.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1055219881' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:58:51.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/247215244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1055219881' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:58:51.800 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/247215244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:58:51.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:58:51.836 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:58:51.862 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.862 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T09:58:51.862 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T09:58:51.862 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1055219881' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:58:51.862 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T09:58:51.862 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: failed to encode map e96 with expected crc 2023-10-09T09:58:51.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/247215244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:58:51.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:58:51.864 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:51.898 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:58:51.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:51.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:58:52.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:58:52.029 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:52.061 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:58:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:52.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:58:52.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:58:52.192 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:52.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T09:58:52.504 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:58:52.504 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:58:52.504 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:58:52.513 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:58:52.531 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:58:52.549 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:58:52.568 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:58:52.587 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:58:52.589 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:58:52.589 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:58:52.589 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:58:52.589 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:58:52.589 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:58:52.589 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:58:52.589 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:58:52.589 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1179941243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.785 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.785 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1179941243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:58:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:58:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T09:58:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:58:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T09:58:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: failed to encode map e97 with expected crc 2023-10-09T09:58:52.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1179941243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:58:53.078 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:58:53.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:53.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:58:53.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T09:58:53.184 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:53.197 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:53.211 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:53.245 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:53.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:53.349 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:58:53.377 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:53.408 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:58:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:53.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:58:53.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:58:53.540 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:53.574 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:58:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:53.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:58:53.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:58:53.692 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:53.713 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:53.894 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:58:53.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[147926]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[147926]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T09:58:53.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[147926]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[147926]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:58:53.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[147926]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[152263]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[152263]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T09:58:53.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[152263]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[152263]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:58:53.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:53 smithi012 ceph-mon[152263]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:53.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:58:53.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:53 smithi110 ceph-mon[136032]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:53 smithi110 ceph-mon[136032]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T09:58:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:53 smithi110 ceph-mon[136032]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:53 smithi110 ceph-mon[136032]: failed to encode map e98 with expected crc 2023-10-09T09:58:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:53 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T09:58:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:53 smithi110 ceph-mon[136032]: failed to encode map e98 with expected crc 2023-10-09T09:58:54.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:58:54.029 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:54.162 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:58:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:54.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:58:54.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:54.295 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:54.298 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:54.327 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:54.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:54.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:54.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:54.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:54.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:58:54.538 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:54.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:54.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:54.642 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:54.669 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:54.702 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:54.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:54.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:54.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:58:54.832 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[147926]: pgmap v233: 89 pgs: 1 creating+activating, 1 creating+peering, 12 unknown, 75 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:54.832 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[147926]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.832 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[147926]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.832 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[147926]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T09:58:54.833 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[147926]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.833 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[152263]: pgmap v233: 89 pgs: 1 creating+activating, 1 creating+peering, 12 unknown, 75 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:54.833 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[152263]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.833 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[152263]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.833 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[152263]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T09:58:54.833 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:54 smithi012 ceph-mon[152263]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.835 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:54 smithi110 ceph-mon[136032]: pgmap v233: 89 pgs: 1 creating+activating, 1 creating+peering, 12 unknown, 75 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:54 smithi110 ceph-mon[136032]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:54 smithi110 ceph-mon[136032]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:54 smithi110 ceph-mon[136032]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T09:58:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:54 smithi110 ceph-mon[136032]: failed to encode map e99 with expected crc 2023-10-09T09:58:54.970 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:58:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:54.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:58:55.074 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:58:55.104 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:55.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:58:55.290 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:58:55.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:55.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:58:55.395 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:55.421 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:55.454 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:55.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:55.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:58:55.596 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:55.633 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T09:58:55.686 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T09:58:55.686 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:58:55.687 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:58:55.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:55.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:58:55.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1765780165' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:58:55.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:55 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:58:55.709 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1765780165' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:58:55.709 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:55 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:58:55.709 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1765780165' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:58:55.709 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:55 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T09:58:55.791 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:55.817 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:55.849 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:55.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:55.952 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:55.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:56.011 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:56.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:56.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:56.117 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:58:56.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:56.176 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:58:56.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:56.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:58:56.280 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:56.309 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:56.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:56.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:56.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:56.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T09:58:56.429 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:56.444 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:56.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T09:58:56.468 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:56.552 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:56.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:56.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:56.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:56.681 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:56.713 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:56.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:56.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:56.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:58:56.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[147926]: pgmap v235: 89 pgs: 1 creating+activating, 1 creating+peering, 2 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:56.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[147926]: failed to encode map e100 with expected crc 2023-10-09T09:58:56.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:58:56.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[147926]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T09:58:56.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[147926]: failed to encode map e100 with expected crc 2023-10-09T09:58:56.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[147926]: failed to encode map e100 with expected crc 2023-10-09T09:58:56.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[152263]: pgmap v235: 89 pgs: 1 creating+activating, 1 creating+peering, 2 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:56.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[152263]: failed to encode map e100 with expected crc 2023-10-09T09:58:56.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:58:56.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[152263]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T09:58:56.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[152263]: failed to encode map e100 with expected crc 2023-10-09T09:58:56.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:56 smithi012 ceph-mon[152263]: failed to encode map e100 with expected crc 2023-10-09T09:58:56.845 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:56.850 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:56 smithi110 ceph-mon[136032]: pgmap v235: 89 pgs: 1 creating+activating, 1 creating+peering, 2 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:56 smithi110 ceph-mon[136032]: failed to encode map e100 with expected crc 2023-10-09T09:58:56.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:56 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T09:58:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:56 smithi110 ceph-mon[136032]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T09:58:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:56 smithi110 ceph-mon[136032]: failed to encode map e100 with expected crc 2023-10-09T09:58:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:56 smithi110 ceph-mon[136032]: failed to encode map e100 with expected crc 2023-10-09T09:58:57.216 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:58:57.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:57.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:58:57.321 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:57.348 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:57.379 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:57.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:57.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:57.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:58:57.523 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-mon[147926]: failed to encode map e100 with expected crc 2023-10-09T09:58:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2770001685' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:58:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-mon[152263]: failed to encode map e100 with expected crc 2023-10-09T09:58:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2770001685' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:58:57.719 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:58:57.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:57.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:58:57.823 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:57.850 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:57.883 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:57.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:57 smithi110 ceph-mon[136032]: failed to encode map e100 with expected crc 2023-10-09T09:58:57.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:57 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:57.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:57 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:57.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2770001685' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T09:58:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:57.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:57.987 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:58.013 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:58.022 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:58:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:58:57] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-10-09T09:58:58.046 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:58.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:58.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:58.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:58:58.180 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:58.214 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:58:58.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:58.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:58:58.317 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:58:58.344 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:58.377 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:58:58.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:58.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:58:58.414 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:58.483 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:58:58.510 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:58.541 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:58:58.544 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:58:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:58.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:58:58.567 INFO:teuthology.orchestra.run.smithi012.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T09:58:58.645 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:58:58.672 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:58.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:58 smithi012 ceph-mon[147926]: pgmap v237: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2770001685' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:58:58.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:58 smithi012 ceph-mon[147926]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T09:58:58.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:58 smithi012 ceph-mon[152263]: pgmap v237: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:58.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2770001685' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:58:58.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:58 smithi012 ceph-mon[152263]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T09:58:58.704 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:58:58.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:58.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:58:58.808 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:58.835 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:58.868 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:58.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:58.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:58 smithi110 ceph-mon[136032]: pgmap v237: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2770001685' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T09:58:58.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:58 smithi110 ceph-mon[136032]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T09:58:58.971 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:58:58.998 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:59.033 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:58:59.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:59.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:58:59.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:59.165 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:59.197 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:59.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:59.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:59.303 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:58:59.330 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:59.367 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:58:59.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:59.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:58:59.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:58:59.507 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:58:59.844 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:58:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:58:59.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:58:59.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[147926]: failed to encode map e101 with expected crc 2023-10-09T09:58:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[147926]: failed to encode map e101 with expected crc 2023-10-09T09:58:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[147926]: pgmap v239: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[147926]: failed to encode map e101 with expected crc 2023-10-09T09:58:59.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[152263]: failed to encode map e101 with expected crc 2023-10-09T09:58:59.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[152263]: failed to encode map e101 with expected crc 2023-10-09T09:58:59.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:58:59.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:58:59.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[152263]: pgmap v239: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:58:59.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:58:59 smithi012 ceph-mon[152263]: failed to encode map e101 with expected crc 2023-10-09T09:58:59.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:58:59.991 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:00.028 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:00.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:00.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:00.160 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:59 smithi110 ceph-mon[136032]: failed to encode map e101 with expected crc 2023-10-09T09:59:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:59 smithi110 ceph-mon[136032]: failed to encode map e101 with expected crc 2023-10-09T09:59:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:59 smithi110 ceph-mon[136032]: pgmap v239: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:59:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:58:59 smithi110 ceph-mon[136032]: failed to encode map e101 with expected crc 2023-10-09T09:59:00.246 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:00.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:00.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:00.280 DEBUG:teuthology.orchestra.run.smithi012:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T09:59:00.295 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T09:59:00.295 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T09:59:00.345 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:hints = 1 2023-10-09T09:59:00.345 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T09:59:00.345 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object prefix: benchmark_data_smithi012_182576 2023-10-09T09:59:00.346 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:59:00.346 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 0 0 0 0 0 0 - 0 2023-10-09T09:59:00.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:00.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:00.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:00.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:00.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:00.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:00.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:00.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:59:00.387 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:59:00.416 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:00.775 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:00.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:00.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:00.885 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:00.925 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:01.003 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T09:59:01.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:01.004 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:01.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:01.105 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:01.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4034475254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:01.119 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:00 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:01.119 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1990059700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:01.119 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4034475254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:01.119 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:00 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:01.119 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1990059700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:01.143 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4034475254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:01.144 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:00 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:01.144 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1990059700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:01.147 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:01.196 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:01.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:01.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:01.300 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:01.343 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:01.345 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 1 16 3372 3356 209.731 209.75 0.0052573 0.00475064 2023-10-09T09:59:01.398 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:01.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:01.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:01.503 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:01.562 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:01.738 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:01.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:01.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:01.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:01.880 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:01.925 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:01.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:01.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:02.029 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:02.066 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:02.106 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:02.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:02.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:02.145 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:59:02.145 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T09:59:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T09:59:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T09:59:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:242 6.9 6:9ae961b1:::benchmark_data_smithi012_182576_object241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T09:59:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:00.415677+0000", 2023-10-09T09:59:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.7257075209999999, 2023-10-09T09:59:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013398466, 2023-10-09T09:59:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 242 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.415677+0000", 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.415677+0000", 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.415677+0000", 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.415701+0000", 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3921e-05 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.415710+0000", 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0990000000000005e-06 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.415716+0000", 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.759e-06 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.416707+0000", 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099034199999999996 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.416830+0000", 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012333999999999999 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.416980+0000", 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000149944 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.417710+0000", 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00073033500000000003 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.418417+0000", 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00070645500000000004 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.428982+0000", 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010564631 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.429039+0000", 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7397e-05 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.429076+0000", 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6835000000000002e-05 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:998 6.9 6:98d45dc0:::benchmark_data_smithi012_182576_object997:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e101)", 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:00.623109+0000", 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.518275297, 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012650260999999999, 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.154 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 998 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.623109+0000", 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.623109+0000", 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.623108+0000", 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.623133+0000", 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5009999999999999e-05 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.623150+0000", 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7037e-05 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.623158+0000", 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1410000000000005e-06 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.623735+0000", 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00057665699999999995 2023-10-09T09:59:02.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.623865+0000", 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130677 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.624000+0000", 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013498199999999999 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.627484+0000", 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003483341 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.627516+0000", 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2546000000000003e-05 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.635656+0000", 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0081394640000000008 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.635720+0000", 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4188000000000003e-05 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.635760+0000", 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9916999999999997e-05 2023-10-09T09:59:02.160 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:1968 6.d e102/97 6:b59ff674:::benchmark_data_smithi012_182576_object1967:head v 102'111, mlcod=101'109)", 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:00.920004+0000", 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.221380691, 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011521892000000001, 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.920004+0000", 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.920004+0000", 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.919999+0000", 2023-10-09T09:59:02.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999952 2023-10-09T09:59:02.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.920207+0000", 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000207204 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.920226+0000", 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9409999999999999e-05 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.920235+0000", 2023-10-09T09:59:02.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5709999999999998e-06 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.920306+0000", 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0872999999999997e-05 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.920343+0000", 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7916000000000001e-05 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.931485+0000", 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011141852000000001 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:00.931526+0000", 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0677999999999999e-05 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.166 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:2437 6.0 6:0768ca38:::benchmark_data_smithi012_182576_object2436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.046369+0000", 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.095016118, 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014351144, 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.167 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 2437 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046369+0000", 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046369+0000", 2023-10-09T09:59:02.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046367+0000", 2023-10-09T09:59:02.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:02.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046398+0000", 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0960999999999997e-05 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046424+0000", 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6508000000000001e-05 2023-10-09T09:59:02.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046435+0000", 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0876000000000001e-05 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046644+0000", 2023-10-09T09:59:02.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020923599999999999 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046773+0000", 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012871099999999999 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046884+0000", 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011085400000000001 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.048931+0000", 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020471299999999999 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050385+0000", 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014538489999999999 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060641+0000", 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010256042999999999 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060682+0000", 2023-10-09T09:59:02.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1142e-05 2023-10-09T09:59:02.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060720+0000", 2023-10-09T09:59:02.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7774000000000001e-05 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:2440 6.9 6:9914abd1:::benchmark_data_smithi012_182576_object2439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.046451+0000", 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.0949341079999999, 2023-10-09T09:59:02.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014127845, 2023-10-09T09:59:02.177 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.177 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.177 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.177 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.177 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.177 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 2440 2023-10-09T09:59:02.177 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.177 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.178 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046451+0000", 2023-10-09T09:59:02.178 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.178 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.178 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046451+0000", 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046450+0000", 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046481+0000", 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1427000000000003e-05 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046491+0000", 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0620000000000007e-06 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046497+0000", 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3949999999999998e-06 2023-10-09T09:59:02.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046546+0000", 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9092999999999999e-05 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046683+0000", 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013669800000000001 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046820+0000", 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013741400000000001 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.047845+0000", 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010249860000000001 2023-10-09T09:59:02.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.048326+0000", 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00048129299999999999 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060497+0000", 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012170488 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060546+0000", 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8742000000000002e-05 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060579+0000", 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2873999999999997e-05 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:2441 6.0 6:0a92b35c:::benchmark_data_smithi012_182576_object2440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.046511+0000", 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.0948740100000001, 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014323600000000001, 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 2441 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046511+0000", 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.186 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046511+0000", 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046510+0000", 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046533+0000", 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2382999999999999e-05 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046541+0000", 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0679999999999997e-06 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046547+0000", 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4230000000000002e-06 2023-10-09T09:59:02.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.047247+0000", 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00070023299999999998 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.047370+0000", 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012280099999999999 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.047511+0000", 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014126399999999999 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.048981+0000", 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014699520000000001 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.190 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060754+0000", 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011773067 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060772+0000", 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7436e-05 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060801+0000", 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8991000000000001e-05 2023-10-09T09:59:02.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060834+0000", 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3541000000000003e-05 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:2442 6.0 6:0828419c:::benchmark_data_smithi012_182576_object2441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.192 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.046557+0000", 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.094827488, 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014420985000000001, 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 2442 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.193 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046557+0000", 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046557+0000", 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046557+0000", 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046581+0000", 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3867999999999999e-05 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046589+0000", 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8469999999999996e-06 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.046595+0000", 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4389999999999997e-06 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.047797+0000", 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001201967 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.047909+0000", 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011157799999999999 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.048048+0000", 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013919 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.048988+0000", 2023-10-09T09:59:02.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094019600000000002 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060868+0000", 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011880352 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060897+0000", 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9210999999999999e-05 2023-10-09T09:59:02.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060944+0000", 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6354999999999999e-05 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.060978+0000", 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4443000000000002e-05 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.199 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:2447 6.9 6:9895deb8:::benchmark_data_smithi012_182576_object2446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.050191+0000", 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.0911935230000001, 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017832071000000001, 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.200 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 2447 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050191+0000", 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050191+0000", 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050190+0000", 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050215+0000", 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5429e-05 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050231+0000", 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5858999999999998e-05 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050240+0000", 2023-10-09T09:59:02.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.414e-06 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050292+0000", 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1514999999999997e-05 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050454+0000", 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016241300000000001 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.204 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.050629+0000", 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017437 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.051453+0000", 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082390399999999998 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.052309+0000", 2023-10-09T09:59:02.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00085600400000000005 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.067935+0000", 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015626699000000001 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.067992+0000", 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6730000000000001e-05 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.068023+0000", 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1253e-05 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:2874 6.0 6:0852a036:::benchmark_data_smithi012_182576_object2873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.194571+0000", 2023-10-09T09:59:02.207 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.94681353400000001, 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011305004, 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 2874 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.208 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.194571+0000", 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.194571+0000", 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.194570+0000", 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:02.209 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.194596+0000", 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6619e-05 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.194609+0000", 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2554999999999999e-05 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.210 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.194617+0000", 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6059999999999997e-06 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.194910+0000", 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029295699999999999 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.211 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.195025+0000", 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115665 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.195161+0000", 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135969 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.204442+0000", 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092808370000000001 2023-10-09T09:59:02.212 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.204637+0000", 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000195038 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.205790+0000", 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001152775 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.213 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.205841+0000", 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0958000000000002e-05 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.205876+0000", 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5496e-05 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.214 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:3004 6.9 6:9a0dd870:::benchmark_data_smithi012_182576_object3003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.244045+0000", 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.89733979100000005, 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012116892000000001, 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 3004 2023-10-09T09:59:02.215 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244045+0000", 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244045+0000", 2023-10-09T09:59:02.216 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244043+0000", 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244069+0000", 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5927000000000001e-05 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.217 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244082+0000", 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2745000000000001e-05 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244090+0000", 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1829999999999994e-06 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.218 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244812+0000", 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00072196499999999998 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244941+0000", 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012926300000000001 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:59:02.219 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.245054+0000", 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112818 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.245577+0000", 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000522909 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.246056+0000", 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00047879599999999997 2023-10-09T09:59:02.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.256075+0000", 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01001881 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.256127+0000", 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2772999999999998e-05 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.221 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.256162+0000", 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4588999999999997e-05 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:3005 6.0 6:0a2445d0:::benchmark_data_smithi012_182576_object3004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.244105+0000", 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.89727943600000004, 2023-10-09T09:59:02.222 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011044157000000001, 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 3005 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244105+0000", 2023-10-09T09:59:02.223 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244105+0000", 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244105+0000", 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.224 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244125+0000", 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0789999999999999e-05 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244133+0000", 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9389999999999997e-06 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.225 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244139+0000", 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1120000000000001e-06 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244189+0000", 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9076999999999998e-05 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.226 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244323+0000", 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013487699999999999 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.244463+0000", 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014007399999999999 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.227 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.245168+0000", 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00070450700000000005 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.247905+0000", 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027374669999999999 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.255056+0000", 2023-10-09T09:59:02.228 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071506549999999997 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.255106+0000", 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9941000000000002e-05 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.255150+0000", 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3460000000000001e-05 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.229 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:3948 6.0 6:0a39467c:::benchmark_data_smithi012_182576_object3947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.500772+0000", 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.64061274099999999, 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017223525999999999, 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.230 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 3948 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.500772+0000", 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.231 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.500772+0000", 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.500770+0000", 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.232 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.500795+0000", 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5043000000000001e-05 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.500807+0000", 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1505000000000001e-05 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.500815+0000", 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3119999999999994e-06 2023-10-09T09:59:02.233 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.500867+0000", 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2337999999999999e-05 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.500998+0000", 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013026100000000001 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.234 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.501108+0000", 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110154 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.501813+0000", 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00070556000000000004 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.502507+0000", 2023-10-09T09:59:02.235 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00069303599999999996 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.517899+0000", 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015392874000000001 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.517951+0000", 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1634000000000001e-05 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.517996+0000", 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4564000000000002e-05 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:3955 6.0 6:01164350:::benchmark_data_smithi012_182576_object3954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.502964+0000", 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.63842075200000004, 2023-10-09T09:59:02.237 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015174528, 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 3955 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.238 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.502964+0000", 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.502964+0000", 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.502963+0000", 2023-10-09T09:59:02.239 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.502987+0000", 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3756000000000001e-05 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.502998+0000", 2023-10-09T09:59:02.240 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1433999999999999e-05 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.503006+0000", 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4780000000000002e-06 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.503043+0000", 2023-10-09T09:59:02.241 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6903000000000002e-05 2023-10-09T09:59:02.242 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.242 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.242 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.242 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.503115+0000", 2023-10-09T09:59:02.242 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2519000000000001e-05 2023-10-09T09:59:02.242 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.242 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.242 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.503210+0000", 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5025999999999994e-05 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.503685+0000", 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000475358 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504003+0000", 2023-10-09T09:59:02.243 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031800399999999999 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.518036+0000", 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014032854000000001 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.518091+0000", 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4737999999999997e-05 2023-10-09T09:59:02.244 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.518139+0000", 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7508e-05 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:3956 6.0 6:0b400126:::benchmark_data_smithi012_182576_object3955:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.245 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.504030+0000", 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.63735504700000001, 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01424158, 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 3956 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.246 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504030+0000", 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504030+0000", 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.247 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504029+0000", 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504053+0000", 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3853999999999999e-05 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.248 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504063+0000", 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0559e-05 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504071+0000", 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5209999999999997e-06 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.249 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504104+0000", 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2662e-05 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504180+0000", 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6658999999999999e-05 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504258+0000", 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7903000000000003e-05 2023-10-09T09:59:02.250 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504646+0000", 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038839499999999998 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.504916+0000", 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026907500000000002 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.251 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.518181+0000", 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013265844000000001 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.518236+0000", 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5009000000000001e-05 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.518271+0000", 2023-10-09T09:59:02.252 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4898999999999999e-05 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:4088 6.9 6:9f187c0b:::benchmark_data_smithi012_182576_object4087:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.561691+0000", 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.57969410399999999, 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010957692999999999, 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.253 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 4088 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.561691+0000", 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.254 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.561691+0000", 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.561690+0000", 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:02.255 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.561730+0000", 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0314000000000001e-05 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.561741+0000", 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0154e-05 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.256 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.561748+0000", 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.182e-06 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.562507+0000", 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00075970300000000003 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.562653+0000", 2023-10-09T09:59:02.257 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014537099999999999 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.562791+0000", 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013806800000000001 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.563452+0000", 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00066080800000000003 2023-10-09T09:59:02.258 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.565409+0000", 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019575650000000001 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.572533+0000", 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071236629999999997 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.259 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.572609+0000", 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6211000000000006e-05 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.572648+0000", 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9307999999999997e-05 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.260 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:4241 6.0 6:06fd2fff:::benchmark_data_smithi012_182576_object4240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.622411+0000", 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.51897336100000002, 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012919547999999999, 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.261 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 4241 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622411+0000", 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.262 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622411+0000", 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622410+0000", 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.263 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622432+0000", 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2183000000000001e-05 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622443+0000", 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0392e-05 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622450+0000", 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6629999999999993e-06 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622514+0000", 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3596999999999994e-05 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622642+0000", 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012847600000000001 2023-10-09T09:59:02.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.622772+0000", 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012922500000000001 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.625512+0000", 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027403359999999999 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.266 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.631217+0000", 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005705112 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.635207+0000", 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039897270000000002 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.267 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.635271+0000", 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4325000000000005e-05 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.635331+0000", 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9883000000000002e-05 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:4260 6.0 6:07121064:::benchmark_data_smithi012_182576_object4259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.636462+0000", 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.50492268399999995, 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013127796000000001, 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 4260 2023-10-09T09:59:02.269 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636462+0000", 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636462+0000", 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.270 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636462+0000", 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636484+0000", 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2175e-05 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636496+0000", 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1960000000000001e-05 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636502+0000", 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2060000000000004e-06 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.272 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636556+0000", 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3922000000000003e-05 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636694+0000", 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013856 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.636879+0000", 2023-10-09T09:59:02.273 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018470699999999999 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.637707+0000", 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082757000000000002 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.640212+0000", 2023-10-09T09:59:02.274 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025057500000000002 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.649507+0000", 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092942960000000005 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.649553+0000", 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6362000000000001e-05 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.275 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.649590+0000", 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6847999999999997e-05 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:4387 6.0 6:09469dc1:::benchmark_data_smithi012_182576_object4386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.687733+0000", 2023-10-09T09:59:02.276 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.45365165800000001, 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012052958000000001, 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 4387 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.277 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.687733+0000", 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.687733+0000", 2023-10-09T09:59:02.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.687732+0000", 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.279 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.687790+0000", 2023-10-09T09:59:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7985000000000001e-05 2023-10-09T09:59:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.687803+0000", 2023-10-09T09:59:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2624e-05 2023-10-09T09:59:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.687813+0000", 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0241000000000001e-05 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.691489+0000", 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0036753010000000002 2023-10-09T09:59:02.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.691609+0000", 2023-10-09T09:59:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012026100000000001 2023-10-09T09:59:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:02.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.691744+0000", 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000134809 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.699436+0000", 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076923429999999999 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.283 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.699695+0000", 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000258948 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.699702+0000", 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3200000000000002e-06 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.699751+0000", 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8414000000000001e-05 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.699786+0000", 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5462e-05 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:4581 6.9 6:96956291:::benchmark_data_smithi012_182576_object4580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.769254+0000", 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.37213053800000001, 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012591896999999999, 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 4581 2023-10-09T09:59:02.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769254+0000", 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769254+0000", 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769252+0000", 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769298+0000", 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5843999999999997e-05 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769336+0000", 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8064e-05 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769348+0000", 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1683e-05 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769599+0000", 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00025067500000000001 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769764+0000", 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016555899999999999 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.290 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769961+0000", 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019661099999999999 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.771630+0000", 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001669369 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.775504+0000", 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038738589999999999 2023-10-09T09:59:02.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.781725+0000", 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062208619999999997 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.781794+0000", 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9121000000000004e-05 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.781846+0000", 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2141000000000001e-05 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:4586 6.9 6:9bcd2f92:::benchmark_data_smithi012_182576_object4585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:01.769427+0000", 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.37195749, 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012578331999999999, 2023-10-09T09:59:02.293 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 4586 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:02.294 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769427+0000", 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769427+0000", 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769427+0000", 2023-10-09T09:59:02.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769447+0000", 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0329000000000001e-05 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769456+0000", 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8640000000000002e-06 2023-10-09T09:59:02.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.769463+0000", 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3139999999999998e-06 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.770516+0000", 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001052982 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.770661+0000", 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014500099999999999 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.770837+0000", 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000175794 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.775494+0000", 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046570689999999998 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.775511+0000", 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7139e-05 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.781898+0000", 2023-10-09T09:59:02.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0063869349999999998 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.781959+0000", 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0618999999999997e-05 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:02.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:01.782006+0000", 2023-10-09T09:59:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6844000000000001e-05 2023-10-09T09:59:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:02.301 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:02.301 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:59:02.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:02.302 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:01 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:02.302 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:01 smithi110 ceph-mon[136032]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:01 smithi110 ceph-mon[136032]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:01 smithi110 ceph-mon[136032]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:01 smithi110 ceph-mon[136032]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:01 smithi110 ceph-mon[136032]: pgmap v241: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[147926]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[147926]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[147926]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[147926]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.304 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[147926]: pgmap v241: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:59:02.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:02.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[152263]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:02.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[152263]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T09:59:02.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[152263]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[152263]: failed to encode map e102 with expected crc 2023-10-09T09:59:02.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:01 smithi012 ceph-mon[152263]: pgmap v241: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T09:59:02.345 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 2 16 6030 6014 187.92 166.125 0.00551671 0.00530387 2023-10-09T09:59:02.375 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:02.560 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:02.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:02.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:02.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T09:59:02.665 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:02.700 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:02.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:02.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:02.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:02.851 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:02.888 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:02.938 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:02.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:02.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:02.963 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T09:59:03.043 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:03.075 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2135153176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:59:03.075 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:59:03.076 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:03.076 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:03.078 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:03.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:59:03.136 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:03.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:03.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:03.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:03.338 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2135153176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:59:03.338 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:59:03.338 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:03.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:03.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2135153176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:59:03.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T09:59:03.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:03.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:03.342 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:03.345 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 3 15 8655 8640 179.979 164.125 0.00448841 0.00554741 2023-10-09T09:59:03.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T09:59:03.467 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:03.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:03.574 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:03.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:59:03.761 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:03.762 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:03.852 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:03.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:03.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:03.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:04.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: pgmap v242: 89 pgs: 89 active+clean; 136 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s wr, 297 op/s 2023-10-09T09:59:04.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:59:04.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:59:04.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T09:59:04.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.062 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:59:04.062 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: pgmap v242: 89 pgs: 89 active+clean; 136 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s wr, 297 op/s 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:59:04.063 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:03 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:04.066 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:04.078 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:04.162 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:04.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:04.271 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:04.309 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: pgmap v242: 89 pgs: 89 active+clean; 136 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s wr, 297 op/s 2023-10-09T09:59:04.309 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T09:59:04.309 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.309 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T09:59:04.310 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T09:59:04.310 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.310 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.310 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: failed to encode map e103 with expected crc 2023-10-09T09:59:04.310 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T09:59:04.310 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:03 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:04.313 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:04.346 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 4 16 10852 10836 169.292 137.25 0.00357579 0.0058998 2023-10-09T09:59:04.366 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:04.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:04.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:04.536 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:04.702 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:04.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:04.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:04.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:04.853 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:04.894 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:04.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:04.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:05.052 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:05 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:05.052 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:05 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:05.060 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:05.128 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:05.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:05.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:05.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:05 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T09:59:05.239 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:05.277 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:05.330 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:05.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:05.346 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 5 16 13396 13380 167.228 159 0.00380545 0.00597349 2023-10-09T09:59:05.401 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:05.435 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:05.474 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:05.528 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:05.529 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:05.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:05.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:05.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:05.688 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:05.867 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:05.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:05.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:05.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14740 6.4 6:29d13c3e:::benchmark_data_smithi012_182576_object14739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.984645+0000", 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00068528800000000004, 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00073262800000000003, 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:05.994 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14740 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984645+0000", 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984645+0000", 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984644+0000", 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984666+0000", 2023-10-09T09:59:05.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2272e-05 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984679+0000", 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2992e-05 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984687+0000", 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9000000000000006e-06 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984731+0000", 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3958999999999997e-05 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984875+0000", 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000144413 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.985046+0000", 2023-10-09T09:59:05.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000170106 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14745 6.4 6.a5e430f4 (undecoded) ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.984822+0000", 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00050861299999999999, 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00063947199999999998, 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:05.999 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14745 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984822+0000", 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984822+0000", 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984821+0000", 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:06.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984854+0000", 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3052999999999997e-05 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984866+0000", 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1216999999999999e-05 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984873+0000", 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1210000000000001e-06 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.002 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14746 6.1 6:82d7aec7:::benchmark_data_smithi012_182576_object14745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.984885+0000", 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00044584300000000002, 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00067160799999999997, 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:06.003 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14746 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984885+0000", 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984885+0000", 2023-10-09T09:59:06.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984884+0000", 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984935+0000", 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0504000000000003e-05 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984943+0000", 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5960000000000004e-06 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984950+0000", 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5899999999999996e-06 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:06.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984995+0000", 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5392000000000002e-05 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.985124+0000", 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012887599999999999 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.985264+0000", 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013928200000000001 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.007 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14728 6.4 6:272f97e4:::benchmark_data_smithi012_182576_object14727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.976538+0000", 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0087922009999999995, 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091263549999999992, 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:06.008 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14728 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976538+0000", 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976538+0000", 2023-10-09T09:59:06.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976537+0000", 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976568+0000", 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1486000000000003e-05 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976585+0000", 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6849000000000001e-05 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976594+0000", 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4870000000000008e-06 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.977632+0000", 2023-10-09T09:59:06.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010378710000000001 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.977751+0000", 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118874 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.977889+0000", 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013801300000000001 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.982593+0000", 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0047042339999999998 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984381+0000", 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001788137 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14729 6.4 6:2853401d:::benchmark_data_smithi012_182576_object14728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.976614+0000", 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0087163850000000001, 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091397709999999997, 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14729 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.014 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976614+0000", 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976614+0000", 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:06.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976614+0000", 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976639+0000", 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5004e-05 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976647+0000", 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.473e-06 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976653+0000", 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2570000000000004e-06 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.978606+0000", 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019522879999999999 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.978708+0000", 2023-10-09T09:59:06.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102346 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.978829+0000", 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121274 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.983394+0000", 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045648659999999999 2023-10-09T09:59:06.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984388+0000", 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099422099999999999 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14731 6.4 6:24f53c99:::benchmark_data_smithi012_182576_object14730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:06.019 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.976659+0000", 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0086715679999999993, 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091842, 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14731 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:06.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976659+0000", 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976659+0000", 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976659+0000", 2023-10-09T09:59:06.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976684+0000", 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5452e-05 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976691+0000", 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2309999999999999e-06 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976697+0000", 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.874e-06 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980645+0000", 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039480970000000002 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:06.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980765+0000", 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119642 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980894+0000", 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012929799999999999 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984393+0000", 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034990239999999999 2023-10-09T09:59:06.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984602+0000", 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000208968 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14735 6.1 6:89854dd2:::benchmark_data_smithi012_182576_object14734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.976702+0000", 2023-10-09T09:59:06.025 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0086282939999999999, 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092461939999999992, 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14735 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:06.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976702+0000", 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976702+0000", 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976702+0000", 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976723+0000", 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1078e-05 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976730+0000", 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3350000000000001e-06 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:06.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976736+0000", 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9699999999999996e-06 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.976924+0000", 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000188164 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.977031+0000", 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106535 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.977124+0000", 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2573999999999998e-05 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.979799+0000", 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026749600000000001 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:06.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984209+0000", 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044108580000000001 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14737 6.1 6:87e690b6:::benchmark_data_smithi012_182576_object14736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.980466+0000", 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0048647209999999998, 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055967810000000003, 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:06.031 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14737 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980466+0000", 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980466+0000", 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980465+0000", 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980509+0000", 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4910000000000002e-05 2023-10-09T09:59:06.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980526+0000", 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6341000000000002e-05 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980534+0000", 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.303e-06 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980740+0000", 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020589 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980876+0000", 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013605799999999999 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.981054+0000", 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017809600000000001 2023-10-09T09:59:06.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.982114+0000", 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010601639999999999 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984409+0000", 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022948019999999999 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:14739 6.4 6:2b603e0f:::benchmark_data_smithi012_182576_object14738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.980549+0000", 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0047819289999999999, 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055868480000000002, 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 14739 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.037 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980549+0000", 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980549+0000", 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980548+0000", 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980579+0000", 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1086e-05 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980587+0000", 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0830000000000004e-06 2023-10-09T09:59:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.980594+0000", 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.477e-06 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.981905+0000", 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013107450000000001 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.981980+0000", 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5444000000000005e-05 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.982114+0000", 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013417600000000001 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984399+0000", 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022848980000000001 2023-10-09T09:59:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.984630+0000", 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023056300000000001 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 9 2023-10-09T09:59:06.042 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:59:06.043 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:06.114 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:06.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:06.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:06.224 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:06.255 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:06 smithi110 ceph-mon[136032]: pgmap v244: 89 pgs: 89 active+clean; 581 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:59:06.259 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:06.310 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:06.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:06.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:06.346 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 6 16 15553 15537 161.823 134.812 0.0113912 0.00617283 2023-10-09T09:59:06.402 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:06 smithi012 ceph-mon[147926]: pgmap v244: 89 pgs: 89 active+clean; 581 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:59:06.402 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:06 smithi012 ceph-mon[152263]: pgmap v244: 89 pgs: 89 active+clean; 581 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T09:59:06.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:06.452 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:06.499 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:06.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:06.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:06.604 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:06.646 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:06.695 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:06.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:06.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:06.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:06.831 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:06.855 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:06.860 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:06.883 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:06.903 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:06.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T09:59:06.927 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:06.958 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:06.977 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:06.994 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:07.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:07.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:07.021 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:07.029 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:59:07.029 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:59:07.029 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:59:07.030 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:59:07.030 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:59:07.030 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:59:07.030 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:59:07.030 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:59:07.105 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:07.142 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:07.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1017470917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:07.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1017470917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:07.193 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:07.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:07.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:07.346 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 7 16 17534 17518 156.392 123.812 0.00961995 0.00638348 2023-10-09T09:59:07.366 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:07.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1017470917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:07.646 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:07.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:07.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:07.753 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:07.788 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:07.842 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:07.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:07.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:07.916 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:59:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:59:07] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.33.4" 2023-10-09T09:59:07.947 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:07.990 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:08.046 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:08.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:08.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:08.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:08.241 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:08.329 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:08.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:08.346 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 8 16 19856 19840 154.982 145.125 0.00562484 0.00644669 2023-10-09T09:59:08.439 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:08.473 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:08.519 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:08.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:08.566 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:08 smithi012 ceph-mon[152263]: pgmap v245: 89 pgs: 89 active+clean; 724 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T09:59:08.566 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:08 smithi012 ceph-mon[147926]: pgmap v245: 89 pgs: 89 active+clean; 724 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T09:59:08.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:08.702 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:08.732 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:08.775 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:08.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:08.881 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:08.917 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:08 smithi110 ceph-mon[136032]: pgmap v245: 89 pgs: 89 active+clean; 724 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T09:59:08.920 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:08.965 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:08.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:09.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:09.130 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:09.347 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 9 16 21847 21831 151.587 124.438 0.00589851 0.00659104 2023-10-09T09:59:09.398 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:09.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:09.506 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:09.546 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:09.596 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:09.597 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T09:59:09.597 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T09:59:09.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:09.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:09.634 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1307167471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:09.635 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:09 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:09.635 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2501142356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:59:09.635 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:09 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:59:09.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1307167471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:09.636 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:09 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:09.636 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2501142356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:59:09.636 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:09 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:59:09.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:09.773 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:09.835 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:09.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:09.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:09.942 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:09.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1307167471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:09 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2501142356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:59:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:09 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T09:59:10.013 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:10.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:59:10.276 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:10.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:10.346 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 10 16 24088 24072 150.434 140.062 0.00564149 0.00664167 2023-10-09T09:59:10.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:10.448 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:10.584 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: pgmap v246: 89 pgs: 89 active+clean; 946 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T09:59:10.585 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:10.585 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.585 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.585 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.585 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:59:10.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:59:10.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T09:59:10.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[147926]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.586 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: pgmap v246: 89 pgs: 89 active+clean; 946 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T09:59:10.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:10.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.588 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:59:10.588 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:59:10.588 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T09:59:10.588 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:10 smithi012 ceph-mon[152263]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.721 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:10.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:10.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:10.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T09:59:10.828 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:10.858 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: pgmap v246: 89 pgs: 89 active+clean; 946 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T09:59:10.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:10.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:59:10.860 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T09:59:10.860 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T09:59:10.860 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:10 smithi110 ceph-mon[136032]: failed to encode map e104 with expected crc 2023-10-09T09:59:10.861 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:10.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:10.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:10.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:11.007 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:11.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:11.084 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:11.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:11.188 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:11.196 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T09:59:11.196 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:59:11.234 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:11.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:11.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:11.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:11.279 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:11.347 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 11 16 25923 25907 147.18 114.688 0.0121694 0.00678881 2023-10-09T09:59:11.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:11.444 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:11.512 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:11.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:11.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:11.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T09:59:11.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:11.706 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:11.762 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:11.801 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:11.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:11.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:11.975 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:12.092 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:12.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:12.201 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:12.242 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:12.294 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:12.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:12.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:12.347 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 12 16 27746 27730 144.409 113.938 0.00490324 0.00691995 2023-10-09T09:59:12.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:12.468 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:12.652 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:12.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:12.763 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:12.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:59:12.799 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:12.828 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:12 smithi012 ceph-mon[147926]: pgmap v248: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T09:59:12.829 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:12 smithi012 ceph-mon[152263]: pgmap v248: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T09:59:12.845 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:12.850 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:12 smithi110 ceph-mon[136032]: pgmap v248: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T09:59:12.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:12.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:13.018 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:13.262 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:13.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:13.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:13.281 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:13.350 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 13 16 29529 29513 141.835 111.438 0.0145271 0.00704427 2023-10-09T09:59:13.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:13.445 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:13.501 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:13.517 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:13.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:13.550 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:13.596 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:13.598 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:13.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:13.644 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:13.686 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:13.698 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3170040401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:13.699 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3170040401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:13.711 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:13.727 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:13.775 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:13.814 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:13.827 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:59:13.827 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:59:13.828 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:59:13.828 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:59:13.828 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:59:13.828 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:59:13.828 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:59:13.828 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:59:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3170040401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:14.036 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:14.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:14.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:14.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:14.208 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:14.351 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 14 16 31080 31064 138.627 96.9375 0.00813048 0.00720878 2023-10-09T09:59:14.539 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:14.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:14.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:14.620 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T09:59:14.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:14.682 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:14.729 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:14.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:14.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:14.840 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:14 smithi012 ceph-mon[147926]: pgmap v249: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T09:59:14.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:14.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4212010993' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:59:14.841 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:14 smithi012 ceph-mon[152263]: pgmap v249: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T09:59:14.841 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:14.841 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4212010993' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:59:14.896 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:14 smithi110 ceph-mon[136032]: pgmap v249: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T09:59:14.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:14.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4212010993' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T09:59:15.251 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:15.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:15.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:15.352 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 15 16 32993 32977 137.347 119.562 0.00807453 0.00727453 2023-10-09T09:59:15.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:15.433 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:15.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[147926]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4212010993' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:59:15.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[147926]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T09:59:15.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[147926]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[147926]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[147926]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[152263]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4212010993' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:59:15.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[152263]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T09:59:15.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[152263]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[152263]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:15 smithi012 ceph-mon[152263]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:15 smithi110 ceph-mon[136032]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4212010993' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T09:59:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:15 smithi110 ceph-mon[136032]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T09:59:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:15 smithi110 ceph-mon[136032]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:15 smithi110 ceph-mon[136032]: failed to encode map e105 with expected crc 2023-10-09T09:59:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:15 smithi110 ceph-mon[136032]: failed to encode map e105 with expected crc 2023-10-09T09:59:16.353 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 16 16 34992 34976 136.555 124.938 0.00714737 0.00731793 2023-10-09T09:59:16.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:59:16.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:16 smithi012 ceph-mon[147926]: pgmap v251: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2023-10-09T09:59:16.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:16 smithi012 ceph-mon[152263]: pgmap v251: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2023-10-09T09:59:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:16 smithi110 ceph-mon[136032]: pgmap v251: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2023-10-09T09:59:17.354 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 17 16 36753 36737 134.992 110.062 0.00849609 0.00740311 2023-10-09T09:59:17.479 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:17.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:17.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:17.584 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:17.615 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:17.661 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:17.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:17.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:17.812 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:17.862 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:17.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:17.969 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:18.011 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:18.056 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:18.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:18.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:18.076 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T09:59:18.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:59:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:59:17] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T09:59:18.255 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:18.356 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 18 16 38430 38414 133.301 104.812 0.0120346 0.00749618 2023-10-09T09:59:18.571 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:18 smithi012 ceph-mon[147926]: pgmap v252: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-09T09:59:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:18 smithi012 ceph-mon[152263]: pgmap v252: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-09T09:59:18.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:18 smithi110 ceph-mon[136032]: pgmap v252: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-09T09:59:19.053 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:19.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:19.162 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:19.197 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:19.248 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:19.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:19.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:19.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:19.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 19 16 40250 40234 132.272 113.75 0.0162546 0.00755567 2023-10-09T09:59:19.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:19.440 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:19.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:19.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:19.477 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:19.544 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:19.574 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:19.620 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:19.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:19.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:19.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:19.809 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:19.908 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:59:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:9537 6.4 6:2df8d729:::benchmark_data_smithi012_182576_object9536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:03.744217+0000", 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 16.150898089000002, 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023534552, 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 9537 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744217+0000", 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744217+0000", 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744215+0000", 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744250+0000", 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4879000000000003e-05 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744261+0000", 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1e-05 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744268+0000", 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9990000000000002e-06 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744533+0000", 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026496899999999999 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744629+0000", 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5747000000000003e-05 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.744710+0000", 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0751e-05 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.746275+0000", 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001565428 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.746578+0000", 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00030321900000000003 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.767656+0000", 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021077433 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.767711+0000", 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5791000000000002e-05 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:03.767751+0000", 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9675999999999998e-05 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:10288 6.1 6:8ce443a6:::benchmark_data_smithi012_182576_object10287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:04.133397+0000", 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.761717487, 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024089701000000002, 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 10288 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.133397+0000", 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.133397+0000", 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.133395+0000", 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.133423+0000", 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7640999999999998e-05 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.133438+0000", 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4927e-05 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.133446+0000", 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8220000000000007e-06 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.134237+0000", 2023-10-09T09:59:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00079108200000000003 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.134408+0000", 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000170831 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.134580+0000", 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017199699999999999 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.135569+0000", 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00098945700000000001 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.135961+0000", 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039201900000000002 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.157389+0000", 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021427649999999999 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.157445+0000", 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5766000000000001e-05 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.157487+0000", 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2216999999999999e-05 2023-10-09T09:59:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:12303 6.4 6:24be0a69:::benchmark_data_smithi012_182576_object12302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:04.880586+0000", 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.014528728, 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062269318999999997, 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 12303 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880586+0000", 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880586+0000", 2023-10-09T09:59:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880584+0000", 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880615+0000", 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0661999999999999e-05 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880636+0000", 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1044999999999999e-05 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880650+0000", 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4596000000000001e-05 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880707+0000", 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6609e-05 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880859+0000", 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151948 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.881032+0000", 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017333999999999999 2023-10-09T09:59:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.883186+0000", 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002154151 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.883592+0000", 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000405287 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.942764+0000", 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059172484999999997 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.942809+0000", 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4527999999999998e-05 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.942855+0000", 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6465000000000003e-05 2023-10-09T09:59:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:12306 6.4 6:283562b3:::benchmark_data_smithi012_182576_object12305:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:04.880669+0000", 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.014445616, 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062808126000000006, 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 12306 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880669+0000", 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880669+0000", 2023-10-09T09:59:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880669+0000", 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880694+0000", 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5352999999999999e-05 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880702+0000", 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5269999999999998e-06 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880709+0000", 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.956e-06 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.881785+0000", 2023-10-09T09:59:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010758849999999999 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.881895+0000", 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109714 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.882021+0000", 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012641400000000001 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.883790+0000", 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001768181 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.885583+0000", 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017935679999999999 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.943379+0000", 2023-10-09T09:59:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057795671999999999 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.943423+0000", 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4057999999999997e-05 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.943477+0000", 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4327999999999999e-05 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:12308 6.4 6:21076667:::benchmark_data_smithi012_182576_object12307:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:04.880716+0000", 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.014398804000001, 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063447982, 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 12308 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880716+0000", 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880716+0000", 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880715+0000", 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880737+0000", 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1974e-05 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880747+0000", 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1880000000000008e-06 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.880753+0000", 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0039999999999996e-06 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.882562+0000", 2023-10-09T09:59:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018091369999999999 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.882678+0000", 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011611 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.882815+0000", 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137033 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.883801+0000", 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00098655399999999995 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.885888+0000", 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020862369999999999 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944097+0000", 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058208946999999997 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944136+0000", 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9063000000000002e-05 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944164+0000", 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8200000000000001e-05 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:12316 6.4 6:2e3dfd86:::benchmark_data_smithi012_182576_object12315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:04.884281+0000", 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.010833871999999, 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060013770000000001, 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 12316 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884281+0000", 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884281+0000", 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884280+0000", 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884302+0000", 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1801e-05 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884311+0000", 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5869999999999993e-06 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884335+0000", 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4922e-05 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884397+0000", 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1948999999999999e-05 2023-10-09T09:59:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884520+0000", 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000122965 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884664+0000", 2023-10-09T09:59:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014393899999999999 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.885497+0000", 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00083277899999999996 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.885935+0000", 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043814499999999997 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944209+0000", 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058274096999999997 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944256+0000", 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6938999999999999e-05 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944295+0000", 2023-10-09T09:59:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8272999999999998e-05 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:13926 6.1 6:8de8341f:::benchmark_data_smithi012_182576_object13925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.562632+0000", 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 14.332482449, 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.094049454000000005, 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 13926 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562632+0000", 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562632+0000", 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562631+0000", 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562650+0000", 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8715999999999998e-05 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562659+0000", 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8990000000000007e-06 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562664+0000", 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2660000000000001e-06 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562717+0000", 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2908000000000002e-05 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562848+0000", 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013092 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562987+0000", 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013950799999999999 2023-10-09T09:59:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.564697+0000", 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001709225 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.566058+0000", 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013610720000000001 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.656584+0000", 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.090526628999999997 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.656644+0000", 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9478999999999998e-05 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.656682+0000", 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7802999999999997e-05 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:30046 6.1 6:8ca33e20:::benchmark_data_smithi012_182576_object30045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e104)", 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:13.687678+0000", 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 6.2074363229999996, 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020035635999999999, 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 30046 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687678+0000", 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687678+0000", 2023-10-09T09:59:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687677+0000", 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687706+0000", 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9009999999999998e-05 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687737+0000", 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1415000000000003e-05 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687747+0000", 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0339e-05 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687823+0000", 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5726e-05 2023-10-09T09:59:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.688003+0000", 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017943000000000001 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.688171+0000", 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000168541 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.692854+0000", 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004683319 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.693506+0000", 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00065169700000000002 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.707605+0000", 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014098941 2023-10-09T09:59:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.707663+0000", 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8180000000000002e-05 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.707714+0000", 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0751000000000002e-05 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:30047 6.1 6:8360da2f:::benchmark_data_smithi012_182576_object30046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e104)", 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:13.687874+0000", 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 6.207240648, 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020117563000000001, 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 30047 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687874+0000", 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687874+0000", 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687873+0000", 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687895+0000", 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2138999999999999e-05 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687905+0000", 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6670000000000008e-06 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.687912+0000", 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0119999999999999e-06 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.690359+0000", 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002447233 2023-10-09T09:59:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.690487+0000", 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012782000000000001 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.690638+0000", 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015096 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.693485+0000", 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028476389999999999 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.693512+0000", 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6242999999999999e-05 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.707756+0000", 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014244263 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.707940+0000", 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018384099999999999 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:13.707992+0000", 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1780000000000002e-05 2023-10-09T09:59:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:30682 6.4 6:2e6b24bb:::benchmark_data_smithi012_182576_object30681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e104)", 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:14.095356+0000", 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 5.7997590109999999, 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024546510000000001, 2023-10-09T09:59:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 30682 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095356+0000", 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095356+0000", 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095355+0000", 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095376+0000", 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1769000000000001e-05 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095389+0000", 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2513e-05 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095396+0000", 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5499999999999997e-06 2023-10-09T09:59:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095454+0000", 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7924999999999999e-05 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095563+0000", 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010864199999999999 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.095685+0000", 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000122019 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.096890+0000", 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001205123 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.097815+0000", 2023-10-09T09:59:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00092447899999999997 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.119800+0000", 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021985615 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.119857+0000", 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7281999999999998e-05 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:14.119902+0000", 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4678999999999997e-05 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:32909 6.4 6:2962b443:::benchmark_data_smithi012_182576_object32908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:15.298294+0000", 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4.5968208480000001, 2023-10-09T09:59:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026007328, 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 32909 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.298294+0000", 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.298294+0000", 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.298292+0000", 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.298322+0000", 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0267999999999999e-05 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.298334+0000", 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1534e-05 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.298342+0000", 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4679999999999992e-06 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.302567+0000", 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042240419999999999 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.302731+0000", 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016435899999999999 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.302878+0000", 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014694799999999999 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.305212+0000", 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023342900000000001 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.305433+0000", 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00022068200000000001 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.324212+0000", 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018779262000000001 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.324264+0000", 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1969999999999999e-05 2023-10-09T09:59:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:15.324301+0000", 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7098000000000003e-05 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:36566 6.4 6:2f5bfd46:::benchmark_data_smithi012_182576_object36565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.198330+0000", 2023-10-09T09:59:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.696784525, 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.078965779, 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 36566 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198330+0000", 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198330+0000", 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198330+0000", 2023-10-09T09:59:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198353+0000", 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3054999999999999e-05 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198363+0000", 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0047000000000001e-05 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198369+0000", 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4320000000000004e-06 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198423+0000", 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3918000000000001e-05 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198522+0000", 2023-10-09T09:59:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9321999999999995e-05 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198604+0000", 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1388000000000001e-05 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.202059+0000", 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034555330000000002 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.202153+0000", 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3955000000000004e-05 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.277165+0000", 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.075011504000000007 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.277252+0000", 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6917999999999997e-05 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.277296+0000", 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4246999999999998e-05 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:36569 6.4 6:25414e53:::benchmark_data_smithi012_182576_object36568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.198385+0000", 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.6967298360000003, 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.079059269000000001, 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 36569 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198385+0000", 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198385+0000", 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198384+0000", 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198408+0000", 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3765e-05 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198416+0000", 2023-10-09T09:59:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6240000000000002e-06 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198421+0000", 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.643e-06 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.200649+0000", 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022281129999999999 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.200762+0000", 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112661 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.200888+0000", 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000125816 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.202580+0000", 2023-10-09T09:59:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001692481 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203591+0000", 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010108129999999999 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.277355+0000", 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073763519 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.277403+0000", 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.863e-05 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.277444+0000", 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0790000000000001e-05 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:37215 6.4 6:272b6405:::benchmark_data_smithi012_182576_object37214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.591809+0000", 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.303305215, 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026187128, 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 37215 2023-10-09T09:59:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.591809+0000", 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.591809+0000", 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.591809+0000", 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.591829+0000", 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0185000000000001e-05 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.591839+0000", 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.063e-05 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.591846+0000", 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.973e-06 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.591891+0000", 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4515999999999999e-05 2023-10-09T09:59:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.591993+0000", 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102408 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.592092+0000", 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8591000000000005e-05 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.593243+0000", 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011515710000000001 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.594297+0000", 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001053785 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.617908+0000", 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023611072 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.617959+0000", 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1112000000000001e-05 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.617997+0000", 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7236000000000001e-05 2023-10-09T09:59:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:38119 6.4 6:2f4d8a36:::benchmark_data_smithi012_182576_object38118:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.099557+0000", 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.795557598, 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055378874000000002, 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 38119 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099557+0000", 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099557+0000", 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099555+0000", 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099590+0000", 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5340999999999999e-05 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099608+0000", 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7499e-05 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099617+0000", 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5149999999999995e-06 2023-10-09T09:59:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099668+0000", 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0850000000000003e-05 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099787+0000", 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118884 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099888+0000", 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101378 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.102661+0000", 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027723040000000002 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.103859+0000", 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011980630000000001 2023-10-09T09:59:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.154857+0000", 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.050997730999999998 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.154901+0000", 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4490999999999998e-05 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.154936+0000", 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4887e-05 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:38129 6.4 6:29367d9b:::benchmark_data_smithi012_182576_object38128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.099790+0000", 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.7953248319999999, 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055415668000000001, 2023-10-09T09:59:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 38129 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099790+0000", 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099790+0000", 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099789+0000", 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099812+0000", 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2558e-05 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099821+0000", 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8360000000000004e-06 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099828+0000", 2023-10-09T09:59:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5050000000000004e-06 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.101310+0000", 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014822850000000001 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.101458+0000", 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000148169 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.101576+0000", 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117218 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.102791+0000", 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001215718 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.103897+0000", 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011058179999999999 2023-10-09T09:59:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.155129+0000", 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.051231978999999997 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.155174+0000", 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4950999999999997e-05 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.155206+0000", 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1390000000000003e-05 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:40381 6.4 6:242777bc:::benchmark_data_smithi012_182576_object40380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:19.444412+0000", 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.45070267400000003, 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019208614999999998, 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40381 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.444412+0000", 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.444412+0000", 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.444410+0000", 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.444449+0000", 2023-10-09T09:59:20.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8964000000000002e-05 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.444475+0000", 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6080000000000001e-05 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.444488+0000", 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3165e-05 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.446075+0000", 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001586694 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.446211+0000", 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013663600000000001 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:20.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.446368+0000", 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015710799999999999 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.448025+0000", 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001656788 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.452670+0000", 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046450099999999998 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.463523+0000", 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010853104000000001 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.463580+0000", 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6923999999999999e-05 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:20.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.463621+0000", 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0364999999999997e-05 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:40390 6.4 6:2e3a792c:::benchmark_data_smithi012_182576_object40389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:19.453082+0000", 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.44203294199999998, 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022679800999999999, 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:20.029 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40390 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453082+0000", 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453082+0000", 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453080+0000", 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453111+0000", 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1371000000000002e-05 2023-10-09T09:59:20.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453128+0000", 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7433000000000001e-05 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453138+0000", 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6229999999999992e-06 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:20.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453197+0000", 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8557999999999997e-05 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453363+0000", 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016679299999999999 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.453499+0000", 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135783 2023-10-09T09:59:20.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.456787+0000", 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032874499999999999 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.459273+0000", 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024867740000000002 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.475673+0000", 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016399950999999999 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.475718+0000", 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4960999999999999e-05 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.475762+0000", 2023-10-09T09:59:20.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3151000000000001e-05 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:40402 6.1 6:881345bb:::benchmark_data_smithi012_182576_object40401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:19.464809+0000", 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.430305247, 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021859923999999999, 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:20.036 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40402 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.464809+0000", 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:20.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.464809+0000", 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.464808+0000", 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.464834+0000", 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6840999999999999e-05 2023-10-09T09:59:20.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.464853+0000", 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8997000000000001e-05 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.464863+0000", 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5340000000000005e-06 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.469058+0000", 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041953349999999997 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.469191+0000", 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013250499999999999 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.469324+0000", 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013300700000000001 2023-10-09T09:59:20.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.473739+0000", 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044148119999999997 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.473747+0000", 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1149999999999994e-06 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.486575+0000", 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012828744 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.486625+0000", 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.956e-05 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.486669+0000", 2023-10-09T09:59:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4326999999999998e-05 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:40798 6.4 6:2d12cdf8:::benchmark_data_smithi012_182576_object40797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:19.764407+0000", 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.13070802500000001, 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.02032165, 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:20.043 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40798 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764407+0000", 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:20.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764407+0000", 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764402+0000", 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999952 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764436+0000", 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4168000000000002e-05 2023-10-09T09:59:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764480+0000", 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3630000000000001e-05 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764493+0000", 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3261999999999999e-05 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:20.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764542+0000", 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9296999999999999e-05 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764659+0000", 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116971 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.764798+0000", 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000138689 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.769542+0000", 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0047439589999999998 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.774980+0000", 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054380110000000004 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:20.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.784634+0000", 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0096544560000000005 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.784688+0000", 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3643999999999997e-05 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:19.784728+0000", 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0209000000000001e-05 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.049 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:20.050 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:20.050 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:59:20.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:20.051 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:20.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:20.152 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:20.197 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:20.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:20.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:20.356 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T09:59:20.357419+0000 min lat: 0.0011673 max lat: 0.113517 avg lat: 0.00768033 2023-10-09T09:59:20.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:59:20.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 20 16 41679 41663 130.125 89.3125 0.00290337 0.00768033 2023-10-09T09:59:20.363 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:20.403 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:20.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T09:59:20.449 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:20.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:20.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:20.537 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:20.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:20.589 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:20 smithi012 ceph-mon[147926]: pgmap v253: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 130 MiB/s wr, 2.07k op/s 2023-10-09T09:59:20.589 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:20 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2334925992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:20.589 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:20 smithi012 ceph-mon[152263]: pgmap v253: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 130 MiB/s wr, 2.07k op/s 2023-10-09T09:59:20.589 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:20 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2334925992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:20.639 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:20.651 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T09:59:20.912 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:20.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:20.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:20 smithi110 ceph-mon[136032]: pgmap v253: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 130 MiB/s wr, 2.07k op/s 2023-10-09T09:59:20.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:20 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2334925992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:21.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:21.053 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:21.104 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:21.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:21.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:21.209 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:21.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:21.295 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:21.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:21.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:21.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 21 16 43182 43166 128.402 93.9375 0.00780737 0.00778428 2023-10-09T09:59:21.399 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:21.440 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:21.486 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:21.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:21.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:21.594 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:21.628 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:21.674 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:21.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:21.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:21.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:21 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:59:21.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:21 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T09:59:21.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2334925992' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:21.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:21 smithi110 ceph-mon[136032]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T09:59:21.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:21 smithi110 ceph-mon[136032]: failed to encode map e106 with expected crc 2023-10-09T09:59:21.779 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:21.813 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:21.820 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:21.851 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:21.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:21.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:59:21.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T09:59:21.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2334925992' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:21.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[147926]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T09:59:21.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[147926]: failed to encode map e106 with expected crc 2023-10-09T09:59:21.859 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:59:21.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T09:59:21.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2334925992' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:21.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[152263]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T09:59:21.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:21 smithi012 ceph-mon[152263]: failed to encode map e106 with expected crc 2023-10-09T09:59:21.944 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:21.956 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:21.992 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:21.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:22.033 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:22.042 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:22.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:22.069 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:22.105 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:22.143 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:22.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:22.182 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:22.221 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:22.236 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:59:22.236 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:59:22.236 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:59:22.236 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:59:22.236 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:59:22.236 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:59:22.236 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:59:22.237 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:59:22.243 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:22.310 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:22.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:22.356 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 22 16 44615 44599 126.637 89.5625 0.0104178 0.00789001 2023-10-09T09:59:22.418 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:22.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:22.504 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:22.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:22.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:22.659 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:22 smithi012 ceph-mon[147926]: pgmap v255: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T09:59:22.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2782802962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:22.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:22 smithi012 ceph-mon[152263]: pgmap v255: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T09:59:22.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2782802962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:22.707 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:22.710 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:22 smithi110 ceph-mon[136032]: pgmap v255: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T09:59:22.710 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2782802962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:22.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:22.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:22.895 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:23.223 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:23.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:23.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:23.335 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:23.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 23 16 46152 46136 125.307 96.0625 0.0105581 0.00797406 2023-10-09T09:59:23.402 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:23.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:59:23.528 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:23.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:23.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:23.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:23.642 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T09:59:23.736 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:23.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:23 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2841977281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:59:23.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:59:23.872 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:23 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2841977281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:59:23.872 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:59:23.909 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T09:59:23.909 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:59:23.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:23 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2841977281' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:59:23.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T09:59:24.230 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:24.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:24.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:24.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T09:59:24.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:24.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 24 16 47541 47525 123.704 86.8125 0.00389995 0.00807904 2023-10-09T09:59:24.384 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:24.433 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:24.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:24.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:24.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T09:59:24.538 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:24.569 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:24.606 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:24.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:24.631 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:24 smithi110 ceph-mon[136032]: pgmap v256: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T09:59:24.631 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:59:24.631 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:24 smithi110 ceph-mon[136032]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T09:59:24.632 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:24 smithi110 ceph-mon[136032]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.632 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:24 smithi110 ceph-mon[136032]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.632 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:24 smithi110 ceph-mon[136032]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.632 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:24 smithi110 ceph-mon[136032]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.710 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:24.710 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:24.742 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:24.787 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:24.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:24.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:59:24.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:24.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[147926]: pgmap v256: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T09:59:24.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:59:24.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[147926]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T09:59:24.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[147926]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[147926]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[147926]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[147926]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[152263]: pgmap v256: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T09:59:24.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T09:59:24.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[152263]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T09:59:24.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[152263]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[152263]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[152263]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:24 smithi012 ceph-mon[152263]: failed to encode map e107 with expected crc 2023-10-09T09:59:24.974 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:25.059 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:25.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:25.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:25.165 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:25.235 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:25.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 25 16 48781 48765 121.856 77.5 0.00801814 0.00820139 2023-10-09T09:59:25.389 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:25.414 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:25.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:25.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:25.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:25.594 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:25.939 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:25.986 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:26.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:26.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:26.104 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:26.145 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:26.198 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:26.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:26.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:26.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:26.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 26 16 50179 50163 120.53 87.375 0.0143031 0.00829025 2023-10-09T09:59:26.388 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:26.455 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:26.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:26.569 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:26.610 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:26.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:26.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:26.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:26.759 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:26.796 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:26.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:26 smithi012 ceph-mon[152263]: pgmap v258: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T09:59:26.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:26 smithi012 ceph-mon[147926]: pgmap v258: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T09:59:26.837 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:26.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:26.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:26.942 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:26 smithi110 ceph-mon[136032]: pgmap v258: 89 pgs: 89 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T09:59:26.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:27.029 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:27.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:27.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:27.134 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:27.206 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:27.227 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:27.268 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:27.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:27.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:27.357 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 27 16 51458 51442 119.027 79.9375 0.0217346 0.00839539 2023-10-09T09:59:27.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:27.418 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:27.464 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:27.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:27.567 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:27.608 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:27.653 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:27.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:27.665 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T09:59:27.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3021589861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:27.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:27.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:27.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3021589861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:27.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:27.817 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3021589861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:27.817 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:27.831 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:27.893 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:27.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:27.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:27.999 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:28.040 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:28.087 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:28.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:28.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:59:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:59:27] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T09:59:28.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:28.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:28.284 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:28.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:28.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:28.359 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 28 16 52776 52760 117.711 82.375 0.00976737 0.00848961 2023-10-09T09:59:28.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:28.454 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:28.665 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: pgmap v259: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T09:59:28.665 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:28.665 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:59:28.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T09:59:28.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4061926695' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:59:28.682 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:28.683 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T09:59:28.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:28.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: pgmap v259: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.716 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4061926695' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:59:28.718 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: pgmap v259: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T09:59:28.718 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:28.718 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.718 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:59:28.718 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T09:59:28.718 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.719 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.719 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: failed to encode map e108 with expected crc 2023-10-09T09:59:28.719 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4061926695' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T09:59:28.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:28.868 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:29.203 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:29.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:29.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T09:59:29.273 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T09:59:29.273 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:59:29.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:29.359 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 29 16 54093 54077 116.491 82.3125 0.023551 0.00857909 2023-10-09T09:59:29.419 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:29.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T09:59:29.825 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4061926695' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:59:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T09:59:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[147926]: pgmap v262: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4061926695' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T09:59:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:29.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:29 smithi012 ceph-mon[152263]: pgmap v262: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:59:29.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:29.928 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:29.937 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4061926695' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T09:59:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T09:59:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: failed to encode map e109 with expected crc 2023-10-09T09:59:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:29 smithi110 ceph-mon[136032]: pgmap v262: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T09:59:29.974 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:30.008 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:30.020 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:30.046 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:30.093 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:30.108 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:30.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:30.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:30.129 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:30.163 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:30.209 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:30.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:30.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:59:30.222 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:59:30.222 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:59:30.222 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:59:30.222 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:59:30.222 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:59:30.222 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:59:30.222 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:59:30.260 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:30.287 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:30.364 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 30 16 55197 55181 114.9 69 0.0159283 0.00869806 2023-10-09T09:59:30.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T09:59:30.746 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:30.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:30.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:30.755 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1526423539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:30.755 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1526423539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:30.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:30.887 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:30.932 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:30.935 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1526423539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:30.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:31.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:31.116 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:31.163 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:31.319 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:31.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:31.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:31.361 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 31 16 56474 56458 113.769 79.8125 0.00807745 0.00878526 2023-10-09T09:59:31.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:31.493 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:31.624 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:31.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:31.730 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:31.761 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:31.811 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:31.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:31.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:31.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:31 smithi012 ceph-mon[152263]: pgmap v263: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T09:59:31.864 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:31 smithi012 ceph-mon[147926]: pgmap v263: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T09:59:31.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:31 smithi110 ceph-mon[136032]: pgmap v263: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T09:59:31.986 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:32.005 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:32.249 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:32.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:32.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:32.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:32.363 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 32 16 57819 57803 112.834 84.0625 0.00581163 0.0088572 2023-10-09T09:59:32.389 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:32.434 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:32.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:32.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:32.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:32.637 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:32.762 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:32.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:32.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:32.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:59:32.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:33.030 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:33.184 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:33.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:33.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:33.289 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:33.306 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:16020 6.f 6:f7331bf8:::benchmark_data_smithi012_182576_object16019:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:06.538073+0000", 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 26.766368331999999, 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055776698, 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.307 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16020 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538073+0000", 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538073+0000", 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538071+0000", 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538102+0000", 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0855000000000002e-05 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538121+0000", 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9273e-05 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538132+0000", 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0772e-05 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538340+0000", 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020801899999999999 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538492+0000", 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015150600000000001 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.538640+0000", 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014828000000000001 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.541177+0000", 2023-10-09T09:59:33.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025372200000000002 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.541307+0000", 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013000899999999999 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.593763+0000", 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052455587999999997 2023-10-09T09:59:33.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.593807+0000", 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4277000000000003e-05 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.593849+0000", 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2558000000000001e-05 2023-10-09T09:59:33.314 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:16038 6.f 6:f6cfc2e5:::benchmark_data_smithi012_182576_object16037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:06.547218+0000", 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 26.75722352, 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.046750541999999999, 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.315 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16038 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547218+0000", 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547218+0000", 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547216+0000", 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547254+0000", 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7849000000000003e-05 2023-10-09T09:59:33.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547269+0000", 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.509e-05 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547275+0000", 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9680000000000003e-06 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547332+0000", 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7620000000000001e-05 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547449+0000", 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011630500000000001 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.547604+0000", 2023-10-09T09:59:33.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015492500000000001 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.548417+0000", 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00081358500000000002 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.549003+0000", 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058616500000000004 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.593886+0000", 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044882787 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.593934+0000", 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.744e-05 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.593968+0000", 2023-10-09T09:59:33.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4483000000000002e-05 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:16046 6.f 6:ffc05458:::benchmark_data_smithi012_182576_object16045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:06.550924+0000", 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 26.753517266999999, 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.043165955999999998, 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.322 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16046 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.550924+0000", 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.550924+0000", 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.550922+0000", 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.550942+0000", 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9572000000000002e-05 2023-10-09T09:59:33.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.550954+0000", 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2516e-05 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.550962+0000", 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4730000000000003e-06 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.551021+0000", 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8745000000000001e-05 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.551178+0000", 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015706800000000001 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.551332+0000", 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000154695 2023-10-09T09:59:33.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.552179+0000", 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00084661199999999995 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555015+0000", 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028358540000000001 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.594011+0000", 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.038995870000000002 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.594054+0000", 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2796999999999997e-05 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.594090+0000", 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6204000000000001e-05 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:16052 6.f 6:f664e8d7:::benchmark_data_smithi012_182576_object16051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:06.555332+0000", 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 26.74910874, 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.039361822999999997, 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.329 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16052 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555332+0000", 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555332+0000", 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555331+0000", 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555356+0000", 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5443999999999999e-05 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555441+0000", 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5494000000000005e-05 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555453+0000", 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1935e-05 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555505+0000", 2023-10-09T09:59:33.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1468999999999998e-05 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555626+0000", 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012126 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.555769+0000", 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0001429 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.556486+0000", 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00071691500000000004 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.558440+0000", 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001953972 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.594612+0000", 2023-10-09T09:59:33.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.036171760999999997 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.594663+0000", 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1004999999999997e-05 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:06.594694+0000", 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1486999999999999e-05 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.335 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:36582 6.f 6:fb3cd064:::benchmark_data_smithi012_182576_object36581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.203526+0000", 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 16.100914701000001, 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073460890000000001, 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 36582 2023-10-09T09:59:33.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203526+0000", 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203526+0000", 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203525+0000", 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203549+0000", 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3615999999999999e-05 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203559+0000", 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8889999999999997e-06 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203564+0000", 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.49e-06 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203606+0000", 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2163999999999998e-05 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203695+0000", 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8806000000000003e-05 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203782+0000", 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6994000000000001e-05 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274531+0000", 2023-10-09T09:59:33.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070748646999999998 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274599+0000", 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7961999999999997e-05 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276902+0000", 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023031810000000001 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276950+0000", 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8232999999999997e-05 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276987+0000", 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7280000000000002e-05 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:36581 6.e e105/97 6:78a07a32:::benchmark_data_smithi012_182576_object36580:head v 105'2309, mlcod=105'2307)", 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.203866+0000", 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 16.100574962, 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.072763919999999996, 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203866+0000", 2023-10-09T09:59:33.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203866+0000", 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203866+0000", 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203954+0000", 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8677999999999994e-05 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203963+0000", 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5129999999999996e-06 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.203974+0000", 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0818999999999999e-05 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.204013+0000", 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9716999999999999e-05 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274728+0000", 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070714595000000005 2023-10-09T09:59:33.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276584+0000", 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001856405 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276630+0000", 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5741000000000001e-05 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.347 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.34797.0:36582 6.f e105/97 ondisk, result = 0)", 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.205233+0000", 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 16.099208492999999, 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069370061999999996, 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.348 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.205233+0000", 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.205233+0000", 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.349 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.205232+0000", 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.205238+0000", 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9009999999999999e-06 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272587+0000", 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067348543999999996 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272596+0000", 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8770000000000005e-06 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274583+0000", 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019863630000000001 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274592+0000", 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3929999999999997e-06 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274603+0000", 2023-10-09T09:59:33.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0541e-05 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:37199 6.f 6:ff567d78:::benchmark_data_smithi012_182576_object37198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.586563+0000", 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.717877724000001, 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030733087999999999, 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.353 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 37199 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586563+0000", 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.354 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586563+0000", 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586562+0000", 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586584+0000", 2023-10-09T09:59:33.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2861999999999999e-05 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586598+0000", 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3326000000000001e-05 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586606+0000", 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4530000000000002e-06 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586662+0000", 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6297e-05 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586786+0000", 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012348799999999999 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.586919+0000", 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000133289 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.590120+0000", 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032008589999999999 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.590226+0000", 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010589100000000001 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.617195+0000", 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026968793000000001 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.617254+0000", 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8958e-05 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.617296+0000", 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2741000000000003e-05 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:38125 6.f 6:f15ec602:::benchmark_data_smithi012_182576_object38124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.098784+0000", 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.205656824, 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053600813999999997, 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.360 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 38125 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098784+0000", 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098784+0000", 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098783+0000", 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098800+0000", 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6894000000000001e-05 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098810+0000", 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0051000000000001e-05 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098816+0000", 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9229999999999999e-06 2023-10-09T09:59:33.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098867+0000", 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1221999999999998e-05 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098988+0000", 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121006 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099117+0000", 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012883 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100568+0000", 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014507509999999999 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.102692+0000", 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021241049999999998 2023-10-09T09:59:33.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.152289+0000", 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049597090000000003 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.152348+0000", 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8770999999999997e-05 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.152385+0000", 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7459999999999997e-05 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:55704 6.f 6:f8db22b5:::benchmark_data_smithi012_182576_object55703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:30.736930+0000", 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.5675110659999998, 2023-10-09T09:59:33.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041702016000000001, 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 55704 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.736930+0000", 2023-10-09T09:59:33.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.736930+0000", 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.736930+0000", 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.736949+0000", 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9769e-05 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.736960+0000", 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0661e-05 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.736966+0000", 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5709999999999997e-06 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.739425+0000", 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024598799999999998 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.739591+0000", 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000165453 2023-10-09T09:59:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.739757+0000", 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016585499999999999 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.742567+0000", 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002809953 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.747839+0000", 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052721460000000001 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.778546+0000", 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030707057999999999 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.778591+0000", 2023-10-09T09:59:33.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4752000000000001e-05 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.778632+0000", 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1374000000000001e-05 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:55724 6.f 6:fd656b98:::benchmark_data_smithi012_182576_object55723:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.374 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:30.755897+0000", 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.5485441460000002, 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.03025098, 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 55724 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755897+0000", 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755897+0000", 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755895+0000", 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:33.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755918+0000", 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2776999999999999e-05 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755930+0000", 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2471e-05 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755938+0000", 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7649999999999993e-06 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755991+0000", 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2819999999999999e-05 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.756114+0000", 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012268700000000001 2023-10-09T09:59:33.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.756252+0000", 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013786700000000001 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.764406+0000", 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0081545769999999997 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.764450+0000", 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4051999999999998e-05 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.786036+0000", 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021585798999999999 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.786099+0000", 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3367999999999994e-05 2023-10-09T09:59:33.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.786148+0000", 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8559e-05 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:55725 6.f 6:f5760d32:::benchmark_data_smithi012_182576_object55724:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:30.755955+0000", 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.5484856689999997, 2023-10-09T09:59:33.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030375777999999999, 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 55725 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755955+0000", 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755955+0000", 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755955+0000", 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755974+0000", 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9046999999999999e-05 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755981+0000", 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9519999999999997e-06 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.755986+0000", 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1100000000000002e-06 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.758163+0000", 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021770240000000001 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.758286+0000", 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012326000000000001 2023-10-09T09:59:33.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.758439+0000", 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000152595 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.764458+0000", 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0060189950000000001 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.764512+0000", 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4221999999999997e-05 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.786222+0000", 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021709728000000001 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.786282+0000", 2023-10-09T09:59:33.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0025000000000002e-05 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:30.786331+0000", 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9289999999999997e-05 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:56970 6.f 6:fc158a74:::benchmark_data_smithi012_182576_object56969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:31.735061+0000", 2023-10-09T09:59:33.388 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.5693806050000001, 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030866547000000001, 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 56970 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.735061+0000", 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.735061+0000", 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.735060+0000", 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:33.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.735080+0000", 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0063000000000001e-05 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.735087+0000", 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0700000000000001e-06 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.735092+0000", 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3680000000000001e-06 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.737175+0000", 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020825290000000001 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.737352+0000", 2023-10-09T09:59:33.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017680899999999999 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.737512+0000", 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016049700000000001 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.740062+0000", 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025494570000000002 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.740439+0000", 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000377248 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.765839+0000", 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025399943000000001 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.765887+0000", 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8165999999999999e-05 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:31.765927+0000", 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9922000000000002e-05 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:57912 6.f 6:f301defe:::benchmark_data_smithi012_182576_object57911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:32.435766+0000", 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.86867472300000004, 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.077923777999999999, 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 57912 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.396 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.435766+0000", 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.435766+0000", 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.435765+0000", 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.435789+0000", 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3867999999999999e-05 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.435806+0000", 2023-10-09T09:59:33.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6776999999999999e-05 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.435814+0000", 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4260000000000003e-06 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.440900+0000", 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050858520000000001 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.441029+0000", 2023-10-09T09:59:33.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012851400000000001 2023-10-09T09:59:33.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.441172+0000", 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000143786 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.445426+0000", 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042532630000000002 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.445893+0000", 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00046742300000000001 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.513604+0000", 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067710554000000006 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.513653+0000", 2023-10-09T09:59:33.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9656e-05 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.513690+0000", 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6934999999999998e-05 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:57914 6.f 6:f23646fd:::benchmark_data_smithi012_182576_object57913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.404 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:32.442806+0000", 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.86163525799999996, 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073699424999999999, 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 57914 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442806+0000", 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442806+0000", 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442805+0000", 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442827+0000", 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2679000000000001e-05 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442838+0000", 2023-10-09T09:59:33.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0548e-05 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442846+0000", 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9869999999999999e-06 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.443234+0000", 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038772399999999999 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.443377+0000", 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014366699999999999 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.443514+0000", 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000136997 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.445611+0000", 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020965150000000002 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.445926+0000", 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031555899999999998 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516406+0000", 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070479926999999998 2023-10-09T09:59:33.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516461+0000", 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5065999999999998e-05 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516505+0000", 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3962999999999999e-05 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:57924 6.f 6:fae149bb:::benchmark_data_smithi012_182576_object57923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:32.447730+0000", 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.85671077699999998, 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068881697000000006, 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 57924 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.412 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.447730+0000", 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.447730+0000", 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.447729+0000", 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.447755+0000", 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6035999999999999e-05 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.447772+0000", 2023-10-09T09:59:33.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7312999999999998e-05 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.447781+0000", 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9809999999999992e-06 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.447833+0000", 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2141000000000001e-05 2023-10-09T09:59:33.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.447967+0000", 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000133425 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.448105+0000", 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013838300000000001 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.453272+0000", 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051670850000000001 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.455076+0000", 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001804118 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516546+0000", 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061470235999999998 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516585+0000", 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8109999999999999e-05 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516612+0000", 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7511000000000001e-05 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:57925 6.f 6:f22e0ae5:::benchmark_data_smithi012_182576_object57924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:32.449611+0000", 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.85482982900000004, 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067102245000000005, 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 57925 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449611+0000", 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449611+0000", 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449610+0000", 2023-10-09T09:59:33.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449636+0000", 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5619e-05 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449649+0000", 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3481e-05 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449658+0000", 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8519999999999993e-06 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449707+0000", 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9109e-05 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449848+0000", 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000140996 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.449985+0000", 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013742999999999999 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.455059+0000", 2023-10-09T09:59:33.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050735149999999998 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.455093+0000", 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4180999999999997e-05 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516645+0000", 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061551996999999997 2023-10-09T09:59:33.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516685+0000", 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9381000000000001e-05 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516714+0000", 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8997e-05 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.425 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:57926 6.f 6:faaf572f:::benchmark_data_smithi012_182576_object57925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:32.450012+0000", 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.854428827, 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066802529999999999, 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.426 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 57926 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450012+0000", 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450012+0000", 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450012+0000", 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450035+0000", 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3564999999999999e-05 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450048+0000", 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2792999999999999e-05 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450056+0000", 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2479999999999996e-06 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.451398+0000", 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001341783 2023-10-09T09:59:33.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.451541+0000", 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014292600000000001 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.451696+0000", 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000154745 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.455103+0000", 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003407434 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.455146+0000", 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3282999999999999e-05 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516747+0000", 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.06160036 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516783+0000", 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5961999999999999e-05 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516815+0000", 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2177e-05 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.432 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:57928 6.f 6:f3e858e3:::benchmark_data_smithi012_182576_object57927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:32.450072+0000", 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.85436891800000003, 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066853940000000001, 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:33.433 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 57928 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450072+0000", 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450072+0000", 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450071+0000", 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450098+0000", 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6639e-05 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450106+0000", 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7870000000000002e-06 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.450112+0000", 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1149999999999999e-06 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.453358+0000", 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032460259999999999 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.453486+0000", 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012815400000000001 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.453639+0000", 2023-10-09T09:59:33.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000152485 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.455217+0000", 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00157829 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.457661+0000", 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002444444 2023-10-09T09:59:33.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516849+0000", 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059187739000000003 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516895+0000", 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5426999999999997e-05 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.516926+0000", 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1541000000000002e-05 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:58521 6.9 e109/97 6:9f684850:::benchmark_data_smithi012_182576_object58520:head v 109'3711, mlcod=109'3709)", 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:33.055444+0000", 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.248996681, 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029977973000000002, 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:33.440 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.055444+0000", 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.055444+0000", 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:33.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.055444+0000", 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.055470+0000", 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5829999999999998e-05 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.055478+0000", 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5959999999999998e-06 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.055483+0000", 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5110000000000003e-06 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.056971+0000", 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014875800000000001 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.056990+0000", 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9658999999999999e-05 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.085384+0000", 2023-10-09T09:59:33.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028393345 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.085422+0000", 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8791999999999997e-05 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:33.445 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:59:33.446 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 33 16 58611 58595 110.915 49.5 0.00319423 0.00895017 2023-10-09T09:59:33.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:33.447 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:33.447 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:33.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:33.548 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:33.591 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:33.625 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T09:59:33.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:33.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:33.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:33.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:33.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2149192975' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:33.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:33 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:33.785 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2149192975' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:33.785 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:33 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:33.791 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:33.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T09:59:33.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2149192975' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:33.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:33 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T09:59:33.961 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:33.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:34.076 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:34.111 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:34.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:34.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:34.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:34.264 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:34.301 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:34.346 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:34.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:34.363 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 34 16 59533 59517 109.348 57.625 0.0102962 0.00913967 2023-10-09T09:59:34.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:34.487 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:34.499 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:34.537 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:34.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:34.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:34.578 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T09:59:34.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:59:34.640 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:34.672 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:34.719 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:34.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:34.775 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[147926]: pgmap v264: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:59:34.775 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[147926]: failed to encode map e110 with expected crc 2023-10-09T09:59:34.775 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:59:34.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[147926]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T09:59:34.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[147926]: failed to encode map e110 with expected crc 2023-10-09T09:59:34.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[152263]: pgmap v264: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:59:34.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[152263]: failed to encode map e110 with expected crc 2023-10-09T09:59:34.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:59:34.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[152263]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T09:59:34.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:34 smithi012 ceph-mon[152263]: failed to encode map e110 with expected crc 2023-10-09T09:59:34.829 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:34.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:34 smithi110 ceph-mon[136032]: pgmap v264: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:59:34.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:34 smithi110 ceph-mon[136032]: failed to encode map e110 with expected crc 2023-10-09T09:59:34.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:34 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T09:59:34.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:34 smithi110 ceph-mon[136032]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T09:59:34.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:34 smithi110 ceph-mon[136032]: failed to encode map e110 with expected crc 2023-10-09T09:59:34.864 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:34.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:34.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:35.013 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:35.051 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:35.105 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:35.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:35.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:35.278 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:35.343 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:35.374 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 35 16 60730 60714 108.362 74.8125 0.0270053 0.00922151 2023-10-09T09:59:35.649 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T09:59:35.652 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:35.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:35.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:35.838 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:35.934 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3996795705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:35.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:35.935 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3996795705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:35.935 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3996795705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:36.250 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:36.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:36.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:36.363 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 36 16 61975 61959 107.513 77.8125 0.0140305 0.00929589 2023-10-09T09:59:36.441 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:36.691 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:36.724 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:36.737 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:36.781 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:36.820 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:36.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:36.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: pgmap v266: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:59:36.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:36.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:36.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T09:59:36.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[147926]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: pgmap v266: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.828 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.828 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.828 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.828 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:36 smithi012 ceph-mon[152263]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.830 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:36.856 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:36.893 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:36.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:36.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: pgmap v266: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T09:59:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T09:59:36.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:36 smithi110 ceph-mon[136032]: failed to encode map e111 with expected crc 2023-10-09T09:59:36.957 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:36.960 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:36.983 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:59:36.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:59:36.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:59:36.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:59:36.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:59:36.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:59:36.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:59:36.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:59:37.005 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:37.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:37.112 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:37.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:37.189 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:37.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:37.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:37.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:37.364 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 37 16 63149 63133 106.591 73.375 0.0167775 0.00937675 2023-10-09T09:59:37.384 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:37.554 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:37.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:37.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:37.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:37.761 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:37.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3190471515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:37.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3190471515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3190471515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:37.971 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:37.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:38.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:38.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:59:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:59:37] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T09:59:38.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:38.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T09:59:38.357 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:38.364 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 38 16 64370 64354 105.794 76.3125 0.00926863 0.0094463 2023-10-09T09:59:38.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:38.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:38.470 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:38.488 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T09:59:38.488 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:59:38.502 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:38.553 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:38.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:38.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:38.657 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:38.768 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:38 smithi012 ceph-mon[152263]: pgmap v268: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T09:59:38.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:38 smithi012 ceph-mon[147926]: pgmap v268: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T09:59:38.773 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:38 smithi110 ceph-mon[136032]: pgmap v268: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T09:59:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:59:39.184 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:39.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:39.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:39.188 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:59:39.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:39.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T09:59:39.365 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 39 16 65417 65401 104.755 65.4375 0.0140089 0.00953973 2023-10-09T09:59:39.380 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:39.561 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:39.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:39.621 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":269,"stamp":"2023-10-09T09:59:38.932742+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4045346720,"num_objects":61742,"num_object_clones":0,"num_object_copies":185226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61742,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":61978,"num_write_kb":3952397,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":61912,"ondisk_log_size":61912,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13254800,"kb_used_data":11816464,"kb_used_omap":126,"kb_used_meta":1438145,"kb_avail":736706416,"statfs":{"total":767960285184,"available":754387369984,"internally_reserved":0,"allocated":12100059136,"data_stored":12096163448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1472661115},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,2,5,2,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":561643520,"num_objects":8570,"num_object_clones":0,"num_object_copies":25710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8570,"num_write_kb":548480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.008346"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":93,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660793+0000","last_change":"2023-10-09T09:57:18.841362+0000","last_active":"2023-10-09T09:59:28.660793+0000","last_peered":"2023-10-09T09:59:28.660793+0000","last_clean":"2023-10-09T09:59:28.660793+0000","last_became_active":"2023-10-09T09:57:18.840951+0000","last_became_peered":"2023-10-09T09:57:18.840951+0000","last_unstale":"2023-10-09T09:59:28.660793+0000","last_undegraded":"2023-10-09T09:59:28.660793+0000","last_fullsized":"2023-10-09T09:59:28.660793+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609376+0000","last_change":"2023-10-09T09:57:49.622753+0000","last_active":"2023-10-09T09:59:33.609376+0000","last_peered":"2023-10-09T09:59:33.609376+0000","last_clean":"2023-10-09T09:59:33.609376+0000","last_became_active":"2023-10-09T09:57:49.622463+0000","last_became_peered":"2023-10-09T09:57:49.622463+0000","last_unstale":"2023-10-09T09:59:33.609376+0000","last_undegraded":"2023-10-09T09:59:33.609376+0000","last_fullsized":"2023-10-09T09:59:33.609376+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:51.020605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.616867+0000","last_change":"2023-10-09T09:57:49.621495+0000","last_active":"2023-10-09T09:59:33.616867+0000","last_peered":"2023-10-09T09:59:33.616867+0000","last_clean":"2023-10-09T09:59:33.616867+0000","last_became_active":"2023-10-09T09:57:49.621144+0000","last_became_peered":"2023-10-09T09:57:49.621144+0000","last_unstale":"2023-10-09T09:59:33.616867+0000","last_undegraded":"2023-10-09T09:59:33.616867+0000","last_fullsized":"2023-10-09T09:59:33.616867+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:11.852424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617643+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T09:59:33.617643+0000","last_peered":"2023-10-09T09:59:33.617643+0000","last_clean":"2023-10-09T09:59:33.617643+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T09:59:33.617643+0000","last_undegraded":"2023-10-09T09:59:33.617643+0000","last_fullsized":"2023-10-09T09:59:33.617643+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609036+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T09:59:33.609036+0000","last_peered":"2023-10-09T09:59:33.609036+0000","last_clean":"2023-10-09T09:59:33.609036+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T09:59:33.609036+0000","last_undegraded":"2023-10-09T09:59:33.609036+0000","last_fullsized":"2023-10-09T09:59:33.609036+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":83,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.671525+0000","last_change":"2023-10-09T09:57:18.838985+0000","last_active":"2023-10-09T09:59:28.671525+0000","last_peered":"2023-10-09T09:59:28.671525+0000","last_clean":"2023-10-09T09:59:28.671525+0000","last_became_active":"2023-10-09T09:57:18.838625+0000","last_became_peered":"2023-10-09T09:57:18.838625+0000","last_unstale":"2023-10-09T09:59:28.671525+0000","last_undegraded":"2023-10-09T09:59:28.671525+0000","last_fullsized":"2023-10-09T09:59:28.671525+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.618045+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T09:59:33.618045+0000","last_peered":"2023-10-09T09:59:33.618045+0000","last_clean":"2023-10-09T09:59:33.618045+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T09:59:33.618045+0000","last_undegraded":"2023-10-09T09:59:33.618045+0000","last_fullsized":"2023-10-09T09:59:33.618045+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608963+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T09:59:33.608963+0000","last_peered":"2023-10-09T09:59:33.608963+0000","last_clean":"2023-10-09T09:59:33.608963+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T09:59:33.608963+0000","last_undegraded":"2023-10-09T09:59:33.608963+0000","last_fullsized":"2023-10-09T09:59:33.608963+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.673100+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T09:59:28.673100+0000","last_peered":"2023-10-09T09:59:28.673100+0000","last_clean":"2023-10-09T09:59:28.673100+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T09:59:28.673100+0000","last_undegraded":"2023-10-09T09:59:28.673100+0000","last_fullsized":"2023-10-09T09:59:28.673100+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608217+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T09:59:33.608217+0000","last_peered":"2023-10-09T09:59:33.608217+0000","last_clean":"2023-10-09T09:59:33.608217+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T09:59:33.608217+0000","last_undegraded":"2023-10-09T09:59:33.608217+0000","last_fullsized":"2023-10-09T09:59:33.608217+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.676979+0000","last_change":"2023-10-09T09:57:49.626281+0000","last_active":"2023-10-09T09:59:28.676979+0000","last_peered":"2023-10-09T09:59:28.676979+0000","last_clean":"2023-10-09T09:59:28.676979+0000","last_became_active":"2023-10-09T09:57:49.625726+0000","last_became_peered":"2023-10-09T09:57:49.625726+0000","last_unstale":"2023-10-09T09:59:28.676979+0000","last_undegraded":"2023-10-09T09:59:28.676979+0000","last_fullsized":"2023-10-09T09:59:28.676979+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617054+0000","last_change":"2023-10-09T09:57:18.840471+0000","last_active":"2023-10-09T09:59:33.617054+0000","last_peered":"2023-10-09T09:59:33.617054+0000","last_clean":"2023-10-09T09:59:33.617054+0000","last_became_active":"2023-10-09T09:57:18.840125+0000","last_became_peered":"2023-10-09T09:57:18.840125+0000","last_unstale":"2023-10-09T09:59:33.617054+0000","last_undegraded":"2023-10-09T09:59:33.617054+0000","last_fullsized":"2023-10-09T09:59:33.617054+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:07.418455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.661354+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T09:59:28.661354+0000","last_peered":"2023-10-09T09:59:28.661354+0000","last_clean":"2023-10-09T09:59:28.661354+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T09:59:28.661354+0000","last_undegraded":"2023-10-09T09:59:28.661354+0000","last_fullsized":"2023-10-09T09:59:28.661354+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.661088+0000","last_change":"2023-10-09T09:56:45.551325+0000","last_active":"2023-10-09T09:59:28.661088+0000","last_peered":"2023-10-09T09:59:28.661088+0000","last_clean":"2023-10-09T09:59:28.661088+0000","last_became_active":"2023-10-09T09:56:45.550946+0000","last_became_peered":"2023-10-09T09:56:45.550946+0000","last_unstale":"2023-10-09T09:59:28.661088+0000","last_undegraded":"2023-10-09T09:59:28.661088+0000","last_fullsized":"2023-10-09T09:59:28.661088+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.676583+0000","last_change":"2023-10-09T09:57:18.843731+0000","last_active":"2023-10-09T09:59:28.676583+0000","last_peered":"2023-10-09T09:59:28.676583+0000","last_clean":"2023-10-09T09:59:28.676583+0000","last_became_active":"2023-10-09T09:57:18.843280+0000","last_became_peered":"2023-10-09T09:57:18.843280+0000","last_unstale":"2023-10-09T09:59:28.676583+0000","last_undegraded":"2023-10-09T09:59:28.676583+0000","last_fullsized":"2023-10-09T09:59:28.676583+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608722+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T09:59:33.608722+0000","last_peered":"2023-10-09T09:59:33.608722+0000","last_clean":"2023-10-09T09:59:33.608722+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T09:59:33.608722+0000","last_undegraded":"2023-10-09T09:59:33.608722+0000","last_fullsized":"2023-10-09T09:59:33.608722+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.618191+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T09:59:33.618191+0000","last_peered":"2023-10-09T09:59:33.618191+0000","last_clean":"2023-10-09T09:59:33.618191+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T09:59:33.618191+0000","last_undegraded":"2023-10-09T09:59:33.618191+0000","last_fullsized":"2023-10-09T09:59:33.618191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.672966+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T09:59:28.672966+0000","last_peered":"2023-10-09T09:59:28.672966+0000","last_clean":"2023-10-09T09:59:28.672966+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T09:59:28.672966+0000","last_undegraded":"2023-10-09T09:59:28.672966+0000","last_fullsized":"2023-10-09T09:59:28.672966+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.676181+0000","last_change":"2023-10-09T09:48:25.523230+0000","last_active":"2023-10-09T09:59:28.676181+0000","last_peered":"2023-10-09T09:59:28.676181+0000","last_clean":"2023-10-09T09:59:28.676181+0000","last_became_active":"2023-10-09T09:48:25.522608+0000","last_became_peered":"2023-10-09T09:48:25.522608+0000","last_unstale":"2023-10-09T09:59:28.676181+0000","last_undegraded":"2023-10-09T09:59:28.676181+0000","last_fullsized":"2023-10-09T09:59:28.676181+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617572+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T09:59:33.617572+0000","last_peered":"2023-10-09T09:59:33.617572+0000","last_clean":"2023-10-09T09:59:33.617572+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T09:59:33.617572+0000","last_undegraded":"2023-10-09T09:59:33.617572+0000","last_fullsized":"2023-10-09T09:59:33.617572+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":83,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.671314+0000","last_change":"2023-10-09T09:56:13.623204+0000","last_active":"2023-10-09T09:59:28.671314+0000","last_peered":"2023-10-09T09:59:28.671314+0000","last_clean":"2023-10-09T09:59:28.671314+0000","last_became_active":"2023-10-09T09:56:13.622853+0000","last_became_peered":"2023-10-09T09:56:13.622853+0000","last_unstale":"2023-10-09T09:59:28.671314+0000","last_undegraded":"2023-10-09T09:59:28.671314+0000","last_fullsized":"2023-10-09T09:59:28.671314+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609579+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T09:59:33.609579+0000","last_peered":"2023-10-09T09:59:33.609579+0000","last_clean":"2023-10-09T09:59:33.609579+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T09:59:33.609579+0000","last_undegraded":"2023-10-09T09:59:33.609579+0000","last_fullsized":"2023-10-09T09:59:33.609579+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"110'3768","reported_seq":3788,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:34.819215+0000","last_change":"2023-10-09T09:58:52.780310+0000","last_active":"2023-10-09T09:59:34.819215+0000","last_peered":"2023-10-09T09:59:34.819215+0000","last_clean":"2023-10-09T09:59:34.819215+0000","last_became_active":"2023-10-09T09:58:52.779910+0000","last_became_peered":"2023-10-09T09:58:52.779910+0000","last_unstale":"2023-10-09T09:59:34.819215+0000","last_undegraded":"2023-10-09T09:59:34.819215+0000","last_fullsized":"2023-10-09T09:59:34.819215+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3768,"log_dups_size":0,"ondisk_log_size":3768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246939648,"num_objects":3768,"num_object_clones":0,"num_object_copies":11304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3768,"num_write_kb":241152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.676447+0000","last_change":"2023-10-09T09:56:13.614485+0000","last_active":"2023-10-09T09:59:28.676447+0000","last_peered":"2023-10-09T09:59:28.676447+0000","last_clean":"2023-10-09T09:59:28.676447+0000","last_became_active":"2023-10-09T09:56:13.614109+0000","last_became_peered":"2023-10-09T09:56:13.614109+0000","last_unstale":"2023-10-09T09:59:28.676447+0000","last_undegraded":"2023-10-09T09:59:28.676447+0000","last_fullsized":"2023-10-09T09:59:28.676447+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.672687+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T09:59:28.672687+0000","last_peered":"2023-10-09T09:59:28.672687+0000","last_clean":"2023-10-09T09:59:28.672687+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T09:59:28.672687+0000","last_undegraded":"2023-10-09T09:59:28.672687+0000","last_fullsized":"2023-10-09T09:59:28.672687+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"111'4113","reported_seq":4136,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:37.624781+0000","last_change":"2023-10-09T09:58:52.625388+0000","last_active":"2023-10-09T09:59:37.624781+0000","last_peered":"2023-10-09T09:59:37.624781+0000","last_clean":"2023-10-09T09:59:37.624781+0000","last_became_active":"2023-10-09T09:58:52.625135+0000","last_became_peered":"2023-10-09T09:58:52.625135+0000","last_unstale":"2023-10-09T09:59:37.624781+0000","last_undegraded":"2023-10-09T09:59:37.624781+0000","last_fullsized":"2023-10-09T09:59:37.624781+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4113,"log_dups_size":0,"ondisk_log_size":4113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:28.713713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269549568,"num_objects":4113,"num_object_clones":0,"num_object_copies":12339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4113,"num_write_kb":263232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.676378+0000","last_change":"2023-10-09T09:56:45.554326+0000","last_active":"2023-10-09T09:59:28.676378+0000","last_peered":"2023-10-09T09:59:28.676378+0000","last_clean":"2023-10-09T09:59:28.676378+0000","last_became_active":"2023-10-09T09:56:45.553876+0000","last_became_peered":"2023-10-09T09:56:45.553876+0000","last_unstale":"2023-10-09T09:59:28.676378+0000","last_undegraded":"2023-10-09T09:59:28.676378+0000","last_fullsized":"2023-10-09T09:59:28.676378+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660424+0000","last_change":"2023-10-09T09:57:49.627073+0000","last_active":"2023-10-09T09:59:28.660424+0000","last_peered":"2023-10-09T09:59:28.660424+0000","last_clean":"2023-10-09T09:59:28.660424+0000","last_became_active":"2023-10-09T09:57:49.626706+0000","last_became_peered":"2023-10-09T09:57:49.626706+0000","last_unstale":"2023-10-09T09:59:28.660424+0000","last_undegraded":"2023-10-09T09:59:28.660424+0000","last_fullsized":"2023-10-09T09:59:28.660424+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"110'3865","reported_seq":3886,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:34.800268+0000","last_change":"2023-10-09T09:58:52.622516+0000","last_active":"2023-10-09T09:59:34.800268+0000","last_peered":"2023-10-09T09:59:34.800268+0000","last_clean":"2023-10-09T09:59:34.800268+0000","last_became_active":"2023-10-09T09:58:52.622112+0000","last_became_peered":"2023-10-09T09:58:52.622112+0000","last_unstale":"2023-10-09T09:59:34.800268+0000","last_undegraded":"2023-10-09T09:59:34.800268+0000","last_fullsized":"2023-10-09T09:59:34.800268+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3865,"log_dups_size":0,"ondisk_log_size":3865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253296640,"num_objects":3865,"num_object_clones":0,"num_object_copies":11595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3865,"num_write_kb":247360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":61,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.677049+0000","last_change":"2023-10-09T09:48:25.514756+0000","last_active":"2023-10-09T09:59:28.677049+0000","last_peered":"2023-10-09T09:59:28.677049+0000","last_clean":"2023-10-09T09:59:28.677049+0000","last_became_active":"2023-10-09T09:48:25.514387+0000","last_became_peered":"2023-10-09T09:48:25.514387+0000","last_unstale":"2023-10-09T09:59:28.677049+0000","last_undegraded":"2023-10-09T09:59:28.677049+0000","last_fullsized":"2023-10-09T09:59:28.677049+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.671588+0000","last_change":"2023-10-09T09:56:45.556365+0000","last_active":"2023-10-09T09:59:28.671588+0000","last_peered":"2023-10-09T09:59:28.671588+0000","last_clean":"2023-10-09T09:59:28.671588+0000","last_became_active":"2023-10-09T09:56:45.555832+0000","last_became_peered":"2023-10-09T09:56:45.555832+0000","last_unstale":"2023-10-09T09:59:28.671588+0000","last_undegraded":"2023-10-09T09:59:28.671588+0000","last_fullsized":"2023-10-09T09:59:28.671588+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"111'3965","reported_seq":3988,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.613908+0000","last_change":"2023-10-09T09:58:52.623147+0000","last_active":"2023-10-09T09:59:36.613908+0000","last_peered":"2023-10-09T09:59:36.613908+0000","last_clean":"2023-10-09T09:59:36.613908+0000","last_became_active":"2023-10-09T09:58:52.622837+0000","last_became_peered":"2023-10-09T09:58:52.622837+0000","last_unstale":"2023-10-09T09:59:36.613908+0000","last_undegraded":"2023-10-09T09:59:36.613908+0000","last_fullsized":"2023-10-09T09:59:36.613908+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3965,"log_dups_size":0,"ondisk_log_size":3965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:20.155378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259850240,"num_objects":3965,"num_object_clones":0,"num_object_copies":11895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3965,"num_write_kb":253760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.672900+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T09:59:28.672900+0000","last_peered":"2023-10-09T09:59:28.672900+0000","last_clean":"2023-10-09T09:59:28.672900+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T09:59:28.672900+0000","last_undegraded":"2023-10-09T09:59:28.672900+0000","last_fullsized":"2023-10-09T09:59:28.672900+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617297+0000","last_change":"2023-10-09T09:56:45.553024+0000","last_active":"2023-10-09T09:59:33.617297+0000","last_peered":"2023-10-09T09:59:33.617297+0000","last_clean":"2023-10-09T09:59:33.617297+0000","last_became_active":"2023-10-09T09:56:45.552670+0000","last_became_peered":"2023-10-09T09:56:45.552670+0000","last_unstale":"2023-10-09T09:59:33.617297+0000","last_undegraded":"2023-10-09T09:59:33.617297+0000","last_fullsized":"2023-10-09T09:59:33.617297+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:58.132008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"110'3737","reported_seq":3758,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:34.818519+0000","last_change":"2023-10-09T09:58:52.626321+0000","last_active":"2023-10-09T09:59:34.818519+0000","last_peered":"2023-10-09T09:59:34.818519+0000","last_clean":"2023-10-09T09:59:34.818519+0000","last_became_active":"2023-10-09T09:58:52.625904+0000","last_became_peered":"2023-10-09T09:58:52.625904+0000","last_unstale":"2023-10-09T09:59:34.818519+0000","last_undegraded":"2023-10-09T09:59:34.818519+0000","last_fullsized":"2023-10-09T09:59:34.818519+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3737,"log_dups_size":0,"ondisk_log_size":3737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244908032,"num_objects":3737,"num_object_clones":0,"num_object_copies":11211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3737,"num_write_kb":239168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.610035+0000","last_change":"2023-10-09T09:57:49.620199+0000","last_active":"2023-10-09T09:59:33.610035+0000","last_peered":"2023-10-09T09:59:33.610035+0000","last_clean":"2023-10-09T09:59:33.610035+0000","last_became_active":"2023-10-09T09:57:49.619695+0000","last_became_peered":"2023-10-09T09:57:49.619695+0000","last_unstale":"2023-10-09T09:59:33.610035+0000","last_undegraded":"2023-10-09T09:59:33.610035+0000","last_fullsized":"2023-10-09T09:59:33.610035+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:48.818881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.661265+0000","last_change":"2023-10-09T09:57:49.621587+0000","last_active":"2023-10-09T09:59:28.661265+0000","last_peered":"2023-10-09T09:59:28.661265+0000","last_clean":"2023-10-09T09:59:28.661265+0000","last_became_active":"2023-10-09T09:57:49.621245+0000","last_became_peered":"2023-10-09T09:57:49.621245+0000","last_unstale":"2023-10-09T09:59:28.661265+0000","last_undegraded":"2023-10-09T09:59:28.661265+0000","last_fullsized":"2023-10-09T09:59:28.661265+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"111'3930","reported_seq":3953,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.818856+0000","last_change":"2023-10-09T09:58:52.624274+0000","last_active":"2023-10-09T09:59:36.818856+0000","last_peered":"2023-10-09T09:59:36.818856+0000","last_clean":"2023-10-09T09:59:36.818856+0000","last_became_active":"2023-10-09T09:58:52.623972+0000","last_became_peered":"2023-10-09T09:58:52.623972+0000","last_unstale":"2023-10-09T09:59:36.818856+0000","last_undegraded":"2023-10-09T09:59:36.818856+0000","last_fullsized":"2023-10-09T09:59:36.818856+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3930,"log_dups_size":0,"ondisk_log_size":3930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:02.499579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257556480,"num_objects":3930,"num_object_clones":0,"num_object_copies":11790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3930,"num_write_kb":251520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.671728+0000","last_change":"2023-10-09T09:56:45.556627+0000","last_active":"2023-10-09T09:59:28.671728+0000","last_peered":"2023-10-09T09:59:28.671728+0000","last_clean":"2023-10-09T09:59:28.671728+0000","last_became_active":"2023-10-09T09:56:45.556265+0000","last_became_peered":"2023-10-09T09:56:45.556265+0000","last_unstale":"2023-10-09T09:59:28.671728+0000","last_undegraded":"2023-10-09T09:59:28.671728+0000","last_fullsized":"2023-10-09T09:59:28.671728+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660999+0000","last_change":"2023-10-09T09:57:18.845748+0000","last_active":"2023-10-09T09:59:28.660999+0000","last_peered":"2023-10-09T09:59:28.660999+0000","last_clean":"2023-10-09T09:59:28.660999+0000","last_became_active":"2023-10-09T09:57:18.845397+0000","last_became_peered":"2023-10-09T09:57:18.845397+0000","last_unstale":"2023-10-09T09:59:28.660999+0000","last_undegraded":"2023-10-09T09:59:28.660999+0000","last_fullsized":"2023-10-09T09:59:28.660999+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"111'3969","reported_seq":3991,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:37.662652+0000","last_change":"2023-10-09T09:58:52.780096+0000","last_active":"2023-10-09T09:59:37.662652+0000","last_peered":"2023-10-09T09:59:37.662652+0000","last_clean":"2023-10-09T09:59:37.662652+0000","last_became_active":"2023-10-09T09:58:52.779754+0000","last_became_peered":"2023-10-09T09:58:52.779754+0000","last_unstale":"2023-10-09T09:59:37.662652+0000","last_undegraded":"2023-10-09T09:59:37.662652+0000","last_fullsized":"2023-10-09T09:59:37.662652+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3969,"log_dups_size":0,"ondisk_log_size":3969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:19:28.304859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260112384,"num_objects":3969,"num_object_clones":0,"num_object_copies":11907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3969,"num_write_kb":254016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660119+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:59:28.660119+0000","last_peered":"2023-10-09T09:59:28.660119+0000","last_clean":"2023-10-09T09:59:28.660119+0000","last_became_active":"2023-10-09T09:57:18.841522+0000","last_became_peered":"2023-10-09T09:57:18.841522+0000","last_unstale":"2023-10-09T09:59:28.660119+0000","last_undegraded":"2023-10-09T09:59:28.660119+0000","last_fullsized":"2023-10-09T09:59:28.660119+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.662240+0000","last_change":"2023-10-09T09:56:45.555100+0000","last_active":"2023-10-09T09:59:28.662240+0000","last_peered":"2023-10-09T09:59:28.662240+0000","last_clean":"2023-10-09T09:59:28.662240+0000","last_became_active":"2023-10-09T09:56:45.554704+0000","last_became_peered":"2023-10-09T09:56:45.554704+0000","last_unstale":"2023-10-09T09:59:28.662240+0000","last_undegraded":"2023-10-09T09:59:28.662240+0000","last_fullsized":"2023-10-09T09:59:28.662240+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"110'3597","reported_seq":3618,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:34.798651+0000","last_change":"2023-10-09T09:58:52.626617+0000","last_active":"2023-10-09T09:59:34.798651+0000","last_peered":"2023-10-09T09:59:34.798651+0000","last_clean":"2023-10-09T09:59:34.798651+0000","last_became_active":"2023-10-09T09:58:52.626409+0000","last_became_peered":"2023-10-09T09:58:52.626409+0000","last_unstale":"2023-10-09T09:59:34.798651+0000","last_undegraded":"2023-10-09T09:59:34.798651+0000","last_fullsized":"2023-10-09T09:59:34.798651+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3597,"log_dups_size":0,"ondisk_log_size":3597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235732992,"num_objects":3597,"num_object_clones":0,"num_object_copies":10791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3597,"num_write_kb":230208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609165+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T09:59:33.609165+0000","last_peered":"2023-10-09T09:59:33.609165+0000","last_clean":"2023-10-09T09:59:33.609165+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T09:59:33.609165+0000","last_undegraded":"2023-10-09T09:59:33.609165+0000","last_fullsized":"2023-10-09T09:59:33.609165+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617726+0000","last_change":"2023-10-09T09:57:49.622055+0000","last_active":"2023-10-09T09:59:33.617726+0000","last_peered":"2023-10-09T09:59:33.617726+0000","last_clean":"2023-10-09T09:59:33.617726+0000","last_became_active":"2023-10-09T09:57:49.621784+0000","last_became_peered":"2023-10-09T09:57:49.621784+0000","last_unstale":"2023-10-09T09:59:33.617726+0000","last_undegraded":"2023-10-09T09:59:33.617726+0000","last_fullsized":"2023-10-09T09:59:33.617726+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:50.025391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.616954+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T09:59:33.616954+0000","last_peered":"2023-10-09T09:59:33.616954+0000","last_clean":"2023-10-09T09:59:33.616954+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T09:59:33.616954+0000","last_undegraded":"2023-10-09T09:59:33.616954+0000","last_fullsized":"2023-10-09T09:59:33.616954+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.673024+0000","last_change":"2023-10-09T09:57:49.628678+0000","last_active":"2023-10-09T09:59:28.673024+0000","last_peered":"2023-10-09T09:59:28.673024+0000","last_clean":"2023-10-09T09:59:28.673024+0000","last_became_active":"2023-10-09T09:57:49.628332+0000","last_became_peered":"2023-10-09T09:57:49.628332+0000","last_unstale":"2023-10-09T09:59:28.673024+0000","last_undegraded":"2023-10-09T09:59:28.673024+0000","last_fullsized":"2023-10-09T09:59:28.673024+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.610080+0000","last_change":"2023-10-09T09:57:18.842325+0000","last_active":"2023-10-09T09:59:33.610080+0000","last_peered":"2023-10-09T09:59:33.610080+0000","last_clean":"2023-10-09T09:59:33.610080+0000","last_became_active":"2023-10-09T09:57:18.841987+0000","last_became_peered":"2023-10-09T09:57:18.841987+0000","last_unstale":"2023-10-09T09:59:33.610080+0000","last_undegraded":"2023-10-09T09:59:33.610080+0000","last_fullsized":"2023-10-09T09:59:33.610080+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:28.615689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.0","version":"111'3812","reported_seq":3835,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:36.574723+0000","last_change":"2023-10-09T09:58:52.622177+0000","last_active":"2023-10-09T09:59:36.574723+0000","last_peered":"2023-10-09T09:59:36.574723+0000","last_clean":"2023-10-09T09:59:36.574723+0000","last_became_active":"2023-10-09T09:58:52.621880+0000","last_became_peered":"2023-10-09T09:58:52.621880+0000","last_unstale":"2023-10-09T09:59:36.574723+0000","last_undegraded":"2023-10-09T09:59:36.574723+0000","last_fullsized":"2023-10-09T09:59:36.574723+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3812,"log_dups_size":0,"ondisk_log_size":3812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:57.847176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249823232,"num_objects":3812,"num_object_clones":0,"num_object_copies":11436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3812,"num_write_kb":243968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"111'3932","reported_seq":3955,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:37.637606+0000","last_change":"2023-10-09T09:58:52.624855+0000","last_active":"2023-10-09T09:59:37.637606+0000","last_peered":"2023-10-09T09:59:37.637606+0000","last_clean":"2023-10-09T09:59:37.637606+0000","last_became_active":"2023-10-09T09:58:52.624556+0000","last_became_peered":"2023-10-09T09:58:52.624556+0000","last_unstale":"2023-10-09T09:59:37.637606+0000","last_undegraded":"2023-10-09T09:59:37.637606+0000","last_fullsized":"2023-10-09T09:59:37.637606+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3932,"log_dups_size":0,"ondisk_log_size":3932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:07.544449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257687552,"num_objects":3932,"num_object_clones":0,"num_object_copies":11796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3932,"num_write_kb":251648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.672796+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T09:59:28.672796+0000","last_peered":"2023-10-09T09:59:28.672796+0000","last_clean":"2023-10-09T09:59:28.672796+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T09:59:28.672796+0000","last_undegraded":"2023-10-09T09:59:28.672796+0000","last_fullsized":"2023-10-09T09:59:28.672796+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660783+0000","last_change":"2023-10-09T09:56:13.613717+0000","last_active":"2023-10-09T09:59:28.660783+0000","last_peered":"2023-10-09T09:59:28.660783+0000","last_clean":"2023-10-09T09:59:28.660783+0000","last_became_active":"2023-10-09T09:56:13.613329+0000","last_became_peered":"2023-10-09T09:56:13.613329+0000","last_unstale":"2023-10-09T09:59:28.660783+0000","last_undegraded":"2023-10-09T09:59:28.660783+0000","last_fullsized":"2023-10-09T09:59:28.660783+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.616882+0000","last_change":"2023-10-09T09:57:49.621680+0000","last_active":"2023-10-09T09:59:33.616882+0000","last_peered":"2023-10-09T09:59:33.616882+0000","last_clean":"2023-10-09T09:59:33.616882+0000","last_became_active":"2023-10-09T09:57:49.621345+0000","last_became_peered":"2023-10-09T09:57:49.621345+0000","last_unstale":"2023-10-09T09:59:33.616882+0000","last_undegraded":"2023-10-09T09:59:33.616882+0000","last_fullsized":"2023-10-09T09:59:33.616882+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:58.483689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"110'3745","reported_seq":3766,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:34.820054+0000","last_change":"2023-10-09T09:58:52.780565+0000","last_active":"2023-10-09T09:59:34.820054+0000","last_peered":"2023-10-09T09:59:34.820054+0000","last_clean":"2023-10-09T09:59:34.820054+0000","last_became_active":"2023-10-09T09:58:52.780260+0000","last_became_peered":"2023-10-09T09:58:52.780260+0000","last_unstale":"2023-10-09T09:59:34.820054+0000","last_undegraded":"2023-10-09T09:59:34.820054+0000","last_fullsized":"2023-10-09T09:59:34.820054+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3745,"log_dups_size":0,"ondisk_log_size":3745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245432320,"num_objects":3745,"num_object_clones":0,"num_object_copies":11235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3745,"num_write_kb":239680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":100,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.610187+0000","last_change":"2023-10-09T09:57:49.621174+0000","last_active":"2023-10-09T09:59:33.610187+0000","last_peered":"2023-10-09T09:59:33.610187+0000","last_clean":"2023-10-09T09:59:33.610187+0000","last_became_active":"2023-10-09T09:57:49.620780+0000","last_became_peered":"2023-10-09T09:57:49.620780+0000","last_unstale":"2023-10-09T09:59:33.610187+0000","last_undegraded":"2023-10-09T09:59:33.610187+0000","last_fullsized":"2023-10-09T09:59:33.610187+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:01.389768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617297+0000","last_change":"2023-10-09T09:57:18.837334+0000","last_active":"2023-10-09T09:59:33.617297+0000","last_peered":"2023-10-09T09:59:33.617297+0000","last_clean":"2023-10-09T09:59:33.617297+0000","last_became_active":"2023-10-09T09:57:18.837037+0000","last_became_peered":"2023-10-09T09:57:18.837037+0000","last_unstale":"2023-10-09T09:59:33.617297+0000","last_undegraded":"2023-10-09T09:59:33.617297+0000","last_fullsized":"2023-10-09T09:59:33.617297+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:11.650333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617815+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T09:59:33.617815+0000","last_peered":"2023-10-09T09:59:33.617815+0000","last_clean":"2023-10-09T09:59:33.617815+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T09:59:33.617815+0000","last_undegraded":"2023-10-09T09:59:33.617815+0000","last_fullsized":"2023-10-09T09:59:33.617815+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"110'3626","reported_seq":3647,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:34.780920+0000","last_change":"2023-10-09T09:58:52.625605+0000","last_active":"2023-10-09T09:59:34.780920+0000","last_peered":"2023-10-09T09:59:34.780920+0000","last_clean":"2023-10-09T09:59:34.780920+0000","last_became_active":"2023-10-09T09:58:52.625332+0000","last_became_peered":"2023-10-09T09:58:52.625332+0000","last_unstale":"2023-10-09T09:59:34.780920+0000","last_undegraded":"2023-10-09T09:59:34.780920+0000","last_fullsized":"2023-10-09T09:59:34.780920+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3626,"log_dups_size":0,"ondisk_log_size":3626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237633536,"num_objects":3626,"num_object_clones":0,"num_object_copies":10878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3626,"num_write_kb":232064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.672805+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:59:28.672805+0000","last_peered":"2023-10-09T09:59:28.672805+0000","last_clean":"2023-10-09T09:59:28.672805+0000","last_became_active":"2023-10-09T09:57:18.841472+0000","last_became_peered":"2023-10-09T09:57:18.841472+0000","last_unstale":"2023-10-09T09:59:28.672805+0000","last_undegraded":"2023-10-09T09:59:28.672805+0000","last_fullsized":"2023-10-09T09:59:28.672805+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":311,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.676165+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T09:59:28.676165+0000","last_peered":"2023-10-09T09:59:28.676165+0000","last_clean":"2023-10-09T09:59:28.676165+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T09:59:28.676165+0000","last_undegraded":"2023-10-09T09:59:28.676165+0000","last_fullsized":"2023-10-09T09:59:28.676165+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608478+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T09:59:33.608478+0000","last_peered":"2023-10-09T09:59:33.608478+0000","last_clean":"2023-10-09T09:59:33.608478+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T09:59:33.608478+0000","last_undegraded":"2023-10-09T09:59:33.608478+0000","last_fullsized":"2023-10-09T09:59:33.608478+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.671632+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T09:59:28.671632+0000","last_peered":"2023-10-09T09:59:28.671632+0000","last_clean":"2023-10-09T09:59:28.671632+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T09:59:28.671632+0000","last_undegraded":"2023-10-09T09:59:28.671632+0000","last_fullsized":"2023-10-09T09:59:28.671632+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":94,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.676248+0000","last_change":"2023-10-09T09:56:45.555907+0000","last_active":"2023-10-09T09:59:28.676248+0000","last_peered":"2023-10-09T09:59:28.676248+0000","last_clean":"2023-10-09T09:59:28.676248+0000","last_became_active":"2023-10-09T09:56:45.555548+0000","last_became_peered":"2023-10-09T09:56:45.555548+0000","last_unstale":"2023-10-09T09:59:28.676248+0000","last_undegraded":"2023-10-09T09:59:28.676248+0000","last_fullsized":"2023-10-09T09:59:28.676248+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.616967+0000","last_change":"2023-10-09T09:57:49.624279+0000","last_active":"2023-10-09T09:59:33.616967+0000","last_peered":"2023-10-09T09:59:33.616967+0000","last_clean":"2023-10-09T09:59:33.616967+0000","last_became_active":"2023-10-09T09:57:49.624010+0000","last_became_peered":"2023-10-09T09:57:49.624010+0000","last_unstale":"2023-10-09T09:59:33.616967+0000","last_undegraded":"2023-10-09T09:59:33.616967+0000","last_fullsized":"2023-10-09T09:59:33.616967+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:47.717690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":87,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617799+0000","last_change":"2023-10-09T09:57:18.840413+0000","last_active":"2023-10-09T09:59:33.617799+0000","last_peered":"2023-10-09T09:59:33.617799+0000","last_clean":"2023-10-09T09:59:33.617799+0000","last_became_active":"2023-10-09T09:57:18.840093+0000","last_became_peered":"2023-10-09T09:57:18.840093+0000","last_unstale":"2023-10-09T09:59:33.617799+0000","last_undegraded":"2023-10-09T09:59:33.617799+0000","last_fullsized":"2023-10-09T09:59:33.617799+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:58.170995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"111'4029","reported_seq":4052,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:38.557200+0000","last_change":"2023-10-09T09:58:52.625649+0000","last_active":"2023-10-09T09:59:38.557200+0000","last_peered":"2023-10-09T09:59:38.557200+0000","last_clean":"2023-10-09T09:59:38.557200+0000","last_became_active":"2023-10-09T09:58:52.625367+0000","last_became_peered":"2023-10-09T09:58:52.625367+0000","last_unstale":"2023-10-09T09:59:38.557200+0000","last_undegraded":"2023-10-09T09:59:38.557200+0000","last_fullsized":"2023-10-09T09:59:38.557200+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4029,"log_dups_size":0,"ondisk_log_size":4029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:37.385560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264044544,"num_objects":4029,"num_object_clones":0,"num_object_copies":12087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4029,"num_write_kb":257856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.677003+0000","last_change":"2023-10-09T09:56:13.614282+0000","last_active":"2023-10-09T09:59:28.677003+0000","last_peered":"2023-10-09T09:59:28.677003+0000","last_clean":"2023-10-09T09:59:28.677003+0000","last_became_active":"2023-10-09T09:56:13.613888+0000","last_became_peered":"2023-10-09T09:56:13.613888+0000","last_unstale":"2023-10-09T09:59:28.677003+0000","last_undegraded":"2023-10-09T09:59:28.677003+0000","last_fullsized":"2023-10-09T09:59:28.677003+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617032+0000","last_change":"2023-10-09T09:57:18.844843+0000","last_active":"2023-10-09T09:59:33.617032+0000","last_peered":"2023-10-09T09:59:33.617032+0000","last_clean":"2023-10-09T09:59:33.617032+0000","last_became_active":"2023-10-09T09:57:18.844530+0000","last_became_peered":"2023-10-09T09:57:18.844530+0000","last_unstale":"2023-10-09T09:59:33.617032+0000","last_undegraded":"2023-10-09T09:59:33.617032+0000","last_fullsized":"2023-10-09T09:59:33.617032+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:33.761788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617143+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T09:59:33.617143+0000","last_peered":"2023-10-09T09:59:33.617143+0000","last_clean":"2023-10-09T09:59:33.617143+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T09:59:33.617143+0000","last_undegraded":"2023-10-09T09:59:33.617143+0000","last_fullsized":"2023-10-09T09:59:33.617143+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"111'4052","reported_seq":4075,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:38.513749+0000","last_change":"2023-10-09T09:58:52.781251+0000","last_active":"2023-10-09T09:59:38.513749+0000","last_peered":"2023-10-09T09:59:38.513749+0000","last_clean":"2023-10-09T09:59:38.513749+0000","last_became_active":"2023-10-09T09:58:52.780842+0000","last_became_peered":"2023-10-09T09:58:52.780842+0000","last_unstale":"2023-10-09T09:59:38.513749+0000","last_undegraded":"2023-10-09T09:59:38.513749+0000","last_fullsized":"2023-10-09T09:59:38.513749+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4052,"log_dups_size":0,"ondisk_log_size":4052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:03.048405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265551872,"num_objects":4052,"num_object_clones":0,"num_object_copies":12156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4052,"num_write_kb":259328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609630+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T09:59:33.609630+0000","last_peered":"2023-10-09T09:59:33.609630+0000","last_clean":"2023-10-09T09:59:33.609630+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T09:59:33.609630+0000","last_undegraded":"2023-10-09T09:59:33.609630+0000","last_fullsized":"2023-10-09T09:59:33.609630+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":49,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617784+0000","last_change":"2023-10-09T09:56:45.549837+0000","last_active":"2023-10-09T09:59:33.617784+0000","last_peered":"2023-10-09T09:59:33.617784+0000","last_clean":"2023-10-09T09:59:33.617784+0000","last_became_active":"2023-10-09T09:56:45.549546+0000","last_became_peered":"2023-10-09T09:56:45.549546+0000","last_unstale":"2023-10-09T09:59:33.617784+0000","last_undegraded":"2023-10-09T09:59:33.617784+0000","last_fullsized":"2023-10-09T09:59:33.617784+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:26.779071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609096+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T09:59:33.609096+0000","last_peered":"2023-10-09T09:59:33.609096+0000","last_clean":"2023-10-09T09:59:33.609096+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T09:59:33.609096+0000","last_undegraded":"2023-10-09T09:59:33.609096+0000","last_fullsized":"2023-10-09T09:59:33.609096+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"110'3839","reported_seq":3860,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:34.822279+0000","last_change":"2023-10-09T09:58:52.627793+0000","last_active":"2023-10-09T09:59:34.822279+0000","last_peered":"2023-10-09T09:59:34.822279+0000","last_clean":"2023-10-09T09:59:34.822279+0000","last_became_active":"2023-10-09T09:58:52.627587+0000","last_became_peered":"2023-10-09T09:58:52.627587+0000","last_unstale":"2023-10-09T09:59:34.822279+0000","last_undegraded":"2023-10-09T09:59:34.822279+0000","last_fullsized":"2023-10-09T09:59:34.822279+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3839,"log_dups_size":0,"ondisk_log_size":3839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251592704,"num_objects":3839,"num_object_clones":0,"num_object_copies":11517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3839,"num_write_kb":245696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.671466+0000","last_change":"2023-10-09T09:57:18.844262+0000","last_active":"2023-10-09T09:59:28.671466+0000","last_peered":"2023-10-09T09:59:28.671466+0000","last_clean":"2023-10-09T09:59:28.671466+0000","last_became_active":"2023-10-09T09:57:18.843935+0000","last_became_peered":"2023-10-09T09:57:18.843935+0000","last_unstale":"2023-10-09T09:59:28.671466+0000","last_undegraded":"2023-10-09T09:59:28.671466+0000","last_fullsized":"2023-10-09T09:59:28.671466+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617725+0000","last_change":"2023-10-09T09:56:45.554759+0000","last_active":"2023-10-09T09:59:33.617725+0000","last_peered":"2023-10-09T09:59:33.617725+0000","last_clean":"2023-10-09T09:59:33.617725+0000","last_became_active":"2023-10-09T09:56:45.554390+0000","last_became_peered":"2023-10-09T09:56:45.554390+0000","last_unstale":"2023-10-09T09:59:33.617725+0000","last_undegraded":"2023-10-09T09:59:33.617725+0000","last_fullsized":"2023-10-09T09:59:33.617725+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:43.005737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":61,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660330+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T09:59:28.660330+0000","last_peered":"2023-10-09T09:59:28.660330+0000","last_clean":"2023-10-09T09:59:28.660330+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T09:59:28.660330+0000","last_undegraded":"2023-10-09T09:59:28.660330+0000","last_fullsized":"2023-10-09T09:59:28.660330+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"110'3739","reported_seq":3760,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:34.804637+0000","last_change":"2023-10-09T09:58:52.780192+0000","last_active":"2023-10-09T09:59:34.804637+0000","last_peered":"2023-10-09T09:59:34.804637+0000","last_clean":"2023-10-09T09:59:34.804637+0000","last_became_active":"2023-10-09T09:58:52.779903+0000","last_became_peered":"2023-10-09T09:58:52.779903+0000","last_unstale":"2023-10-09T09:59:34.804637+0000","last_undegraded":"2023-10-09T09:59:34.804637+0000","last_fullsized":"2023-10-09T09:59:34.804637+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":3739,"log_dups_size":0,"ondisk_log_size":3739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245039104,"num_objects":3739,"num_object_clones":0,"num_object_copies":11217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3739,"num_write_kb":239296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617406+0000","last_change":"2023-10-09T09:56:13.614416+0000","last_active":"2023-10-09T09:59:33.617406+0000","last_peered":"2023-10-09T09:59:33.617406+0000","last_clean":"2023-10-09T09:59:33.617406+0000","last_became_active":"2023-10-09T09:56:13.614050+0000","last_became_peered":"2023-10-09T09:56:13.614050+0000","last_unstale":"2023-10-09T09:59:33.617406+0000","last_undegraded":"2023-10-09T09:59:33.617406+0000","last_fullsized":"2023-10-09T09:59:33.617406+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:17.068217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":98,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.611024+0000","last_change":"2023-10-09T09:57:49.623531+0000","last_active":"2023-10-09T09:59:33.611024+0000","last_peered":"2023-10-09T09:59:33.611024+0000","last_clean":"2023-10-09T09:59:33.611024+0000","last_became_active":"2023-10-09T09:57:49.623045+0000","last_became_peered":"2023-10-09T09:57:49.623045+0000","last_unstale":"2023-10-09T09:59:33.611024+0000","last_undegraded":"2023-10-09T09:59:33.611024+0000","last_fullsized":"2023-10-09T09:59:33.611024+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:49.655085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.672986+0000","last_change":"2023-10-09T09:56:45.554958+0000","last_active":"2023-10-09T09:59:28.672986+0000","last_peered":"2023-10-09T09:59:28.672986+0000","last_clean":"2023-10-09T09:59:28.672986+0000","last_became_active":"2023-10-09T09:56:45.554597+0000","last_became_peered":"2023-10-09T09:56:45.554597+0000","last_unstale":"2023-10-09T09:59:28.672986+0000","last_undegraded":"2023-10-09T09:59:28.672986+0000","last_fullsized":"2023-10-09T09:59:28.672986+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660938+0000","last_change":"2023-10-09T09:57:49.622634+0000","last_active":"2023-10-09T09:59:28.660938+0000","last_peered":"2023-10-09T09:59:28.660938+0000","last_clean":"2023-10-09T09:59:28.660938+0000","last_became_active":"2023-10-09T09:57:49.622247+0000","last_became_peered":"2023-10-09T09:57:49.622247+0000","last_unstale":"2023-10-09T09:59:28.660938+0000","last_undegraded":"2023-10-09T09:59:28.660938+0000","last_fullsized":"2023-10-09T09:59:28.660938+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.673015+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T09:59:28.673015+0000","last_peered":"2023-10-09T09:59:28.673015+0000","last_clean":"2023-10-09T09:59:28.673015+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T09:59:28.673015+0000","last_undegraded":"2023-10-09T09:59:28.673015+0000","last_fullsized":"2023-10-09T09:59:28.673015+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660867+0000","last_change":"2023-10-09T09:57:49.620967+0000","last_active":"2023-10-09T09:59:28.660867+0000","last_peered":"2023-10-09T09:59:28.660867+0000","last_clean":"2023-10-09T09:59:28.660867+0000","last_became_active":"2023-10-09T09:57:49.620567+0000","last_became_peered":"2023-10-09T09:57:49.620567+0000","last_unstale":"2023-10-09T09:59:28.660867+0000","last_undegraded":"2023-10-09T09:59:28.660867+0000","last_fullsized":"2023-10-09T09:59:28.660867+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":84,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608511+0000","last_change":"2023-10-09T09:57:49.623102+0000","last_active":"2023-10-09T09:59:33.608511+0000","last_peered":"2023-10-09T09:59:33.608511+0000","last_clean":"2023-10-09T09:59:33.608511+0000","last_became_active":"2023-10-09T09:57:49.620638+0000","last_became_peered":"2023-10-09T09:57:49.620638+0000","last_unstale":"2023-10-09T09:59:33.608511+0000","last_undegraded":"2023-10-09T09:59:33.608511+0000","last_fullsized":"2023-10-09T09:59:33.608511+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:11.322675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":97,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608285+0000","last_change":"2023-10-09T09:57:49.620527+0000","last_active":"2023-10-09T09:59:33.608285+0000","last_peered":"2023-10-09T09:59:33.608285+0000","last_clean":"2023-10-09T09:59:33.608285+0000","last_became_active":"2023-10-09T09:57:49.620109+0000","last_became_peered":"2023-10-09T09:57:49.620109+0000","last_unstale":"2023-10-09T09:59:33.608285+0000","last_undegraded":"2023-10-09T09:59:33.608285+0000","last_fullsized":"2023-10-09T09:59:33.608285+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:09.284902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617957+0000","last_change":"2023-10-09T09:57:18.842453+0000","last_active":"2023-10-09T09:59:33.617957+0000","last_peered":"2023-10-09T09:59:33.617957+0000","last_clean":"2023-10-09T09:59:33.617957+0000","last_became_active":"2023-10-09T09:57:18.842133+0000","last_became_peered":"2023-10-09T09:57:18.842133+0000","last_unstale":"2023-10-09T09:59:33.617957+0000","last_undegraded":"2023-10-09T09:59:33.617957+0000","last_fullsized":"2023-10-09T09:59:33.617957+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:07.459692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T09:59:28.660065+0000","last_change":"2023-10-09T09:56:45.556456+0000","last_active":"2023-10-09T09:59:28.660065+0000","last_peered":"2023-10-09T09:59:28.660065+0000","last_clean":"2023-10-09T09:59:28.660065+0000","last_became_active":"2023-10-09T09:56:45.555878+0000","last_became_peered":"2023-10-09T09:56:45.555878+0000","last_unstale":"2023-10-09T09:59:28.660065+0000","last_undegraded":"2023-10-09T09:59:28.660065+0000","last_fullsized":"2023-10-09T09:59:28.660065+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4044750848,"num_objects":61718,"num_object_clones":0,"num_object_copies":185154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61718,"num_write_kb":3949952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12159287296,"data_stored":12159287296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":61718,"ondisk_log_size":61718,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430359,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1758848,"kb_used_data":1449336,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91986304,"statfs":{"total":95995035648,"available":94193975296,"internally_reserved":0,"allocated":1484120064,"data_stored":1483653362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.59499999999999997,"15min":0.59399999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55400000000000005,"15min":0.54600000000000004},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":1.012}]},{"osd":1,"last update":"Mon Oct 9 09:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.54600000000000004,"15min":0.54500000000000004},"min":{"1min":0.40100000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.78500000000000003,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.246},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.58599999999999997,"15min":0.58499999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76300000000000001,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":1.244}]},{"osd":2,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.60999999999999999,"15min":0.60299999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.55300000000000005,"15min":0.56100000000000005},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63600000000000001,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":1.1870000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":1.0920000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.312}]},{"osd":4,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.54900000000000004,"15min":0.48699999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.7719999999999998,"5min":2.7719999999999998,"15min":2.7719999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.52000000000000002,"15min":0.48599999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.26500000000000001},"max":{"1min":1.7450000000000001,"5min":1.7450000000000001,"15min":1.7450000000000001},"last":0.33800000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52200000000000002,"15min":0.505},"min":{"1min":0.29999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":1.2589999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.53200000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.429,"5min":1.429,"15min":1.429},"last":0.435}]},{"osd":6,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58899999999999997,"15min":0.55400000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32900000000000001},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58899999999999997,"15min":0.55000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":1.8089999999999999,"5min":1.8089999999999999,"15min":1.8089999999999999},"last":0.46700000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626587,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1501488,"kb_used_data":1196072,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92243664,"statfs":{"total":95995035648,"available":94457511936,"internally_reserved":0,"allocated":1224777728,"data_stored":1224326949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.60899999999999999,"15min":0.57299999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.4239999999999999,"5min":2.4239999999999999,"15min":2.4239999999999999},"last":1.575},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.60699999999999998,"15min":0.53900000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":4.9450000000000003,"5min":4.9450000000000003,"15min":4.9450000000000003},"last":1.73}]},{"osd":1,"last update":"Mon Oct 9 09:58:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.55200000000000005,"15min":0.55000000000000004},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":1.4510000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54100000000000004,"15min":0.53600000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":1.8240000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:59:20 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.56699999999999995,"15min":0.54900000000000004},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.629,"5min":2.629,"15min":2.629},"last":1.7509999999999999},{"interface":"front","average":{"1min":0.873,"5min":0.64400000000000002,"15min":0.60599999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":2.8140000000000001,"5min":2.8140000000000001,"15min":2.8140000000000001},"last":1.702}]},{"osd":3,"last update":"Mon Oct 9 09:58:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":1.9350000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.35199999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.48699999999999999,"15min":0.46000000000000002},"min":{"1min":0.377,"5min":0.27300000000000002,"15min":0.23300000000000001},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.7869999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.5,"15min":0.49299999999999999},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.311},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.32400000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.5,"15min":0.502},"min":{"1min":0.34499999999999997,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":0.66700000000000004,"5min":0.72199999999999998,"15min":0.80600000000000005},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.53800000000000003,"15min":0.51500000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":1.8360000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.55300000000000005,"15min":0.53000000000000003},"min":{"1min":0.38700000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":2.3540000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.61899999999999999,"15min":0.59199999999999997},"min":{"1min":0.438,"5min":0.34899999999999998,"15min":0.34399999999999997},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":2.3929999999999998}]}]},{"osd":1,"up_from":84,"seq":360777252901,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332732,"kb_used_data":1287220,"kb_used_omap":9,"kb_used_meta":45494,"kb_avail":92412420,"statfs":{"total":95995035648,"available":94630318080,"internally_reserved":0,"allocated":1318113280,"data_stored":1317599861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":46586857},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.23400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":1.165},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.38600000000000001,"15min":0.39400000000000002},"min":{"1min":0.23699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.498,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.74299999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:59:21 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.44600000000000001,"15min":0.40799999999999997},"min":{"1min":0.27200000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.121,"5min":2.121,"15min":2.121},"last":6.8280000000000003},{"interface":"front","average":{"1min":0.62,"5min":0.436,"15min":0.40600000000000003},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.248},"max":{"1min":2.2149999999999999,"5min":2.2149999999999999,"15min":2.2149999999999999},"last":5.7249999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.439}]},{"osd":4,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.60499999999999998,"15min":0.60899999999999999},"min":{"1min":0.46300000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.73299999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.65100000000000002,"15min":0.64500000000000002},"min":{"1min":0.40899999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.589,"5min":2.589,"15min":2.589},"last":1.1359999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.61699999999999999,"15min":0.625},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.872,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.64800000000000002,"15min":0.65500000000000003},"min":{"1min":0.47399999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78800000000000003,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.69599999999999995,"15min":0.68999999999999995},"min":{"1min":0.39600000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.6480000000000001,"5min":2.6480000000000001,"15min":2.6480000000000001},"last":1.101},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.66400000000000003,"15min":0.67200000000000004},"min":{"1min":0.45800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.84799999999999998,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.76200000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.63200000000000001,"15min":0.63800000000000001},"min":{"1min":0.432,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83399999999999996,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.69399999999999995,"15min":0.68799999999999994},"min":{"1min":0.38,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.681,"5min":2.681,"15min":2.681},"last":1.2569999999999999}]}]},{"osd":0,"up_from":79,"seq":339302416427,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005320,"kb_used_data":973696,"kb_used_omap":28,"kb_used_meta":31587,"kb_avail":92739832,"statfs":{"total":95995035648,"available":94965587968,"internally_reserved":0,"allocated":997064704,"data_stored":996539374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32345936},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.376,"15min":0.376},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":1.1679999999999999},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.39400000000000002,"15min":0.39600000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.59299999999999997,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.85499999999999998}]},{"osd":2,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.42099999999999999,"15min":0.40200000000000002},"min":{"1min":0.28699999999999998,"5min":0.216,"15min":0.216},"max":{"1min":2.339,"5min":2.339,"15min":2.339},"last":1.8879999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.45300000000000001,"15min":0.435},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.6180000000000001,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":0.67100000000000004}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":1.304},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.64700000000000002}]},{"osd":4,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.66400000000000003,"15min":0.66000000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.69199999999999995,"15min":0.68200000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.667,"5min":1.667,"15min":1.667},"last":0.60499999999999998}]},{"osd":5,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.68100000000000005,"15min":0.65500000000000003},"min":{"1min":0.5,"5min":0.434,"15min":0.434},"max":{"1min":2.0209999999999999,"5min":2.0209999999999999,"15min":2.0209999999999999},"last":1.26},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.65500000000000003,"15min":0.66200000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.0009999999999999,"5min":1.1180000000000001,"15min":1.1180000000000001},"last":0.54900000000000004}]},{"osd":6,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.67000000000000004,"15min":0.68200000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.054,"5min":1.161,"15min":1.161},"last":0.623},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68999999999999995},"min":{"1min":0.438,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.88800000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.68700000000000006,"15min":0.67300000000000004},"min":{"1min":0.45600000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.056,"5min":2.056,"15min":2.056},"last":1.5760000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.67900000000000005,"15min":0.68100000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.093,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":1.1319999999999999}]}]},{"osd":2,"up_from":89,"seq":382252089374,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2077700,"kb_used_data":2014012,"kb_used_omap":81,"kb_used_meta":63598,"kb_avail":91667452,"statfs":{"total":95995035648,"available":93867470848,"internally_reserved":0,"allocated":2062348288,"data_stored":2061826328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":65124570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.45100000000000001,"15min":0.40600000000000003},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.218,"5min":2.218,"15min":2.218},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.47599999999999998,"15min":0.42499999999999999},"min":{"1min":0.317,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.5219999999999998,"5min":2.5219999999999998,"15min":2.5219999999999998},"last":1.2529999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":0.54100000000000004,"15min":0.42499999999999999},"min":{"1min":0.27900000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.6109999999999998,"5min":7.6109999999999998,"15min":7.6109999999999998},"last":1.7010000000000001},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.46200000000000002,"15min":0.42799999999999999},"min":{"1min":0.30099999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.2410000000000001,"5min":2.2410000000000001,"15min":2.2410000000000001},"last":1.464}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":1.431}]},{"osd":4,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.67000000000000004,"15min":0.626},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.5979999999999999,"5min":2.5979999999999999,"15min":2.5979999999999999},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.325,"5min":0.76900000000000002,"15min":0.67600000000000005},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.5229999999999997,"5min":7.5229999999999997,"15min":7.5229999999999997},"last":1.9139999999999999}]},{"osd":5,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8369999999999997,"5min":1.4670000000000001,"15min":0.90500000000000003},"min":{"1min":0.42899999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":79.016000000000005,"5min":79.016000000000005,"15min":79.016000000000005},"last":1.5529999999999999},{"interface":"front","average":{"1min":4.7590000000000003,"5min":1.4550000000000001,"15min":0.90500000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":79.302999999999997,"5min":79.302999999999997,"15min":79.302999999999997},"last":1.29}]},{"osd":6,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.68700000000000006,"15min":0.63800000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.6200000000000001,"5min":2.6200000000000001,"15min":2.6200000000000001},"last":1.3460000000000001},{"interface":"front","average":{"1min":1.288,"5min":0.76400000000000001,"15min":0.67700000000000005},"min":{"1min":0.497,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.569,"5min":7.569,"15min":7.569},"last":1.9970000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.70199999999999996,"15min":0.67000000000000004},"min":{"1min":0.45200000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":2.302,"5min":2.302,"15min":2.302},"last":1.5069999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.70899999999999996,"15min":0.67700000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.6419999999999999,"5min":2.6419999999999999,"15min":2.6419999999999999},"last":1.3740000000000001}]}]},{"osd":3,"up_from":94,"seq":403726925848,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2617656,"kb_used_data":2545904,"kb_used_omap":7,"kb_used_meta":71736,"kb_avail":91127496,"statfs":{"total":95995035648,"available":93314555904,"internally_reserved":0,"allocated":2607005696,"data_stored":2606494318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":73458280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.995,"15min":0.995},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":11.958,"5min":11.958,"15min":11.958},"last":0.434},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":8.8190000000000008}]},{"osd":1,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":3.9060000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.63700000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.046,"15min":1.046},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":13.246,"5min":13.246,"15min":13.246},"last":3.681},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.66100000000000003}]},{"osd":4,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.042,"5min":1.042,"15min":1.042},"last":3.9460000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":4.0540000000000003},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":7.9850000000000003,"5min":7.9850000000000003,"15min":7.9850000000000003},"last":4.0419999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":3.9750000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":8.0410000000000004,"5min":8.0410000000000004,"15min":8.0410000000000004},"last":3.9359999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":8.0719999999999992,"5min":8.0719999999999992,"15min":8.0719999999999992},"last":4.0019999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.46200000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019043,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1484536,"kb_used_data":1183216,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92260616,"statfs":{"total":95995035648,"available":94474870784,"internally_reserved":0,"allocated":1211613184,"data_stored":1211154060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.58299999999999996,"15min":0.53800000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":6.3680000000000003,"5min":6.3680000000000003,"15min":6.3680000000000003},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.60399999999999998,"15min":0.58199999999999996},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.5970000000000004,"5min":4.5970000000000004,"15min":4.5970000000000004},"last":7.0940000000000003}]},{"osd":1,"last update":"Mon Oct 9 09:58:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51800000000000002,"15min":0.51900000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":1.294},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.437,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":2.3100000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":0.61699999999999999,"15min":0.55600000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.8979999999999997,"5min":4.8979999999999997,"15min":4.8979999999999997},"last":2.6709999999999998},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.60599999999999998,"15min":0.57699999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.7629999999999999,"5min":2.7629999999999999,"15min":2.7629999999999999},"last":2.3660000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":2.1120000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:59:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.45700000000000002,"15min":0.436},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.27100000000000002},"max":{"1min":0.72499999999999998,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.49299999999999999,"15min":0.47099999999999997},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.28499999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.83999999999999997},"last":0.39000000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.51400000000000001,"15min":0.47799999999999998},"min":{"1min":0.315,"5min":0.311,"15min":0.26700000000000002},"max":{"1min":1.5509999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":1.415},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.54200000000000004,"15min":0.50700000000000001},"min":{"1min":0.374,"5min":0.318,"15min":0.312},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.35499999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:58:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.55700000000000005,"15min":0.53000000000000003},"min":{"1min":0.44800000000000001,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.84999999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59599999999999997,"15min":0.56200000000000006},"min":{"1min":0.47499999999999998,"5min":0.34599999999999997,"15min":0.316},"max":{"1min":0.72399999999999998,"5min":0.82899999999999996,"15min":0.85199999999999998},"last":0.51000000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822815,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1476520,"kb_used_data":1167008,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92268632,"statfs":{"total":95995035648,"available":94483079168,"internally_reserved":0,"allocated":1195016192,"data_stored":1194569196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.59899999999999998,"15min":0.55800000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.3799999999999999,"5min":3.3799999999999999,"15min":3.3799999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.622,"5min":0.56699999999999995,"15min":0.54700000000000004},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.8260000000000001,"5min":2.4289999999999998,"15min":2.4289999999999998},"last":0.56100000000000005}]},{"osd":1,"last update":"Mon Oct 9 09:58:47 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.58099999999999996,"15min":0.55400000000000005},"min":{"1min":0.38800000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.5979999999999999,"5min":4.5979999999999999,"15min":4.5979999999999999},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.58899999999999997,"15min":0.55900000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":4.4989999999999997,"5min":4.4989999999999997,"15min":4.4989999999999997},"last":0.53700000000000003}]},{"osd":2,"last update":"Mon Oct 9 09:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.69799999999999995,"15min":0.69699999999999995},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.3380000000000001,"5min":3.3380000000000001,"15min":3.3380000000000001},"last":1.028},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.69999999999999996,"15min":0.70199999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.161,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.51200000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:58:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":2.4740000000000002},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.631,"5min":1.631,"15min":1.631},"last":0.58499999999999996}]},{"osd":4,"last update":"Mon Oct 9 09:58:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.49099999999999999,"15min":0.45300000000000001},"min":{"1min":0.33000000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":0.77300000000000002,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.50800000000000001,"15min":0.47399999999999998},"min":{"1min":0.35299999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.69499999999999995,"5min":0.79200000000000004,"15min":0.88200000000000001},"last":0.93899999999999995}]},{"osd":6,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.56699999999999995,"15min":0.52100000000000002},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":2.8250000000000002,"5min":2.8250000000000002,"15min":2.8250000000000002},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.58699999999999997,"15min":0.53400000000000003},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.415,"5min":3.415,"15min":3.415},"last":0.91900000000000004}]},{"osd":7,"last update":"Mon Oct 9 09:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.59699999999999998,"15min":0.56399999999999995},"min":{"1min":0.51200000000000001,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":0.89000000000000001,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.57899999999999996,"15min":0.54600000000000004},"min":{"1min":0.40600000000000003,"5min":0.318,"15min":0.317},"max":{"1min":0.86499999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.65700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1009778688,"data_stored":1009778688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1320222720,"data_stored":1320222720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2064056320,"data_stored":2064056320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2607480832,"data_stored":2607480832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1224867840,"data_stored":1224867840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1210777600,"data_stored":1210777600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1228996608,"data_stored":1228996608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1493106688,"data_stored":1493106688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:59:39.628 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T09:59:39.629 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:39.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:39.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:59:39.760 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:39.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:39 smithi012 ceph-mon[147926]: pgmap v269: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:59:39.860 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:39 smithi012 ceph-mon[147926]: from='client.54670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:59:39.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:39 smithi012 ceph-mon[152263]: pgmap v269: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:59:39.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:39 smithi012 ceph-mon[152263]: from='client.54670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:59:39.944 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:39.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:39.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:40.058 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:40.143 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:40.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:39 smithi110 ceph-mon[136032]: pgmap v269: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T09:59:40.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:39 smithi110 ceph-mon[136032]: from='client.54670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:59:40.259 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:40.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:40.366 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T09:59:40.366532+0000 min lat: 0.0011673 max lat: 0.442574 avg lat: 0.00964273 2023-10-09T09:59:40.366 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T09:59:40.366 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 40 16 66386 66370 103.651 60.5625 0.00901502 0.00964273 2023-10-09T09:59:40.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:40.427 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:40.472 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:40.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:40.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:40.620 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:41.366 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 41 16 67700 67684 103.126 82.125 0.0131858 0.00969272 2023-10-09T09:59:41.447 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:41.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:41.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:41.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:41.631 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:41.655 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:42.366 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 42 16 68882 68866 102.429 73.875 0.0123563 0.00975787 2023-10-09T09:59:42.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:41 smithi012 ceph-mon[147926]: pgmap v270: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:42.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:41 smithi012 ceph-mon[152263]: pgmap v270: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:41 smithi110 ceph-mon[136032]: pgmap v270: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:42.848 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:42.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:42.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:42.983 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:43.225 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:43.232 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:43.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:43.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:43.244 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:43.335 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:43.367 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 43 16 70045 70029 101.735 72.6875 0.0104096 0.00982432 2023-10-09T09:59:43.385 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:43.432 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:43.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:43.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:43.603 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:44.075 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:44.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:44.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:44.263 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:44.367 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 44 16 71154 71138 100.998 69.3125 0.0134021 0.00989531 2023-10-09T09:59:44.409 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:44.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:44.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:44.628 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:44.779 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:44.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:44 smithi012 ceph-mon[152263]: pgmap v271: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:59:44.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:44.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:44.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:44 smithi012 ceph-mon[147926]: pgmap v271: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:59:44.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:44.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:44 smithi110 ceph-mon[136032]: pgmap v271: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T09:59:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:44.968 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:45.047 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:45.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:45.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:45.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:45.197 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:45.253 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:45.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:45.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:45.367 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 45 16 72238 72222 100.259 67.75 0.0079551 0.00996856 2023-10-09T09:59:45.421 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:45.614 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3339011821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:45.614 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:45 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:45.614 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3339011821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:45.614 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:45 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:45.643 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T09:59:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3339011821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:45 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T09:59:46.367 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 46 16 73424 73408 99.691 74.125 0.0135944 0.0100265 2023-10-09T09:59:46.427 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:46.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:46.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:46.537 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:46.572 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:46.615 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:46.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[152263]: pgmap v272: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:59:46.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:46.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:59:46.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[152263]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T09:59:46.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[152263]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[152263]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[152263]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:46.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:46.703 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:46.719 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:46.744 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:46.757 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:46.789 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:46.800 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:46.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:46.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:46.827 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:46.861 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:46.906 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:46.907 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:46.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[147926]: pgmap v272: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:59:46.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:59:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[147926]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T09:59:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[147926]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[147926]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:46 smithi012 ceph-mon[147926]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.933 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:46 smithi110 ceph-mon[136032]: pgmap v272: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:59:46.933 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:46 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:46.933 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:46 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T09:59:46.933 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:46 smithi110 ceph-mon[136032]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T09:59:46.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:46 smithi110 ceph-mon[136032]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:46 smithi110 ceph-mon[136032]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:46 smithi110 ceph-mon[136032]: failed to encode map e112 with expected crc 2023-10-09T09:59:46.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:46.955 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:46.972 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:59:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T09:59:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T09:59:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T09:59:46.972 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:59:46.985 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:46.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:46.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:47.010 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:59:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:59:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:59:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:59:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:59:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:59:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:59:47.031 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:59:47.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:47.194 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:47.368 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 47 16 74402 74386 98.8706 61.125 0.0129658 0.0101085 2023-10-09T09:59:47.456 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T09:59:47.465 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T09:59:47.475 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:47.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:47.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:47.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:47.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2063267026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:47.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2063267026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:47.663 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:47.780 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:47.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:47.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T09:59:47.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:47.919 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:59:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:59:47] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T09:59:47.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2063267026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:47.988 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:48.219 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:48.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:48.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:48.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T09:59:48.342 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":274,"stamp":"2023-10-09T09:59:46.937521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4678162336,"num_objects":71398,"num_object_clones":0,"num_object_copies":214194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71398,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":71634,"num_write_kb":4570381,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71568,"ondisk_log_size":71568,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15117252,"kb_used_data":13664964,"kb_used_omap":126,"kb_used_meta":1452097,"kb_avail":734843964,"statfs":{"total":767960285184,"available":752480219136,"internally_reserved":0,"allocated":13992923136,"data_stored":13989021715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1486947963},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":751960064,"num_objects":11474,"num_object_clones":0,"num_object_copies":34422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11474,"num_write_kb":734336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005279"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.607259+0000","last_change":"2023-10-09T09:57:18.841362+0000","last_active":"2023-10-09T09:59:33.607259+0000","last_peered":"2023-10-09T09:59:33.607259+0000","last_clean":"2023-10-09T09:59:33.607259+0000","last_became_active":"2023-10-09T09:57:18.840951+0000","last_became_peered":"2023-10-09T09:57:18.840951+0000","last_unstale":"2023-10-09T09:59:33.607259+0000","last_undegraded":"2023-10-09T09:59:33.607259+0000","last_fullsized":"2023-10-09T09:59:33.607259+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609376+0000","last_change":"2023-10-09T09:57:49.622753+0000","last_active":"2023-10-09T09:59:33.609376+0000","last_peered":"2023-10-09T09:59:33.609376+0000","last_clean":"2023-10-09T09:59:33.609376+0000","last_became_active":"2023-10-09T09:57:49.622463+0000","last_became_peered":"2023-10-09T09:57:49.622463+0000","last_unstale":"2023-10-09T09:59:33.609376+0000","last_undegraded":"2023-10-09T09:59:33.609376+0000","last_fullsized":"2023-10-09T09:59:33.609376+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:51.020605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.616867+0000","last_change":"2023-10-09T09:57:49.621495+0000","last_active":"2023-10-09T09:59:33.616867+0000","last_peered":"2023-10-09T09:59:33.616867+0000","last_clean":"2023-10-09T09:59:33.616867+0000","last_became_active":"2023-10-09T09:57:49.621144+0000","last_became_peered":"2023-10-09T09:57:49.621144+0000","last_unstale":"2023-10-09T09:59:33.616867+0000","last_undegraded":"2023-10-09T09:59:33.616867+0000","last_fullsized":"2023-10-09T09:59:33.616867+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:11.852424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.641930+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T09:59:35.641930+0000","last_peered":"2023-10-09T09:59:35.641930+0000","last_clean":"2023-10-09T09:59:35.641930+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T09:59:35.641930+0000","last_undegraded":"2023-10-09T09:59:35.641930+0000","last_fullsized":"2023-10-09T09:59:35.641930+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609036+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T09:59:33.609036+0000","last_peered":"2023-10-09T09:59:33.609036+0000","last_clean":"2023-10-09T09:59:33.609036+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T09:59:33.609036+0000","last_undegraded":"2023-10-09T09:59:33.609036+0000","last_fullsized":"2023-10-09T09:59:33.609036+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":84,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.612557+0000","last_change":"2023-10-09T09:57:18.838985+0000","last_active":"2023-10-09T09:59:33.612557+0000","last_peered":"2023-10-09T09:59:33.612557+0000","last_clean":"2023-10-09T09:59:33.612557+0000","last_became_active":"2023-10-09T09:57:18.838625+0000","last_became_peered":"2023-10-09T09:57:18.838625+0000","last_unstale":"2023-10-09T09:59:33.612557+0000","last_undegraded":"2023-10-09T09:59:33.612557+0000","last_fullsized":"2023-10-09T09:59:33.612557+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.643013+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T09:59:35.643013+0000","last_peered":"2023-10-09T09:59:35.643013+0000","last_clean":"2023-10-09T09:59:35.643013+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T09:59:35.643013+0000","last_undegraded":"2023-10-09T09:59:35.643013+0000","last_fullsized":"2023-10-09T09:59:35.643013+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608963+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T09:59:33.608963+0000","last_peered":"2023-10-09T09:59:33.608963+0000","last_clean":"2023-10-09T09:59:33.608963+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T09:59:33.608963+0000","last_undegraded":"2023-10-09T09:59:33.608963+0000","last_fullsized":"2023-10-09T09:59:33.608963+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609493+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T09:59:33.609493+0000","last_peered":"2023-10-09T09:59:33.609493+0000","last_clean":"2023-10-09T09:59:33.609493+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T09:59:33.609493+0000","last_undegraded":"2023-10-09T09:59:33.609493+0000","last_fullsized":"2023-10-09T09:59:33.609493+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608217+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T09:59:33.608217+0000","last_peered":"2023-10-09T09:59:33.608217+0000","last_clean":"2023-10-09T09:59:33.608217+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T09:59:33.608217+0000","last_undegraded":"2023-10-09T09:59:33.608217+0000","last_fullsized":"2023-10-09T09:59:33.608217+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.614582+0000","last_change":"2023-10-09T09:57:49.626281+0000","last_active":"2023-10-09T09:59:33.614582+0000","last_peered":"2023-10-09T09:59:33.614582+0000","last_clean":"2023-10-09T09:59:33.614582+0000","last_became_active":"2023-10-09T09:57:49.625726+0000","last_became_peered":"2023-10-09T09:57:49.625726+0000","last_unstale":"2023-10-09T09:59:33.614582+0000","last_undegraded":"2023-10-09T09:59:33.614582+0000","last_fullsized":"2023-10-09T09:59:33.614582+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617054+0000","last_change":"2023-10-09T09:57:18.840471+0000","last_active":"2023-10-09T09:59:33.617054+0000","last_peered":"2023-10-09T09:59:33.617054+0000","last_clean":"2023-10-09T09:59:33.617054+0000","last_became_active":"2023-10-09T09:57:18.840125+0000","last_became_peered":"2023-10-09T09:57:18.840125+0000","last_unstale":"2023-10-09T09:59:33.617054+0000","last_undegraded":"2023-10-09T09:59:33.617054+0000","last_fullsized":"2023-10-09T09:59:33.617054+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:07.418455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.607936+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T09:59:33.607936+0000","last_peered":"2023-10-09T09:59:33.607936+0000","last_clean":"2023-10-09T09:59:33.607936+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T09:59:33.607936+0000","last_undegraded":"2023-10-09T09:59:33.607936+0000","last_fullsized":"2023-10-09T09:59:33.607936+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.607715+0000","last_change":"2023-10-09T09:56:45.551325+0000","last_active":"2023-10-09T09:59:33.607715+0000","last_peered":"2023-10-09T09:59:33.607715+0000","last_clean":"2023-10-09T09:59:33.607715+0000","last_became_active":"2023-10-09T09:56:45.550946+0000","last_became_peered":"2023-10-09T09:56:45.550946+0000","last_unstale":"2023-10-09T09:59:33.607715+0000","last_undegraded":"2023-10-09T09:59:33.607715+0000","last_fullsized":"2023-10-09T09:59:33.607715+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617594+0000","last_change":"2023-10-09T09:57:18.843731+0000","last_active":"2023-10-09T09:59:33.617594+0000","last_peered":"2023-10-09T09:59:33.617594+0000","last_clean":"2023-10-09T09:59:33.617594+0000","last_became_active":"2023-10-09T09:57:18.843280+0000","last_became_peered":"2023-10-09T09:57:18.843280+0000","last_unstale":"2023-10-09T09:59:33.617594+0000","last_undegraded":"2023-10-09T09:59:33.617594+0000","last_fullsized":"2023-10-09T09:59:33.617594+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608722+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T09:59:33.608722+0000","last_peered":"2023-10-09T09:59:33.608722+0000","last_clean":"2023-10-09T09:59:33.608722+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T09:59:33.608722+0000","last_undegraded":"2023-10-09T09:59:33.608722+0000","last_fullsized":"2023-10-09T09:59:33.608722+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.643538+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T09:59:35.643538+0000","last_peered":"2023-10-09T09:59:35.643538+0000","last_clean":"2023-10-09T09:59:35.643538+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T09:59:35.643538+0000","last_undegraded":"2023-10-09T09:59:35.643538+0000","last_fullsized":"2023-10-09T09:59:35.643538+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609780+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T09:59:33.609780+0000","last_peered":"2023-10-09T09:59:33.609780+0000","last_clean":"2023-10-09T09:59:33.609780+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T09:59:33.609780+0000","last_undegraded":"2023-10-09T09:59:33.609780+0000","last_fullsized":"2023-10-09T09:59:33.609780+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":61,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.614268+0000","last_change":"2023-10-09T09:48:25.523230+0000","last_active":"2023-10-09T09:59:33.614268+0000","last_peered":"2023-10-09T09:59:33.614268+0000","last_clean":"2023-10-09T09:59:33.614268+0000","last_became_active":"2023-10-09T09:48:25.522608+0000","last_became_peered":"2023-10-09T09:48:25.522608+0000","last_unstale":"2023-10-09T09:59:33.614268+0000","last_undegraded":"2023-10-09T09:59:33.614268+0000","last_fullsized":"2023-10-09T09:59:33.614268+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.641836+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T09:59:35.641836+0000","last_peered":"2023-10-09T09:59:35.641836+0000","last_clean":"2023-10-09T09:59:35.641836+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T09:59:35.641836+0000","last_undegraded":"2023-10-09T09:59:35.641836+0000","last_fullsized":"2023-10-09T09:59:35.641836+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":84,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.611954+0000","last_change":"2023-10-09T09:56:13.623204+0000","last_active":"2023-10-09T09:59:33.611954+0000","last_peered":"2023-10-09T09:59:33.611954+0000","last_clean":"2023-10-09T09:59:33.611954+0000","last_became_active":"2023-10-09T09:56:13.622853+0000","last_became_peered":"2023-10-09T09:56:13.622853+0000","last_unstale":"2023-10-09T09:59:33.611954+0000","last_undegraded":"2023-10-09T09:59:33.611954+0000","last_fullsized":"2023-10-09T09:59:33.611954+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609579+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T09:59:33.609579+0000","last_peered":"2023-10-09T09:59:33.609579+0000","last_clean":"2023-10-09T09:59:33.609579+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T09:59:33.609579+0000","last_undegraded":"2023-10-09T09:59:33.609579+0000","last_fullsized":"2023-10-09T09:59:33.609579+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"111'4467","reported_seq":4489,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:44.818946+0000","last_change":"2023-10-09T09:58:52.780310+0000","last_active":"2023-10-09T09:59:44.818946+0000","last_peered":"2023-10-09T09:59:44.818946+0000","last_clean":"2023-10-09T09:59:44.818946+0000","last_became_active":"2023-10-09T09:58:52.779910+0000","last_became_peered":"2023-10-09T09:58:52.779910+0000","last_unstale":"2023-10-09T09:59:44.818946+0000","last_undegraded":"2023-10-09T09:59:44.818946+0000","last_fullsized":"2023-10-09T09:59:44.818946+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4467,"log_dups_size":0,"ondisk_log_size":4467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292749312,"num_objects":4467,"num_object_clones":0,"num_object_copies":13401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4467,"num_write_kb":285888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.614062+0000","last_change":"2023-10-09T09:56:13.614485+0000","last_active":"2023-10-09T09:59:33.614062+0000","last_peered":"2023-10-09T09:59:33.614062+0000","last_clean":"2023-10-09T09:59:33.614062+0000","last_became_active":"2023-10-09T09:56:13.614109+0000","last_became_peered":"2023-10-09T09:56:13.614109+0000","last_unstale":"2023-10-09T09:59:33.614062+0000","last_undegraded":"2023-10-09T09:59:33.614062+0000","last_fullsized":"2023-10-09T09:59:33.614062+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609251+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T09:59:33.609251+0000","last_peered":"2023-10-09T09:59:33.609251+0000","last_clean":"2023-10-09T09:59:33.609251+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T09:59:33.609251+0000","last_undegraded":"2023-10-09T09:59:33.609251+0000","last_fullsized":"2023-10-09T09:59:33.609251+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"111'4472","reported_seq":4493,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:42.636933+0000","last_change":"2023-10-09T09:58:52.625388+0000","last_active":"2023-10-09T09:59:42.636933+0000","last_peered":"2023-10-09T09:59:42.636933+0000","last_clean":"2023-10-09T09:59:42.636933+0000","last_became_active":"2023-10-09T09:58:52.625135+0000","last_became_peered":"2023-10-09T09:58:52.625135+0000","last_unstale":"2023-10-09T09:59:42.636933+0000","last_undegraded":"2023-10-09T09:59:42.636933+0000","last_fullsized":"2023-10-09T09:59:42.636933+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4472,"log_dups_size":0,"ondisk_log_size":4472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:28.713713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293076992,"num_objects":4472,"num_object_clones":0,"num_object_copies":13416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4472,"num_write_kb":286208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.614024+0000","last_change":"2023-10-09T09:56:45.554326+0000","last_active":"2023-10-09T09:59:33.614024+0000","last_peered":"2023-10-09T09:59:33.614024+0000","last_clean":"2023-10-09T09:59:33.614024+0000","last_became_active":"2023-10-09T09:56:45.553876+0000","last_became_peered":"2023-10-09T09:56:45.553876+0000","last_unstale":"2023-10-09T09:59:33.614024+0000","last_undegraded":"2023-10-09T09:59:33.614024+0000","last_fullsized":"2023-10-09T09:59:33.614024+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.606990+0000","last_change":"2023-10-09T09:57:49.627073+0000","last_active":"2023-10-09T09:59:33.606990+0000","last_peered":"2023-10-09T09:59:33.606990+0000","last_clean":"2023-10-09T09:59:33.606990+0000","last_became_active":"2023-10-09T09:57:49.626706+0000","last_became_peered":"2023-10-09T09:57:49.626706+0000","last_unstale":"2023-10-09T09:59:33.606990+0000","last_undegraded":"2023-10-09T09:59:33.606990+0000","last_fullsized":"2023-10-09T09:59:33.606990+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"111'4625","reported_seq":4647,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:44.771085+0000","last_change":"2023-10-09T09:58:52.622516+0000","last_active":"2023-10-09T09:59:44.771085+0000","last_peered":"2023-10-09T09:59:44.771085+0000","last_clean":"2023-10-09T09:59:44.771085+0000","last_became_active":"2023-10-09T09:58:52.622112+0000","last_became_peered":"2023-10-09T09:58:52.622112+0000","last_unstale":"2023-10-09T09:59:44.771085+0000","last_undegraded":"2023-10-09T09:59:44.771085+0000","last_fullsized":"2023-10-09T09:59:44.771085+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4625,"log_dups_size":0,"ondisk_log_size":4625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303104000,"num_objects":4625,"num_object_clones":0,"num_object_copies":13875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4625,"num_write_kb":296000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":62,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.614687+0000","last_change":"2023-10-09T09:48:25.514756+0000","last_active":"2023-10-09T09:59:33.614687+0000","last_peered":"2023-10-09T09:59:33.614687+0000","last_clean":"2023-10-09T09:59:33.614687+0000","last_became_active":"2023-10-09T09:48:25.514387+0000","last_became_peered":"2023-10-09T09:48:25.514387+0000","last_unstale":"2023-10-09T09:59:33.614687+0000","last_undegraded":"2023-10-09T09:59:33.614687+0000","last_fullsized":"2023-10-09T09:59:33.614687+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.614946+0000","last_change":"2023-10-09T09:56:45.556365+0000","last_active":"2023-10-09T09:59:33.614946+0000","last_peered":"2023-10-09T09:59:33.614946+0000","last_clean":"2023-10-09T09:59:33.614946+0000","last_became_active":"2023-10-09T09:56:45.555832+0000","last_became_peered":"2023-10-09T09:56:45.555832+0000","last_unstale":"2023-10-09T09:59:33.614946+0000","last_undegraded":"2023-10-09T09:59:33.614946+0000","last_fullsized":"2023-10-09T09:59:33.614946+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"112'4619","reported_seq":4643,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T09:59:46.619396+0000","last_change":"2023-10-09T09:58:52.623147+0000","last_active":"2023-10-09T09:59:46.619396+0000","last_peered":"2023-10-09T09:59:46.619396+0000","last_clean":"2023-10-09T09:59:46.619396+0000","last_became_active":"2023-10-09T09:58:52.622837+0000","last_became_peered":"2023-10-09T09:58:52.622837+0000","last_unstale":"2023-10-09T09:59:46.619396+0000","last_undegraded":"2023-10-09T09:59:46.619396+0000","last_fullsized":"2023-10-09T09:59:46.619396+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4619,"log_dups_size":0,"ondisk_log_size":4619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:20.155378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302710784,"num_objects":4619,"num_object_clones":0,"num_object_copies":13857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4619,"num_write_kb":295616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609713+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T09:59:33.609713+0000","last_peered":"2023-10-09T09:59:33.609713+0000","last_clean":"2023-10-09T09:59:33.609713+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T09:59:33.609713+0000","last_undegraded":"2023-10-09T09:59:33.609713+0000","last_fullsized":"2023-10-09T09:59:33.609713+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.642817+0000","last_change":"2023-10-09T09:56:45.553024+0000","last_active":"2023-10-09T09:59:35.642817+0000","last_peered":"2023-10-09T09:59:35.642817+0000","last_clean":"2023-10-09T09:59:35.642817+0000","last_became_active":"2023-10-09T09:56:45.552670+0000","last_became_peered":"2023-10-09T09:56:45.552670+0000","last_unstale":"2023-10-09T09:59:35.642817+0000","last_undegraded":"2023-10-09T09:59:35.642817+0000","last_fullsized":"2023-10-09T09:59:35.642817+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:58.132008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"111'4475","reported_seq":4497,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:44.781852+0000","last_change":"2023-10-09T09:58:52.626321+0000","last_active":"2023-10-09T09:59:44.781852+0000","last_peered":"2023-10-09T09:59:44.781852+0000","last_clean":"2023-10-09T09:59:44.781852+0000","last_became_active":"2023-10-09T09:58:52.625904+0000","last_became_peered":"2023-10-09T09:58:52.625904+0000","last_unstale":"2023-10-09T09:59:44.781852+0000","last_undegraded":"2023-10-09T09:59:44.781852+0000","last_fullsized":"2023-10-09T09:59:44.781852+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4475,"log_dups_size":0,"ondisk_log_size":4475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293273600,"num_objects":4475,"num_object_clones":0,"num_object_copies":13425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4475,"num_write_kb":286400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.610035+0000","last_change":"2023-10-09T09:57:49.620199+0000","last_active":"2023-10-09T09:59:33.610035+0000","last_peered":"2023-10-09T09:59:33.610035+0000","last_clean":"2023-10-09T09:59:33.610035+0000","last_became_active":"2023-10-09T09:57:49.619695+0000","last_became_peered":"2023-10-09T09:57:49.619695+0000","last_unstale":"2023-10-09T09:59:33.610035+0000","last_undegraded":"2023-10-09T09:59:33.610035+0000","last_fullsized":"2023-10-09T09:59:33.610035+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:48.818881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.607861+0000","last_change":"2023-10-09T09:57:49.621587+0000","last_active":"2023-10-09T09:59:33.607861+0000","last_peered":"2023-10-09T09:59:33.607861+0000","last_clean":"2023-10-09T09:59:33.607861+0000","last_became_active":"2023-10-09T09:57:49.621245+0000","last_became_peered":"2023-10-09T09:57:49.621245+0000","last_unstale":"2023-10-09T09:59:33.607861+0000","last_undegraded":"2023-10-09T09:59:33.607861+0000","last_fullsized":"2023-10-09T09:59:33.607861+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"112'4599","reported_seq":4623,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T09:59:46.827212+0000","last_change":"2023-10-09T09:58:52.624274+0000","last_active":"2023-10-09T09:59:46.827212+0000","last_peered":"2023-10-09T09:59:46.827212+0000","last_clean":"2023-10-09T09:59:46.827212+0000","last_became_active":"2023-10-09T09:58:52.623972+0000","last_became_peered":"2023-10-09T09:58:52.623972+0000","last_unstale":"2023-10-09T09:59:46.827212+0000","last_undegraded":"2023-10-09T09:59:46.827212+0000","last_fullsized":"2023-10-09T09:59:46.827212+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4599,"log_dups_size":0,"ondisk_log_size":4599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:02.499579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301400064,"num_objects":4599,"num_object_clones":0,"num_object_copies":13797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4599,"num_write_kb":294336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.615049+0000","last_change":"2023-10-09T09:56:45.556627+0000","last_active":"2023-10-09T09:59:33.615049+0000","last_peered":"2023-10-09T09:59:33.615049+0000","last_clean":"2023-10-09T09:59:33.615049+0000","last_became_active":"2023-10-09T09:56:45.556265+0000","last_became_peered":"2023-10-09T09:56:45.556265+0000","last_unstale":"2023-10-09T09:59:33.615049+0000","last_undegraded":"2023-10-09T09:59:33.615049+0000","last_fullsized":"2023-10-09T09:59:33.615049+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608287+0000","last_change":"2023-10-09T09:57:18.845748+0000","last_active":"2023-10-09T09:59:33.608287+0000","last_peered":"2023-10-09T09:59:33.608287+0000","last_clean":"2023-10-09T09:59:33.608287+0000","last_became_active":"2023-10-09T09:57:18.845397+0000","last_became_peered":"2023-10-09T09:57:18.845397+0000","last_unstale":"2023-10-09T09:59:33.608287+0000","last_undegraded":"2023-10-09T09:59:33.608287+0000","last_fullsized":"2023-10-09T09:59:33.608287+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"111'4330","reported_seq":4353,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:42.654447+0000","last_change":"2023-10-09T09:58:52.780096+0000","last_active":"2023-10-09T09:59:42.654447+0000","last_peered":"2023-10-09T09:59:42.654447+0000","last_clean":"2023-10-09T09:59:42.654447+0000","last_became_active":"2023-10-09T09:58:52.779754+0000","last_became_peered":"2023-10-09T09:58:52.779754+0000","last_unstale":"2023-10-09T09:59:42.654447+0000","last_undegraded":"2023-10-09T09:59:42.654447+0000","last_fullsized":"2023-10-09T09:59:42.654447+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4330,"log_dups_size":0,"ondisk_log_size":4330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:19:28.304859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283770880,"num_objects":4330,"num_object_clones":0,"num_object_copies":12990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4330,"num_write_kb":277120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.606708+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:59:33.606708+0000","last_peered":"2023-10-09T09:59:33.606708+0000","last_clean":"2023-10-09T09:59:33.606708+0000","last_became_active":"2023-10-09T09:57:18.841522+0000","last_became_peered":"2023-10-09T09:57:18.841522+0000","last_unstale":"2023-10-09T09:59:33.606708+0000","last_undegraded":"2023-10-09T09:59:33.606708+0000","last_fullsized":"2023-10-09T09:59:33.606708+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609169+0000","last_change":"2023-10-09T09:56:45.555100+0000","last_active":"2023-10-09T09:59:33.609169+0000","last_peered":"2023-10-09T09:59:33.609169+0000","last_clean":"2023-10-09T09:59:33.609169+0000","last_became_active":"2023-10-09T09:56:45.554704+0000","last_became_peered":"2023-10-09T09:56:45.554704+0000","last_unstale":"2023-10-09T09:59:33.609169+0000","last_undegraded":"2023-10-09T09:59:33.609169+0000","last_fullsized":"2023-10-09T09:59:33.609169+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"111'4322","reported_seq":4344,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:44.820608+0000","last_change":"2023-10-09T09:58:52.626617+0000","last_active":"2023-10-09T09:59:44.820608+0000","last_peered":"2023-10-09T09:59:44.820608+0000","last_clean":"2023-10-09T09:59:44.820608+0000","last_became_active":"2023-10-09T09:58:52.626409+0000","last_became_peered":"2023-10-09T09:58:52.626409+0000","last_unstale":"2023-10-09T09:59:44.820608+0000","last_undegraded":"2023-10-09T09:59:44.820608+0000","last_fullsized":"2023-10-09T09:59:44.820608+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4322,"log_dups_size":0,"ondisk_log_size":4322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283246592,"num_objects":4322,"num_object_clones":0,"num_object_copies":12966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4322,"num_write_kb":276608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609165+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T09:59:33.609165+0000","last_peered":"2023-10-09T09:59:33.609165+0000","last_clean":"2023-10-09T09:59:33.609165+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T09:59:33.609165+0000","last_undegraded":"2023-10-09T09:59:33.609165+0000","last_fullsized":"2023-10-09T09:59:33.609165+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617726+0000","last_change":"2023-10-09T09:57:49.622055+0000","last_active":"2023-10-09T09:59:33.617726+0000","last_peered":"2023-10-09T09:59:33.617726+0000","last_clean":"2023-10-09T09:59:33.617726+0000","last_became_active":"2023-10-09T09:57:49.621784+0000","last_became_peered":"2023-10-09T09:57:49.621784+0000","last_unstale":"2023-10-09T09:59:33.617726+0000","last_undegraded":"2023-10-09T09:59:33.617726+0000","last_fullsized":"2023-10-09T09:59:33.617726+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:50.025391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.616954+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T09:59:33.616954+0000","last_peered":"2023-10-09T09:59:33.616954+0000","last_clean":"2023-10-09T09:59:33.616954+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T09:59:33.616954+0000","last_undegraded":"2023-10-09T09:59:33.616954+0000","last_fullsized":"2023-10-09T09:59:33.616954+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609863+0000","last_change":"2023-10-09T09:57:49.628678+0000","last_active":"2023-10-09T09:59:33.609863+0000","last_peered":"2023-10-09T09:59:33.609863+0000","last_clean":"2023-10-09T09:59:33.609863+0000","last_became_active":"2023-10-09T09:57:49.628332+0000","last_became_peered":"2023-10-09T09:57:49.628332+0000","last_unstale":"2023-10-09T09:59:33.609863+0000","last_undegraded":"2023-10-09T09:59:33.609863+0000","last_fullsized":"2023-10-09T09:59:33.609863+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":46,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.641816+0000","last_change":"2023-10-09T09:57:18.842325+0000","last_active":"2023-10-09T09:59:35.641816+0000","last_peered":"2023-10-09T09:59:35.641816+0000","last_clean":"2023-10-09T09:59:35.641816+0000","last_became_active":"2023-10-09T09:57:18.841987+0000","last_became_peered":"2023-10-09T09:57:18.841987+0000","last_unstale":"2023-10-09T09:59:35.641816+0000","last_undegraded":"2023-10-09T09:59:35.641816+0000","last_fullsized":"2023-10-09T09:59:35.641816+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:28.615689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.0","version":"112'4528","reported_seq":4552,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T09:59:46.623720+0000","last_change":"2023-10-09T09:58:52.622177+0000","last_active":"2023-10-09T09:59:46.623720+0000","last_peered":"2023-10-09T09:59:46.623720+0000","last_clean":"2023-10-09T09:59:46.623720+0000","last_became_active":"2023-10-09T09:58:52.621880+0000","last_became_peered":"2023-10-09T09:58:52.621880+0000","last_unstale":"2023-10-09T09:59:46.623720+0000","last_undegraded":"2023-10-09T09:59:46.623720+0000","last_fullsized":"2023-10-09T09:59:46.623720+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4528,"log_dups_size":0,"ondisk_log_size":4528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:57.847176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296747008,"num_objects":4528,"num_object_clones":0,"num_object_copies":13584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4528,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"111'4283","reported_seq":4306,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:42.650373+0000","last_change":"2023-10-09T09:58:52.624855+0000","last_active":"2023-10-09T09:59:42.650373+0000","last_peered":"2023-10-09T09:59:42.650373+0000","last_clean":"2023-10-09T09:59:42.650373+0000","last_became_active":"2023-10-09T09:58:52.624556+0000","last_became_peered":"2023-10-09T09:58:52.624556+0000","last_unstale":"2023-10-09T09:59:42.650373+0000","last_undegraded":"2023-10-09T09:59:42.650373+0000","last_fullsized":"2023-10-09T09:59:42.650373+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4283,"log_dups_size":0,"ondisk_log_size":4283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:07.544449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280690688,"num_objects":4283,"num_object_clones":0,"num_object_copies":12849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4283,"num_write_kb":274112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609729+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T09:59:33.609729+0000","last_peered":"2023-10-09T09:59:33.609729+0000","last_clean":"2023-10-09T09:59:33.609729+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T09:59:33.609729+0000","last_undegraded":"2023-10-09T09:59:33.609729+0000","last_fullsized":"2023-10-09T09:59:33.609729+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.607370+0000","last_change":"2023-10-09T09:56:13.613717+0000","last_active":"2023-10-09T09:59:33.607370+0000","last_peered":"2023-10-09T09:59:33.607370+0000","last_clean":"2023-10-09T09:59:33.607370+0000","last_became_active":"2023-10-09T09:56:13.613329+0000","last_became_peered":"2023-10-09T09:56:13.613329+0000","last_unstale":"2023-10-09T09:59:33.607370+0000","last_undegraded":"2023-10-09T09:59:33.607370+0000","last_fullsized":"2023-10-09T09:59:33.607370+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.616882+0000","last_change":"2023-10-09T09:57:49.621680+0000","last_active":"2023-10-09T09:59:33.616882+0000","last_peered":"2023-10-09T09:59:33.616882+0000","last_clean":"2023-10-09T09:59:33.616882+0000","last_became_active":"2023-10-09T09:57:49.621345+0000","last_became_peered":"2023-10-09T09:57:49.621345+0000","last_unstale":"2023-10-09T09:59:33.616882+0000","last_undegraded":"2023-10-09T09:59:33.616882+0000","last_fullsized":"2023-10-09T09:59:33.616882+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:58.483689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"111'4469","reported_seq":4491,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:44.825801+0000","last_change":"2023-10-09T09:58:52.780565+0000","last_active":"2023-10-09T09:59:44.825801+0000","last_peered":"2023-10-09T09:59:44.825801+0000","last_clean":"2023-10-09T09:59:44.825801+0000","last_became_active":"2023-10-09T09:58:52.780260+0000","last_became_peered":"2023-10-09T09:58:52.780260+0000","last_unstale":"2023-10-09T09:59:44.825801+0000","last_undegraded":"2023-10-09T09:59:44.825801+0000","last_fullsized":"2023-10-09T09:59:44.825801+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4469,"log_dups_size":0,"ondisk_log_size":4469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292880384,"num_objects":4469,"num_object_clones":0,"num_object_copies":13407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4469,"num_write_kb":286016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":101,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.642154+0000","last_change":"2023-10-09T09:57:49.621174+0000","last_active":"2023-10-09T09:59:35.642154+0000","last_peered":"2023-10-09T09:59:35.642154+0000","last_clean":"2023-10-09T09:59:35.642154+0000","last_became_active":"2023-10-09T09:57:49.620780+0000","last_became_peered":"2023-10-09T09:57:49.620780+0000","last_unstale":"2023-10-09T09:59:35.642154+0000","last_undegraded":"2023-10-09T09:59:35.642154+0000","last_fullsized":"2023-10-09T09:59:35.642154+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:01.389768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617297+0000","last_change":"2023-10-09T09:57:18.837334+0000","last_active":"2023-10-09T09:59:33.617297+0000","last_peered":"2023-10-09T09:59:33.617297+0000","last_clean":"2023-10-09T09:59:33.617297+0000","last_became_active":"2023-10-09T09:57:18.837037+0000","last_became_peered":"2023-10-09T09:57:18.837037+0000","last_unstale":"2023-10-09T09:59:33.617297+0000","last_undegraded":"2023-10-09T09:59:33.617297+0000","last_fullsized":"2023-10-09T09:59:33.617297+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:11.650333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617815+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T09:59:33.617815+0000","last_peered":"2023-10-09T09:59:33.617815+0000","last_clean":"2023-10-09T09:59:33.617815+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T09:59:33.617815+0000","last_undegraded":"2023-10-09T09:59:33.617815+0000","last_fullsized":"2023-10-09T09:59:33.617815+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"111'4334","reported_seq":4356,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:44.816014+0000","last_change":"2023-10-09T09:58:52.625605+0000","last_active":"2023-10-09T09:59:44.816014+0000","last_peered":"2023-10-09T09:59:44.816014+0000","last_clean":"2023-10-09T09:59:44.816014+0000","last_became_active":"2023-10-09T09:58:52.625332+0000","last_became_peered":"2023-10-09T09:58:52.625332+0000","last_unstale":"2023-10-09T09:59:44.816014+0000","last_undegraded":"2023-10-09T09:59:44.816014+0000","last_fullsized":"2023-10-09T09:59:44.816014+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4334,"log_dups_size":0,"ondisk_log_size":4334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284033024,"num_objects":4334,"num_object_clones":0,"num_object_copies":13002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4334,"num_write_kb":277376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609407+0000","last_change":"2023-10-09T09:57:18.841830+0000","last_active":"2023-10-09T09:59:33.609407+0000","last_peered":"2023-10-09T09:59:33.609407+0000","last_clean":"2023-10-09T09:59:33.609407+0000","last_became_active":"2023-10-09T09:57:18.841472+0000","last_became_peered":"2023-10-09T09:57:18.841472+0000","last_unstale":"2023-10-09T09:59:33.609407+0000","last_undegraded":"2023-10-09T09:59:33.609407+0000","last_fullsized":"2023-10-09T09:59:33.609407+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":312,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.613738+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T09:59:33.613738+0000","last_peered":"2023-10-09T09:59:33.613738+0000","last_clean":"2023-10-09T09:59:33.613738+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T09:59:33.613738+0000","last_undegraded":"2023-10-09T09:59:33.613738+0000","last_fullsized":"2023-10-09T09:59:33.613738+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608478+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T09:59:33.608478+0000","last_peered":"2023-10-09T09:59:33.608478+0000","last_clean":"2023-10-09T09:59:33.608478+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T09:59:33.608478+0000","last_undegraded":"2023-10-09T09:59:33.608478+0000","last_fullsized":"2023-10-09T09:59:33.608478+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.613057+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T09:59:33.613057+0000","last_peered":"2023-10-09T09:59:33.613057+0000","last_clean":"2023-10-09T09:59:33.613057+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T09:59:33.613057+0000","last_undegraded":"2023-10-09T09:59:33.613057+0000","last_fullsized":"2023-10-09T09:59:33.613057+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":95,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.613816+0000","last_change":"2023-10-09T09:56:45.555907+0000","last_active":"2023-10-09T09:59:33.613816+0000","last_peered":"2023-10-09T09:59:33.613816+0000","last_clean":"2023-10-09T09:59:33.613816+0000","last_became_active":"2023-10-09T09:56:45.555548+0000","last_became_peered":"2023-10-09T09:56:45.555548+0000","last_unstale":"2023-10-09T09:59:33.613816+0000","last_undegraded":"2023-10-09T09:59:33.613816+0000","last_fullsized":"2023-10-09T09:59:33.613816+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.644711+0000","last_change":"2023-10-09T09:57:49.624279+0000","last_active":"2023-10-09T09:59:35.644711+0000","last_peered":"2023-10-09T09:59:35.644711+0000","last_clean":"2023-10-09T09:59:35.644711+0000","last_became_active":"2023-10-09T09:57:49.624010+0000","last_became_peered":"2023-10-09T09:57:49.624010+0000","last_unstale":"2023-10-09T09:59:35.644711+0000","last_undegraded":"2023-10-09T09:59:35.644711+0000","last_fullsized":"2023-10-09T09:59:35.644711+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:47.717690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.642108+0000","last_change":"2023-10-09T09:57:18.840413+0000","last_active":"2023-10-09T09:59:35.642108+0000","last_peered":"2023-10-09T09:59:35.642108+0000","last_clean":"2023-10-09T09:59:35.642108+0000","last_became_active":"2023-10-09T09:57:18.840093+0000","last_became_peered":"2023-10-09T09:57:18.840093+0000","last_unstale":"2023-10-09T09:59:35.642108+0000","last_undegraded":"2023-10-09T09:59:35.642108+0000","last_fullsized":"2023-10-09T09:59:35.642108+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:58.170995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"111'4397","reported_seq":4419,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:43.568430+0000","last_change":"2023-10-09T09:58:52.625649+0000","last_active":"2023-10-09T09:59:43.568430+0000","last_peered":"2023-10-09T09:59:43.568430+0000","last_clean":"2023-10-09T09:59:43.568430+0000","last_became_active":"2023-10-09T09:58:52.625367+0000","last_became_peered":"2023-10-09T09:58:52.625367+0000","last_unstale":"2023-10-09T09:59:43.568430+0000","last_undegraded":"2023-10-09T09:59:43.568430+0000","last_fullsized":"2023-10-09T09:59:43.568430+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4397,"log_dups_size":0,"ondisk_log_size":4397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:37.385560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288161792,"num_objects":4397,"num_object_clones":0,"num_object_copies":13191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4397,"num_write_kb":281408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.614655+0000","last_change":"2023-10-09T09:56:13.614282+0000","last_active":"2023-10-09T09:59:33.614655+0000","last_peered":"2023-10-09T09:59:33.614655+0000","last_clean":"2023-10-09T09:59:33.614655+0000","last_became_active":"2023-10-09T09:56:13.613888+0000","last_became_peered":"2023-10-09T09:56:13.613888+0000","last_unstale":"2023-10-09T09:59:33.614655+0000","last_undegraded":"2023-10-09T09:59:33.614655+0000","last_fullsized":"2023-10-09T09:59:33.614655+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617032+0000","last_change":"2023-10-09T09:57:18.844843+0000","last_active":"2023-10-09T09:59:33.617032+0000","last_peered":"2023-10-09T09:59:33.617032+0000","last_clean":"2023-10-09T09:59:33.617032+0000","last_became_active":"2023-10-09T09:57:18.844530+0000","last_became_peered":"2023-10-09T09:57:18.844530+0000","last_unstale":"2023-10-09T09:59:33.617032+0000","last_undegraded":"2023-10-09T09:59:33.617032+0000","last_fullsized":"2023-10-09T09:59:33.617032+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:33.761788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.642734+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T09:59:35.642734+0000","last_peered":"2023-10-09T09:59:35.642734+0000","last_clean":"2023-10-09T09:59:35.642734+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T09:59:35.642734+0000","last_undegraded":"2023-10-09T09:59:35.642734+0000","last_fullsized":"2023-10-09T09:59:35.642734+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"111'4424","reported_seq":4447,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:43.551104+0000","last_change":"2023-10-09T09:58:52.781251+0000","last_active":"2023-10-09T09:59:43.551104+0000","last_peered":"2023-10-09T09:59:43.551104+0000","last_clean":"2023-10-09T09:59:43.551104+0000","last_became_active":"2023-10-09T09:58:52.780842+0000","last_became_peered":"2023-10-09T09:58:52.780842+0000","last_unstale":"2023-10-09T09:59:43.551104+0000","last_undegraded":"2023-10-09T09:59:43.551104+0000","last_fullsized":"2023-10-09T09:59:43.551104+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4424,"log_dups_size":0,"ondisk_log_size":4424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:03.048405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289931264,"num_objects":4424,"num_object_clones":0,"num_object_copies":13272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4424,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609630+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T09:59:33.609630+0000","last_peered":"2023-10-09T09:59:33.609630+0000","last_clean":"2023-10-09T09:59:33.609630+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T09:59:33.609630+0000","last_undegraded":"2023-10-09T09:59:33.609630+0000","last_fullsized":"2023-10-09T09:59:33.609630+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":49,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.617784+0000","last_change":"2023-10-09T09:56:45.549837+0000","last_active":"2023-10-09T09:59:33.617784+0000","last_peered":"2023-10-09T09:59:33.617784+0000","last_clean":"2023-10-09T09:59:33.617784+0000","last_became_active":"2023-10-09T09:56:45.549546+0000","last_became_peered":"2023-10-09T09:56:45.549546+0000","last_unstale":"2023-10-09T09:59:33.617784+0000","last_undegraded":"2023-10-09T09:59:33.617784+0000","last_fullsized":"2023-10-09T09:59:33.617784+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:26.779071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609096+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T09:59:33.609096+0000","last_peered":"2023-10-09T09:59:33.609096+0000","last_clean":"2023-10-09T09:59:33.609096+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T09:59:33.609096+0000","last_undegraded":"2023-10-09T09:59:33.609096+0000","last_fullsized":"2023-10-09T09:59:33.609096+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"111'4549","reported_seq":4571,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:44.821078+0000","last_change":"2023-10-09T09:58:52.627793+0000","last_active":"2023-10-09T09:59:44.821078+0000","last_peered":"2023-10-09T09:59:44.821078+0000","last_clean":"2023-10-09T09:59:44.821078+0000","last_became_active":"2023-10-09T09:58:52.627587+0000","last_became_peered":"2023-10-09T09:58:52.627587+0000","last_unstale":"2023-10-09T09:59:44.821078+0000","last_undegraded":"2023-10-09T09:59:44.821078+0000","last_fullsized":"2023-10-09T09:59:44.821078+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4549,"log_dups_size":0,"ondisk_log_size":4549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298123264,"num_objects":4549,"num_object_clones":0,"num_object_copies":13647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4549,"num_write_kb":291136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.612482+0000","last_change":"2023-10-09T09:57:18.844262+0000","last_active":"2023-10-09T09:59:33.612482+0000","last_peered":"2023-10-09T09:59:33.612482+0000","last_clean":"2023-10-09T09:59:33.612482+0000","last_became_active":"2023-10-09T09:57:18.843935+0000","last_became_peered":"2023-10-09T09:57:18.843935+0000","last_unstale":"2023-10-09T09:59:33.612482+0000","last_undegraded":"2023-10-09T09:59:33.612482+0000","last_fullsized":"2023-10-09T09:59:33.612482+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.642028+0000","last_change":"2023-10-09T09:56:45.554759+0000","last_active":"2023-10-09T09:59:35.642028+0000","last_peered":"2023-10-09T09:59:35.642028+0000","last_clean":"2023-10-09T09:59:35.642028+0000","last_became_active":"2023-10-09T09:56:45.554390+0000","last_became_peered":"2023-10-09T09:56:45.554390+0000","last_unstale":"2023-10-09T09:59:35.642028+0000","last_undegraded":"2023-10-09T09:59:35.642028+0000","last_fullsized":"2023-10-09T09:59:35.642028+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:43.005737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":62,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.606925+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T09:59:33.606925+0000","last_peered":"2023-10-09T09:59:33.606925+0000","last_clean":"2023-10-09T09:59:33.606925+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T09:59:33.606925+0000","last_undegraded":"2023-10-09T09:59:33.606925+0000","last_fullsized":"2023-10-09T09:59:33.606925+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"111'4481","reported_seq":4503,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:44.784169+0000","last_change":"2023-10-09T09:58:52.780192+0000","last_active":"2023-10-09T09:59:44.784169+0000","last_peered":"2023-10-09T09:59:44.784169+0000","last_clean":"2023-10-09T09:59:44.784169+0000","last_became_active":"2023-10-09T09:58:52.779903+0000","last_became_peered":"2023-10-09T09:58:52.779903+0000","last_unstale":"2023-10-09T09:59:44.784169+0000","last_undegraded":"2023-10-09T09:59:44.784169+0000","last_fullsized":"2023-10-09T09:59:44.784169+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":4481,"log_dups_size":0,"ondisk_log_size":4481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293666816,"num_objects":4481,"num_object_clones":0,"num_object_copies":13443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4481,"num_write_kb":286784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":58,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.642864+0000","last_change":"2023-10-09T09:56:13.614416+0000","last_active":"2023-10-09T09:59:35.642864+0000","last_peered":"2023-10-09T09:59:35.642864+0000","last_clean":"2023-10-09T09:59:35.642864+0000","last_became_active":"2023-10-09T09:56:13.614050+0000","last_became_peered":"2023-10-09T09:56:13.614050+0000","last_unstale":"2023-10-09T09:59:35.642864+0000","last_undegraded":"2023-10-09T09:59:35.642864+0000","last_fullsized":"2023-10-09T09:59:35.642864+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:17.068217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":100,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-10-09T09:59:45.638682+0000","last_change":"2023-10-09T09:57:49.623531+0000","last_active":"2023-10-09T09:59:45.638682+0000","last_peered":"2023-10-09T09:59:45.638682+0000","last_clean":"2023-10-09T09:59:45.638682+0000","last_became_active":"2023-10-09T09:57:49.623045+0000","last_became_peered":"2023-10-09T09:57:49.623045+0000","last_unstale":"2023-10-09T09:59:45.638682+0000","last_undegraded":"2023-10-09T09:59:45.638682+0000","last_fullsized":"2023-10-09T09:59:45.638682+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:49.655085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609848+0000","last_change":"2023-10-09T09:56:45.554958+0000","last_active":"2023-10-09T09:59:33.609848+0000","last_peered":"2023-10-09T09:59:33.609848+0000","last_clean":"2023-10-09T09:59:33.609848+0000","last_became_active":"2023-10-09T09:56:45.554597+0000","last_became_peered":"2023-10-09T09:56:45.554597+0000","last_unstale":"2023-10-09T09:59:33.609848+0000","last_undegraded":"2023-10-09T09:59:33.609848+0000","last_fullsized":"2023-10-09T09:59:33.609848+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.607493+0000","last_change":"2023-10-09T09:57:49.622634+0000","last_active":"2023-10-09T09:59:33.607493+0000","last_peered":"2023-10-09T09:59:33.607493+0000","last_clean":"2023-10-09T09:59:33.607493+0000","last_became_active":"2023-10-09T09:57:49.622247+0000","last_became_peered":"2023-10-09T09:57:49.622247+0000","last_unstale":"2023-10-09T09:59:33.607493+0000","last_undegraded":"2023-10-09T09:59:33.607493+0000","last_fullsized":"2023-10-09T09:59:33.607493+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.609946+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T09:59:33.609946+0000","last_peered":"2023-10-09T09:59:33.609946+0000","last_clean":"2023-10-09T09:59:33.609946+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T09:59:33.609946+0000","last_undegraded":"2023-10-09T09:59:33.609946+0000","last_fullsized":"2023-10-09T09:59:33.609946+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":92,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.607339+0000","last_change":"2023-10-09T09:57:49.620967+0000","last_active":"2023-10-09T09:59:33.607339+0000","last_peered":"2023-10-09T09:59:33.607339+0000","last_clean":"2023-10-09T09:59:33.607339+0000","last_became_active":"2023-10-09T09:57:49.620567+0000","last_became_peered":"2023-10-09T09:57:49.620567+0000","last_unstale":"2023-10-09T09:59:33.607339+0000","last_undegraded":"2023-10-09T09:59:33.607339+0000","last_fullsized":"2023-10-09T09:59:33.607339+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":84,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608511+0000","last_change":"2023-10-09T09:57:49.623102+0000","last_active":"2023-10-09T09:59:33.608511+0000","last_peered":"2023-10-09T09:59:33.608511+0000","last_clean":"2023-10-09T09:59:33.608511+0000","last_became_active":"2023-10-09T09:57:49.620638+0000","last_became_peered":"2023-10-09T09:57:49.620638+0000","last_unstale":"2023-10-09T09:59:33.608511+0000","last_undegraded":"2023-10-09T09:59:33.608511+0000","last_fullsized":"2023-10-09T09:59:33.608511+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:11.322675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":97,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.608285+0000","last_change":"2023-10-09T09:57:49.620527+0000","last_active":"2023-10-09T09:59:33.608285+0000","last_peered":"2023-10-09T09:59:33.608285+0000","last_clean":"2023-10-09T09:59:33.608285+0000","last_became_active":"2023-10-09T09:57:49.620109+0000","last_became_peered":"2023-10-09T09:57:49.620109+0000","last_unstale":"2023-10-09T09:59:33.608285+0000","last_undegraded":"2023-10-09T09:59:33.608285+0000","last_fullsized":"2023-10-09T09:59:33.608285+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:09.284902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-10-09T09:59:35.642969+0000","last_change":"2023-10-09T09:57:18.842453+0000","last_active":"2023-10-09T09:59:35.642969+0000","last_peered":"2023-10-09T09:59:35.642969+0000","last_clean":"2023-10-09T09:59:35.642969+0000","last_became_active":"2023-10-09T09:57:18.842133+0000","last_became_peered":"2023-10-09T09:57:18.842133+0000","last_unstale":"2023-10-09T09:59:35.642969+0000","last_undegraded":"2023-10-09T09:59:35.642969+0000","last_fullsized":"2023-10-09T09:59:35.642969+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:07.459692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T09:59:33.606688+0000","last_change":"2023-10-09T09:56:45.556456+0000","last_active":"2023-10-09T09:59:33.606688+0000","last_peered":"2023-10-09T09:59:33.606688+0000","last_clean":"2023-10-09T09:59:33.606688+0000","last_became_active":"2023-10-09T09:56:45.555878+0000","last_became_peered":"2023-10-09T09:56:45.555878+0000","last_unstale":"2023-10-09T09:59:33.606688+0000","last_undegraded":"2023-10-09T09:59:33.606688+0000","last_fullsized":"2023-10-09T09:59:33.606688+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4677566464,"num_objects":71374,"num_object_clones":0,"num_object_copies":214122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":71374,"num_write_kb":4567936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14057668608,"data_stored":14057668608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71374,"ondisk_log_size":71374,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430361,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2033800,"kb_used_data":1728384,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91711352,"statfs":{"total":95995035648,"available":93912424448,"internally_reserved":0,"allocated":1769865216,"data_stored":1769398186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.59499999999999997,"15min":0.59399999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":1.544},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55400000000000005,"15min":0.54600000000000004},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":2.2999999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.54600000000000004,"15min":0.54500000000000004},"min":{"1min":0.40100000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.78500000000000003,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":2.4430000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.58599999999999997,"15min":0.58499999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76300000000000001,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":1.236}]},{"osd":2,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.60999999999999999,"15min":0.60299999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.55300000000000005,"15min":0.56100000000000005},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63600000000000001,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":2.411}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":2.617}]},{"osd":4,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.54900000000000004,"15min":0.48699999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.7719999999999998,"5min":2.7719999999999998,"15min":2.7719999999999998},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.52000000000000002,"15min":0.48599999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.26500000000000001},"max":{"1min":1.7450000000000001,"5min":1.7450000000000001,"15min":1.7450000000000001},"last":0.75600000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52200000000000002,"15min":0.505},"min":{"1min":0.29999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.53200000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.429,"5min":1.429,"15min":1.429},"last":0.46400000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58899999999999997,"15min":0.55400000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32900000000000001},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58899999999999997,"15min":0.55000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":1.8089999999999999,"5min":1.8089999999999999,"15min":1.8089999999999999},"last":0.79700000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626589,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1730424,"kb_used_data":1429104,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92014728,"statfs":{"total":95995035648,"available":94223081472,"internally_reserved":0,"allocated":1463402496,"data_stored":1462951389,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.60899999999999999,"15min":0.57299999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.4239999999999999,"5min":2.4239999999999999,"15min":2.4239999999999999},"last":2.8660000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.60699999999999998,"15min":0.53900000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":4.9450000000000003,"5min":4.9450000000000003,"15min":4.9450000000000003},"last":2.2930000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:58:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.55200000000000005,"15min":0.55000000000000004},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54100000000000004,"15min":0.53600000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.67500000000000004}]},{"osd":2,"last update":"Mon Oct 9 09:59:20 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.56699999999999995,"15min":0.54900000000000004},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.629,"5min":2.629,"15min":2.629},"last":1.589},{"interface":"front","average":{"1min":0.873,"5min":0.64400000000000002,"15min":0.60599999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":2.8140000000000001,"5min":2.8140000000000001,"15min":2.8140000000000001},"last":0.73399999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:58:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.98499999999999999}]},{"osd":4,"last update":"Mon Oct 9 09:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.48699999999999999,"15min":0.46000000000000002},"min":{"1min":0.377,"5min":0.27300000000000002,"15min":0.23300000000000001},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.252},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.5,"15min":0.49299999999999999},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.311},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.26400000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.5,"15min":0.502},"min":{"1min":0.34499999999999997,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":0.66700000000000004,"5min":0.72199999999999998,"15min":0.80600000000000005},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.53800000000000003,"15min":0.51500000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":0.38900000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.56699999999999995,"15min":0.53900000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.3540000000000001,"5min":2.3540000000000001,"15min":2.3540000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.63100000000000001,"15min":0.60199999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34399999999999997},"max":{"1min":2.3929999999999998,"5min":2.3929999999999998,"15min":2.3929999999999998},"last":0.43099999999999999}]}]},{"osd":1,"up_from":84,"seq":360777252902,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455484,"kb_used_data":1401780,"kb_used_omap":9,"kb_used_meta":53686,"kb_avail":92289668,"statfs":{"total":95995035648,"available":94504620032,"internally_reserved":0,"allocated":1435422720,"data_stored":1434909301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":54975465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.23400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.38600000000000001,"15min":0.39400000000000002},"min":{"1min":0.23699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.498,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.63900000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:59:21 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.44600000000000001,"15min":0.40799999999999997},"min":{"1min":0.27200000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.121,"5min":2.121,"15min":2.121},"last":9.6839999999999993},{"interface":"front","average":{"1min":0.62,"5min":0.436,"15min":0.40600000000000003},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.248},"max":{"1min":2.2149999999999999,"5min":2.2149999999999999,"15min":2.2149999999999999},"last":9.5519999999999996}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":3.3610000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":9.3740000000000006}]},{"osd":4,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.60499999999999998,"15min":0.60899999999999999},"min":{"1min":0.46300000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.73299999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":9.5139999999999993},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.65100000000000002,"15min":0.64500000000000002},"min":{"1min":0.40899999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.589,"5min":2.589,"15min":2.589},"last":0.78100000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.61699999999999999,"15min":0.625},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.872,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.64800000000000002,"15min":0.65500000000000003},"min":{"1min":0.47399999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78800000000000003,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.755}]},{"osd":6,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.69599999999999995,"15min":0.68999999999999995},"min":{"1min":0.39600000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.6480000000000001,"5min":2.6480000000000001,"15min":2.6480000000000001},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.66400000000000003,"15min":0.67200000000000004},"min":{"1min":0.45800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.84799999999999998,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.93700000000000006}]},{"osd":7,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.63200000000000001,"15min":0.63800000000000001},"min":{"1min":0.432,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83399999999999996,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":9.4329999999999998},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.69399999999999995,"15min":0.68799999999999994},"min":{"1min":0.38,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.681,"5min":2.681,"15min":2.681},"last":0.443}]}]},{"osd":0,"up_from":79,"seq":339302416429,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1212756,"kb_used_data":1154892,"kb_used_omap":28,"kb_used_meta":57827,"kb_avail":92532396,"statfs":{"total":95995035648,"available":94753173504,"internally_reserved":0,"allocated":1182609408,"data_stored":1182079985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":59215696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.376,"15min":0.376},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":3.0139999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.39400000000000002,"15min":0.39600000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.59299999999999997,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":2.778}]},{"osd":2,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.42099999999999999,"15min":0.40200000000000002},"min":{"1min":0.28699999999999998,"5min":0.216,"15min":0.216},"max":{"1min":2.339,"5min":2.339,"15min":2.339},"last":3.8530000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.45300000000000001,"15min":0.435},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.6180000000000001,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":2.9079999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":4.3600000000000003},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":3.8420000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.66400000000000003,"15min":0.66000000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":3.0830000000000002},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.69199999999999995,"15min":0.68200000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.667,"5min":1.667,"15min":1.667},"last":2.9340000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.68100000000000005,"15min":0.65500000000000003},"min":{"1min":0.5,"5min":0.434,"15min":0.434},"max":{"1min":2.0209999999999999,"5min":2.0209999999999999,"15min":2.0209999999999999},"last":4.3029999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.65500000000000003,"15min":0.66200000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.0009999999999999,"5min":1.1180000000000001,"15min":1.1180000000000001},"last":3.0579999999999998}]},{"osd":6,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.67000000000000004,"15min":0.68200000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.054,"5min":1.161,"15min":1.161},"last":3.1059999999999999},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68999999999999995},"min":{"1min":0.438,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":2.8050000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.68700000000000006,"15min":0.67300000000000004},"min":{"1min":0.45600000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.056,"5min":2.056,"15min":2.056},"last":4.3360000000000003},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.67900000000000005,"15min":0.68100000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.093,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":3.1309999999999998}]}]},{"osd":2,"up_from":89,"seq":382252089376,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2436172,"kb_used_data":2372484,"kb_used_omap":81,"kb_used_meta":63598,"kb_avail":91308980,"statfs":{"total":95995035648,"available":93500395520,"internally_reserved":0,"allocated":2429423616,"data_stored":2428901328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":65124570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.45100000000000001,"15min":0.40600000000000003},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.218,"5min":2.218,"15min":2.218},"last":1.679},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.47599999999999998,"15min":0.42499999999999999},"min":{"1min":0.317,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.5219999999999998,"5min":2.5219999999999998,"15min":2.5219999999999998},"last":1.6679999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":0.54100000000000004,"15min":0.42499999999999999},"min":{"1min":0.27900000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.6109999999999998,"5min":7.6109999999999998,"15min":7.6109999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.46200000000000002,"15min":0.42799999999999999},"min":{"1min":0.30099999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.2410000000000001,"5min":2.2410000000000001,"15min":2.2410000000000001},"last":6.1479999999999997}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":2.169},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":1.881}]},{"osd":4,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.67000000000000004,"15min":0.626},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.5979999999999999,"5min":2.5979999999999999,"15min":2.5979999999999999},"last":1.788},{"interface":"front","average":{"1min":1.325,"5min":0.76900000000000002,"15min":0.67600000000000005},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.5229999999999997,"5min":7.5229999999999997,"15min":7.5229999999999997},"last":0.70999999999999996}]},{"osd":5,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8369999999999997,"5min":1.4670000000000001,"15min":0.90500000000000003},"min":{"1min":0.42899999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":79.016000000000005,"5min":79.016000000000005,"15min":79.016000000000005},"last":1.74},{"interface":"front","average":{"1min":4.7590000000000003,"5min":1.4550000000000001,"15min":0.90500000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":79.302999999999997,"5min":79.302999999999997,"15min":79.302999999999997},"last":1.8400000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.68700000000000006,"15min":0.63800000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.6200000000000001,"5min":2.6200000000000001,"15min":2.6200000000000001},"last":1.861},{"interface":"front","average":{"1min":1.288,"5min":0.76400000000000001,"15min":0.67700000000000005},"min":{"1min":0.497,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.569,"5min":7.569,"15min":7.569},"last":0.73499999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.70199999999999996,"15min":0.67000000000000004},"min":{"1min":0.45200000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":2.302,"5min":2.302,"15min":2.302},"last":1.7669999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.70899999999999996,"15min":0.67700000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.6419999999999999,"5min":2.6419999999999999,"15min":2.6419999999999999},"last":1.8180000000000001}]}]},{"osd":3,"up_from":94,"seq":403726925849,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2835384,"kb_used_data":2771824,"kb_used_omap":7,"kb_used_meta":63544,"kb_avail":90909768,"statfs":{"total":95995035648,"available":93091602432,"internally_reserved":0,"allocated":2838347776,"data_stored":2837836398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":65069672},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.995,"15min":0.995},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":11.958,"5min":11.958,"15min":11.958},"last":3.7090000000000001},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":3.9039999999999999}]},{"osd":1,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":3.9279999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":3.734}]},{"osd":2,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.046,"15min":1.046},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":13.246,"5min":13.246,"15min":13.246},"last":3.8700000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.1120000000000001,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":3.5139999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":3.569},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.042,"5min":1.042,"15min":1.042},"last":3.8130000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":3.7730000000000001},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":7.9850000000000003,"5min":7.9850000000000003,"15min":7.9850000000000003},"last":1.5880000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":4.0970000000000004},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":8.0410000000000004,"5min":8.0410000000000004,"15min":8.0410000000000004},"last":1.6240000000000001}]},{"osd":7,"last update":"Mon Oct 9 09:58:49 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":8.0719999999999992,"5min":8.0719999999999992,"15min":8.0719999999999992},"last":1.649},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":3.8250000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019045,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1709824,"kb_used_data":1408504,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92035328,"statfs":{"total":95995035648,"available":94244175872,"internally_reserved":0,"allocated":1442308096,"data_stored":1441848644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.58299999999999996,"15min":0.53800000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":6.3680000000000003,"5min":6.3680000000000003,"15min":6.3680000000000003},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.60399999999999998,"15min":0.58199999999999996},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.5970000000000004,"5min":4.5970000000000004,"15min":4.5970000000000004},"last":2.2890000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:58:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51800000000000002,"15min":0.51900000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":2.0449999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.437,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":2.1160000000000001}]},{"osd":2,"last update":"Mon Oct 9 09:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":0.61699999999999999,"15min":0.55600000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.8979999999999997,"5min":4.8979999999999997,"15min":4.8979999999999997},"last":2.2770000000000001},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.60599999999999998,"15min":0.57699999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.7629999999999999,"5min":2.7629999999999999,"15min":2.7629999999999999},"last":1.4079999999999999}]},{"osd":3,"last update":"Mon Oct 9 09:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.317},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":4.9370000000000003}]},{"osd":5,"last update":"Mon Oct 9 09:59:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.45700000000000002,"15min":0.436},"min":{"1min":0.29899999999999999,"5min":0.28399999999999997,"15min":0.27100000000000002},"max":{"1min":0.72499999999999998,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.49299999999999999,"15min":0.47099999999999997},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.28499999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.83999999999999997},"last":1.72}]},{"osd":6,"last update":"Mon Oct 9 09:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.51400000000000001,"15min":0.47799999999999998},"min":{"1min":0.315,"5min":0.311,"15min":0.26700000000000002},"max":{"1min":1.5509999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.54200000000000004,"15min":0.50700000000000001},"min":{"1min":0.374,"5min":0.318,"15min":0.312},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.52700000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:58:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.55700000000000005,"15min":0.53000000000000003},"min":{"1min":0.44800000000000001,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.84999999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59599999999999997,"15min":0.56200000000000006},"min":{"1min":0.47499999999999998,"5min":0.34599999999999997,"15min":0.316},"max":{"1min":0.72399999999999998,"5min":0.82899999999999996,"15min":0.85199999999999998},"last":0.51300000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822817,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1703408,"kb_used_data":1397992,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92041744,"statfs":{"total":95995035648,"available":94250745856,"internally_reserved":0,"allocated":1431543808,"data_stored":1431096484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.59899999999999998,"15min":0.55800000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.3799999999999999,"5min":3.3799999999999999,"15min":3.3799999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.622,"5min":0.56699999999999995,"15min":0.54700000000000004},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.8260000000000001,"5min":2.4289999999999998,"15min":2.4289999999999998},"last":0.45900000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:58:47 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.58099999999999996,"15min":0.55400000000000005},"min":{"1min":0.38800000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.5979999999999999,"5min":4.5979999999999999,"15min":4.5979999999999999},"last":7.6929999999999996},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.58899999999999997,"15min":0.55900000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":4.4989999999999997,"5min":4.4989999999999997,"15min":4.4989999999999997},"last":1.02}]},{"osd":2,"last update":"Mon Oct 9 09:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.69799999999999995,"15min":0.69699999999999995},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.3380000000000001,"5min":3.3380000000000001,"15min":3.3380000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.69999999999999996,"15min":0.70199999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.161,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.374}]},{"osd":3,"last update":"Mon Oct 9 09:58:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":2.4740000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.631,"5min":1.631,"15min":1.631},"last":1.0780000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:58:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.49099999999999999,"15min":0.45300000000000001},"min":{"1min":0.33000000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":0.77300000000000002,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.50800000000000001,"15min":0.47399999999999998},"min":{"1min":0.35299999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.69499999999999995,"5min":0.79200000000000004,"15min":0.88200000000000001},"last":0.39000000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:59:18 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.56699999999999995,"15min":0.52100000000000002},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":2.8250000000000002,"5min":2.8250000000000002,"15min":2.8250000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.58699999999999997,"15min":0.53400000000000003},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.415,"5min":3.415,"15min":3.415},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 09:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.63600000000000001,"15min":0.57999999999999996},"min":{"1min":0.318,"5min":0.318,"15min":0.309},"max":{"1min":3.536,"5min":3.536,"15min":3.536},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.60799999999999998,"15min":0.55700000000000005},"min":{"1min":0.33200000000000002,"5min":0.318,"15min":0.317},"max":{"1min":2.8620000000000001,"5min":2.8620000000000001,"15min":2.8620000000000001},"last":0.41099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1192755200,"data_stored":1192755200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1442054144,"data_stored":1442054144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2433024000,"data_stored":2433024000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2841051136,"data_stored":2841051136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1458438144,"data_stored":1458438144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1445593088,"data_stored":1445593088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1466236928,"data_stored":1466236928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1778515968,"data_stored":1778515968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T09:59:48.349 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T09:59:48.349 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:59:48.349 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T09:59:48.349 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:59:48.349 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:59:48.350 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T09:59:48.363 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:48.368 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 48 16 75452 75436 98.1782 65.625 0.021261 0.0101798 2023-10-09T09:59:48.414 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:48.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:48.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:48.608 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:48.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:48 smithi110 ceph-mon[136032]: pgmap v274: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:59:48.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:48 smithi110 ceph-mon[136032]: from='client.44783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:59:48.683 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:48.684 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:48.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:48.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:48.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:48.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:48 smithi012 ceph-mon[147926]: pgmap v274: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:59:48.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:48 smithi012 ceph-mon[147926]: from='client.44783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:59:48.850 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:48 smithi012 ceph-mon[152263]: pgmap v274: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T09:59:48.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:48 smithi012 ceph-mon[152263]: from='client.44783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T09:59:48.858 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:48.972 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:48.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:49.063 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:59:49.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:49.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:49.212 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:49.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:49.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:49.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:49.368 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 49 16 76467 76451 97.4693 63.4375 0.0191458 0.0102542 2023-10-09T09:59:49.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:49.598 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:49.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:49.673 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:49.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:49.796 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:49.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T09:59:49.884 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:49.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:49.996 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:50.051 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:50.094 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:50.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:50.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:50.200 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:50.237 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:50.337 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:50.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:50.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:50.368 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 50 15 77491 77476 96.8013 64.0625 0.0234776 0.0103255 2023-10-09T09:59:50.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:50.457 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T09:59:50.457 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:59:50.457 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T09:59:50.590 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:50.590 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:12317 6.6 6:6717de4d:::benchmark_data_smithi012_182576_object12316:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:04.884650+0000", 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 45.694913, 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059443178999999999, 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.591 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 12317 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884650+0000", 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884650+0000", 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884648+0000", 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884676+0000", 2023-10-09T09:59:50.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8138999999999999e-05 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884693+0000", 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.696e-05 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884701+0000", 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2630000000000003e-06 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884755+0000", 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3829000000000002e-05 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.884916+0000", 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016098999999999999 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.885082+0000", 2023-10-09T09:59:50.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016562799999999999 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.885897+0000", 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00081518499999999995 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.887629+0000", 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001731425 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.943997+0000", 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.056368268999999999 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944053+0000", 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6294e-05 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944093+0000", 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9968000000000001e-05 2023-10-09T09:59:50.597 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:12318 6.6 6:635ceacc:::benchmark_data_smithi012_182576_object12317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:04.886855+0000", 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 45.692707841999997, 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057436543, 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.598 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 12318 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.886855+0000", 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.886855+0000", 2023-10-09T09:59:50.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.886853+0000", 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.886874+0000", 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0069e-05 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.886886+0000", 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2149e-05 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.886891+0000", 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7610000000000004e-06 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.886928+0000", 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6108000000000001e-05 2023-10-09T09:59:50.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.887029+0000", 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100919 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.887128+0000", 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9679999999999994e-05 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.887751+0000", 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062329799999999999 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.887909+0000", 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015787900000000001 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944191+0000", 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.056281439000000003 2023-10-09T09:59:50.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944254+0000", 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3563000000000001e-05 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:04.944292+0000", 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7228999999999999e-05 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.604 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:13899 6.c 6:39dbd6a1:::benchmark_data_smithi012_182576_object13898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.544296+0000", 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 45.035267333999997, 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11120894000000001, 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.605 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 13899 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544296+0000", 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544296+0000", 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544295+0000", 2023-10-09T09:59:50.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:50.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544346+0000", 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0963e-05 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544355+0000", 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7390000000000005e-06 2023-10-09T09:59:50.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544370+0000", 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5468000000000002e-05 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544424+0000", 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3727999999999998e-05 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544551+0000", 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012757400000000001 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.544679+0000", 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012737999999999999 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.546908+0000", 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022288070000000002 2023-10-09T09:59:50.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.546999+0000", 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1494000000000002e-05 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.655412+0000", 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.108412789 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.655456+0000", 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3990999999999999e-05 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.655504+0000", 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8659000000000003e-05 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.613 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:13929 6.c 6:39e98511:::benchmark_data_smithi012_182576_object13928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:05.562720+0000", 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 45.016842660999998, 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.094359981999999995, 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 13929 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562720+0000", 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562720+0000", 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562719+0000", 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:50.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562743+0000", 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3266000000000001e-05 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562753+0000", 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0434e-05 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562758+0000", 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2950000000000001e-06 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562812+0000", 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.321e-05 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.562945+0000", 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013293800000000001 2023-10-09T09:59:50.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.563090+0000", 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000145637 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.565937+0000", 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028466899999999998 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.566168+0000", 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000230852 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.656986+0000", 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.090818572 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.657039+0000", 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2954999999999997e-05 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:05.657080+0000", 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0878999999999999e-05 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:36565 6.6 6:6e18ae43:::benchmark_data_smithi012_182576_object36564:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.196555+0000", 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 33.383007804999998, 2023-10-09T09:59:50.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.076842649999999998, 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 36565 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196555+0000", 2023-10-09T09:59:50.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196555+0000", 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196554+0000", 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196576+0000", 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2768999999999998e-05 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196589+0000", 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2388999999999999e-05 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196597+0000", 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8380000000000002e-06 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196642+0000", 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5664000000000002e-05 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196782+0000", 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013941499999999999 2023-10-09T09:59:50.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196915+0000", 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013309699999999999 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272641+0000", 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.075726435999999994 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273277+0000", 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00063542099999999995 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273306+0000", 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9141000000000001e-05 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273363+0000", 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7568999999999997e-05 2023-10-09T09:59:50.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273398+0000", 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4301999999999998e-05 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:36567 6.b 6:df2d5787:::benchmark_data_smithi012_182576_object36566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.196613+0000", 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 33.382949617999998, 2023-10-09T09:59:50.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.076571608999999999, 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 36567 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196613+0000", 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196613+0000", 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196613+0000", 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196632+0000", 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9456000000000001e-05 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196639+0000", 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0650000000000001e-06 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196645+0000", 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3199999999999999e-06 2023-10-09T09:59:50.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196683+0000", 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8602000000000001e-05 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196785+0000", 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102048 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.196895+0000", 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010927 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273036+0000", 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.076141206000000003 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273065+0000", 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9764000000000002e-05 2023-10-09T09:59:50.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273081+0000", 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.56e-05 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273129+0000", 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7620999999999997e-05 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273185+0000", 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6180000000000001e-05 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:36575 6.c 6:3157ee00:::benchmark_data_smithi012_182576_object36574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.197722+0000", 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 33.381840762000003, 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.078423557000000005, 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.634 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 36575 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.197722+0000", 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.197722+0000", 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.197721+0000", 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.197745+0000", 2023-10-09T09:59:50.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3998999999999998e-05 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.197757+0000", 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1837e-05 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.197764+0000", 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7360000000000001e-06 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.197813+0000", 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9091999999999997e-05 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272690+0000", 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.074876208999999999 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272845+0000", 2023-10-09T09:59:50.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000155678 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274760+0000", 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019148259999999999 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274802+0000", 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2101999999999998e-05 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276061+0000", 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012584930000000001 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276106+0000", 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4858999999999999e-05 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276146+0000", 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0015999999999998e-05 2023-10-09T09:59:50.640 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:36566 6.4 e105/97 6:2f5bfd46:::benchmark_data_smithi012_182576_object36565:head v 105'2326, mlcod=105'2324)", 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.198682+0000", 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 33.380880644999998, 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.078236930999999996, 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198682+0000", 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198682+0000", 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198681+0000", 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.198702+0000", 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0958000000000002e-05 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272646+0000", 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073944116000000004 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272657+0000", 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0788000000000001e-05 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273459+0000", 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00080196100000000001 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273478+0000", 2023-10-09T09:59:50.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8991000000000002e-05 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276885+0000", 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034068929999999998 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276919+0000", 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4079999999999999e-05 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.645 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:36574 6.7 e105/97 6:e5d9ae2e:::benchmark_data_smithi012_182576_object36573:head v 105'2190, mlcod=105'2188)", 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.199512+0000", 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 33.380050447000002, 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.077104048999999994, 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.199512+0000", 2023-10-09T09:59:50.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.199512+0000", 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.199512+0000", 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.199788+0000", 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027670100000000002 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272785+0000", 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.072996365999999993 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272793+0000", 2023-10-09T09:59:50.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0839999999999992e-06 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274332+0000", 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001539519 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.274353+0000", 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0271e-05 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276563+0000", 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022105369999999998 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.276616+0000", 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3347999999999998e-05 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.34797.0:36565 6.6 e105/97 ondisk, result = 0)", 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:17.200059+0000", 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 33.379504134000001, 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073220719000000004, 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.200059+0000", 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.200059+0000", 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.200058+0000", 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.200064+0000", 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0419999999999999e-06 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272827+0000", 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.072762504000000006 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.272831+0000", 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.622e-06 2023-10-09T09:59:50.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273264+0000", 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043300400000000002 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273271+0000", 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2200000000000003e-06 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:17.273279+0000", 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1179999999999992e-06 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:38117 6.b 6:d4c989e1:::benchmark_data_smithi012_182576_object38116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.097256+0000", 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.482306905999998, 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062931829999999994, 2023-10-09T09:59:50.655 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 38117 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097256+0000", 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097256+0000", 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097254+0000", 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097286+0000", 2023-10-09T09:59:50.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1476000000000002e-05 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097298+0000", 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2293000000000001e-05 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097306+0000", 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2429999999999999e-06 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097397+0000", 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1413e-05 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097491+0000", 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4010000000000003e-05 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2023-10-09T09:59:50.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097573+0000", 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1960999999999997e-05 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100243+0000", 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026703809999999999 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150685+0000", 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.050441930000000003 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.160095+0000", 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094099950000000009 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.160143+0000", 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8062999999999997e-05 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.160188+0000", 2023-10-09T09:59:50.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4555000000000003e-05 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:38118 6.b 6:da104d7c:::benchmark_data_smithi012_182576_object38117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.097331+0000", 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.482231802999998, 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062986799999999996, 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.662 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 38118 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097331+0000", 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097331+0000", 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097330+0000", 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097351+0000", 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0843999999999999e-05 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097359+0000", 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0970000000000006e-06 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.097365+0000", 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6230000000000002e-06 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098856+0000", 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014906450000000001 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.098937+0000", 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1262999999999998e-05 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2023-10-09T09:59:50.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099019+0000", 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1808e-05 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150092+0000", 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.051073407000000001 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150721+0000", 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062891599999999998 2023-10-09T09:59:50.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.160232+0000", 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095108640000000008 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.160278+0000", 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5634999999999999e-05 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.160318+0000", 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0327999999999997e-05 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:38125 6.f e105/97 6:f15ec602:::benchmark_data_smithi012_182576_object38124:head v 105'2376, mlcod=105'2374)", 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.099286+0000", 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.480276965999998, 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.051277706999999999, 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.669 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099286+0000", 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099286+0000", 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099285+0000", 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099308+0000", 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2895e-05 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099329+0000", 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1310000000000001e-05 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099337+0000", 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.153e-06 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099395+0000", 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7975e-05 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.099413+0000", 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.825e-05 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150535+0000", 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.051121840000000002 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150564+0000", 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8938e-05 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:38113 6.0 e105/97 6:0e71af63:::benchmark_data_smithi012_182576_object38112:head v 105'2355, mlcod=105'2352)", 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.100270+0000", 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.479292487000002, 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052167456000000001, 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.674 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100270+0000", 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100270+0000", 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100270+0000", 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100295+0000", 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5618000000000001e-05 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100305+0000", 2023-10-09T09:59:50.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3209999999999994e-06 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100311+0000", 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0630000000000002e-06 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100355+0000", 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3915000000000002e-05 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100371+0000", 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6633000000000002e-05 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.152397+0000", 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052026056000000001 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.152438+0000", 2023-10-09T09:59:50.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0484000000000001e-05 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:38122 6.c 6:385bfe97:::benchmark_data_smithi012_182576_object38121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.100349+0000", 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.479214267000003, 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053734524999999998, 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.679 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 38122 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100349+0000", 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100349+0000", 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100347+0000", 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100369+0000", 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2069000000000002e-05 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100378+0000", 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2220000000000002e-06 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100384+0000", 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1070000000000001e-06 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100440+0000", 2023-10-09T09:59:50.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6230000000000002e-05 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150126+0000", 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049686091000000002 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150311+0000", 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000184699 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.152599+0000", 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022876849999999998 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.153909+0000", 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00131036 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.154007+0000", 2023-10-09T09:59:50.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7884e-05 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.154048+0000", 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1455000000000002e-05 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.154083+0000", 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4934000000000001e-05 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.685 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:38124 6.6 6:64c976da:::benchmark_data_smithi012_182576_object38123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.100402+0000", 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.479160544000003, 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053300254999999998, 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.686 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 38124 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100402+0000", 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100402+0000", 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100402+0000", 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100423+0000", 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.126e-05 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100430+0000", 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.545e-06 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100435+0000", 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.95e-06 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150101+0000", 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049666488000000002 2023-10-09T09:59:50.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150253+0000", 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015219300000000001 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150415+0000", 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000161378 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.153560+0000", 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031456489999999999 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.153588+0000", 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7979000000000001e-05 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.153624+0000", 2023-10-09T09:59:50.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5871999999999998e-05 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.153667+0000", 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2666000000000001e-05 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.153703+0000", 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5722999999999997e-05 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.692 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:38119 6.4 e105/97 6:2f4d8a36:::benchmark_data_smithi012_182576_object38118:head v 105'2396, mlcod=105'2394)", 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:18.100458+0000", 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.479105087000001, 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054155551000000003, 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100458+0000", 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100458+0000", 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100457+0000", 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:50.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.100477+0000", 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9575000000000001e-05 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150265+0000", 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049788405000000001 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150273+0000", 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0949999999999998e-06 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150675+0000", 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040229399999999998 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.150695+0000", 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9819000000000001e-05 2023-10-09T09:59:50.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.154578+0000", 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038834439999999998 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:18.154613+0000", 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5238999999999999e-05 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:57915 6.6 6:6ac90c7c:::benchmark_data_smithi012_182576_object57914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:32.442807+0000", 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 18.136755874999999, 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069825416000000001, 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 57915 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.698 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442807+0000", 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442807+0000", 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442806+0000", 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442827+0000", 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1759999999999998e-05 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442842+0000", 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4788e-05 2023-10-09T09:59:50.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442850+0000", 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4780000000000002e-06 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.442902+0000", 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2061999999999997e-05 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.443010+0000", 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010846 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,7", 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.443115+0000", 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010439599999999999 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.445134+0000", 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020190780000000001 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.445724+0000", 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00059041699999999998 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.512527+0000", 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066803468000000005 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.512587+0000", 2023-10-09T09:59:50.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9752e-05 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:32.512632+0000", 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5213000000000002e-05 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:58586 6.c 6:36672524:::benchmark_data_smithi012_182576_object58585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:33.138643+0000", 2023-10-09T09:59:50.704 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 17.440919607000001, 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.411976964, 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 58586 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.138643+0000", 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.138643+0000", 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.138642+0000", 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.138667+0000", 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5486999999999998e-05 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.138682+0000", 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4843e-05 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.138689+0000", 2023-10-09T09:59:50.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1430000000000001e-06 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.138768+0000", 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8805000000000002e-05 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.138908+0000", 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014003799999999999 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.139047+0000", 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013831999999999999 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.141556+0000", 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025088839999999999 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.143010+0000", 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001454393 2023-10-09T09:59:50.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.550518+0000", 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.407507705 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.550574+0000", 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6375999999999997e-05 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.710 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.550620+0000", 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6267000000000003e-05 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:58611 6.c 6:373b1a2d:::benchmark_data_smithi012_182576_object58610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T09:59:33.170958+0000", 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 17.408605189999999, 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38295153300000001, 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T09:59:50.711 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 58611 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.170958+0000", 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.170958+0000", 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.170956+0000", 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.170978+0000", 2023-10-09T09:59:50.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1900999999999999e-05 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.170992+0000", 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4003e-05 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.171000+0000", 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2169999999999994e-06 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.171043+0000", 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3084000000000003e-05 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.171161+0000", 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011772700000000001 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.171267+0000", 2023-10-09T09:59:50.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106269 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.173048+0000", 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001781007 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.173072+0000", 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3512000000000001e-05 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.553808+0000", 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38073633200000001 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.553871+0000", 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2692000000000002e-05 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T09:59:33.553909+0000", 2023-10-09T09:59:50.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8637999999999998e-05 2023-10-09T09:59:50.718 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.718 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.718 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.718 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T09:59:50.718 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T09:59:50.718 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T09:59:50.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:50 smithi012 ceph-mon[147926]: pgmap v275: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:50.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:50 smithi012 ceph-mon[152263]: pgmap v275: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:50.936 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:50.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:50 smithi110 ceph-mon[136032]: pgmap v275: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:50.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:51.044 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:51.088 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:51.140 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:51.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:51.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:51.229 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T09:59:51.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:51.332 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:51.368 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 51 16 78531 78515 96.1766 64.9375 0.00711301 0.0103915 2023-10-09T09:59:51.406 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:51.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:51.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:51.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:51.578 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:51.631 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:51.647 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:51.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:51.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:51.760 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:51.816 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:51.884 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:51.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:51.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:51.968 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:59:51.989 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:52.081 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:52.211 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:52.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:52.316 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:52.368 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 52 16 79527 79511 95.5243 62.25 0.00934296 0.0104633 2023-10-09T09:59:52.369 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:52.480 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:52.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:52.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:52.659 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:52.683 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T09:59:52.766 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:52.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:52.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:52.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:52.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:52 smithi110 ceph-mon[136032]: pgmap v276: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T09:59:52.886 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2459642655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:52.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:52 smithi012 ceph-mon[152263]: pgmap v276: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T09:59:52.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2459642655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:52.890 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:52 smithi012 ceph-mon[147926]: pgmap v276: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T09:59:52.890 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2459642655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T09:59:52.965 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:53.186 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:53.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:53.299 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:53.349 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:53.359 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T09:59:53.360 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T09:59:53.360 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T09:59:53.360 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T09:59:53.368 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 53 16 80588 80572 94.9732 66.3125 0.0183469 0.0105246 2023-10-09T09:59:53.400 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:53.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:53.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:53.483 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:53.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:53.520 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:53.561 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:53.567 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:53.606 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:53.645 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:53.655 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:53.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:53 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:59:53.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:53 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T09:59:53.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2459642655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:53.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:53 smithi110 ceph-mon[136032]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T09:59:53.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:53.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:53.690 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:53.720 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:53.735 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T09:59:53.766 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:53.768 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T09:59:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T09:59:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T09:59:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T09:59:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T09:59:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T09:59:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T09:59:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T09:59:53.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:59:53.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T09:59:53.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2459642655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:53.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[147926]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T09:59:53.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[147926]: failed to encode map e113 with expected crc 2023-10-09T09:59:53.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[147926]: failed to encode map e113 with expected crc 2023-10-09T09:59:53.815 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/367254360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:53.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T09:59:53.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T09:59:53.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2459642655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T09:59:53.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[152263]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T09:59:53.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[152263]: failed to encode map e113 with expected crc 2023-10-09T09:59:53.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[152263]: failed to encode map e113 with expected crc 2023-10-09T09:59:53.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/367254360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:53.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:53.889 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:53.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:53.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:53 smithi110 ceph-mon[136032]: failed to encode map e113 with expected crc 2023-10-09T09:59:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:53 smithi110 ceph-mon[136032]: failed to encode map e113 with expected crc 2023-10-09T09:59:53.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/367254360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:53.999 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:54.055 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:54.183 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:54.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:54.287 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:54.331 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:54.368 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 54 16 81640 81624 94.4321 65.75 0.0137854 0.0105846 2023-10-09T09:59:54.385 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:59:54.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:54.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:54.497 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:54.523 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:54.664 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:54 smithi110 ceph-mon[136032]: pgmap v278: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-09T09:59:54.829 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:54.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:54.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:54 smithi012 ceph-mon[147926]: pgmap v278: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-09T09:59:54.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:54 smithi012 ceph-mon[152263]: pgmap v278: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-09T09:59:54.934 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:55.002 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:55.064 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:55.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:55.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:55.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T09:59:55.250 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:55.313 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:55.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:55.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:55.372 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 55 16 82783 82767 94.0073 71.4375 0.00978046 0.0106321 2023-10-09T09:59:55.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T09:59:55.513 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:55.578 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T09:59:55.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:55.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T09:59:55.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:55.742 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:55.843 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:55.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:55.851 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T09:59:55.851 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T09:59:55.851 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T09:59:55.851 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T09:59:55.851 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T09:59:55.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T09:59:55.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:56.021 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:56.084 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:56.098 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:56.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:56.141 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T09:59:56.141 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T09:59:56.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:56.260 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:56.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T09:59:56.316 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:56.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:56.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:56.375 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 56 16 83813 83797 93.4737 64.375 0.0162668 0.0106926 2023-10-09T09:59:56.427 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:56.470 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:56.629 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:56.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:56.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:56.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T09:59:56.721 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:56 smithi110 ceph-mon[136032]: pgmap v279: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:56.736 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:56.812 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:56.823 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:59:56.872 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:56.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:56.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:56 smithi012 ceph-mon[147926]: pgmap v279: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:56.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:56 smithi012 ceph-mon[152263]: pgmap v279: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T09:59:56.980 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T09:59:57.017 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:57.115 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T09:59:57.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:57.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T09:59:57.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:57.275 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T09:59:57.292 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:57.375 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 57 16 84765 84749 92.8778 59.5 0.0169016 0.0107613 2023-10-09T09:59:57.631 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:57.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:57.730 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T09:59:57.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T09:59:57.812 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:57.993 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T09:59:58.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:58.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T09:59:58.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T09:59:58.170 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 09:59:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:09:59:57] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T09:59:58.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:58.359 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T09:59:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:58.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T09:59:58.375 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 58 16 85748 85732 92.3357 61.4375 0.0192914 0.0108247 2023-10-09T09:59:58.437 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T09:59:58.437 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T09:59:58.437 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T09:59:58.463 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T09:59:58.528 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:58 smithi110 ceph-mon[136032]: pgmap v280: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:59:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1104410963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:58.725 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T09:59:58.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:58 smithi012 ceph-mon[147926]: pgmap v280: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:59:58.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1104410963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:58.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:58.884 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:58 smithi012 ceph-mon[152263]: pgmap v280: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T09:59:58.884 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1104410963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:58.885 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T09:59:58.956 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T09:59:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:58.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T09:59:59.066 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T09:59:59.122 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:59.208 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T09:59:59.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:59.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T09:59:59.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T09:59:59.328 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T09:59:59.365 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T09:59:59.376 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 59 16 86728 86712 91.809 61.25 0.0123162 0.0108871 2023-10-09T09:59:59.383 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:59.406 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T09:59:59.443 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T09:59:59.461 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T09:59:59.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:59.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T09:59:59.488 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T09:59:59.532 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T09:59:59.569 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T09:59:59.571 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T09:59:59.590 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T09:59:59.590 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T09:59:59.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T09:59:59.590 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T09:59:59.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.590 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:16015 6.e 6:7f78ae73:::benchmark_data_smithi012_182576_object16014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:59.590 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:06.535759+0000", 2023-10-09T09:59:59.590 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.051428819999998, 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055727141000000001, 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 16015 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.535759+0000", 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.535759+0000", 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.535757+0000", 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030560100000000002 2023-10-09T09:59:59.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.536063+0000", 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2099000000000001e-05 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.536075+0000", 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6189999999999996e-06 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.536082+0000", 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7052000000000003e-05 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.536129+0000", 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121981 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.536251+0000", 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000152263 2023-10-09T09:59:59.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.536403+0000", 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021635769999999999 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538566+0000", 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058986200000000003 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539156+0000", 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052242907999999998 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591399+0000", 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8201000000000001e-05 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591447+0000", 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8492999999999999e-05 2023-10-09T09:59:59.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591486+0000", 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055727141000000001 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:16021 6.2 6:47105632:::benchmark_data_smithi012_182576_object16020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:06.537449+0000", 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.049738785000002, 2023-10-09T09:59:59.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055179621999999998, 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 16021 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537449+0000", 2023-10-09T09:59:59.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537449+0000", 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537447+0000", 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6188e-05 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537484+0000", 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1829e-05 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537495+0000", 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6239999999999996e-06 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537502+0000", 2023-10-09T09:59:59.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2580999999999997e-05 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537555+0000", 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000134177 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537689+0000", 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000166372 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.537855+0000", 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085218000000000002 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538707+0000", 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052691806000000001 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591399+0000", 2023-10-09T09:59:59.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001142924 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.592542+0000", 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0856000000000002e-05 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.592593+0000", 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5553999999999999e-05 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.592628+0000", 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055179621999999998 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:16016 6.7 e103/97 6:eeca39e7:::benchmark_data_smithi012_182576_object16015:head v 103'975, mlcod=103'971)", 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:06.538594+0000", 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.048593969000002, 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053888024999999999, 2023-10-09T09:59:59.604 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538594+0000", 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538594+0000", 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T09:59:59.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538593+0000", 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4105999999999999e-05 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538617+0000", 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6139999999999992e-06 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538625+0000", 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.502e-06 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538631+0000", 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3795000000000003e-05 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538675+0000", 2023-10-09T09:59:59.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.207e-05 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.538697+0000", 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053748470999999999 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.592445+0000", 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6368999999999997e-05 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.592482+0000", 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053888024999999999 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:16020 6.f e103/97 6:f7331bf8:::benchmark_data_smithi012_182576_object16019:head v 103'996, mlcod=103'994)", 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:06.539123+0000", 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.048064689, 2023-10-09T09:59:59.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054397269999999998, 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539123+0000", 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539123+0000", 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539123+0000", 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0679999999999998e-05 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539153+0000", 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9480000000000008e-06 2023-10-09T09:59:59.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539161+0000", 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7199999999999997e-06 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539166+0000", 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8212999999999999e-05 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539194+0000", 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5038999999999999e-05 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.539209+0000", 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054277956000000002 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.593487+0000", 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3065999999999998e-05 2023-10-09T09:59:59.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.593520+0000", 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054397269999999998 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.34797.0:16015 6.e e103/97 ondisk, result = 0)", 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:06.540134+0000", 2023-10-09T09:59:59.614 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.047053103000003, 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051356470000000001, 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.540134+0000", 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.540134+0000", 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.540134+0000", 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4619999999999998e-06 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.540138+0000", 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051105755000000003 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591244+0000", 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3074e-05 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591267+0000", 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114423 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591382+0000", 2023-10-09T09:59:59.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0141e-05 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591392+0000", 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9238000000000001e-05 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591491+0000", 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051356470000000001 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.618 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.34797.0:16021 6.2 e103/97 ondisk, result = 0)", 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:06.540969+0000", 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 53.046218986, 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.050433082999999997, 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.540969+0000", 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.540969+0000", 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.540968+0000", 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5210000000000002e-06 2023-10-09T09:59:59.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.540973+0000", 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.050266536000000001 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591240+0000", 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1812999999999999e-05 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591262+0000", 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119139 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591381+0000", 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9399999999999997e-06 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591391+0000", 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.078e-05 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:06.591402+0000", 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.050433082999999997 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:36581 6.e 6:78a07a32:::benchmark_data_smithi012_182576_object36580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T09:59:59.623 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:17.203437+0000", 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 42.383750782, 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.073460469, 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36581 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203437+0000", 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203437+0000", 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203435+0000", 2023-10-09T09:59:59.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7761000000000004e-05 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203503+0000", 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6582999999999999e-05 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203530+0000", 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9340000000000003e-06 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203536+0000", 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5926000000000001e-05 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203572+0000", 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9265999999999995e-05 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203661+0000", 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104348 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203765+0000", 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001083606 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.204849+0000", 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028363999999999999 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.205132+0000", 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.071660868000000003 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.276793+0000", 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0739999999999998e-05 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.276854+0000", 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3216000000000001e-05 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.276897+0000", 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.073460469 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.630 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:57899 6.2 6:4ae5067f:::benchmark_data_smithi012_182576_object57898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:32.431948+0000", 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 27.155239615999999, 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076099255000000005, 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 57899 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.431948+0000", 2023-10-09T09:59:59.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.431948+0000", 2023-10-09T09:59:59.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:59.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.431946+0000", 2023-10-09T09:59:59.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2293999999999999e-05 2023-10-09T09:59:59.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.431988+0000", 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4036000000000001e-05 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432002+0000", 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8179999999999998e-06 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432009+0000", 2023-10-09T09:59:59.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7142999999999999e-05 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432056+0000", 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103881 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432160+0000", 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136062 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432296+0000", 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001651764 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.433948+0000", 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015851750000000001 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435533+0000", 2023-10-09T09:59:59.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072430576999999996 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507964+0000", 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6255999999999999e-05 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.508010+0000", 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7110999999999998e-05 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.508047+0000", 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076099255000000005 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:57909 6.e 6:77ce4db4:::benchmark_data_smithi012_182576_object57908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:32.435413+0000", 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 27.151774664000001, 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079633525999999996, 2023-10-09T09:59:59.638 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 57909 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435413+0000", 2023-10-09T09:59:59.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435413+0000", 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435412+0000", 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3256999999999994e-05 2023-10-09T09:59:59.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435475+0000", 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1637000000000001e-05 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435486+0000", 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.037e-06 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435492+0000", 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072267670000000006 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507760+0000", 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014061400000000001 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507901+0000", 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015924400000000001 2023-10-09T09:59:59.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.508060+0000", 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036826929999999999 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.511743+0000", 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001201964 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.512945+0000", 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020054690000000002 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.514950+0000", 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3223000000000002e-05 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.515003+0000", 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3075999999999999e-05 2023-10-09T09:59:59.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.515046+0000", 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079633525999999996 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:57903 6.7 e109/97 6:e524b57e:::benchmark_data_smithi012_182576_object57902:head v 109'3509, mlcod=109'3507)", 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:32.435800+0000", 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 27.151387229000001, 2023-10-09T09:59:59.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076747323000000006, 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435800+0000", 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435800+0000", 2023-10-09T09:59:59.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435800+0000", 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3377e-05 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435823+0000", 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.071937872 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507761+0000", 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2327000000000001e-05 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507773+0000", 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9439e-05 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507813+0000", 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5551999999999999e-05 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507828+0000", 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046742590000000001 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.512502+0000", 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5252e-05 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.512548+0000", 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076747323000000006 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:57912 6.f e109/97 6:f301defe:::benchmark_data_smithi012_182576_object57911:head v 109'3645, mlcod=109'3643)", 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:32.441362+0000", 2023-10-09T09:59:59.650 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 27.145825541000001, 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072083867999999995, 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.441362+0000", 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.441362+0000", 2023-10-09T09:59:59.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.441361+0000", 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8736000000000002e-05 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.441410+0000", 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.066370543000000004 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507780+0000", 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0382e-05 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507791+0000", 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3578000000000002e-05 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507834+0000", 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0423000000000001e-05 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507855+0000", 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055641529999999996 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.513419+0000", 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7073e-05 2023-10-09T09:59:59.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.513446+0000", 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072083867999999995 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:58222 6.2 6:4e0cdcba:::benchmark_data_smithi012_182576_object58221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:32.751100+0000", 2023-10-09T09:59:59.655 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 26.836087379999999, 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028560823999999999, 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 58222 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.751100+0000", 2023-10-09T09:59:59.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.751100+0000", 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.751100+0000", 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7678000000000002e-05 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.751127+0000", 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3039999999999999e-06 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.751135+0000", 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5719999999999996e-06 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.751139+0000", 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006202717 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.757342+0000", 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014331200000000001 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.757485+0000", 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000166126 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.757651+0000", 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001643858 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.759295+0000", 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062945589999999999 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.765590+0000", 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013966063000000001 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.779556+0000", 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1941999999999997e-05 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.779618+0000", 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3343999999999997e-05 2023-10-09T09:59:59.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.779661+0000", 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028560823999999999 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:58488 6.e 6:79631276:::benchmark_data_smithi012_182576_object58487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:33.004372+0000", 2023-10-09T09:59:59.662 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 26.582815457999999, 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029581486000000001, 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 58488 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.004372+0000", 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.004372+0000", 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.004372+0000", 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1617999999999999e-05 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.004393+0000", 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6490000000000002e-06 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.004400+0000", 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7180000000000001e-06 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.004403+0000", 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036956060000000002 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.008098+0000", 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012804 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.008226+0000", 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018384400000000001 2023-10-09T09:59:59.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.008410+0000", 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002152686 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.010563+0000", 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010764602999999999 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.021327+0000", 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012571636000000001 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.033899+0000", 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4054000000000003e-05 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.033933+0000", 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0567000000000001e-05 2023-10-09T09:59:59.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.033954+0000", 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029581486000000001 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:73931 6.e 6:7fa12923:::benchmark_data_smithi012_182576_object73930:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T09:59:59.669 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:46.871233+0000", 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 12.715954642, 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028574550000000001, 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 73931 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871233+0000", 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871233+0000", 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871231+0000", 2023-10-09T09:59:59.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9330999999999998e-05 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871291+0000", 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3067e-05 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871304+0000", 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4180000000000002e-06 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871310+0000", 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4724999999999997e-05 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871345+0000", 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118168 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871463+0000", 2023-10-09T09:59:59.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000178777 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.871642+0000", 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035236120000000002 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.875166+0000", 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013628675999999999 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.888794+0000", 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010912392 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.899707+0000", 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8279999999999998e-05 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.899765+0000", 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2622e-05 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:46.899807+0000", 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028574550000000001 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:76967 6.2 6:410a2270:::benchmark_data_smithi012_182576_object76966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:49.823152+0000", 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 9.7640355860000003, 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028973375999999999, 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 76967 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.677 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823152+0000", 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823152+0000", 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823150+0000", 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0340000000000003e-05 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823201+0000", 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.199e-05 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823213+0000", 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7100000000000001e-06 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823219+0000", 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8532000000000002e-05 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823268+0000", 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9258000000000006e-05 2023-10-09T09:59:59.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823347+0000", 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015548299999999999 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.823503+0000", 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030745920000000001 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.826577+0000", 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081960420000000006 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.834773+0000", 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017286642000000001 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.852060+0000", 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8132e-05 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.852098+0000", 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7370000000000001e-05 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:49.852125+0000", 2023-10-09T09:59:59.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028973375999999999 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:79050 6.2 6:48a7c353:::benchmark_data_smithi012_182576_object79049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:51.894368+0000", 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 7.6928200589999998, 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028186948, 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.684 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79050 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894368+0000", 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894368+0000", 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894365+0000", 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8152000000000007e-05 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894443+0000", 2023-10-09T09:59:59.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6710000000000001e-05 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894460+0000", 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9080000000000001e-06 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894469+0000", 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8152999999999997e-05 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894527+0000", 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000141538 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894668+0000", 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000176804 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.894845+0000", 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024292720000000001 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.897274+0000", 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019029310000000001 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.899177+0000", 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023303879 2023-10-09T09:59:59.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.922481+0000", 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8606999999999997e-05 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.922530+0000", 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4556999999999999e-05 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:51.922554+0000", 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028186948 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:79848 6.2 6:46e3aa5b:::benchmark_data_smithi012_182576_object79847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:52.664557+0000", 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 6.9226306119999999, 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.032666215999999998, 2023-10-09T09:59:59.691 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79848 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.664557+0000", 2023-10-09T09:59:59.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.664557+0000", 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.664555+0000", 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.606e-05 2023-10-09T09:59:59.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.664631+0000", 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6265999999999999e-05 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.664658+0000", 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2490000000000001e-06 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.664666+0000", 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7141000000000001e-05 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.664723+0000", 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014636799999999999 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.664869+0000", 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018987499999999999 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.665059+0000", 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051690019999999998 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.670228+0000", 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070331769999999998 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.677261+0000", 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019885371999999998 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.697147+0000", 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4542999999999997e-05 2023-10-09T09:59:59.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.697191+0000", 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1918000000000002e-05 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.697223+0000", 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.032666215999999998 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.698 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:79859 6.2 6:417ddb6e:::benchmark_data_smithi012_182576_object79858:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:52.665754+0000", 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 6.9214336210000003, 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031584279, 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.699 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79859 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.665754+0000", 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.665754+0000", 2023-10-09T09:59:59.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.665753+0000", 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1081999999999996e-05 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.665814+0000", 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3760999999999999e-05 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.665828+0000", 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4479999999999999e-06 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.665834+0000", 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001761814 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.667596+0000", 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000145823 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.667742+0000", 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016520799999999999 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.667907+0000", 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024630709999999998 2023-10-09T09:59:59.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.670370+0000", 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070360819999999999 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.677406+0000", 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019857535999999999 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.697264+0000", 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8228000000000001e-05 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.697302+0000", 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6145e-05 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.697338+0000", 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031584279 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:79864 6.2 6:4cfdbf3a:::benchmark_data_smithi012_182576_object79863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:52.673547+0000", 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 6.9136401310000002, 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031843477000000002, 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.706 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 79864 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.673547+0000", 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.673547+0000", 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.673546+0000", 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0325000000000003e-05 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.673606+0000", 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6901999999999998e-05 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.673623+0000", 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7400000000000004e-06 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.673631+0000", 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5779000000000003e-05 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.673677+0000", 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000157245 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.673834+0000", 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000177861 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.674012+0000", 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002309654 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.676321+0000", 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001436215 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.677758+0000", 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027572287000000001 2023-10-09T09:59:59.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.705330+0000", 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7261999999999997e-05 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.705367+0000", 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3771999999999999e-05 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:52.705391+0000", 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031843477000000002 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:80749 6.2 6:43fd11ad:::benchmark_data_smithi012_182576_object80748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:53.526857+0000", 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 6.0603308419999999, 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028096034999999998, 2023-10-09T09:59:59.713 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 80749 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.526857+0000", 2023-10-09T09:59:59.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.526857+0000", 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.526856+0000", 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7572999999999998e-05 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.526884+0000", 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9580000000000007e-06 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.526892+0000", 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5820000000000003e-06 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T09:59:59.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.526896+0000", 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023335180000000001 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.529230+0000", 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000131693 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.529361+0000", 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000163556 2023-10-09T09:59:59.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.529525+0000", 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048125529999999998 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.534338+0000", 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040027370000000001 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.538340+0000", 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016509375999999999 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.554850+0000", 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8158999999999997e-05 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.554908+0000", 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4969000000000003e-05 2023-10-09T09:59:59.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:53.554953+0000", 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028096034999999998 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T09:59:59.720 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T09:59:59.721 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T09:59:59.721 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:59.721 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:59:59.721 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T09:59:59.721 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.721 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.722 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.722 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.722 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.722 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:59.722 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 09:59:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2968679585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:59.722 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T09:59:59.722 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T09:59:59.722 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T09:59:59.722 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T09:59:59.722 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T09:59:59.723 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T09:59:59.723 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T09:59:59.723 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T09:59:59.723 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T09:59:59.887 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T09:59:59.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T09:59:59.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T09:59:59.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2968679585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:59.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T09:59:59.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T09:59:59.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T09:59:59.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: failed to encode map e114 with expected crc 2023-10-09T09:59:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T09:59:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 09:59:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2968679585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T09:59:59.990 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:00.038 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:00.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:00:00.107 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:00.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:00.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:00.267 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:00.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:00.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:00.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:00.376 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:00:00.376595+0000 min lat: 0.0011673 max lat: 0.442574 avg lat: 0.010942 2023-10-09T10:00:00.376 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:00:00.376 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 60 16 87754 87738 91.3477 64.125 0.0152445 0.010942 2023-10-09T10:00:00.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:00.498 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:00.569 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:00.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:00.679 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:00.744 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:00.817 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:00.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:00.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:00.877 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:00.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:00.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:00 smithi012 ceph-mon[147926]: pgmap v282: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T10:00:00.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:00 smithi012 ceph-mon[147926]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T10:00:00.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:00 smithi012 ceph-mon[147926]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T10:00:00.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:00 smithi012 ceph-mon[152263]: pgmap v282: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T10:00:00.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:00 smithi012 ceph-mon[152263]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T10:00:00.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:00 smithi012 ceph-mon[152263]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T10:00:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:00 smithi110 ceph-mon[136032]: pgmap v282: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T10:00:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:00 smithi110 ceph-mon[136032]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T10:00:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:00 smithi110 ceph-mon[136032]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T10:00:00.988 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:01.055 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:01.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:01.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:01.231 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:01.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:00:01.376 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 61 16 88809 88793 90.9312 65.9375 0.0155741 0.0109909 2023-10-09T10:00:02.009 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:02.196 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:02.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:02.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:00:02.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:02.376 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 62 16 89827 89811 90.4908 63.625 0.0091698 0.0110451 2023-10-09T10:00:02.407 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:02.490 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:02.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:02.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:02.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:02.618 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:02 smithi110 ceph-mon[136032]: pgmap v283: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T10:00:02.686 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:02.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:02 smithi012 ceph-mon[147926]: pgmap v283: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T10:00:02.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:02 smithi012 ceph-mon[152263]: pgmap v283: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T10:00:03.021 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:03.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:03.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:03.183 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:03.244 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:03.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:03.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:03.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:03.360 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:03.379 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 63 16 90732 90716 89.9481 56.5625 0.0182924 0.0111129 2023-10-09T10:00:03.419 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:03.634 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:03 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:00:03.791 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:03.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:03.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:03.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:03.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:03 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:00:03.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:03 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:00:03.961 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:04.038 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:00:04.038 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:00:04.039 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:00:04.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T10:00:04.276 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:00:04.276 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:00:04.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:91452 6.7 6:e09b2ecd:::benchmark_data_smithi012_182576_object91451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:04.263482+0000", 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.012195498000000001, 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012231158000000001, 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 91452 2023-10-09T10:00:04.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263482+0000", 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263482+0000", 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999957 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263478+0000", 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3903999999999998e-05 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263532+0000", 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1656000000000001e-05 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263553+0000", 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5080000000000005e-06 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263562+0000", 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7022999999999998e-05 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263629+0000", 2023-10-09T10:00:04.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017550899999999999 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263804+0000", 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000190323 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.263995+0000", 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033565779999999998 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.267351+0000", 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054747219999999996 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.272826+0000", 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0093438429999999992 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:91463 6.4 e114/97 6:2b51f941:::benchmark_data_smithi012_182576_object91462:head v 114'5704, mlcod=114'5702)", 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:04.274835+0000", 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00084257699999999995, 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000960935, 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.274835+0000", 2023-10-09T10:00:04.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.274835+0000", 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.274835+0000", 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3497999999999997e-05 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.274868+0000", 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3320000000000003e-06 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.274875+0000", 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3080000000000001e-06 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.274880+0000", 2023-10-09T10:00:04.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5913000000000004e-05 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.274946+0000", 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8961e-05 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:04.274965+0000", 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000129584 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:04.286 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:04.287 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:04.287 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:00:04.287 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 2 2023-10-09T10:00:04.287 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:00:04.370 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:04.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:04.379 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 64 16 91552 91536 89.3437 51.25 0.0199739 0.0111857 2023-10-09T10:00:04.478 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:04.515 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:04.644 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:04 smithi110 ceph-mon[136032]: pgmap v284: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:00:04.644 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/862470008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:04.644 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:04 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:04.675 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:00:04.719 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:04.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:04.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:04.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:00:04.822 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:04.899 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:04 smithi012 ceph-mon[152263]: pgmap v284: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:00:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/862470008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:04 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:04.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:04 smithi012 ceph-mon[147926]: pgmap v284: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:00:04.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/862470008' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:04.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:04 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:04.962 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:04.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:05.070 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:05.118 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:00:05.138 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:05.158 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:00:05.194 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:00:05.239 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:00:05.273 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:00:05.303 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:05.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:05.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:05.327 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:00:05.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:00:05.379 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 65 16 92444 92428 88.827 55.75 0.00835516 0.0112527 2023-10-09T10:00:05.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:05.408 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:00:05.418 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:00:05.418 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:00:05.419 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:00:05.419 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:00:05.419 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:00:05.419 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:00:05.419 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:00:05.419 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:00:05.481 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:05.606 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[147926]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[147926]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[147926]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[147926]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/238906259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:05.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:05.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[152263]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T10:00:05.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[152263]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[152263]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[152263]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/238906259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:05.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:05 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:05.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:05 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:05.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:05 smithi110 ceph-mon[136032]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T10:00:05.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:05 smithi110 ceph-mon[136032]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:05 smithi110 ceph-mon[136032]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.934 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:05 smithi110 ceph-mon[136032]: failed to encode map e115 with expected crc 2023-10-09T10:00:05.935 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/238906259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:06.261 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:06.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:06.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:06.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:06.384 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 66 16 93397 93381 88.3768 59.5625 0.00825795 0.0113101 2023-10-09T10:00:06.477 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:06.556 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:06.560 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:06.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:06.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:06.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:06 smithi110 ceph-mon[136032]: pgmap v286: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T10:00:06.716 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:06.790 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:06 smithi012 ceph-mon[147926]: pgmap v286: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T10:00:06.791 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:06 smithi012 ceph-mon[152263]: pgmap v286: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T10:00:06.877 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:06.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:06.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:07.052 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:07.115 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:07.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:07.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:07.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:07.286 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:07.287 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:07.287 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:07.385 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 67 16 94280 94264 87.8816 55.1875 0.00903675 0.011373 2023-10-09T10:00:07.466 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:07.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:07.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:07.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:07.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:07.711 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:00:07.782 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:07.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:07.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:07.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:07.955 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:08.040 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:08.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:08.049 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:00:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:00:07] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T10:00:08.150 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:08.209 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:08.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:00:08.280 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:08.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:08.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:08.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:00:08.386 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:08.402 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 68 16 95124 95108 87.3651 52.75 0.0227453 0.0114408 2023-10-09T10:00:08.417 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:08.468 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:08.517 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:08.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:08.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:08.602 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:00:08.499+0000 7f85a99d3700 -1 mon.a@0(leader).osd e115 definitely_dead 0 2023-10-09T10:00:08.602 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-mon[147926]: pgmap v287: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T10:00:08.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1148790542' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:00:08.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:00:08.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-mon[147926]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:00:08.603 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-mon[152263]: pgmap v287: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T10:00:08.603 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1148790542' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:00:08.604 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:00:08.604 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:08 smithi012 ceph-mon[152263]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:00:08.636 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:08.678 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:08.728 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.2. 2023-10-09T10:00:08.826 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:08.827 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:08 smithi110 ceph-mon[136032]: pgmap v287: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T10:00:08.827 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1148790542' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:00:08.827 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:00:08.828 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:08 smithi110 ceph-mon[136032]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:00:08.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:08.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:08.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:08.957 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:09.125 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:09.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:09.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:09.229 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:09.257 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:09.385 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 69 16 95366 95350 86.3186 15.125 0.00498467 0.0114688 2023-10-09T10:00:09.393 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:09.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:09.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:09.432 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:09.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:09.529 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:09.613 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:09.766 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:09.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:09.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:09.780 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:09 smithi110 ceph-mon[136032]: failed to encode map e116 with expected crc 2023-10-09T10:00:09.780 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:09 smithi110 ceph-mon[136032]: failed to encode map e116 with expected crc 2023-10-09T10:00:09.780 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:09 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:00:09.781 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:09 smithi110 ceph-mon[136032]: osdmap e116: 8 total, 8 up, 7 in 2023-10-09T10:00:09.781 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:09 smithi110 ceph-mon[136032]: pgmap v289: 89 pgs: 89 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T10:00:09.871 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:09.899 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:09.944 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:09.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:09.996 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:09.996 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:10.001 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[147926]: failed to encode map e116 with expected crc 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[147926]: failed to encode map e116 with expected crc 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[147926]: osdmap e116: 8 total, 8 up, 7 in 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[147926]: pgmap v289: 89 pgs: 89 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[152263]: failed to encode map e116 with expected crc 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[152263]: failed to encode map e116 with expected crc 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[152263]: osdmap e116: 8 total, 8 up, 7 in 2023-10-09T10:00:10.002 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:09 smithi012 ceph-mon[152263]: pgmap v289: 89 pgs: 89 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T10:00:10.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:10.083 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:10.220 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:10.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:10.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:10.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:10.351 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:10.385 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 70 16 95366 95350 85.0861 0 - 0.0114688 2023-10-09T10:00:10.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:00:10.661 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:10.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:10.767 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:10.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:10.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: osdmap e117: 8 total, 8 up, 7 in 2023-10-09T10:00:10.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:10.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:10.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: osdmap e117: 8 total, 8 up, 7 in 2023-10-09T10:00:10.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:10.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:10 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:10.956 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:10.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:10.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: osdmap e117: 8 total, 8 up, 7 in 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: failed to encode map e117 with expected crc 2023-10-09T10:00:10.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:10.978 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:10 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:11.076 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:11.120 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:11.138 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:11.309 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:11.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:11.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:11.386 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 71 16 95582 95566 84.0767 6.75 0.0291534 0.0118878 2023-10-09T10:00:11.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:11.545 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:11.593 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:00:11.617 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:11.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:11.637 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:00:11.689 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:00:11.728 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:11.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:00:11.796 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:11.805 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:00:11.865 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:00:11.931 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: failed to encode map e118 with expected crc 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: failed to encode map e118 with expected crc 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: failed to encode map e118 with expected crc 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: osdmap e118: 8 total, 8 up, 7 in 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: failed to encode map e118 with expected crc 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: failed to encode map e118 with expected crc 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: pgmap v292: 89 pgs: 4 remapped+peering, 85 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 644 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:11.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2848891695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:11.984 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:00:11.992 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:00:11.993 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:00:11.993 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:00:11.993 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:00:11.993 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:00:11.993 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:00:11.993 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:00:11.993 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:00:11.994 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:12.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:12.031 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: osdmap e118: 8 total, 8 up, 7 in 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: pgmap v292: 89 pgs: 4 remapped+peering, 85 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 644 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2848891695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:12.032 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: osdmap e118: 8 total, 8 up, 7 in 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: failed to encode map e118 with expected crc 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: pgmap v292: 89 pgs: 4 remapped+peering, 85 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 644 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:12.033 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2848891695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:12.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:12.161 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:12.384 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:12.386 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 72 16 96183 96167 83.4309 37.5625 0.0443755 0.0119804 2023-10-09T10:00:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:12.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:12.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:12.568 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:12.616 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:12.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96391 6.2 e118/117 6:4b593417:::benchmark_data_smithi012_182576_object96390:head v 118'6103, mlcod=0'0)", 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:12.690994+0000", 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0086260799999999995, 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086579059999999999, 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:12.702 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.690994+0000", 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.690994+0000", 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.690993+0000", 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5055999999999997e-05 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691038+0000", 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6382e-05 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691054+0000", 2023-10-09T10:00:12.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6080000000000001e-06 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691061+0000", 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7023e-05 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691108+0000", 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8881999999999999e-05 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691137+0000", 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000142965 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96383 6.8 e118/117 6:157c85ce:::benchmark_data_smithi012_182576_object96382:head v 118'6137, mlcod=0'0)", 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:12.691114+0000", 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0085055489999999994, 2023-10-09T10:00:12.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086062059999999999, 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691114+0000", 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691114+0000", 2023-10-09T10:00:12.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691114+0000", 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1124999999999998e-05 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691135+0000", 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6170000000000001e-06 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691143+0000", 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5789999999999997e-06 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691147+0000", 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3930999999999995e-05 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691211+0000", 2023-10-09T10:00:12.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2849000000000001e-05 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.691234+0000", 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011957 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96382 6.f e118/117 6:fee656a3:::benchmark_data_smithi012_182576_object96381:head v 118'5993, mlcod=0'0)", 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:12.698316+0000", 2023-10-09T10:00:12.710 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00130405, 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014657959999999999, 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698316+0000", 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698316+0000", 2023-10-09T10:00:12.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698315+0000", 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1094999999999998e-05 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698366+0000", 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0192000000000002e-05 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698396+0000", 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9140000000000008e-06 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698404+0000", 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020073500000000001 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698605+0000", 2023-10-09T10:00:12.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8331e-05 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698633+0000", 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000317027 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96384 6.f e118/117)", 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:12.698418+0000", 2023-10-09T10:00:12.714 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0012018319999999999, 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014196860000000001, 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698418+0000", 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:12.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698418+0000", 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698418+0000", 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4288000000000001e-05 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698452+0000", 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698458+0000", 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8290000000000001e-06 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698462+0000", 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3887999999999997e-05 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96385 6.f e118/117)", 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:12.698466+0000", 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.001154012, 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014196759999999999, 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698466+0000", 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:12.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698466+0000", 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698466+0000", 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4958999999999999e-05 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:12.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698491+0000", 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0780000000000003e-06 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698496+0000", 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3189999999999999e-06 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.698499+0000", 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3104e-05 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.720 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:00:12.721 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 5 2023-10-09T10:00:12.721 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:00:12.729 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:12.775 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:12.830 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:12.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:12.934 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:12.997 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:12 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:00:12.997 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:12 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:00:12.997 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:12.997 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3233234025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:12.997 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:12 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:13.007 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:00:13.013 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:13.038 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:00:13.039 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:00:13.039 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:13.039 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3233234025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:13.039 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:13.039 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:00:13.039 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:00:13.039 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:13.039 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3233234025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:13.040 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:12 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:13.105 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:13.113 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:00:13.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:13.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:13.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:13.295 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:13.386 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 73 16 96800 96784 82.8165 38.5625 0.00521393 0.0120652 2023-10-09T10:00:13.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:13.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:13.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:13.502 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:13.616 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:13.688 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:13.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:13.802 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:13.903 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:13 smithi110 ceph-mon[136032]: pgmap v293: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 82 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 11850/287064 objects degraded (4.128%); 5878/287064 objects misplaced (2.048%); 256 KiB/s, 1 keys/s, 4 objects/s recovering 2023-10-09T10:00:13.904 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:13 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:13.904 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:13 smithi110 ceph-mon[136032]: failed to encode map e119 with expected crc 2023-10-09T10:00:13.904 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:00:13.904 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:13 smithi110 ceph-mon[136032]: osdmap e119: 8 total, 8 up, 7 in 2023-10-09T10:00:13.904 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:13 smithi110 ceph-mon[136032]: failed to encode map e119 with expected crc 2023-10-09T10:00:13.925 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:13.976 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:14.025 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:14.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:14.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:14.134 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:14.172 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[147926]: pgmap v293: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 82 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 11850/287064 objects degraded (4.128%); 5878/287064 objects misplaced (2.048%); 256 KiB/s, 1 keys/s, 4 objects/s recovering 2023-10-09T10:00:14.172 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:14.172 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[147926]: failed to encode map e119 with expected crc 2023-10-09T10:00:14.172 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:00:14.172 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[147926]: osdmap e119: 8 total, 8 up, 7 in 2023-10-09T10:00:14.172 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[147926]: failed to encode map e119 with expected crc 2023-10-09T10:00:14.173 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[152263]: pgmap v293: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 82 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 11850/287064 objects degraded (4.128%); 5878/287064 objects misplaced (2.048%); 256 KiB/s, 1 keys/s, 4 objects/s recovering 2023-10-09T10:00:14.173 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:14.173 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[152263]: failed to encode map e119 with expected crc 2023-10-09T10:00:14.173 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:00:14.173 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[152263]: osdmap e119: 8 total, 8 up, 7 in 2023-10-09T10:00:14.173 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:13 smithi012 ceph-mon[152263]: failed to encode map e119 with expected crc 2023-10-09T10:00:14.175 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:14.205 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:00:14.205 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:00:14.206 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:00:14.206 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.611993 2023-10-09T10:00:14.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0.6119925725472329 2023-10-09T10:00:14.238 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:14.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:14.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:14.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:14.387 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 74 16 97399 97383 82.2035 37.4375 0.0552341 0.0121601 2023-10-09T10:00:14.412 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:14.448 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:00:14.491 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:14.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:14.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:14.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:14.662 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:14.747 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:14.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:14.760 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:14 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 11850/287064 objects degraded (4.128%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:14.760 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:14.760 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:14.760 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:14 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 11850/287064 objects degraded (4.128%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:14.760 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:14.760 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:14.860 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:14.921 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:14 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 11850/287064 objects degraded (4.128%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:14.921 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:14.921 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:14.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:14.995 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:15.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:15.047 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:15.113 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:15.190 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:15.200 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:15.258 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:15.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:15.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:15.368 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:15.386 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 75 16 98014 97998 81.6202 38.4375 0.0146921 0.0122453 2023-10-09T10:00:15.441 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:00:15.443 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:15.518 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:15.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:15.581 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:00:15.581 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:00:15.582 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:00:15.582 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:00:15.582 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:00:15.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:15.682 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:15.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:15 smithi012 ceph-mon[147926]: pgmap v295: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 35935/291126 objects degraded (12.343%); 11771/291126 objects misplaced (4.043%); 1.2 MiB/s, 1 keys/s, 19 objects/s recovering 2023-10-09T10:00:15.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:15 smithi012 ceph-mon[152263]: pgmap v295: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 35935/291126 objects degraded (12.343%); 11771/291126 objects misplaced (4.043%); 1.2 MiB/s, 1 keys/s, 19 objects/s recovering 2023-10-09T10:00:16.031 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:15 smithi110 ceph-mon[136032]: pgmap v295: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 35935/291126 objects degraded (12.343%); 11771/291126 objects misplaced (4.043%); 1.2 MiB/s, 1 keys/s, 19 objects/s recovering 2023-10-09T10:00:16.079 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:00:16.387 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 76 16 98622 98606 81.0465 38 0.0116316 0.0123336 2023-10-09T10:00:16.457 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:16.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:16.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:16.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:16.690 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:16.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:16.737 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:16.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:16.871 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:17.028 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:17.264 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:17.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:17.347 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:00:17.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:17.387 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:00:17.388 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 77 16 99199 99183 80.4613 36.0625 0.0286138 0.0124213 2023-10-09T10:00:17.415 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:00:17.437 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:17.449 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:00:17.479 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:00:17.511 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:17.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:17.540 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:00:17.570 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:00:17.610 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:00:17.620 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:00:17.620 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:00:17.620 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:00:17.620 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:00:17.620 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:00:17.620 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:00:17.620 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:00:17.620 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:00:17.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:17.683 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:18.063 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:18.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:18.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:18.162 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:18 smithi012 ceph-mon[147926]: pgmap v296: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 588 op/s; 35935/292227 objects degraded (12.297%); 11572/292227 objects misplaced (3.960%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-10-09T10:00:18.162 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4029470341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:18.162 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1162204996' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]: dispatch 2023-10-09T10:00:18.162 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:18 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]: dispatch 2023-10-09T10:00:18.162 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:00:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:00:17] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T10:00:18.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:18 smithi012 ceph-mon[152263]: pgmap v296: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 588 op/s; 35935/292227 objects degraded (12.297%); 11572/292227 objects misplaced (3.960%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-10-09T10:00:18.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4029470341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:18.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1162204996' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]: dispatch 2023-10-09T10:00:18.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:18 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]: dispatch 2023-10-09T10:00:18.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:18.227 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:18.298 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:18.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:18.312 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:18 smithi110 ceph-mon[136032]: pgmap v296: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 588 op/s; 35935/292227 objects degraded (12.297%); 11572/292227 objects misplaced (3.960%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-10-09T10:00:18.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4029470341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:18.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1162204996' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]: dispatch 2023-10-09T10:00:18.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:18 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]: dispatch 2023-10-09T10:00:18.390 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 78 16 99803 99787 79.9125 37.75 0.0408187 0.0125076 2023-10-09T10:00:18.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:18.479 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:18.479 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:18.528 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:18.632 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:18.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:18.651 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.7 primary-affinity to 0.611993 (8401072) 2023-10-09T10:00:18.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:18.800 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:18.982 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:00:19.026 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:19.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:19.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:19.130 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:19.156 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:19.191 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:19.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:19.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:19.295 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:19.322 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:19.390 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 79 16 100023 100007 79.0753 13.75 0.0040933 0.0125331 2023-10-09T10:00:19.409 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:19.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:19.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:19.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:19.543 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:19.582 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:19.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:19.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:19.600 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:19.635 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:00:19.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:19.724 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:19.759 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:19.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:19.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:19.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[152263]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]': finished 2023-10-09T10:00:19.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[152263]: osdmap e120: 8 total, 8 up, 7 in 2023-10-09T10:00:19.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[152263]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[152263]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1389477210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:19.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:19.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[147926]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]': finished 2023-10-09T10:00:19.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[147926]: osdmap e120: 8 total, 8 up, 7 in 2023-10-09T10:00:19.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[147926]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[147926]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1389477210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:19.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:19 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:19.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:19.892 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:19 smithi110 ceph-mon[136032]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:19 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6119925725472329}]': finished 2023-10-09T10:00:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:19 smithi110 ceph-mon[136032]: osdmap e120: 8 total, 8 up, 7 in 2023-10-09T10:00:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:19 smithi110 ceph-mon[136032]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:19 smithi110 ceph-mon[136032]: failed to encode map e120 with expected crc 2023-10-09T10:00:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1389477210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:19 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:20.011 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:20.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:20.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:20.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:20.147 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:20.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:00:20.377 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:20.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:20.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:20.389 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:00:20.390498+0000 min lat: 0.0011673 max lat: 2.36676 avg lat: 0.0125331 2023-10-09T10:00:20.389 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:00:20.390 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 80 16 100023 100007 78.0874 0 - 0.0125331 2023-10-09T10:00:20.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:20.511 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:20.546 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:20.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:20.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: pgmap v298: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 458 op/s; 35935/295665 objects degraded (12.154%); 11572/295665 objects misplaced (3.914%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 35935/295665 objects degraded (12.154%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.635 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.636 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:20 smithi110 ceph-mon[136032]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.650 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:20.681 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:20.722 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:20.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:20.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:20.828 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:20.830 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:20.853 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:20.853 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: pgmap v298: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 458 op/s; 35935/295665 objects degraded (12.154%); 11572/295665 objects misplaced (3.914%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 35935/295665 objects degraded (12.154%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[147926]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: pgmap v298: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 458 op/s; 35935/295665 objects degraded (12.154%); 11572/295665 objects misplaced (3.914%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 35935/295665 objects degraded (12.154%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:20 smithi012 ceph-mon[152263]: failed to encode map e121 with expected crc 2023-10-09T10:00:20.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:21.015 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:21.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:21.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:21.182 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:21.235 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:21.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:21.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:21.265 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:00:21.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:21.389 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 81 16 100408 100392 77.4207 12.0312 0.0202679 0.0129098 2023-10-09T10:00:21.406 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:21.643 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:21 smithi110 ceph-mon[136032]: failed to encode map e122 with expected crc 2023-10-09T10:00:21.643 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:21 smithi110 ceph-mon[136032]: osdmap e122: 8 total, 8 up, 7 in 2023-10-09T10:00:21.643 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:21 smithi110 ceph-mon[136032]: failed to encode map e122 with expected crc 2023-10-09T10:00:21.643 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:21 smithi110 ceph-mon[136032]: failed to encode map e122 with expected crc 2023-10-09T10:00:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:21 smithi012 ceph-mon[147926]: failed to encode map e122 with expected crc 2023-10-09T10:00:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:21 smithi012 ceph-mon[147926]: osdmap e122: 8 total, 8 up, 7 in 2023-10-09T10:00:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:21 smithi012 ceph-mon[147926]: failed to encode map e122 with expected crc 2023-10-09T10:00:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:21 smithi012 ceph-mon[147926]: failed to encode map e122 with expected crc 2023-10-09T10:00:21.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:21 smithi012 ceph-mon[152263]: failed to encode map e122 with expected crc 2023-10-09T10:00:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:21 smithi012 ceph-mon[152263]: osdmap e122: 8 total, 8 up, 7 in 2023-10-09T10:00:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:21 smithi012 ceph-mon[152263]: failed to encode map e122 with expected crc 2023-10-09T10:00:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:21 smithi012 ceph-mon[152263]: failed to encode map e122 with expected crc 2023-10-09T10:00:22.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:00:22.149 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:22.390 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 82 16 101052 101036 76.9675 40.25 0.0169511 0.0129863 2023-10-09T10:00:22.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:22 smithi012 ceph-mon[147926]: pgmap v301: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 26 MiB/s wr, 420 op/s; 35935/298689 objects degraded (12.031%); 11400/298689 objects misplaced (3.817%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T10:00:22.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:22 smithi012 ceph-mon[152263]: pgmap v301: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 26 MiB/s wr, 420 op/s; 35935/298689 objects degraded (12.031%); 11400/298689 objects misplaced (3.817%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T10:00:22.949 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:22 smithi110 ceph-mon[136032]: pgmap v301: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 26 MiB/s wr, 420 op/s; 35935/298689 objects degraded (12.031%); 11400/298689 objects misplaced (3.817%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T10:00:23.392 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 83 16 101680 101664 76.5113 39.25 0.013097 0.0130638 2023-10-09T10:00:23.688 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:23.688 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:23.796 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:23.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:23.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:23.972 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:24.060 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:00:24.060 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:00:24.060 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T10:00:24.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T10:00:24.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:00:24.395 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 84 16 102298 102282 76.0574 38.625 0.0284854 0.0131411 2023-10-09T10:00:24.760 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:24.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:24.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:24.866 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:24.875 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:00:24.908 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:24.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:24 smithi012 ceph-mon[147926]: pgmap v302: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32 MiB/s wr, 516 op/s; 35935/301518 objects degraded (11.918%); 11136/301518 objects misplaced (3.693%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:00:24.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:24 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:24.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:24 smithi012 ceph-mon[152263]: pgmap v302: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32 MiB/s wr, 516 op/s; 35935/301518 objects degraded (11.918%); 11136/301518 objects misplaced (3.693%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:00:24.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:24 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:24.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:24 smithi110 ceph-mon[136032]: pgmap v302: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32 MiB/s wr, 516 op/s; 35935/301518 objects degraded (11.918%); 11136/301518 objects misplaced (3.693%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:00:24.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:24 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:00:25.034 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:25.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:25.142 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:25.189 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:25.233 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:25.312 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:25.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:25.396 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 85 16 102903 102887 75.6077 37.8125 0.0137425 0.0132201 2023-10-09T10:00:25.425 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:25.484 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:25.592 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:25.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:25.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:25.705 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:25.786 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:25.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:25.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:25.908 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:25 smithi110 ceph-mon[136032]: pgmap v303: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s; 35935/305550 objects degraded (11.761%); 10973/305550 objects misplaced (3.591%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T10:00:25.908 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:25 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 35935/305550 objects degraded (11.761%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:26.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:26.038 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:25 smithi012 ceph-mon[147926]: pgmap v303: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s; 35935/305550 objects degraded (11.761%); 10973/305550 objects misplaced (3.591%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T10:00:26.038 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:25 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 35935/305550 objects degraded (11.761%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:26.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:25 smithi012 ceph-mon[152263]: pgmap v303: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s; 35935/305550 objects degraded (11.761%); 10973/305550 objects misplaced (3.591%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T10:00:26.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:25 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 35935/305550 objects degraded (11.761%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:26.081 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:26.082 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:00:26.115 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:00:26.149 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:00:26.177 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:00:26.186 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:26.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:26.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:26.223 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:00:26.267 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:00:26.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:26.329 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:00:26.366 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:00:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:00:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:00:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:00:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:00:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:00:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:00:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:00:26.368 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:00:26.396 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:26.401 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 86 16 103499 103483 75.1568 37.25 0.0357194 0.0132992 2023-10-09T10:00:26.509 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:00:26.509 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:00:26.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:00:26.509 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:00:26.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.509 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13895 6.3 6:cb4c122f:::benchmark_data_smithi012_182576_object13894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.959247477999995, 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.111765246, 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13895 2023-10-09T10:00:26.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544303+0000", 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7165999999999999e-05 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544380+0000", 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1870000000000003e-06 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544389+0000", 2023-10-09T10:00:26.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6619999999999996e-06 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544394+0000", 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2271999999999998e-05 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544436+0000", 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138376 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544575+0000", 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000172697 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544747+0000", 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019732679999999998 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546721+0000", 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00053507399999999997 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.547256+0000", 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10869904900000001 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655955+0000", 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1564999999999998e-05 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656016+0000", 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2889999999999997e-05 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656069+0000", 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.111765246 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.516 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13902 6.5 6:a6447830:::benchmark_data_smithi012_182576_object13901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.959146795999999, 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.112868624, 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.517 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.520 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13902 2023-10-09T10:00:26.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.520 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544404+0000", 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103757 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544508+0000", 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3230999999999999e-05 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544521+0000", 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6330000000000004e-06 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544526+0000", 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4892000000000003e-05 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544570+0000", 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107476 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544678+0000", 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010674400000000001 2023-10-09T10:00:26.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544785+0000", 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022269719999999998 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.547012+0000", 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001912513 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.548924+0000", 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108253376 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657178+0000", 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4778000000000003e-05 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657232+0000", 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0898e-05 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657273+0000", 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.112868624 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13907 6.3 6:c0d71b15:::benchmark_data_smithi012_182576_object13906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.953834416000007, 2023-10-09T10:00:26.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108893135, 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13907 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549716+0000", 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9034999999999999e-05 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549745+0000", 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1456e-05 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549756+0000", 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0289999999999996e-06 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549761+0000", 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6547000000000001e-05 2023-10-09T10:00:26.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549798+0000", 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012402 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549922+0000", 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021892400000000001 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550141+0000", 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017843049999999999 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551925+0000", 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031652469999999999 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555090+0000", 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10346385299999999 2023-10-09T10:00:26.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658554+0000", 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1267999999999999e-05 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658585+0000", 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4669e-05 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658610+0000", 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108893135 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13908 6.5 6:aed1463a:::benchmark_data_smithi012_182576_object13907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.953781833999997, 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.107611733, 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.533 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13908 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549769+0000", 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7673000000000001e-05 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549797+0000", 2023-10-09T10:00:26.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4170000000000003e-06 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549804+0000", 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.826e-06 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549809+0000", 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9627999999999999e-05 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549869+0000", 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000173229 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550042+0000", 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019173700000000001 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550234+0000", 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099897300000000001 2023-10-09T10:00:26.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551233+0000", 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021422310000000001 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.553375+0000", 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.103950349 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657325+0000", 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2319e-05 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657358+0000", 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3770000000000001e-05 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657381+0000", 2023-10-09T10:00:26.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.107611733 2023-10-09T10:00:26.540 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.540 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.540 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.541 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13911 6.3 6:c7c65153:::benchmark_data_smithi012_182576_object13910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.541 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:00:26.541 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.953730734999994, 2023-10-09T10:00:26.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108882675, 2023-10-09T10:00:26.541 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13911 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:00:26.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549820+0000", 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3742000000000003e-05 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549854+0000", 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7170000000000002e-06 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549860+0000", 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5469999999999998e-06 2023-10-09T10:00:26.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549865+0000", 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00089393100000000004 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550759+0000", 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0737999999999998e-05 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550850+0000", 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012563299999999999 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550975+0000", 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00095688000000000001 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551932+0000", 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032028109999999999 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555135+0000", 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10350596099999999 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658641+0000", 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2475000000000003e-05 2023-10-09T10:00:26.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658673+0000", 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9924999999999999e-05 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658703+0000", 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108882675 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13918 6.5 6:a8d8ff46:::benchmark_data_smithi012_182576_object13917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.949546806000001, 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104220092, 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.549 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13918 2023-10-09T10:00:26.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.550 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554003+0000", 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4011000000000006e-05 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554087+0000", 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8400000000000001e-06 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554096+0000", 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4819999999999996e-06 2023-10-09T10:00:26.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554100+0000", 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3441e-05 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554144+0000", 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126305 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554270+0000", 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016421900000000001 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554434+0000", 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000721037 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555155+0000", 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078462779999999999 2023-10-09T10:00:26.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563002+0000", 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095134755000000001 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658136+0000", 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0183999999999999e-05 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658187+0000", 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8050000000000003e-05 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658225+0000", 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104220092 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.556 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13922 6.3 6:cfff9caa:::benchmark_data_smithi012_182576_object13921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.944951175, 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10019183399999999, 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13922 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558599+0000", 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7776999999999997e-05 2023-10-09T10:00:26.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558687+0000", 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3254999999999999e-05 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558700+0000", 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.977e-06 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558705+0000", 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0094e-05 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558735+0000", 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8771999999999997e-05 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558814+0000", 2023-10-09T10:00:26.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9957999999999995e-05 2023-10-09T10:00:26.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:00:26.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558904+0000", 2023-10-09T10:00:26.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070560600000000001 2023-10-09T10:00:26.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.559610+0000", 2023-10-09T10:00:26.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002049763 2023-10-09T10:00:26.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.561659+0000", 2023-10-09T10:00:26.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.097076735999999997 2023-10-09T10:00:26.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658736+0000", 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.726e-05 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658763+0000", 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8696000000000001e-05 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658792+0000", 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10019183399999999 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13930 6.3 6:cba38fc5:::benchmark_data_smithi012_182576_object13929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.938769530000002, 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094088516999999997, 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.565 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13930 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564781+0000", 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0343000000000002e-05 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564831+0000", 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7879999999999995e-06 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564840+0000", 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4070000000000003e-06 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564844+0000", 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0728999999999997e-05 2023-10-09T10:00:26.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564875+0000", 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9303999999999997e-05 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564964+0000", 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107612 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:00:26.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.565072+0000", 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050480800000000004 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.565577+0000", 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090698299999999999 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566484+0000", 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092340803999999999 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658824+0000", 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6945e-05 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658851+0000", 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9034000000000001e-05 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658870+0000", 2023-10-09T10:00:26.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094088516999999997 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13932 6.3 6:cf3c9e78:::benchmark_data_smithi012_182576_object13931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.937101182999996, 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092527188999999996, 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.572 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13932 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:00:26.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:00:26.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566449+0000", 2023-10-09T10:00:26.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6433000000000003e-05 2023-10-09T10:00:26.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566546+0000", 2023-10-09T10:00:26.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2155e-05 2023-10-09T10:00:26.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566558+0000", 2023-10-09T10:00:26.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2870000000000001e-06 2023-10-09T10:00:26.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566564+0000", 2023-10-09T10:00:26.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1313000000000002e-05 2023-10-09T10:00:26.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566606+0000", 2023-10-09T10:00:26.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2416000000000005e-05 2023-10-09T10:00:26.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566688+0000", 2023-10-09T10:00:26.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9309999999999996e-05 2023-10-09T10:00:26.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:00:26.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566787+0000", 2023-10-09T10:00:26.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043576999999999998 2023-10-09T10:00:26.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.567223+0000", 2023-10-09T10:00:26.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00077269100000000005 2023-10-09T10:00:26.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.567996+0000", 2023-10-09T10:00:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.090905342 2023-10-09T10:00:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658901+0000", 2023-10-09T10:00:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3823000000000003e-05 2023-10-09T10:00:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658955+0000", 2023-10-09T10:00:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2500000000000001e-05 2023-10-09T10:00:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658977+0000", 2023-10-09T10:00:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092527188999999996 2023-10-09T10:00:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13936 6.5 6:acd4ecb9:::benchmark_data_smithi012_182576_object13935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:00:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.575103+0000", 2023-10-09T10:00:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.928448106999994, 2023-10-09T10:00:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083239002000000006, 2023-10-09T10:00:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13936 2023-10-09T10:00:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575103+0000", 2023-10-09T10:00:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575103+0000", 2023-10-09T10:00:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:00:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575103+0000", 2023-10-09T10:00:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4441000000000003e-05 2023-10-09T10:00:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575157+0000", 2023-10-09T10:00:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0647e-05 2023-10-09T10:00:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575168+0000", 2023-10-09T10:00:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0260000000000004e-06 2023-10-09T10:00:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575174+0000", 2023-10-09T10:00:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2377999999999999e-05 2023-10-09T10:00:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575226+0000", 2023-10-09T10:00:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000135566 2023-10-09T10:00:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575362+0000", 2023-10-09T10:00:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016899800000000001 2023-10-09T10:00:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:00:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575531+0000", 2023-10-09T10:00:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00061649199999999995 2023-10-09T10:00:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.576147+0000", 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000487092 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.576634+0000", 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081631312999999997 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658266+0000", 2023-10-09T10:00:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3992999999999997e-05 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658310+0000", 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2771999999999997e-05 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658342+0000", 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083239002000000006 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:36563 6.8 6:10f4a19b:::benchmark_data_smithi012_182576_object36562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:17.197546+0000", 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 69.306005294000002, 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079528249999999995, 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36563 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197546+0000", 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197546+0000", 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197545+0000", 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9709000000000001e-05 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197594+0000", 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3861e-05 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197608+0000", 2023-10-09T10:00:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8610000000000003e-06 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197614+0000", 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2370000000000003e-05 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197656+0000", 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019276800000000001 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197849+0000", 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017279 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.198022+0000", 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028725869999999998 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.200895+0000", 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085708599999999996 2023-10-09T10:00:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.201752+0000", 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.075220750000000003 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.276972+0000", 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9431000000000001e-05 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.277032+0000", 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2469999999999998e-05 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.277074+0000", 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079528249999999995 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96730 6.3 6:cd9f8e90:::benchmark_data_smithi012_182576_object96729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.264896+0000", 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 13.238655736, 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.078131232999999994, 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96730 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.264896+0000", 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.264896+0000", 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.264895+0000", 2023-10-09T10:00:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6108000000000001e-05 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.264931+0000", 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8300000000000001e-06 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.264937+0000", 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1130000000000003e-06 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.264941+0000", 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017765609000000002 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.282707+0000", 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020383900000000001 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.282911+0000", 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025926299999999997 2023-10-09T10:00:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.283170+0000", 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078983460000000005 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.291068+0000", 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071009380000000002 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298169+0000", 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040752661000000003 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338922+0000", 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5679000000000006e-05 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338988+0000", 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040393590000000002 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.343027+0000", 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.078131232999999994 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96742 6.3 6:ccfab877:::benchmark_data_smithi012_182576_object96741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 13.224297647, 2023-10-09T10:00:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11727006399999999, 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96742 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279251+0000", 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6714e-05 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279298+0000", 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3504999999999999e-05 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279321+0000", 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0139999999999999e-05 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279331+0000", 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018866076999999998 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298197+0000", 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018085900000000001 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298378+0000", 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022614600000000001 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298604+0000", 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033326643000000003 2023-10-09T10:00:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.331931+0000", 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5298e-05 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.331966+0000", 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060961738000000001 2023-10-09T10:00:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.392928+0000", 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4974000000000005e-05 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.392993+0000", 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035309400000000002 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396524+0000", 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11727006399999999 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96748 6.3 6:ce4562bc:::benchmark_data_smithi012_182576_object96747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 13.215780625000001, 2023-10-09T10:00:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11285076300000001, 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96748 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:00:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287768+0000", 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8236999999999999e-05 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287816+0000", 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2195999999999999e-05 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287838+0000", 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0397000000000001e-05 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287849+0000", 2023-10-09T10:00:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014132728000000001 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.301981+0000", 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021475899999999999 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.302196+0000", 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024190600000000001 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.302438+0000", 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029561812 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332000+0000", 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2532e-05 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332032+0000", 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.064502819000000003 2023-10-09T10:00:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396535+0000", 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4147999999999998e-05 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396579+0000", 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040421540000000001 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400622+0000", 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11285076300000001 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96749 6.3 6:c755d6fb:::benchmark_data_smithi012_182576_object96748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 13.213542907000001, 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11442171800000001, 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96749 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290007+0000", 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9950000000000001e-05 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290056+0000", 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9181000000000002e-05 2023-10-09T10:00:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290076+0000", 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8350000000000004e-06 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290083+0000", 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015899986000000001 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.305983+0000", 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000202692 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.306186+0000", 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023536100000000001 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.306422+0000", 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025644121999999998 2023-10-09T10:00:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332066+0000", 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068449019999999999 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338911+0000", 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.061723756999999997 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400634+0000", 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9935e-05 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400704+0000", 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003725983 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.404430+0000", 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11442171800000001 2023-10-09T10:00:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.34797.0:96749 6.3 e119/117 ondisk, result = 0)", 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.317299+0000", 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 13.186252701000001, 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.087135486999999998, 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317299+0000", 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317299+0000", 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317298+0000", 2023-10-09T10:00:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.54e-06 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317302+0000", 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1517e-05 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317313+0000", 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8250000000000001e-06 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317319+0000", 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021573518999999999 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338893+0000", 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9680000000000001e-06 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338903+0000", 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.065531639000000003 2023-10-09T10:00:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.404434+0000", 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.087135486999999998 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96765 6.b e119/97 6:d2083b06:::benchmark_data_smithi012_182576_object96764:head v 119'6152, mlcod=119'6148)", 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.325246+0000", 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 13.178305754, 2023-10-09T10:00:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081827742999999994, 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325246+0000", 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325246+0000", 2023-10-09T10:00:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325245+0000", 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2183e-05 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325277+0000", 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.363e-05 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325291+0000", 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8780000000000003e-06 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325297+0000", 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067885280000000003 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332085+0000", 2023-10-09T10:00:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6765999999999998e-05 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332102+0000", 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.074909911999999995 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.407012+0000", 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1492e-05 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.407073+0000", 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081827742999999994 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:97077 6.3 6:c1ae7079:::benchmark_data_smithi012_182576_object97076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.867733+0000", 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 12.635818848, 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079050519999999999, 2023-10-09T10:00:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97077 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.867733+0000", 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.867733+0000", 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.867731+0000", 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022912240000000001 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.870022+0000", 2023-10-09T10:00:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5792999999999999e-05 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.870038+0000", 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9460000000000001e-06 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.870045+0000", 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026542884999999999 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.896588+0000", 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000145069 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.896733+0000", 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017958899999999999 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.896912+0000", 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024225915000000001 2023-10-09T10:00:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.921138+0000", 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012575622 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.933714+0000", 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090923250000000001 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.942806+0000", 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4181000000000003e-05 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.942850+0000", 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039328480000000001 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.946783+0000", 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079050519999999999 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:97524 6.3 6:c7f23124:::benchmark_data_smithi012_182576_object97523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:14.603054+0000", 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 11.90049767, 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081403381999999996, 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97524 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603054+0000", 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603054+0000", 2023-10-09T10:00:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603053+0000", 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7908000000000001e-05 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603081+0000", 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0090000000000004e-06 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603087+0000", 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1999999999999996e-06 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603091+0000", 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018937381 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.622029+0000", 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017725299999999999 2023-10-09T10:00:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.622206+0000", 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000194873 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.622401+0000", 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017724859999999999 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.640126+0000", 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6192999999999997e-05 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.640162+0000", 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040276379000000001 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.680438+0000", 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8034999999999995e-05 2023-10-09T10:00:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.680506+0000", 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039506890000000003 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.684457+0000", 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081403381999999996 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:97526 6.3 6:cd3b62b4:::benchmark_data_smithi012_182576_object97525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:14.604333+0000", 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 11.899218738, 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084183229999999998, 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97526 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604333+0000", 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604333+0000", 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:00:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604331+0000", 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6183999999999998e-05 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604367+0000", 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6690000000000001e-05 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604384+0000", 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604390+0000", 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021184410000000001 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.625575+0000", 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012946799999999999 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:00:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.625704+0000", 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015452700000000001 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.625859+0000", 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014272895000000001 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.640132+0000", 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9092000000000003e-05 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.640191+0000", 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044416529000000003 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.684607+0000", 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4169000000000002e-05 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.684672+0000", 2023-10-09T10:00:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038443459999999998 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.688516+0000", 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084183229999999998 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:00:26.662 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:00:26.796 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:26.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:26.876 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/338247126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:26.882 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:26.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:26.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:00:26.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/338247126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:26.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/338247126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:26.966 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:27.166 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:27.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:27.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:27.277 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:27.346 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:27.402 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 87 16 104107 104091 74.7299 38 0.0232508 0.0133751 2023-10-09T10:00:27.413 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:27.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:27.521 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:27.592 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:27.646 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:27.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:27.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:27.818 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:27.911 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:27.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:27.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:28.018 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:28.046 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:28.101 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:28.172 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:00:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:00:27] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T10:00:28.180 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:28.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:28.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:28.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:28.342 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:28.403 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 88 16 104702 104686 74.3027 37.1875 0.0287368 0.0134527 2023-10-09T10:00:28.480 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:28.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:28.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:28.585 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:28.635 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:28.688 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:28.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:28.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:28.758 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:00:28.797 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:28.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:28 smithi012 ceph-mon[147926]: pgmap v304: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 458 op/s; 35935/306669 objects degraded (11.718%); 10766/306669 objects misplaced (3.511%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T10:00:28.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2054894870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:00:28.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:28 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:00:28.810 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:28 smithi012 ceph-mon[152263]: pgmap v304: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 458 op/s; 35935/306669 objects degraded (11.718%); 10766/306669 objects misplaced (3.511%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T10:00:28.810 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2054894870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:00:28.810 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:28 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:00:28.828 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:28.897 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:28.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:28 smithi110 ceph-mon[136032]: pgmap v304: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 458 op/s; 35935/306669 objects degraded (11.718%); 10766/306669 objects misplaced (3.511%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T10:00:28.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2054894870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:00:28.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:28 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:00:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:28.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:28.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:00:29.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:29.052 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:29.124 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:29.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:29.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:00:28.883+0000 7f85a99d3700 -1 mon.a@0(leader).osd e123 definitely_dead 0 2023-10-09T10:00:29.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:29.316 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:29.403 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 89 16 105288 105272 73.8794 36.625 0.0296924 0.0135285 2023-10-09T10:00:29.475 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:29.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:29.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:29.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:29.637 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:29.804 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.2. 2023-10-09T10:00:29.816 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:29.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:00:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[147926]: osdmap e123: 8 total, 8 up, 7 in 2023-10-09T10:00:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[147926]: failed to encode map e123 with expected crc 2023-10-09T10:00:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[147926]: failed to encode map e123 with expected crc 2023-10-09T10:00:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3253446879' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:00:29.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:00:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:00:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[152263]: osdmap e123: 8 total, 8 up, 7 in 2023-10-09T10:00:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[152263]: failed to encode map e123 with expected crc 2023-10-09T10:00:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[152263]: failed to encode map e123 with expected crc 2023-10-09T10:00:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3253446879' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:00:29.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:00:29.908 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:00:29.909 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:00:29.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.909 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:105505 6.9 6:94a116b5:::benchmark_data_smithi012_182576_object105504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:00:29.909 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:29.762401+0000", 2023-10-09T10:00:29.910 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.145824272, 2023-10-09T10:00:29.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.145861307, 2023-10-09T10:00:29.910 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:29.910 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "delayed", 2023-10-09T10:00:29.910 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:29.911 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:29.911 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:29.911 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 105505 2023-10-09T10:00:29.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.912 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:29.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:29.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.762401+0000", 2023-10-09T10:00:29.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:29.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:29.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.762401+0000", 2023-10-09T10:00:29.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:29.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:29.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.762399+0000", 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.762428+0000", 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9598999999999999e-05 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.762445+0000", 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6991999999999999e-05 2023-10-09T10:00:29.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.762460+0000", 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4401e-05 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.773887+0000", 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011427379 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.774087+0000", 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000200306 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.774280+0000", 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000192416 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.793620+0000", 2023-10-09T10:00:29.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019340539 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.793633+0000", 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2908000000000001e-05 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:105507 6.0 6:03d92862:::benchmark_data_smithi012_182576_object105506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:00:29.918 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:29.775823+0000", 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.13240175500000001, 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.132514455, 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "delayed", 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 105507 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.775823+0000", 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.775823+0000", 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.775822+0000", 2023-10-09T10:00:29.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.775846+0000", 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3631000000000001e-05 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.775855+0000", 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6590000000000002e-06 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.775864+0000", 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1589999999999999e-06 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.776063+0000", 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000198586 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.776250+0000", 2023-10-09T10:00:29.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018708700000000001 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.776444+0000", 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019371300000000001 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.792328+0000", 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015884680000000002 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.792349+0000", 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0157e-05 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:105515 6.9 6.c496eb49 (undecoded) ondisk+write+known_if_redirected e124)", 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:29.804690+0000", 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.103535159, 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.103699402, 2023-10-09T10:00:29.924 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "delayed", 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 105515 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.804690+0000", 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.804690+0000", 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.804688+0000", 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:29.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.804716+0000", 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7937e-05 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.804733+0000", 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7439e-05 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.804745+0000", 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1571e-05 2023-10-09T10:00:29.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.805759+0000", 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001014455 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.805777+0000", 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8218000000000001e-05 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:29.928 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:29.929 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:29.929 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:00:29.929 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 3 2023-10-09T10:00:29.929 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:00:29.929 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:29 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:00:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:29 smithi110 ceph-mon[136032]: osdmap e123: 8 total, 8 up, 7 in 2023-10-09T10:00:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:29 smithi110 ceph-mon[136032]: failed to encode map e123 with expected crc 2023-10-09T10:00:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:29 smithi110 ceph-mon[136032]: failed to encode map e123 with expected crc 2023-10-09T10:00:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3253446879' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:00:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:00:29.955 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:29.987 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:29.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:30.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:30.121 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:30.206 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:30.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:30.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:30.310 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:30.340 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:30.374 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:30.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:30.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:30.399 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T10:00:30.403 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 90 16 105518 105502 73.2185 14.375 0.00867873 0.0135582 2023-10-09T10:00:30.433 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:30.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:00:30.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:30.536 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:30.578 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:30.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:30.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:30.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:30.712 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:30.749 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:30.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:30.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:30.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:30.888 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:30.923 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:30.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:30 smithi110 ceph-mon[136032]: pgmap v306: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 35935/310554 objects degraded (11.571%); 10766/310554 objects misplaced (3.467%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:00:30.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:00:30.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:30 smithi110 ceph-mon[136032]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T10:00:30.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:30 smithi110 ceph-mon[136032]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:30 smithi110 ceph-mon[136032]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:30 smithi110 ceph-mon[136032]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:30 smithi110 ceph-mon[136032]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:30 smithi110 ceph-mon[136032]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[147926]: pgmap v306: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 35935/310554 objects degraded (11.571%); 10766/310554 objects misplaced (3.467%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:00:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:00:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[147926]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T10:00:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[147926]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[147926]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[147926]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[147926]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[147926]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:30.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:30.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[152263]: pgmap v306: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 35935/310554 objects degraded (11.571%); 10766/310554 objects misplaced (3.467%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:00:30.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:00:30.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[152263]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T10:00:30.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[152263]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[152263]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[152263]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[152263]: failed to encode map e124 with expected crc 2023-10-09T10:00:30.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:30 smithi012 ceph-mon[152263]: failed to encode map e124 with expected crc 2023-10-09T10:00:31.029 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:31.056 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:31.091 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:31.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:31.195 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:31.220 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:31.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:31.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:31.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:31.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:31.383 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:31.403 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 91 6 105518 105512 72.4212 0.625 1.21484 0.0136677 2023-10-09T10:00:31.414 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:31.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:31.433 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:00:31.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:31.544 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:31.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:31.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:31.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:31.648 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:31.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:31.721 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:31.750 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:00:31.775 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:00:31.779 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:31.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:31.796 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:00:31.819 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:00:31.845 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:00:31.865 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:00:31.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:31.910 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:00:31.916 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:00:31.920 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:00:31.920 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:00:31.920 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:00:31.920 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:00:31.920 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:00:31.920 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:00:31.920 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:00:31.920 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:00:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T10:00:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: failed to encode map e125 with expected crc 2023-10-09T10:00:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: pgmap v309: 89 pgs: 3 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 74 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s; 23961/314664 objects degraded (7.615%); 5208/314664 objects misplaced (1.655%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:00:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:31 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3510975940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:31.943 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:32.033 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T10:00:32.033 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.033 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.033 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.033 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.033 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.033 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: pgmap v309: 89 pgs: 3 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 74 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s; 23961/314664 objects degraded (7.615%); 5208/314664 objects misplaced (1.655%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3510975940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.034 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: failed to encode map e125 with expected crc 2023-10-09T10:00:32.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: pgmap v309: 89 pgs: 3 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 74 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s; 23961/314664 objects degraded (7.615%); 5208/314664 objects misplaced (1.655%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:00:32.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:31 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3510975940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:32.132 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:32.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:32.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:32.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:32.265 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:32.350 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:32.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:32.403 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total time run: 91.6096 2023-10-09T10:00:32.403 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total writes made: 105518 2023-10-09T10:00:32.403 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Write size: 65536 2023-10-09T10:00:32.403 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object size: 65536 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Bandwidth (MB/sec): 71.9889 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Bandwidth: 39.4248 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max bandwidth (MB/sec): 209.75 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min bandwidth (MB/sec): 0 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average IOPS: 1151 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev IOPS: 630.806 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max IOPS: 3356 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min IOPS: 0 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average Latency(s): 0.0137905 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Latency(s): 0.0440573 2023-10-09T10:00:32.404 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max latency(s): 2.36676 2023-10-09T10:00:32.405 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min latency(s): 0.0011673 2023-10-09T10:00:32.407 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:00:32.454 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:32.531 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:32.614 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:32.615 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:32.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:32.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:32.734 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:32.775 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:32.828 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:32.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:32.906 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 23961/314664 objects degraded (7.615%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:32.906 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[152263]: failed to encode map e126 with expected crc 2023-10-09T10:00:32.906 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[152263]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T10:00:32.906 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[152263]: failed to encode map e126 with expected crc 2023-10-09T10:00:32.906 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[152263]: failed to encode map e126 with expected crc 2023-10-09T10:00:32.906 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[152263]: failed to encode map e126 with expected crc 2023-10-09T10:00:32.906 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 23961/314664 objects degraded (7.615%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:32.907 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[147926]: failed to encode map e126 with expected crc 2023-10-09T10:00:32.907 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[147926]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T10:00:32.907 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[147926]: failed to encode map e126 with expected crc 2023-10-09T10:00:32.907 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[147926]: failed to encode map e126 with expected crc 2023-10-09T10:00:32.907 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:32 smithi012 ceph-mon[147926]: failed to encode map e126 with expected crc 2023-10-09T10:00:32.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:32.994 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:32 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 23961/314664 objects degraded (7.615%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:32 smithi110 ceph-mon[136032]: failed to encode map e126 with expected crc 2023-10-09T10:00:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:32 smithi110 ceph-mon[136032]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T10:00:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:32 smithi110 ceph-mon[136032]: failed to encode map e126 with expected crc 2023-10-09T10:00:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:32 smithi110 ceph-mon[136032]: failed to encode map e126 with expected crc 2023-10-09T10:00:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:32 smithi110 ceph-mon[136032]: failed to encode map e126 with expected crc 2023-10-09T10:00:33.286 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:33.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:33.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:33.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:33.498 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:33.605 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:33.723 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:33.731 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:33.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:33.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:33.836 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:33.934 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:34.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:33 smithi012 ceph-mon[152263]: pgmap v311: 89 pgs: 3 remapped+peering, 7 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 73 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 478 op/s; 8554/316128 objects degraded (2.706%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T10:00:34.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:33 smithi012 ceph-mon[147926]: pgmap v311: 89 pgs: 3 remapped+peering, 7 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 73 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 478 op/s; 8554/316128 objects degraded (2.706%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T10:00:34.068 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:34.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:34.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:34.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:34.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:33 smithi110 ceph-mon[136032]: pgmap v311: 89 pgs: 3 remapped+peering, 7 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 73 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 478 op/s; 8554/316128 objects degraded (2.706%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T10:00:34.229 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:00:34.273 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:34.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:00:34.373 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:34.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:34.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:34.488 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:34.524 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:34.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:34.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:34.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:34.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:34.698 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:34.736 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:34.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:34.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:34.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:34.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:34.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:34.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:34.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:35.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:35.112 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:35.154 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:35.203 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:35.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:35.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:35.313 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:35.341 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:35.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:35.400 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:00:35.401 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:00:35.401 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T10:00:35.401 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:00:35.402 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:35.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:35.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:35.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:35.541 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:35.589 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:35.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:35.694 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:35.727 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:35.768 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:35.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:35.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:35.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:35.949 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:36.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:36 smithi110 ceph-mon[136032]: pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 170 B/s rd, 21 MiB/s wr, 496 op/s; 4519/313749 objects degraded (1.440%); 100/313749 objects misplaced (0.032%); 8.3 MiB/s, 1 keys/s, 132 objects/s recovering 2023-10-09T10:00:36.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2201023792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:36.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:36.038 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:36 smithi012 ceph-mon[147926]: pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 170 B/s rd, 21 MiB/s wr, 496 op/s; 4519/313749 objects degraded (1.440%); 100/313749 objects misplaced (0.032%); 8.3 MiB/s, 1 keys/s, 132 objects/s recovering 2023-10-09T10:00:36.038 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2201023792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:36.038 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:36.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:36 smithi012 ceph-mon[152263]: pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 170 B/s rd, 21 MiB/s wr, 496 op/s; 4519/313749 objects degraded (1.440%); 100/313749 objects misplaced (0.032%); 8.3 MiB/s, 1 keys/s, 132 objects/s recovering 2023-10-09T10:00:36.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2201023792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:36.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:36.083 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:00:36.200 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:36.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:36.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:36.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:36.339 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:36.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:36.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:36.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:36.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:36.503 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:36.536 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:36.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:36.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:36.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:36.670 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:37.214 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:37.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:37.222 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:37.222 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:37.222 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[147926]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T10:00:37.222 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[147926]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.222 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[147926]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.223 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[147926]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.223 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:37.223 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:37.223 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[152263]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T10:00:37.223 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[152263]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.223 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[152263]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.223 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-mon[152263]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.323 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:37.344 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:37.345 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:37.407 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:37.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:37 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:37.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:37.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:37 smithi110 ceph-mon[136032]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T10:00:37.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:37 smithi110 ceph-mon[136032]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:37 smithi110 ceph-mon[136032]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:37 smithi110 ceph-mon[136032]: failed to encode map e127 with expected crc 2023-10-09T10:00:37.466 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:37.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:37.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:37.574 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:37.609 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:37.664 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:37.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:37.768 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:37.800 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:37.850 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:37.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:37.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:37.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:37.991 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:38.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:00:38.010 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:38.032 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:38.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:38.136 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:38.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[147926]: failed to encode map e127 with expected crc 2023-10-09T10:00:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[147926]: pgmap v314: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 165 B/s rd, 6.6 MiB/s wr, 320 op/s; 3897/312639 objects degraded (1.246%); 614/312639 objects misplaced (0.196%); 7.8 MiB/s, 1 keys/s, 125 objects/s recovering 2023-10-09T10:00:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3897/312639 objects degraded (1.246%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[147926]: failed to encode map e128 with expected crc 2023-10-09T10:00:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[147926]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T10:00:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[147926]: failed to encode map e128 with expected crc 2023-10-09T10:00:38.181 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:00:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:00:37] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.33.4" 2023-10-09T10:00:38.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[152263]: failed to encode map e127 with expected crc 2023-10-09T10:00:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[152263]: pgmap v314: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 165 B/s rd, 6.6 MiB/s wr, 320 op/s; 3897/312639 objects degraded (1.246%); 614/312639 objects misplaced (0.196%); 7.8 MiB/s, 1 keys/s, 125 objects/s recovering 2023-10-09T10:00:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3897/312639 objects degraded (1.246%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[152263]: failed to encode map e128 with expected crc 2023-10-09T10:00:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[152263]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T10:00:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:38 smithi012 ceph-mon[152263]: failed to encode map e128 with expected crc 2023-10-09T10:00:38.218 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:38.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:38.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:38.322 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:38.350 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:38 smithi110 ceph-mon[136032]: failed to encode map e127 with expected crc 2023-10-09T10:00:38.351 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:38 smithi110 ceph-mon[136032]: pgmap v314: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 165 B/s rd, 6.6 MiB/s wr, 320 op/s; 3897/312639 objects degraded (1.246%); 614/312639 objects misplaced (0.196%); 7.8 MiB/s, 1 keys/s, 125 objects/s recovering 2023-10-09T10:00:38.351 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3897/312639 objects degraded (1.246%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:38.351 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:38 smithi110 ceph-mon[136032]: failed to encode map e128 with expected crc 2023-10-09T10:00:38.351 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:38 smithi110 ceph-mon[136032]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T10:00:38.351 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:38 smithi110 ceph-mon[136032]: failed to encode map e128 with expected crc 2023-10-09T10:00:38.353 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:38.392 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:38.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:38.459 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:00:38.497 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:38.506 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:00:38.529 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:38.549 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:00:38.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:38.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:38.589 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:00:38.621 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:00:38.672 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:00:38.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:38.710 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:00:38.750 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:00:38.753 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:38.775 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:00:38.775 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:00:38.775 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:00:38.775 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:00:38.776 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:00:38.776 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:00:38.776 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:00:38.776 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:00:38.838 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:38.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:38.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:38.945 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:38.984 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:39.032 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:39.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:39.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:39.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:39.225 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:39.317 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:39.318 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:39.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:39.338 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:39 smithi012 ceph-mon[147926]: failed to encode map e128 with expected crc 2023-10-09T10:00:39.338 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/23619505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:39.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:39 smithi012 ceph-mon[152263]: failed to encode map e128 with expected crc 2023-10-09T10:00:39.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/23619505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:39.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:39.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:39 smithi110 ceph-mon[136032]: failed to encode map e128 with expected crc 2023-10-09T10:00:39.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/23619505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:39.499 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:39.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:00:39.831 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:39.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:39.938 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:39.970 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:40.021 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:40.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:40.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:40.127 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:40.156 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:40.199 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:40.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:40.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:40.308 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:40.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:40 smithi110 ceph-mon[136032]: pgmap v316: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 142 B/s rd, 3.8 MiB/s wr, 438 op/s; 3897/308475 objects degraded (1.263%); 614/308475 objects misplaced (0.199%); 5.8 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T10:00:40.343 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:40.377 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:40 smithi012 ceph-mon[152263]: pgmap v316: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 142 B/s rd, 3.8 MiB/s wr, 438 op/s; 3897/308475 objects degraded (1.263%); 614/308475 objects misplaced (0.199%); 5.8 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T10:00:40.379 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:40 smithi012 ceph-mon[147926]: pgmap v316: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 142 B/s rd, 3.8 MiB/s wr, 438 op/s; 3897/308475 objects degraded (1.263%); 614/308475 objects misplaced (0.199%); 5.8 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T10:00:40.381 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:40.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:40.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:40.577 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:00:40.580 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:40.591 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:00:40.670 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:40.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:40.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:40.895 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:41.113 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:41 smithi012 ceph-mon[147926]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:00:41.115 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:41 smithi012 ceph-mon[152263]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:00:41.164 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:41.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:41.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:41.269 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:41.274 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:41.285 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":316,"stamp":"2023-10-09T10:00:38.968644+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6737696696,"num_objects":102825,"num_object_clones":0,"num_object_copies":308475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3897,"num_objects_misplaced":614,"num_objects_unfound":0,"num_objects_dirty":102825,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":108497,"num_write_kb":6755598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2030,"num_bytes_recovered":132579950,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108431,"ondisk_log_size":108431,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":277,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21482744,"kb_used_data":19941816,"kb_used_omap":126,"kb_used_meta":1540673,"kb_avail":728478472,"statfs":{"total":767960285184,"available":745961955328,"internally_reserved":0,"allocated":20420419584,"data_stored":20416339958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1577649787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-149422056,"num_objects":-2279,"num_object_clones":0,"num_object_copies":-6837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-15397,"num_objects_misplaced":614,"num_objects_unfound":0,"num_objects_dirty":-2279,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3157,"num_write_kb":28033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":668,"num_bytes_recovered":43647010,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.195152"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":127,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.807104+0000","last_change":"2023-10-09T10:00:30.806831+0000","last_active":"2023-10-09T10:00:30.807104+0000","last_peered":"2023-10-09T10:00:30.807104+0000","last_clean":"2023-10-09T10:00:30.807104+0000","last_became_active":"2023-10-09T10:00:30.806279+0000","last_became_peered":"2023-10-09T10:00:30.806279+0000","last_unstale":"2023-10-09T10:00:30.807104+0000","last_undegraded":"2023-10-09T10:00:30.807104+0000","last_fullsized":"2023-10-09T10:00:30.807104+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":96,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.054405+0000","last_change":"2023-10-09T09:57:49.622753+0000","last_active":"2023-10-09T10:00:36.054405+0000","last_peered":"2023-10-09T10:00:36.054405+0000","last_clean":"2023-10-09T10:00:36.054405+0000","last_became_active":"2023-10-09T09:57:49.622463+0000","last_became_peered":"2023-10-09T09:57:49.622463+0000","last_unstale":"2023-10-09T10:00:36.054405+0000","last_undegraded":"2023-10-09T10:00:36.054405+0000","last_fullsized":"2023-10-09T10:00:36.054405+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:51.020605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.058927+0000","last_change":"2023-10-09T09:57:49.621495+0000","last_active":"2023-10-09T10:00:36.058927+0000","last_peered":"2023-10-09T10:00:36.058927+0000","last_clean":"2023-10-09T10:00:36.058927+0000","last_became_active":"2023-10-09T09:57:49.621144+0000","last_became_peered":"2023-10-09T09:57:49.621144+0000","last_unstale":"2023-10-09T10:00:36.058927+0000","last_undegraded":"2023-10-09T10:00:36.058927+0000","last_fullsized":"2023-10-09T10:00:36.058927+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:11.852424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.993683+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T10:00:31.993683+0000","last_peered":"2023-10-09T10:00:31.993683+0000","last_clean":"2023-10-09T10:00:31.993683+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T10:00:31.993683+0000","last_undegraded":"2023-10-09T10:00:31.993683+0000","last_fullsized":"2023-10-09T10:00:31.993683+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":112,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.063566+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T10:00:36.063566+0000","last_peered":"2023-10-09T10:00:36.063566+0000","last_clean":"2023-10-09T10:00:36.063566+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T10:00:36.063566+0000","last_undegraded":"2023-10-09T10:00:36.063566+0000","last_fullsized":"2023-10-09T10:00:36.063566+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":108,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.777300+0000","last_change":"2023-10-09T10:00:09.714491+0000","last_active":"2023-10-09T10:00:30.777300+0000","last_peered":"2023-10-09T10:00:30.777300+0000","last_clean":"2023-10-09T10:00:30.777300+0000","last_became_active":"2023-10-09T10:00:09.714161+0000","last_became_peered":"2023-10-09T10:00:09.714161+0000","last_unstale":"2023-10-09T10:00:30.777300+0000","last_undegraded":"2023-10-09T10:00:30.777300+0000","last_fullsized":"2023-10-09T10:00:30.777300+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":84,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.994008+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T10:00:31.994008+0000","last_peered":"2023-10-09T10:00:31.994008+0000","last_clean":"2023-10-09T10:00:31.994008+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T10:00:31.994008+0000","last_undegraded":"2023-10-09T10:00:31.994008+0000","last_fullsized":"2023-10-09T10:00:31.994008+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":96,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.063464+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T10:00:36.063464+0000","last_peered":"2023-10-09T10:00:36.063464+0000","last_clean":"2023-10-09T10:00:36.063464+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T10:00:36.063464+0000","last_undegraded":"2023-10-09T10:00:36.063464+0000","last_fullsized":"2023-10-09T10:00:36.063464+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.781354+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T10:00:30.781354+0000","last_peered":"2023-10-09T10:00:30.781354+0000","last_clean":"2023-10-09T10:00:30.781354+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T10:00:30.781354+0000","last_undegraded":"2023-10-09T10:00:30.781354+0000","last_fullsized":"2023-10-09T10:00:30.781354+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":57,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.053639+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T10:00:36.053639+0000","last_peered":"2023-10-09T10:00:36.053639+0000","last_clean":"2023-10-09T10:00:36.053639+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T10:00:36.053639+0000","last_undegraded":"2023-10-09T10:00:36.053639+0000","last_fullsized":"2023-10-09T10:00:36.053639+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.778856+0000","last_change":"2023-10-09T09:57:49.626281+0000","last_active":"2023-10-09T10:00:30.778856+0000","last_peered":"2023-10-09T10:00:30.778856+0000","last_clean":"2023-10-09T10:00:30.778856+0000","last_became_active":"2023-10-09T09:57:49.625726+0000","last_became_peered":"2023-10-09T09:57:49.625726+0000","last_unstale":"2023-10-09T10:00:30.778856+0000","last_undegraded":"2023-10-09T10:00:30.778856+0000","last_fullsized":"2023-10-09T10:00:30.778856+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":100,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.056970+0000","last_change":"2023-10-09T10:00:30.790050+0000","last_active":"2023-10-09T10:00:36.056970+0000","last_peered":"2023-10-09T10:00:36.056970+0000","last_clean":"2023-10-09T10:00:36.056970+0000","last_became_active":"2023-10-09T10:00:30.789740+0000","last_became_peered":"2023-10-09T10:00:30.789740+0000","last_unstale":"2023-10-09T10:00:36.056970+0000","last_undegraded":"2023-10-09T10:00:36.056970+0000","last_fullsized":"2023-10-09T10:00:36.056970+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:36.206642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":100,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.774592+0000","last_change":"2023-10-09T09:48:25.517046+0000","last_active":"2023-10-09T10:00:30.774592+0000","last_peered":"2023-10-09T10:00:30.774592+0000","last_clean":"2023-10-09T10:00:30.774592+0000","last_became_active":"2023-10-09T09:48:25.516727+0000","last_became_peered":"2023-10-09T09:48:25.516727+0000","last_unstale":"2023-10-09T10:00:30.774592+0000","last_undegraded":"2023-10-09T10:00:30.774592+0000","last_fullsized":"2023-10-09T10:00:30.774592+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.774788+0000","last_change":"2023-10-09T09:56:45.551325+0000","last_active":"2023-10-09T10:00:30.774788+0000","last_peered":"2023-10-09T10:00:30.774788+0000","last_clean":"2023-10-09T10:00:30.774788+0000","last_became_active":"2023-10-09T09:56:45.550946+0000","last_became_peered":"2023-10-09T09:56:45.550946+0000","last_unstale":"2023-10-09T10:00:30.774788+0000","last_undegraded":"2023-10-09T10:00:30.774788+0000","last_fullsized":"2023-10-09T10:00:30.774788+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":123,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.874906+0000","last_change":"2023-10-09T10:00:30.831802+0000","last_active":"2023-10-09T10:00:30.874906+0000","last_peered":"2023-10-09T10:00:30.874906+0000","last_clean":"2023-10-09T10:00:30.874906+0000","last_became_active":"2023-10-09T10:00:30.799747+0000","last_became_peered":"2023-10-09T10:00:30.799747+0000","last_unstale":"2023-10-09T10:00:30.874906+0000","last_undegraded":"2023-10-09T10:00:30.874906+0000","last_fullsized":"2023-10-09T10:00:30.874906+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":96,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.055451+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T10:00:36.055451+0000","last_peered":"2023-10-09T10:00:36.055451+0000","last_clean":"2023-10-09T10:00:36.055451+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T10:00:36.055451+0000","last_undegraded":"2023-10-09T10:00:36.055451+0000","last_fullsized":"2023-10-09T10:00:36.055451+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":84,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.993093+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T10:00:31.993093+0000","last_peered":"2023-10-09T10:00:31.993093+0000","last_clean":"2023-10-09T10:00:31.993093+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T10:00:31.993093+0000","last_undegraded":"2023-10-09T10:00:31.993093+0000","last_fullsized":"2023-10-09T10:00:31.993093+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.781035+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T10:00:30.781035+0000","last_peered":"2023-10-09T10:00:30.781035+0000","last_clean":"2023-10-09T10:00:30.781035+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T10:00:30.781035+0000","last_undegraded":"2023-10-09T10:00:30.781035+0000","last_fullsized":"2023-10-09T10:00:30.781035+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":17,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.774221+0000","last_change":"2023-10-09T10:00:19.642139+0000","last_active":"2023-10-09T10:00:30.774221+0000","last_peered":"2023-10-09T10:00:30.774221+0000","last_clean":"2023-10-09T10:00:30.774221+0000","last_became_active":"2023-10-09T10:00:19.641857+0000","last_became_peered":"2023-10-09T10:00:19.641857+0000","last_unstale":"2023-10-09T10:00:30.774221+0000","last_undegraded":"2023-10-09T10:00:30.774221+0000","last_fullsized":"2023-10-09T10:00:30.774221+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.993641+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T10:00:31.993641+0000","last_peered":"2023-10-09T10:00:31.993641+0000","last_clean":"2023-10-09T10:00:31.993641+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T10:00:31.993641+0000","last_undegraded":"2023-10-09T10:00:31.993641+0000","last_fullsized":"2023-10-09T10:00:31.993641+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":99,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.778498+0000","last_change":"2023-10-09T09:56:13.623204+0000","last_active":"2023-10-09T10:00:30.778498+0000","last_peered":"2023-10-09T10:00:30.778498+0000","last_clean":"2023-10-09T10:00:30.778498+0000","last_became_active":"2023-10-09T09:56:13.622853+0000","last_became_peered":"2023-10-09T09:56:13.622853+0000","last_unstale":"2023-10-09T10:00:30.778498+0000","last_undegraded":"2023-10-09T10:00:30.778498+0000","last_fullsized":"2023-10-09T10:00:30.778498+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":96,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.054574+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T10:00:36.054574+0000","last_peered":"2023-10-09T10:00:36.054574+0000","last_clean":"2023-10-09T10:00:36.054574+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T10:00:36.054574+0000","last_undegraded":"2023-10-09T10:00:36.054574+0000","last_fullsized":"2023-10-09T10:00:36.054574+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"126'6725","reported_seq":6762,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:34.762670+0000","last_change":"2023-10-09T09:58:52.780310+0000","last_active":"2023-10-09T10:00:34.762670+0000","last_peered":"2023-10-09T10:00:34.762670+0000","last_clean":"2023-10-09T10:00:34.762670+0000","last_became_active":"2023-10-09T09:58:52.779910+0000","last_became_peered":"2023-10-09T09:58:52.779910+0000","last_unstale":"2023-10-09T10:00:34.762670+0000","last_undegraded":"2023-10-09T10:00:34.762670+0000","last_fullsized":"2023-10-09T10:00:34.762670+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425787392,"num_objects":6497,"num_object_clones":0,"num_object_copies":19491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6725,"num_write_kb":423104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.779410+0000","last_change":"2023-10-09T09:56:13.614485+0000","last_active":"2023-10-09T10:00:30.779410+0000","last_peered":"2023-10-09T10:00:30.779410+0000","last_clean":"2023-10-09T10:00:30.779410+0000","last_became_active":"2023-10-09T09:56:13.614109+0000","last_became_peered":"2023-10-09T09:56:13.614109+0000","last_unstale":"2023-10-09T10:00:30.779410+0000","last_undegraded":"2023-10-09T10:00:30.779410+0000","last_fullsized":"2023-10-09T10:00:30.779410+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.780708+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T10:00:30.780708+0000","last_peered":"2023-10-09T10:00:30.780708+0000","last_clean":"2023-10-09T10:00:30.780708+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T10:00:30.780708+0000","last_undegraded":"2023-10-09T10:00:30.780708+0000","last_fullsized":"2023-10-09T10:00:30.780708+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"128'6897","reported_seq":6937,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-09T10:00:37.760459+0000","last_change":"2023-10-09T09:58:52.625388+0000","last_active":"2023-10-09T10:00:37.760459+0000","last_peered":"2023-10-09T10:00:37.760459+0000","last_clean":"2023-10-09T10:00:37.760459+0000","last_became_active":"2023-10-09T09:58:52.625135+0000","last_became_peered":"2023-10-09T09:58:52.625135+0000","last_unstale":"2023-10-09T10:00:37.760459+0000","last_undegraded":"2023-10-09T10:00:37.760459+0000","last_fullsized":"2023-10-09T10:00:37.760459+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:28.713713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424214528,"num_objects":6473,"num_object_clones":0,"num_object_copies":19419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6897,"num_write_kb":427840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.779614+0000","last_change":"2023-10-09T09:56:45.554326+0000","last_active":"2023-10-09T10:00:30.779614+0000","last_peered":"2023-10-09T10:00:30.779614+0000","last_clean":"2023-10-09T10:00:30.779614+0000","last_became_active":"2023-10-09T09:56:45.553876+0000","last_became_peered":"2023-10-09T09:56:45.553876+0000","last_unstale":"2023-10-09T10:00:30.779614+0000","last_undegraded":"2023-10-09T10:00:30.779614+0000","last_fullsized":"2023-10-09T10:00:30.779614+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.774125+0000","last_change":"2023-10-09T09:57:49.627073+0000","last_active":"2023-10-09T10:00:30.774125+0000","last_peered":"2023-10-09T10:00:30.774125+0000","last_clean":"2023-10-09T10:00:30.774125+0000","last_became_active":"2023-10-09T09:57:49.626706+0000","last_became_peered":"2023-10-09T09:57:49.626706+0000","last_unstale":"2023-10-09T10:00:30.774125+0000","last_undegraded":"2023-10-09T10:00:30.774125+0000","last_fullsized":"2023-10-09T10:00:30.774125+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"128'6973","reported_seq":7023,"reported_epoch":128,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:00:37.750429+0000","last_change":"2023-10-09T10:00:31.943789+0000","last_active":"2023-10-09T10:00:37.750429+0000","last_peered":"2023-10-09T10:00:37.750429+0000","last_clean":"2023-10-09T10:00:08.698812+0000","last_became_active":"2023-10-09T10:00:31.937597+0000","last_became_peered":"2023-10-09T10:00:31.937597+0000","last_unstale":"2023-10-09T10:00:37.750429+0000","last_undegraded":"2023-10-09T10:00:31.780262+0000","last_fullsized":"2023-10-09T10:00:31.773813+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6973,"log_dups_size":0,"ondisk_log_size":6973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:11.309049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429195264,"num_objects":6549,"num_object_clones":0,"num_object_copies":19647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":668,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6973,"num_write_kb":432704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,6,7","objects":667},{"shards":"3,7","objects":5881}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":28,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.780960+0000","last_change":"2023-10-09T10:00:19.641304+0000","last_active":"2023-10-09T10:00:30.780960+0000","last_peered":"2023-10-09T10:00:30.780960+0000","last_clean":"2023-10-09T10:00:30.780960+0000","last_became_active":"2023-10-09T10:00:19.640958+0000","last_became_peered":"2023-10-09T10:00:19.640958+0000","last_unstale":"2023-10-09T10:00:30.780960+0000","last_undegraded":"2023-10-09T10:00:30.780960+0000","last_fullsized":"2023-10-09T10:00:30.780960+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":90,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.777451+0000","last_change":"2023-10-09T09:56:45.556365+0000","last_active":"2023-10-09T10:00:30.777451+0000","last_peered":"2023-10-09T10:00:30.777451+0000","last_clean":"2023-10-09T10:00:30.777451+0000","last_became_active":"2023-10-09T09:56:45.555832+0000","last_became_peered":"2023-10-09T09:56:45.555832+0000","last_unstale":"2023-10-09T10:00:30.777451+0000","last_undegraded":"2023-10-09T10:00:30.777451+0000","last_fullsized":"2023-10-09T10:00:30.777451+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"127'6729","reported_seq":8386,"reported_epoch":127,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:00:36.297761+0000","last_change":"2023-10-09T10:00:30.970926+0000","last_active":"2023-10-09T10:00:36.297761+0000","last_peered":"2023-10-09T10:00:36.297761+0000","last_clean":"2023-10-09T10:00:08.693890+0000","last_became_active":"2023-10-09T10:00:30.965575+0000","last_became_peered":"2023-10-09T10:00:30.965575+0000","last_unstale":"2023-10-09T10:00:36.297761+0000","last_undegraded":"2023-10-09T10:00:30.849530+0000","last_fullsized":"2023-10-09T10:00:30.775686+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:25.945898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415301632,"num_objects":6337,"num_object_clones":0,"num_object_copies":19011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6729,"num_write_kb":418112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":801,"num_bytes_recovered":52494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,3,7","objects":629},{"shards":"0,7","objects":5708}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.781772+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T10:00:30.781772+0000","last_peered":"2023-10-09T10:00:30.781772+0000","last_clean":"2023-10-09T10:00:30.781772+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T10:00:30.781772+0000","last_undegraded":"2023-10-09T10:00:30.781772+0000","last_fullsized":"2023-10-09T10:00:30.781772+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.993406+0000","last_change":"2023-10-09T09:56:45.553024+0000","last_active":"2023-10-09T10:00:31.993406+0000","last_peered":"2023-10-09T10:00:31.993406+0000","last_clean":"2023-10-09T10:00:31.993406+0000","last_became_active":"2023-10-09T09:56:45.552670+0000","last_became_peered":"2023-10-09T09:56:45.552670+0000","last_unstale":"2023-10-09T10:00:31.993406+0000","last_undegraded":"2023-10-09T10:00:31.993406+0000","last_fullsized":"2023-10-09T10:00:31.993406+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:58.132008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"126'6742","reported_seq":7910,"reported_epoch":126,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:00:34.833623+0000","last_change":"2023-10-09T10:00:30.925252+0000","last_active":"2023-10-09T10:00:34.833623+0000","last_peered":"2023-10-09T10:00:34.833623+0000","last_clean":"2023-10-09T10:00:08.644218+0000","last_became_active":"2023-10-09T10:00:30.897002+0000","last_became_peered":"2023-10-09T10:00:30.897002+0000","last_unstale":"2023-10-09T10:00:34.833623+0000","last_undegraded":"2023-10-09T10:00:34.833623+0000","last_fullsized":"2023-10-09T10:00:30.774596+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424148992,"num_objects":6472,"num_object_clones":0,"num_object_copies":19416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":100,"num_objects_unfound":0,"num_objects_dirty":6472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6742,"num_write_kb":422848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":558,"num_bytes_recovered":36569088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":6372},{"shards":"1,4,5","objects":100}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.060591+0000","last_change":"2023-10-09T09:57:49.620199+0000","last_active":"2023-10-09T10:00:36.060591+0000","last_peered":"2023-10-09T10:00:36.060591+0000","last_clean":"2023-10-09T10:00:36.060591+0000","last_became_active":"2023-10-09T09:57:49.619695+0000","last_became_peered":"2023-10-09T09:57:49.619695+0000","last_unstale":"2023-10-09T10:00:36.060591+0000","last_undegraded":"2023-10-09T10:00:36.060591+0000","last_fullsized":"2023-10-09T10:00:36.060591+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:48.818881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.774476+0000","last_change":"2023-10-09T09:57:49.621587+0000","last_active":"2023-10-09T10:00:30.774476+0000","last_peered":"2023-10-09T10:00:30.774476+0000","last_clean":"2023-10-09T10:00:30.774476+0000","last_became_active":"2023-10-09T09:57:49.621245+0000","last_became_peered":"2023-10-09T09:57:49.621245+0000","last_unstale":"2023-10-09T10:00:30.774476+0000","last_undegraded":"2023-10-09T10:00:30.774476+0000","last_fullsized":"2023-10-09T10:00:30.774476+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"128'6773","reported_seq":6828,"reported_epoch":128,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:00:37.666004+0000","last_change":"2023-10-09T10:00:30.874282+0000","last_active":"2023-10-09T10:00:37.666004+0000","last_peered":"2023-10-09T10:00:37.666004+0000","last_clean":"2023-10-09T10:00:08.654106+0000","last_became_active":"2023-10-09T10:00:30.867149+0000","last_became_peered":"2023-10-09T10:00:30.867149+0000","last_unstale":"2023-10-09T10:00:37.666004+0000","last_undegraded":"2023-10-09T10:00:30.780366+0000","last_fullsized":"2023-10-09T10:00:30.771408+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6773,"log_dups_size":0,"ondisk_log_size":6773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:04.232992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414121984,"num_objects":6319,"num_object_clones":0,"num_object_copies":18957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6773,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":5705},{"shards":"3,4,6","objects":614}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.777344+0000","last_change":"2023-10-09T09:56:45.556627+0000","last_active":"2023-10-09T10:00:30.777344+0000","last_peered":"2023-10-09T10:00:30.777344+0000","last_clean":"2023-10-09T10:00:30.777344+0000","last_became_active":"2023-10-09T09:56:45.556265+0000","last_became_peered":"2023-10-09T09:56:45.556265+0000","last_unstale":"2023-10-09T10:00:30.777344+0000","last_undegraded":"2023-10-09T10:00:30.777344+0000","last_fullsized":"2023-10-09T10:00:30.777344+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.798744+0000","last_change":"2023-10-09T10:00:30.793147+0000","last_active":"2023-10-09T10:00:30.798744+0000","last_peered":"2023-10-09T10:00:30.798744+0000","last_clean":"2023-10-09T10:00:30.798744+0000","last_became_active":"2023-10-09T10:00:30.792779+0000","last_became_peered":"2023-10-09T10:00:30.792779+0000","last_unstale":"2023-10-09T10:00:30.798744+0000","last_undegraded":"2023-10-09T10:00:30.798744+0000","last_fullsized":"2023-10-09T10:00:30.798744+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"128'6831","reported_seq":6871,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-09T10:00:37.735026+0000","last_change":"2023-10-09T09:58:52.780096+0000","last_active":"2023-10-09T10:00:37.735026+0000","last_peered":"2023-10-09T10:00:37.735026+0000","last_clean":"2023-10-09T10:00:37.735026+0000","last_became_active":"2023-10-09T09:58:52.779754+0000","last_became_peered":"2023-10-09T09:58:52.779754+0000","last_unstale":"2023-10-09T10:00:37.735026+0000","last_undegraded":"2023-10-09T10:00:37.735026+0000","last_fullsized":"2023-10-09T10:00:37.735026+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6831,"log_dups_size":0,"ondisk_log_size":6831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:19:28.304859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418578432,"num_objects":6387,"num_object_clones":0,"num_object_copies":19161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6831,"num_write_kb":422976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":111,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.789259+0000","last_change":"2023-10-09T10:00:30.787189+0000","last_active":"2023-10-09T10:00:30.789259+0000","last_peered":"2023-10-09T10:00:30.789259+0000","last_clean":"2023-10-09T10:00:30.789259+0000","last_became_active":"2023-10-09T10:00:30.786846+0000","last_became_peered":"2023-10-09T10:00:30.786846+0000","last_unstale":"2023-10-09T10:00:30.789259+0000","last_undegraded":"2023-10-09T10:00:30.789259+0000","last_fullsized":"2023-10-09T10:00:30.789259+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.811863+0000","last_change":"2023-10-09T09:56:45.555100+0000","last_active":"2023-10-09T10:00:30.811863+0000","last_peered":"2023-10-09T10:00:30.811863+0000","last_clean":"2023-10-09T10:00:30.811863+0000","last_became_active":"2023-10-09T09:56:45.554704+0000","last_became_peered":"2023-10-09T09:56:45.554704+0000","last_unstale":"2023-10-09T10:00:30.811863+0000","last_undegraded":"2023-10-09T10:00:30.811863+0000","last_fullsized":"2023-10-09T10:00:30.811863+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"126'6574","reported_seq":6612,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:34.762684+0000","last_change":"2023-10-09T09:58:52.626617+0000","last_active":"2023-10-09T10:00:34.762684+0000","last_peered":"2023-10-09T10:00:34.762684+0000","last_clean":"2023-10-09T10:00:34.762684+0000","last_became_active":"2023-10-09T09:58:52.626409+0000","last_became_peered":"2023-10-09T09:58:52.626409+0000","last_unstale":"2023-10-09T10:00:34.762684+0000","last_undegraded":"2023-10-09T10:00:34.762684+0000","last_fullsized":"2023-10-09T10:00:34.762684+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6574,"log_dups_size":0,"ondisk_log_size":6574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416874520,"num_objects":6362,"num_object_clones":0,"num_object_copies":19086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6362,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6574,"num_write_kb":413889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.063743+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T10:00:36.063743+0000","last_peered":"2023-10-09T10:00:36.063743+0000","last_clean":"2023-10-09T10:00:36.063743+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T10:00:36.063743+0000","last_undegraded":"2023-10-09T10:00:36.063743+0000","last_fullsized":"2023-10-09T10:00:36.063743+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":82,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.056578+0000","last_change":"2023-10-09T09:57:49.622055+0000","last_active":"2023-10-09T10:00:36.056578+0000","last_peered":"2023-10-09T10:00:36.056578+0000","last_clean":"2023-10-09T10:00:36.056578+0000","last_became_active":"2023-10-09T09:57:49.621784+0000","last_became_peered":"2023-10-09T09:57:49.621784+0000","last_unstale":"2023-10-09T10:00:36.056578+0000","last_undegraded":"2023-10-09T10:00:36.056578+0000","last_fullsized":"2023-10-09T10:00:36.056578+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:50.025391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.057236+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T10:00:36.057236+0000","last_peered":"2023-10-09T10:00:36.057236+0000","last_clean":"2023-10-09T10:00:36.057236+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T10:00:36.057236+0000","last_undegraded":"2023-10-09T10:00:36.057236+0000","last_fullsized":"2023-10-09T10:00:36.057236+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":125,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.786607+0000","last_change":"2023-10-09T10:00:30.785799+0000","last_active":"2023-10-09T10:00:30.786607+0000","last_peered":"2023-10-09T10:00:30.786607+0000","last_clean":"2023-10-09T10:00:30.786607+0000","last_became_active":"2023-10-09T10:00:30.785396+0000","last_became_peered":"2023-10-09T10:00:30.785396+0000","last_unstale":"2023-10-09T10:00:30.786607+0000","last_undegraded":"2023-10-09T10:00:30.786607+0000","last_fullsized":"2023-10-09T10:00:30.786607+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":22,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.056735+0000","last_change":"2023-10-09T10:00:09.712845+0000","last_active":"2023-10-09T10:00:36.056735+0000","last_peered":"2023-10-09T10:00:36.056735+0000","last_clean":"2023-10-09T10:00:36.056735+0000","last_became_active":"2023-10-09T10:00:09.712505+0000","last_became_peered":"2023-10-09T10:00:09.712505+0000","last_unstale":"2023-10-09T10:00:36.056735+0000","last_undegraded":"2023-10-09T10:00:36.056735+0000","last_fullsized":"2023-10-09T10:00:36.056735+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:11:00.069568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"127'6716","reported_seq":6989,"reported_epoch":127,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:00:36.681570+0000","last_change":"2023-10-09T10:00:36.046937+0000","last_active":"2023-10-09T10:00:36.681570+0000","last_peered":"2023-10-09T10:00:36.681570+0000","last_clean":"2023-10-09T10:00:08.678113+0000","last_became_active":"2023-10-09T10:00:30.901163+0000","last_became_peered":"2023-10-09T10:00:30.901163+0000","last_unstale":"2023-10-09T10:00:36.681570+0000","last_undegraded":"2023-10-09T10:00:36.681570+0000","last_fullsized":"2023-10-09T10:00:30.775176+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6716,"log_dups_size":0,"ondisk_log_size":6716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:19.302626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417333248,"num_objects":6368,"num_object_clones":0,"num_object_copies":19104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":514,"num_objects_unfound":0,"num_objects_dirty":6368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6716,"num_write_kb":418688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":7143424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":5854},{"shards":"0,3,5","objects":514}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"128'6830","reported_seq":6870,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-09T10:00:37.733307+0000","last_change":"2023-10-09T09:58:52.624855+0000","last_active":"2023-10-09T10:00:37.733307+0000","last_peered":"2023-10-09T10:00:37.733307+0000","last_clean":"2023-10-09T10:00:37.733307+0000","last_became_active":"2023-10-09T09:58:52.624556+0000","last_became_peered":"2023-10-09T09:58:52.624556+0000","last_unstale":"2023-10-09T10:00:37.733307+0000","last_undegraded":"2023-10-09T10:00:37.733307+0000","last_fullsized":"2023-10-09T10:00:37.733307+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:07.544449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417464320,"num_objects":6370,"num_object_clones":0,"num_object_copies":19110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6830,"num_write_kb":422400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.780807+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T10:00:30.780807+0000","last_peered":"2023-10-09T10:00:30.780807+0000","last_clean":"2023-10-09T10:00:30.780807+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T10:00:30.780807+0000","last_undegraded":"2023-10-09T10:00:30.780807+0000","last_fullsized":"2023-10-09T10:00:30.780807+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.773879+0000","last_change":"2023-10-09T09:56:13.613717+0000","last_active":"2023-10-09T10:00:30.773879+0000","last_peered":"2023-10-09T10:00:30.773879+0000","last_clean":"2023-10-09T10:00:30.773879+0000","last_became_active":"2023-10-09T09:56:13.613329+0000","last_became_peered":"2023-10-09T09:56:13.613329+0000","last_unstale":"2023-10-09T10:00:30.773879+0000","last_undegraded":"2023-10-09T10:00:30.773879+0000","last_fullsized":"2023-10-09T10:00:30.773879+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.057172+0000","last_change":"2023-10-09T10:00:30.782106+0000","last_active":"2023-10-09T10:00:36.057172+0000","last_peered":"2023-10-09T10:00:36.057172+0000","last_clean":"2023-10-09T10:00:36.057172+0000","last_became_active":"2023-10-09T10:00:30.781623+0000","last_became_peered":"2023-10-09T10:00:30.781623+0000","last_unstale":"2023-10-09T10:00:36.057172+0000","last_undegraded":"2023-10-09T10:00:36.057172+0000","last_fullsized":"2023-10-09T10:00:36.057172+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:14.657623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"126'6633","reported_seq":6670,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:34.853821+0000","last_change":"2023-10-09T09:58:52.780565+0000","last_active":"2023-10-09T10:00:34.853821+0000","last_peered":"2023-10-09T10:00:34.853821+0000","last_clean":"2023-10-09T10:00:34.853821+0000","last_became_active":"2023-10-09T09:58:52.780260+0000","last_became_peered":"2023-10-09T09:58:52.780260+0000","last_unstale":"2023-10-09T10:00:34.853821+0000","last_undegraded":"2023-10-09T10:00:34.853821+0000","last_fullsized":"2023-10-09T10:00:34.853821+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6633,"log_dups_size":0,"ondisk_log_size":6633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421199872,"num_objects":6427,"num_object_clones":0,"num_object_copies":19281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6633,"num_write_kb":417920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.915836+0000","last_change":"2023-10-09T10:00:30.892640+0000","last_active":"2023-10-09T10:00:31.915836+0000","last_peered":"2023-10-09T10:00:31.915836+0000","last_clean":"2023-10-09T10:00:31.915836+0000","last_became_active":"2023-10-09T10:00:30.892119+0000","last_became_peered":"2023-10-09T10:00:30.892119+0000","last_unstale":"2023-10-09T10:00:31.915836+0000","last_undegraded":"2023-10-09T10:00:31.915836+0000","last_fullsized":"2023-10-09T10:00:31.915836+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:37:45.627291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":99,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.056641+0000","last_change":"2023-10-09T10:00:30.786721+0000","last_active":"2023-10-09T10:00:36.056641+0000","last_peered":"2023-10-09T10:00:36.056641+0000","last_clean":"2023-10-09T10:00:36.056641+0000","last_became_active":"2023-10-09T10:00:30.786375+0000","last_became_peered":"2023-10-09T10:00:30.786375+0000","last_unstale":"2023-10-09T10:00:36.056641+0000","last_undegraded":"2023-10-09T10:00:36.056641+0000","last_fullsized":"2023-10-09T10:00:36.056641+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:19:38.052691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.056656+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T10:00:36.056656+0000","last_peered":"2023-10-09T10:00:36.056656+0000","last_clean":"2023-10-09T10:00:36.056656+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T10:00:36.056656+0000","last_undegraded":"2023-10-09T10:00:36.056656+0000","last_fullsized":"2023-10-09T10:00:36.056656+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"126'6544","reported_seq":6581,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:34.712659+0000","last_change":"2023-10-09T09:58:52.625605+0000","last_active":"2023-10-09T10:00:34.712659+0000","last_peered":"2023-10-09T10:00:34.712659+0000","last_clean":"2023-10-09T10:00:34.712659+0000","last_became_active":"2023-10-09T09:58:52.625332+0000","last_became_peered":"2023-10-09T09:58:52.625332+0000","last_unstale":"2023-10-09T10:00:34.712659+0000","last_undegraded":"2023-10-09T10:00:34.712659+0000","last_fullsized":"2023-10-09T10:00:34.712659+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6544,"log_dups_size":0,"ondisk_log_size":6544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415629312,"num_objects":6342,"num_object_clones":0,"num_object_copies":19026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6544,"num_write_kb":412352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":114,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.790494+0000","last_change":"2023-10-09T10:00:30.790235+0000","last_active":"2023-10-09T10:00:30.790494+0000","last_peered":"2023-10-09T10:00:30.790494+0000","last_clean":"2023-10-09T10:00:30.790494+0000","last_became_active":"2023-10-09T10:00:30.789866+0000","last_became_peered":"2023-10-09T10:00:30.789866+0000","last_unstale":"2023-10-09T10:00:30.790494+0000","last_undegraded":"2023-10-09T10:00:30.790494+0000","last_fullsized":"2023-10-09T10:00:30.790494+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":327,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.779745+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T10:00:30.779745+0000","last_peered":"2023-10-09T10:00:30.779745+0000","last_clean":"2023-10-09T10:00:30.779745+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T10:00:30.779745+0000","last_undegraded":"2023-10-09T10:00:30.779745+0000","last_fullsized":"2023-10-09T10:00:30.779745+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":57,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.054056+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T10:00:36.054056+0000","last_peered":"2023-10-09T10:00:36.054056+0000","last_clean":"2023-10-09T10:00:36.054056+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T10:00:36.054056+0000","last_undegraded":"2023-10-09T10:00:36.054056+0000","last_fullsized":"2023-10-09T10:00:36.054056+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":75,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.777444+0000","last_change":"2023-10-09T09:48:27.153459+0000","last_active":"2023-10-09T10:00:30.777444+0000","last_peered":"2023-10-09T10:00:30.777444+0000","last_clean":"2023-10-09T10:00:30.777444+0000","last_became_active":"2023-10-09T09:48:27.153136+0000","last_became_peered":"2023-10-09T09:48:27.153136+0000","last_unstale":"2023-10-09T10:00:30.777444+0000","last_undegraded":"2023-10-09T10:00:30.777444+0000","last_fullsized":"2023-10-09T10:00:30.777444+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.056251+0000","last_change":"2023-10-09T10:00:19.650114+0000","last_active":"2023-10-09T10:00:36.056251+0000","last_peered":"2023-10-09T10:00:36.056251+0000","last_clean":"2023-10-09T10:00:36.056251+0000","last_became_active":"2023-10-09T10:00:19.649776+0000","last_became_peered":"2023-10-09T10:00:19.649776+0000","last_unstale":"2023-10-09T10:00:36.056251+0000","last_undegraded":"2023-10-09T10:00:36.056251+0000","last_fullsized":"2023-10-09T10:00:36.056251+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:09:03.140113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.994205+0000","last_change":"2023-10-09T09:57:49.624279+0000","last_active":"2023-10-09T10:00:31.994205+0000","last_peered":"2023-10-09T10:00:31.994205+0000","last_clean":"2023-10-09T10:00:31.994205+0000","last_became_active":"2023-10-09T09:57:49.624010+0000","last_became_peered":"2023-10-09T09:57:49.624010+0000","last_unstale":"2023-10-09T10:00:31.994205+0000","last_undegraded":"2023-10-09T10:00:31.994205+0000","last_fullsized":"2023-10-09T10:00:31.994205+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:47.717690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":122,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.993805+0000","last_change":"2023-10-09T10:00:30.785844+0000","last_active":"2023-10-09T10:00:31.993805+0000","last_peered":"2023-10-09T10:00:31.993805+0000","last_clean":"2023-10-09T10:00:31.993805+0000","last_became_active":"2023-10-09T10:00:30.785565+0000","last_became_peered":"2023-10-09T10:00:30.785565+0000","last_unstale":"2023-10-09T10:00:31.993805+0000","last_undegraded":"2023-10-09T10:00:31.993805+0000","last_fullsized":"2023-10-09T10:00:31.993805+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:56.229142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"128'6812","reported_seq":6852,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-09T10:00:38.567701+0000","last_change":"2023-10-09T09:58:52.625649+0000","last_active":"2023-10-09T10:00:38.567701+0000","last_peered":"2023-10-09T10:00:38.567701+0000","last_clean":"2023-10-09T10:00:38.567701+0000","last_became_active":"2023-10-09T09:58:52.625367+0000","last_became_peered":"2023-10-09T09:58:52.625367+0000","last_unstale":"2023-10-09T10:00:38.567701+0000","last_undegraded":"2023-10-09T10:00:38.567701+0000","last_fullsized":"2023-10-09T10:00:38.567701+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6812,"log_dups_size":0,"ondisk_log_size":6812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:37.385560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414580736,"num_objects":6326,"num_object_clones":0,"num_object_copies":18978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6812,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.778947+0000","last_change":"2023-10-09T09:56:13.614282+0000","last_active":"2023-10-09T10:00:30.778947+0000","last_peered":"2023-10-09T10:00:30.778947+0000","last_clean":"2023-10-09T10:00:30.778947+0000","last_became_active":"2023-10-09T09:56:13.613888+0000","last_became_peered":"2023-10-09T09:56:13.613888+0000","last_unstale":"2023-10-09T10:00:30.778947+0000","last_undegraded":"2023-10-09T10:00:30.778947+0000","last_fullsized":"2023-10-09T10:00:30.778947+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":100,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.057308+0000","last_change":"2023-10-09T10:00:30.786812+0000","last_active":"2023-10-09T10:00:36.057308+0000","last_peered":"2023-10-09T10:00:36.057308+0000","last_clean":"2023-10-09T10:00:36.057308+0000","last_became_active":"2023-10-09T10:00:30.786500+0000","last_became_peered":"2023-10-09T10:00:30.786500+0000","last_unstale":"2023-10-09T10:00:36.057308+0000","last_undegraded":"2023-10-09T10:00:36.057308+0000","last_fullsized":"2023-10-09T10:00:36.057308+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:25.060464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":71,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.993251+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T10:00:31.993251+0000","last_peered":"2023-10-09T10:00:31.993251+0000","last_clean":"2023-10-09T10:00:31.993251+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T10:00:31.993251+0000","last_undegraded":"2023-10-09T10:00:31.993251+0000","last_fullsized":"2023-10-09T10:00:31.993251+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"128'6883","reported_seq":6936,"reported_epoch":128,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:00:38.567626+0000","last_change":"2023-10-09T10:00:30.916336+0000","last_active":"2023-10-09T10:00:38.567626+0000","last_peered":"2023-10-09T10:00:38.567626+0000","last_clean":"2023-10-09T10:00:08.717617+0000","last_became_active":"2023-10-09T10:00:30.909470+0000","last_became_peered":"2023-10-09T10:00:30.909470+0000","last_unstale":"2023-10-09T10:00:38.567626+0000","last_undegraded":"2023-10-09T10:00:30.786197+0000","last_fullsized":"2023-10-09T10:00:30.775548+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6883,"log_dups_size":0,"ondisk_log_size":6883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:50.983066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421199872,"num_objects":6427,"num_object_clones":0,"num_object_copies":19281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":634,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6883,"num_write_kb":425920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":5793},{"shards":"1,6,7","objects":634}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.054498+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T10:00:36.054498+0000","last_peered":"2023-10-09T10:00:36.054498+0000","last_clean":"2023-10-09T10:00:36.054498+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T10:00:36.054498+0000","last_undegraded":"2023-10-09T10:00:36.054498+0000","last_fullsized":"2023-10-09T10:00:36.054498+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":66,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.065279+0000","last_change":"2023-10-09T09:56:45.549837+0000","last_active":"2023-10-09T10:00:36.065279+0000","last_peered":"2023-10-09T10:00:36.065279+0000","last_clean":"2023-10-09T10:00:36.065279+0000","last_became_active":"2023-10-09T09:56:45.549546+0000","last_became_peered":"2023-10-09T09:56:45.549546+0000","last_unstale":"2023-10-09T10:00:36.065279+0000","last_undegraded":"2023-10-09T10:00:36.065279+0000","last_fullsized":"2023-10-09T10:00:36.065279+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:26.779071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.063651+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T10:00:36.063651+0000","last_peered":"2023-10-09T10:00:36.063651+0000","last_clean":"2023-10-09T10:00:36.063651+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T10:00:36.063651+0000","last_undegraded":"2023-10-09T10:00:36.063651+0000","last_fullsized":"2023-10-09T10:00:36.063651+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"126'6817","reported_seq":6869,"reported_epoch":126,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:00:34.808533+0000","last_change":"2023-10-09T10:00:30.896934+0000","last_active":"2023-10-09T10:00:34.808533+0000","last_peered":"2023-10-09T10:00:34.808533+0000","last_clean":"2023-10-09T10:00:08.575217+0000","last_became_active":"2023-10-09T10:00:30.890988+0000","last_became_peered":"2023-10-09T10:00:30.890988+0000","last_unstale":"2023-10-09T10:00:34.808533+0000","last_undegraded":"2023-10-09T10:00:30.786060+0000","last_fullsized":"2023-10-09T10:00:30.773768+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6817,"log_dups_size":0,"ondisk_log_size":6817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432603136,"num_objects":6601,"num_object_clones":0,"num_object_copies":19803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":684,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6817,"num_write_kb":429376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":5917},{"shards":"3,4,6","objects":684}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":113,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.851875+0000","last_change":"2023-10-09T10:00:30.851245+0000","last_active":"2023-10-09T10:00:30.851875+0000","last_peered":"2023-10-09T10:00:30.851875+0000","last_clean":"2023-10-09T10:00:30.851875+0000","last_became_active":"2023-10-09T10:00:30.850854+0000","last_became_peered":"2023-10-09T10:00:30.850854+0000","last_unstale":"2023-10-09T10:00:30.851875+0000","last_undegraded":"2023-10-09T10:00:30.851875+0000","last_fullsized":"2023-10-09T10:00:30.851875+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":89,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.993728+0000","last_change":"2023-10-09T09:56:45.554759+0000","last_active":"2023-10-09T10:00:31.993728+0000","last_peered":"2023-10-09T10:00:31.993728+0000","last_clean":"2023-10-09T10:00:31.993728+0000","last_became_active":"2023-10-09T09:56:45.554390+0000","last_became_peered":"2023-10-09T09:56:45.554390+0000","last_unstale":"2023-10-09T10:00:31.993728+0000","last_undegraded":"2023-10-09T10:00:31.993728+0000","last_fullsized":"2023-10-09T10:00:31.993728+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:43.005737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.773949+0000","last_change":"2023-10-09T09:48:26.525162+0000","last_active":"2023-10-09T10:00:30.773949+0000","last_peered":"2023-10-09T10:00:30.773949+0000","last_clean":"2023-10-09T10:00:30.773949+0000","last_became_active":"2023-10-09T09:48:26.524864+0000","last_became_peered":"2023-10-09T09:48:26.524864+0000","last_unstale":"2023-10-09T10:00:30.773949+0000","last_undegraded":"2023-10-09T10:00:30.773949+0000","last_fullsized":"2023-10-09T10:00:30.773949+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"126'6758","reported_seq":7922,"reported_epoch":126,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:00:34.825732+0000","last_change":"2023-10-09T10:00:30.883947+0000","last_active":"2023-10-09T10:00:34.825732+0000","last_peered":"2023-10-09T10:00:34.825732+0000","last_clean":"2023-10-09T10:00:08.677832+0000","last_became_active":"2023-10-09T10:00:30.877685+0000","last_became_peered":"2023-10-09T10:00:30.877685+0000","last_unstale":"2023-10-09T10:00:34.825732+0000","last_undegraded":"2023-10-09T10:00:30.787418+0000","last_fullsized":"2023-10-09T10:00:30.778863+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428867584,"num_objects":6544,"num_object_clones":0,"num_object_copies":19632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":667,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6758,"num_write_kb":425664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":555,"num_bytes_recovered":36372480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"1,6,7","objects":667},{"shards":"6,7","objects":5877}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":73,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.993505+0000","last_change":"2023-10-09T09:56:13.614416+0000","last_active":"2023-10-09T10:00:31.993505+0000","last_peered":"2023-10-09T10:00:31.993505+0000","last_clean":"2023-10-09T10:00:31.993505+0000","last_became_active":"2023-10-09T09:56:13.614050+0000","last_became_peered":"2023-10-09T09:56:13.614050+0000","last_unstale":"2023-10-09T10:00:31.993505+0000","last_undegraded":"2023-10-09T10:00:31.993505+0000","last_fullsized":"2023-10-09T10:00:31.993505+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:17.068217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T10:00:31.916075+0000","last_change":"2023-10-09T10:00:30.779150+0000","last_active":"2023-10-09T10:00:31.916075+0000","last_peered":"2023-10-09T10:00:31.916075+0000","last_clean":"2023-10-09T10:00:31.916075+0000","last_became_active":"2023-10-09T10:00:30.778817+0000","last_became_peered":"2023-10-09T10:00:30.778817+0000","last_unstale":"2023-10-09T10:00:31.916075+0000","last_undegraded":"2023-10-09T10:00:31.916075+0000","last_fullsized":"2023-10-09T10:00:31.916075+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:04.893208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.781917+0000","last_change":"2023-10-09T09:56:45.554958+0000","last_active":"2023-10-09T10:00:30.781917+0000","last_peered":"2023-10-09T10:00:30.781917+0000","last_clean":"2023-10-09T10:00:30.781917+0000","last_became_active":"2023-10-09T09:56:45.554597+0000","last_became_peered":"2023-10-09T09:56:45.554597+0000","last_unstale":"2023-10-09T10:00:30.781917+0000","last_undegraded":"2023-10-09T10:00:30.781917+0000","last_fullsized":"2023-10-09T10:00:30.781917+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.774012+0000","last_change":"2023-10-09T09:57:49.622634+0000","last_active":"2023-10-09T10:00:30.774012+0000","last_peered":"2023-10-09T10:00:30.774012+0000","last_clean":"2023-10-09T10:00:30.774012+0000","last_became_active":"2023-10-09T09:57:49.622247+0000","last_became_peered":"2023-10-09T09:57:49.622247+0000","last_unstale":"2023-10-09T10:00:30.774012+0000","last_undegraded":"2023-10-09T10:00:30.774012+0000","last_fullsized":"2023-10-09T10:00:30.774012+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.781132+0000","last_change":"2023-10-09T09:48:25.516226+0000","last_active":"2023-10-09T10:00:30.781132+0000","last_peered":"2023-10-09T10:00:30.781132+0000","last_clean":"2023-10-09T10:00:30.781132+0000","last_became_active":"2023-10-09T09:48:25.515866+0000","last_became_peered":"2023-10-09T09:48:25.515866+0000","last_unstale":"2023-10-09T10:00:30.781132+0000","last_undegraded":"2023-10-09T10:00:30.781132+0000","last_fullsized":"2023-10-09T10:00:30.781132+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.787325+0000","last_change":"2023-10-09T09:57:49.620967+0000","last_active":"2023-10-09T10:00:30.787325+0000","last_peered":"2023-10-09T10:00:30.787325+0000","last_clean":"2023-10-09T10:00:30.787325+0000","last_became_active":"2023-10-09T09:57:49.620567+0000","last_became_peered":"2023-10-09T09:57:49.620567+0000","last_unstale":"2023-10-09T10:00:30.787325+0000","last_undegraded":"2023-10-09T10:00:30.787325+0000","last_fullsized":"2023-10-09T10:00:30.787325+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":120,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.053915+0000","last_change":"2023-10-09T10:00:30.779348+0000","last_active":"2023-10-09T10:00:36.053915+0000","last_peered":"2023-10-09T10:00:36.053915+0000","last_clean":"2023-10-09T10:00:36.053915+0000","last_became_active":"2023-10-09T10:00:30.778963+0000","last_became_peered":"2023-10-09T10:00:30.778963+0000","last_unstale":"2023-10-09T10:00:36.053915+0000","last_undegraded":"2023-10-09T10:00:36.053915+0000","last_fullsized":"2023-10-09T10:00:36.053915+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.112454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T10:00:36.053553+0000","last_change":"2023-10-09T10:00:30.865002+0000","last_active":"2023-10-09T10:00:36.053553+0000","last_peered":"2023-10-09T10:00:36.053553+0000","last_clean":"2023-10-09T10:00:36.053553+0000","last_became_active":"2023-10-09T10:00:30.864670+0000","last_became_peered":"2023-10-09T10:00:30.864670+0000","last_unstale":"2023-10-09T10:00:36.053553+0000","last_undegraded":"2023-10-09T10:00:36.053553+0000","last_fullsized":"2023-10-09T10:00:36.053553+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:16.562219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":116,"reported_epoch":126,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:00:31.993924+0000","last_change":"2023-10-09T10:00:30.921710+0000","last_active":"2023-10-09T10:00:31.993924+0000","last_peered":"2023-10-09T10:00:31.993924+0000","last_clean":"2023-10-09T10:00:28.748714+0000","last_became_active":"2023-10-09T10:00:30.861856+0000","last_became_peered":"2023-10-09T10:00:30.861856+0000","last_unstale":"2023-10-09T10:00:31.993924+0000","last_undegraded":"2023-10-09T10:00:30.921596+0000","last_fullsized":"2023-10-09T10:00:31.993924+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:34.634782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,2,5","objects":1},{"shards":"0,5,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-10-09T10:00:30.773682+0000","last_change":"2023-10-09T09:56:45.556456+0000","last_active":"2023-10-09T10:00:30.773682+0000","last_peered":"2023-10-09T10:00:30.773682+0000","last_clean":"2023-10-09T10:00:30.773682+0000","last_became_active":"2023-10-09T09:56:45.555878+0000","last_became_peered":"2023-10-09T09:56:45.555878+0000","last_unstale":"2023-10-09T10:00:30.773682+0000","last_undegraded":"2023-10-09T10:00:30.773682+0000","last_fullsized":"2023-10-09T10:00:30.773682+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6737100824,"num_objects":102801,"num_object_clones":0,"num_object_copies":308403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3896,"num_objects_misplaced":614,"num_objects_unfound":0,"num_objects_dirty":102801,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108237,"num_write_kb":6753153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2023,"num_bytes_recovered":132579328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20366962688,"data_stored":20366950472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108237,"ondisk_log_size":108237,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":622,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430371,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2853184,"kb_used_data":2547768,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90891968,"statfs":{"total":95995035648,"available":93073375232,"internally_reserved":0,"allocated":2608914432,"data_stored":2608444272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":0.82699999999999996,"15min":0.67100000000000004},"min":{"1min":0.251,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5519999999999996,"5min":4.5519999999999996,"15min":4.5519999999999996},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.9870000000000001,"5min":0.84299999999999997,"15min":0.64200000000000002},"min":{"1min":0.25900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":8.0259999999999998,"5min":8.0259999999999998,"15min":8.0259999999999998},"last":0.29099999999999998}]},{"osd":1,"last update":"Mon Oct 9 09:59:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":0.66100000000000003,"15min":0.58299999999999996},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":2.4430000000000001,"5min":2.4430000000000001,"15min":2.4430000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.4039999999999999,"5min":0.75,"15min":0.64000000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.6189999999999998,"5min":5.6189999999999998,"15min":5.6189999999999998},"last":1.1299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":0.70199999999999996,"15min":0.63400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.806,"5min":2.806,"15min":2.806},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.1160000000000001,"5min":0.86299999999999999,"15min":0.66400000000000003},"min":{"1min":0.29999999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.7400000000000002,"5min":6.7400000000000002,"15min":6.7400000000000002},"last":6.0590000000000002}]},{"osd":3,"last update":"Mon Oct 9 09:59:52 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":0.69899999999999995,"15min":0.623},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.9380000000000002,"5min":2.9380000000000002,"15min":2.9380000000000002},"last":3.4660000000000002},{"interface":"front","average":{"1min":1.323,"5min":0.73499999999999999,"15min":0.63700000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.351,"5min":4.351,"15min":4.351},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.54900000000000004,"15min":0.48699999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.7719999999999998,"5min":2.7719999999999998,"15min":2.7719999999999998},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.52000000000000002,"15min":0.48599999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.26500000000000001},"max":{"1min":1.7450000000000001,"5min":1.7450000000000001,"15min":1.7450000000000001},"last":0.46300000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52200000000000002,"15min":0.505},"min":{"1min":0.29999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.53200000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.429,"5min":1.429,"15min":1.429},"last":2.6800000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:59:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58899999999999997,"15min":0.55400000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32900000000000001},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.375},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58899999999999997,"15min":0.55000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":1.8089999999999999,"5min":1.8089999999999999,"15min":1.8089999999999999},"last":0.41699999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626599,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2519476,"kb_used_data":2222252,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91225676,"statfs":{"total":95995035648,"available":93415092224,"internally_reserved":0,"allocated":2275586048,"data_stored":2275128269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":0.91300000000000003,"15min":0.67500000000000004},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":8.8650000000000002,"5min":8.8650000000000002,"15min":8.8650000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.843,"5min":0.875,"15min":0.629},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.246},"max":{"1min":12.439,"5min":12.439,"15min":12.439},"last":0.45200000000000001}]},{"osd":1,"last update":"Mon Oct 9 09:59:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":0.66900000000000004,"15min":0.58899999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.1470000000000002,"5min":5.1470000000000002,"15min":5.1470000000000002},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.625,"15min":0.56399999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.0179999999999998,"5min":3.0179999999999998,"15min":3.0179999999999998},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:00:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8210000000000002,"5min":1.0229999999999999,"15min":0.70099999999999996},"min":{"1min":0.41799999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":18.800000000000001,"5min":18.800000000000001,"15min":18.800000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.7570000000000001,"5min":1.0780000000000001,"15min":0.75},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":12.413,"5min":12.413,"15min":12.413},"last":0.63100000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:59:52 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":0.78300000000000003,"15min":0.67100000000000004},"min":{"1min":0.52900000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":4.0279999999999996,"5min":4.0279999999999996,"15min":4.0279999999999996},"last":0.504},{"interface":"front","average":{"1min":1.2310000000000001,"5min":0.72599999999999998,"15min":0.64200000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.0140000000000002,"5min":4.0140000000000002,"15min":4.0140000000000002},"last":0.51100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:00:12 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":0.622,"15min":0.50600000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":6.6219999999999999,"5min":6.6219999999999999,"15min":6.6219999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.2849999999999999,"5min":0.66500000000000004,"15min":0.54600000000000004},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.2379999999999995,"5min":8.2379999999999995,"15min":8.2379999999999995},"last":0.47199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:00:12 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.55900000000000005,"15min":0.51900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.8920000000000003,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.63200000000000001,"15min":0.54600000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7069999999999999,"5min":5.7069999999999999,"15min":5.7069999999999999},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.56699999999999995,"15min":0.53900000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.3540000000000001,"5min":2.3540000000000001,"15min":2.3540000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.63100000000000001,"15min":0.60199999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34399999999999997},"max":{"1min":2.3929999999999998,"5min":2.3929999999999998,"15min":2.3929999999999998},"last":0.66200000000000003}]}]},{"osd":1,"up_from":84,"seq":360777252913,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2180836,"kb_used_data":2113180,"kb_used_omap":9,"kb_used_meta":67638,"kb_avail":91564316,"statfs":{"total":95995035648,"available":93761859584,"internally_reserved":0,"allocated":2163896320,"data_stored":2163349612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":69262313},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:55 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":0.68100000000000005,"15min":0.46100000000000002},"min":{"1min":0.23300000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.5469999999999997,"5min":6.5469999999999997,"15min":6.5469999999999997},"last":2.347},{"interface":"front","average":{"1min":1.4630000000000001,"5min":0.59899999999999998,"15min":0.46500000000000002},"min":{"1min":0.32400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.3499999999999996,"5min":7.3499999999999996,"15min":7.3499999999999996},"last":0.83399999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:00:23 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":0.95899999999999996,"15min":0.57899999999999996},"min":{"1min":0.30099999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.6839999999999993,"5min":9.6839999999999993,"15min":9.6839999999999993},"last":4.6929999999999996},{"interface":"front","average":{"1min":3.3340000000000001,"5min":1.0249999999999999,"15min":0.60199999999999998},"min":{"1min":0.32100000000000001,"5min":0.248,"15min":0.248},"max":{"1min":12.069000000000001,"5min":12.069000000000001,"15min":12.069000000000001},"last":8.6999999999999993}]},{"osd":3,"last update":"Mon Oct 9 09:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":0.84399999999999997,"15min":0.53300000000000003},"min":{"1min":0.34599999999999997,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.1289999999999996,"5min":8.1289999999999996,"15min":8.1289999999999996},"last":2.786},{"interface":"front","average":{"1min":1.9239999999999999,"5min":0.69999999999999996,"15min":0.496},"min":{"1min":0.30399999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.3740000000000006,"5min":9.3740000000000006,"15min":9.3740000000000006},"last":8.6389999999999993}]},{"osd":4,"last update":"Mon Oct 9 09:59:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":0.85599999999999998,"15min":0.69199999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":9.5139999999999993,"5min":9.5139999999999993,"15min":9.5139999999999993},"last":1.5629999999999999},{"interface":"front","average":{"1min":1.6599999999999999,"5min":0.85499999999999998,"15min":0.71299999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.2539999999999996,"5min":6.2539999999999996,"15min":6.2539999999999996},"last":2.3860000000000001}]},{"osd":5,"last update":"Mon Oct 9 09:59:55 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":0.877,"15min":0.71099999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.7670000000000003,"5min":6.7670000000000003,"15min":6.7670000000000003},"last":1.883},{"interface":"front","average":{"1min":1.341,"5min":0.78400000000000003,"15min":0.69999999999999996},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.585,"5min":5.585,"15min":5.585},"last":1.6000000000000001}]},{"osd":6,"last update":"Mon Oct 9 09:59:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":0.878,"15min":0.751},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.3040000000000003,"5min":6.3040000000000003,"15min":6.3040000000000003},"last":3.8559999999999999},{"interface":"front","average":{"1min":1.367,"5min":0.80200000000000005,"15min":0.71799999999999997},"min":{"1min":0.38300000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.6399999999999997,"5min":5.6399999999999997,"15min":5.6399999999999997},"last":0.86499999999999999}]},{"osd":7,"last update":"Mon Oct 9 09:59:55 2023","interfaces":[{"interface":"back","average":{"1min":1.9710000000000001,"5min":0.89800000000000002,"15min":0.72699999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":9.4329999999999998,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":1.6299999999999999},{"interface":"front","average":{"1min":1.8260000000000001,"5min":0.92200000000000004,"15min":0.76400000000000001},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.7759999999999998,"5min":6.7759999999999998,"15min":6.7759999999999998},"last":2.5070000000000001}]}]},{"osd":0,"up_from":79,"seq":339302416439,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1691948,"kb_used_data":1634084,"kb_used_omap":28,"kb_used_meta":57827,"kb_avail":92053204,"statfs":{"total":95995035648,"available":94262480896,"internally_reserved":0,"allocated":1673302016,"data_stored":1672732354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":59215696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 09:59:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":0.63800000000000001,"15min":0.46300000000000002},"min":{"1min":0.36799999999999999,"5min":0.191,"15min":0.191},"max":{"1min":6.5149999999999997,"5min":6.5149999999999997,"15min":6.5149999999999997},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.782,"5min":0.67100000000000004,"15min":0.48899999999999999},"min":{"1min":0.311,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.9539999999999997,"5min":6.9539999999999997,"15min":6.9539999999999997},"last":0.26900000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.4529999999999998,"5min":0.83299999999999996,"15min":0.53900000000000003},"min":{"1min":0.23000000000000001,"5min":0.216,"15min":0.216},"max":{"1min":10.175000000000001,"5min":10.175000000000001,"15min":10.175000000000001},"last":0.224},{"interface":"front","average":{"1min":3.6930000000000001,"5min":1.107,"15min":0.65300000000000002},"min":{"1min":0.29599999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":17.920999999999999,"5min":17.920999999999999,"15min":17.920999999999999},"last":0.52900000000000003}]},{"osd":3,"last update":"Mon Oct 9 09:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":0.80700000000000005,"15min":0.52300000000000002},"min":{"1min":0.34300000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.9380000000000006,"5min":9.9380000000000006,"15min":9.9380000000000006},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.766,"5min":0.90500000000000003,"15min":0.59499999999999997},"min":{"1min":0.441,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.7450000000000001,"5min":6.7450000000000001,"15min":6.7450000000000001},"last":0.496}]},{"osd":4,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":0.94199999999999995,"15min":0.752},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.035,"5min":10.035,"15min":10.035},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.6499999999999999,"5min":1.0860000000000001,"15min":0.81299999999999994},"min":{"1min":0.40899999999999997,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":8.6370000000000005,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":0.55100000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2349999999999999,"5min":0.999,"15min":0.76100000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":6.5010000000000003,"5min":6.5010000000000003,"15min":6.5010000000000003},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.1749999999999998,"5min":0.95699999999999996,"15min":0.76300000000000001},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":10.083,"5min":10.083,"15min":10.083},"last":0.47699999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.1060000000000001,"15min":0.82799999999999996},"min":{"1min":0.432,"5min":0.379,"15min":0.379},"max":{"1min":10.115,"5min":10.115,"15min":10.115},"last":0.437},{"interface":"front","average":{"1min":2.9689999999999999,"5min":1.143,"15min":0.84199999999999997},"min":{"1min":0.57099999999999995,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":8.7940000000000005,"5min":8.7940000000000005,"15min":8.7940000000000005},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.038,"15min":0.79000000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.3179999999999996,"5min":7.3179999999999996,"15min":7.3179999999999996},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.7189999999999999,"5min":1.0860000000000001,"15min":0.81699999999999995},"min":{"1min":0.46600000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":10.147,"5min":10.147,"15min":10.147},"last":0.40200000000000002}]}]},{"osd":2,"up_from":89,"seq":382252089386,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3106908,"kb_used_data":3016980,"kb_used_omap":81,"kb_used_meta":89838,"kb_avail":90638244,"statfs":{"total":95995035648,"available":92813561856,"internally_reserved":0,"allocated":3089387520,"data_stored":3088832625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":91994330},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":4.1399999999999997,"5min":1.2030000000000001,"15min":0.65700000000000003},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":56.883000000000003,"5min":56.883000000000003,"15min":56.883000000000003},"last":1.1970000000000001},{"interface":"front","average":{"1min":4.6769999999999996,"5min":1.331,"15min":0.70999999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":57.595999999999997,"5min":57.595999999999997,"15min":57.595999999999997},"last":1.1200000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.0189999999999999,"15min":0.58399999999999996},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.035,"5min":12.035,"15min":12.035},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.2330000000000001,"5min":0.82599999999999996,"15min":0.54900000000000004},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":13.217000000000001,"5min":13.217000000000001,"15min":13.217000000000001},"last":1.0980000000000001}]},{"osd":3,"last update":"Mon Oct 9 09:59:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8940000000000001,"5min":1.3140000000000001,"15min":0.71699999999999997},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":58.067,"5min":58.067,"15min":58.067},"last":1.246},{"interface":"front","average":{"1min":5.3600000000000003,"5min":1.3939999999999999,"15min":0.73299999999999998},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":60.456000000000003,"5min":60.456000000000003,"15min":60.456000000000003},"last":1.403}]},{"osd":4,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":4.2789999999999999,"5min":1.405,"15min":0.871},"min":{"1min":0.39300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":57.646999999999998,"5min":57.646999999999998,"15min":57.646999999999998},"last":1.486},{"interface":"front","average":{"1min":4.5570000000000004,"5min":1.554,"15min":0.93799999999999994},"min":{"1min":0.38800000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":57.024000000000001,"5min":57.024000000000001,"15min":57.024000000000001},"last":0.64800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":3.96,"5min":2.1339999999999999,"15min":1.1279999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":56.969999999999999,"5min":79.016000000000005,"15min":79.016000000000005},"last":1.349},{"interface":"front","average":{"1min":4.5199999999999996,"5min":2.2330000000000001,"15min":1.1639999999999999},"min":{"1min":0.438,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":57.731999999999999,"5min":79.302999999999997,"15min":79.302999999999997},"last":1.4259999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":4.7309999999999999,"5min":1.5109999999999999,"15min":0.91300000000000003},"min":{"1min":0.46600000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":57.698,"5min":57.698,"15min":57.698},"last":1.3620000000000001},{"interface":"front","average":{"1min":4.6559999999999997,"5min":1.569,"15min":0.94499999999999995},"min":{"1min":0.42599999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":57.058,"5min":57.058,"15min":57.058},"last":0.67300000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0549999999999997,"5min":1.3819999999999999,"15min":0.89700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":57.003,"5min":57.003,"15min":57.003},"last":1.3},{"interface":"front","average":{"1min":4.4290000000000003,"5min":1.4630000000000001,"15min":0.92800000000000005},"min":{"1min":0.48999999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":57.676000000000002,"5min":57.676000000000002,"15min":57.676000000000002},"last":1.462}]}]},{"osd":3,"up_from":94,"seq":403726925860,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4295796,"kb_used_data":4183788,"kb_used_omap":7,"kb_used_meta":111928,"kb_avail":89449356,"statfs":{"total":95995035648,"available":91596140544,"internally_reserved":0,"allocated":4284198912,"data_stored":4283635598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":114614888},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 09:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.1799999999999999,"15min":1.056},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.4239999999999995,"5min":11.958,"15min":11.958},"last":2.3999999999999999},{"interface":"front","average":{"1min":2.4489999999999998,"5min":0.84999999999999998,"15min":0.58399999999999996},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":8.8190000000000008,"5min":8.8190000000000008,"15min":8.8190000000000008},"last":7.9080000000000004}]},{"osd":1,"last update":"Mon Oct 9 09:59:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":0.83699999999999997,"15min":0.55800000000000005},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":8.0600000000000005,"5min":8.0600000000000005,"15min":8.0600000000000005},"last":8.0370000000000008},{"interface":"front","average":{"1min":2.6269999999999998,"5min":0.85699999999999998,"15min":0.56200000000000006},"min":{"1min":0.30099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.3699999999999992,"5min":8.3699999999999992,"15min":8.3699999999999992},"last":7.4359999999999999}]},{"osd":2,"last update":"Mon Oct 9 09:59:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5630000000000002,"5min":1.349,"15min":1.147},"min":{"1min":0.313,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":11.173,"5min":13.246,"15min":13.246},"last":2.7509999999999999},{"interface":"front","average":{"1min":2.98,"5min":0.96099999999999997,"15min":0.625},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.496,"5min":11.496,"15min":11.496},"last":2.5350000000000001}]},{"osd":4,"last update":"Mon Oct 9 09:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":0.84299999999999997,"15min":0.71099999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.3650000000000002,"5min":5.3650000000000002,"15min":5.3650000000000002},"last":2.3679999999999999},{"interface":"front","average":{"1min":1.766,"5min":0.90300000000000002,"15min":0.75900000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.2439999999999998,"5min":4.2439999999999998,"15min":4.2439999999999998},"last":7.9740000000000002}]},{"osd":5,"last update":"Mon Oct 9 09:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":0.92500000000000004,"15min":0.76700000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.4470000000000001,"5min":5.4470000000000001,"15min":5.4470000000000001},"last":8.0069999999999997},{"interface":"front","average":{"1min":1.849,"5min":1.212,"15min":1.1060000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.6989999999999998,"5min":8.6989999999999998,"15min":8.6989999999999998},"last":2.8490000000000002}]},{"osd":6,"last update":"Mon Oct 9 09:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":0.91700000000000004,"15min":0.76200000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.883,"5min":4.883,"15min":4.883},"last":8.0649999999999995},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.1579999999999999,"15min":1.0900000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":8.7390000000000008,"5min":8.7390000000000008,"15min":8.7390000000000008},"last":2.8279999999999998}]},{"osd":7,"last update":"Mon Oct 9 09:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":1.222,"15min":1.103},"min":{"1min":0.52100000000000002,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":8.7650000000000006,"5min":8.7650000000000006,"15min":8.7650000000000006},"last":2.802},{"interface":"front","average":{"1min":2.5099999999999998,"5min":1.026,"15min":0.77900000000000003},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":8.282,"5min":8.282,"15min":8.282},"last":2.3319999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019055,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2392248,"kb_used_data":2082736,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91352904,"statfs":{"total":95995035648,"available":93545373696,"internally_reserved":0,"allocated":2132721664,"data_stored":2132259082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":0.88600000000000001,"15min":0.63900000000000001},"min":{"1min":0.26300000000000001,"5min":0.252,"15min":0.252},"max":{"1min":7.7939999999999996,"5min":7.7939999999999996,"15min":7.7939999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.786,"5min":0.84699999999999998,"15min":0.66300000000000003},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.9570000000000007,"5min":8.9570000000000007,"15min":8.9570000000000007},"last":0.58499999999999996}]},{"osd":1,"last update":"Mon Oct 9 09:59:47 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":0.61399999999999999,"15min":0.55100000000000005},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.7000000000000002,"5min":3.7000000000000002,"15min":3.7000000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.296,"5min":0.72399999999999998,"15min":0.622},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0410000000000004,"5min":6.0410000000000004,"15min":6.0410000000000004},"last":0.64900000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.371,"5min":0.98599999999999999,"15min":0.67900000000000005},"min":{"1min":0.434,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.4640000000000004,"5min":9.4640000000000004,"15min":9.4640000000000004},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.3980000000000001,"5min":0.97299999999999998,"15min":0.69899999999999995},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.9800000000000004,"5min":8.9800000000000004,"15min":8.9800000000000004},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 09:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":0.77400000000000002,"15min":0.60599999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":8.9429999999999996,"5min":8.9429999999999996,"15min":8.9429999999999996},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.6990000000000001,"5min":0.77500000000000002,"15min":0.621},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":9.0389999999999997,"5min":9.0389999999999997,"15min":9.0389999999999997},"last":0.501}]},{"osd":5,"last update":"Mon Oct 9 10:00:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.47399999999999998,"15min":0.44500000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.133,"5min":2.133,"15min":2.133},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.56200000000000006,"15min":0.497},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.60199999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.53900000000000003,"15min":0.48799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":1.9430000000000001,"5min":1.9430000000000001,"15min":1.9430000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.198,"5min":0.69199999999999995,"15min":0.55600000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.312},"max":{"1min":9.0109999999999992,"5min":9.0109999999999992,"15min":9.0109999999999992},"last":0.68600000000000005}]},{"osd":7,"last update":"Mon Oct 9 09:59:47 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":0.60199999999999998,"15min":0.54900000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.6259999999999999,"5min":2.6259999999999999,"15min":2.6259999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59799999999999998,"15min":0.56799999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.316},"max":{"1min":1.9039999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"last":0.57599999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822827,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2442348,"kb_used_data":2141028,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91302804,"statfs":{"total":95995035648,"available":93494071296,"internally_reserved":0,"allocated":2192412672,"data_stored":2191958146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1,"15min":0.69199999999999995},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":13.112,"5min":13.112,"15min":13.112},"last":1.7310000000000001},{"interface":"front","average":{"1min":2.4489999999999998,"5min":0.94999999999999996,"15min":0.67400000000000004},"min":{"1min":0.32700000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.8529999999999998,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":0.33200000000000002}]},{"osd":1,"last update":"Mon Oct 9 09:59:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":0.80100000000000005,"15min":0.627},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.6929999999999996,"5min":7.6929999999999996,"15min":7.6929999999999996},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.71099999999999997,"15min":0.59999999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.9480000000000004,"5min":4.9480000000000004,"15min":4.9480000000000004},"last":3.6640000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:00:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":1.1379999999999999,"15min":0.84399999999999997},"min":{"1min":0.30099999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.582000000000001,"5min":12.582000000000001,"15min":12.582000000000001},"last":2.1120000000000001},{"interface":"front","average":{"1min":2.3490000000000002,"5min":1.0289999999999999,"15min":0.81200000000000006},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.438000000000001,"5min":12.438000000000001,"15min":12.438000000000001},"last":3.8239999999999998}]},{"osd":3,"last update":"Mon Oct 9 09:59:54 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":0.82899999999999996,"15min":0.72899999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.4630000000000001,"5min":4.4630000000000001,"15min":4.4630000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.544,"5min":0.82499999999999996,"15min":0.70499999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.2910000000000004,"5min":4.2910000000000004,"15min":4.2910000000000004},"last":3.7879999999999998}]},{"osd":4,"last update":"Mon Oct 9 09:59:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.53600000000000003,"15min":0.47199999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.6819999999999999,"5min":2.6819999999999999,"15min":2.6819999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.032,"5min":0.61499999999999999,"15min":0.51400000000000001},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.839,"5min":2.839,"15min":2.839},"last":0.52500000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:00:19 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":1.0429999999999999,"15min":0.68000000000000005},"min":{"1min":0.34599999999999997,"5min":0.317,"15min":0.27600000000000002},"max":{"1min":27.716000000000001,"5min":27.716000000000001,"15min":27.716000000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.6819999999999999,"5min":0.82799999999999996,"15min":0.61499999999999999},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.173999999999999,"5min":10.173999999999999,"15min":10.173999999999999},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 09:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.63600000000000001,"15min":0.57999999999999996},"min":{"1min":0.318,"5min":0.318,"15min":0.309},"max":{"1min":3.536,"5min":3.536,"15min":3.536},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.60799999999999998,"15min":0.55700000000000005},"min":{"1min":0.33200000000000002,"5min":0.318,"15min":0.317},"max":{"1min":2.8620000000000001,"5min":2.8620000000000001,"15min":2.8620000000000001},"last":0.35899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1666322432,"data_stored":1666318360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2161250304,"data_stored":2161246232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3093299200,"data_stored":3093299200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4276879360,"data_stored":4276879360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2117861376,"data_stored":2117861376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2190610432,"data_stored":2190606360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2258763776,"data_stored":2258763776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2601975808,"data_stored":2601975808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:00:41.292 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:00:41.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 26 2023-10-09T10:00:41.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:00:41.304 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:41.310 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:00:41.310 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:00:41.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.310 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:111506 6.4 6:2a1f6246:::benchmark_data_smithi012_182576_object5985:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-10-09T10:00:41.310 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:41.289544+0000", 2023-10-09T10:00:41.311 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.016525842999999998, 2023-10-09T10:00:41.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016552599000000001, 2023-10-09T10:00:41.311 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:41.311 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:00:41.311 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:41.311 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:41.311 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:41.312 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 111506 2023-10-09T10:00:41.312 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.312 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:41.312 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.312 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:41.312 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289544+0000", 2023-10-09T10:00:41.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:41.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289544+0000", 2023-10-09T10:00:41.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:41.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289542+0000", 2023-10-09T10:00:41.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:00:41.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:41.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289553+0000", 2023-10-09T10:00:41.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1555e-05 2023-10-09T10:00:41.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:41.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289581+0000", 2023-10-09T10:00:41.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7634e-05 2023-10-09T10:00:41.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289593+0000", 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1963999999999999e-05 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.290922+0000", 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001328973 2023-10-09T10:00:41.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.291071+0000", 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014919600000000001 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5", 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.291174+0000", 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103441 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:41.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.300429+0000", 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092549769999999993 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:111505 6.e e128/127 6:788282cc:::benchmark_data_smithi012_182576_object5984:head v 128'6970, mlcod=128'6968)", 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:41.289625+0000", 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.016444685000000001, 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016527472000000001, 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:41.318 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289625+0000", 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289625+0000", 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289624+0000", 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289633+0000", 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.799e-06 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289642+0000", 2023-10-09T10:00:41.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6489999999999997e-06 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289647+0000", 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7740000000000001e-06 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.292680+0000", 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003032121 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.292703+0000", 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3828e-05 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:111503 6.d e128/97 6:b218ea3a:::benchmark_data_smithi012_182576_object5982:head v 128'6828, mlcod=128'6826)", 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:41.289698+0000", 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.01637137, 2023-10-09T10:00:41.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016491543000000001, 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289698+0000", 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289698+0000", 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289698+0000", 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289706+0000", 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8310000000000007e-06 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289716+0000", 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6439999999999995e-06 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289722+0000", 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9669999999999998e-06 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289790+0000", 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7673000000000001e-05 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.289814+0000", 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4045000000000001e-05 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:111514 6.e e128/127 6:7fece64a:::benchmark_data_smithi012_182576_object5993:head v 128'6971, mlcod=128'6968)", 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:41.303044+0000", 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0030254539999999999, 2023-10-09T10:00:41.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00317965, 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303044+0000", 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303044+0000", 2023-10-09T10:00:41.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303043+0000", 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303058+0000", 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4985e-05 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303080+0000", 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1503000000000001e-05 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303089+0000", 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7960000000000007e-06 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303778+0000", 2023-10-09T10:00:41.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00068879100000000001 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303798+0000", 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0034000000000002e-05 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:111513 6.d e128/97 6:bc1f2aee:::benchmark_data_smithi012_182576_object5992:head v 128'6829, mlcod=128'6826)", 2023-10-09T10:00:41.330 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:41.303144+0000", 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0029255800000000001, 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031150219999999998, 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303144+0000", 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303144+0000", 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303144+0000", 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:41.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303153+0000", 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8030000000000008e-06 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303163+0000", 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5710000000000004e-06 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303170+0000", 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7499999999999997e-06 2023-10-09T10:00:41.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303234+0000", 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4241999999999999e-05 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:41.303256+0000", 2023-10-09T10:00:41.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2365e-05 2023-10-09T10:00:41.335 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.335 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:41.335 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.335 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:41.335 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:00:41.335 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 5 2023-10-09T10:00:41.335 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:00:41.355 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:41.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:41.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:41.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:41 smithi110 ceph-mon[136032]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:00:41.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:41.598 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:41.975 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:41.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:41.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:42.081 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:42.125 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:42.170 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:42.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:42.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:42.355 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:42 smithi012 ceph-mon[147926]: pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 663 KiB/s wr, 504 op/s; 3897/304506 objects degraded (1.280%); 514/304506 objects misplaced (0.169%); 6.0 MiB/s, 1 keys/s, 95 objects/s recovering 2023-10-09T10:00:42.355 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:42 smithi012 ceph-mon[152263]: pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 663 KiB/s wr, 504 op/s; 3897/304506 objects degraded (1.280%); 514/304506 objects misplaced (0.169%); 6.0 MiB/s, 1 keys/s, 95 objects/s recovering 2023-10-09T10:00:42.363 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:42 smithi110 ceph-mon[136032]: pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 663 KiB/s wr, 504 op/s; 3897/304506 objects degraded (1.280%); 514/304506 objects misplaced (0.169%); 6.0 MiB/s, 1 keys/s, 95 objects/s recovering 2023-10-09T10:00:42.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:00:43.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3897/304506 objects degraded (1.280%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:43.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3897/304506 objects degraded (1.280%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3897/304506 objects degraded (1.280%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:44.346 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:44.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:44.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:44.490 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:44.543 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:44.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:44.647 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:44.714 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:44.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:44 smithi012 ceph-mon[152263]: pgmap v318: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 3897/299652 objects degraded (1.301%); 166/299652 objects misplaced (0.055%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:00:44.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:44 smithi012 ceph-mon[147926]: pgmap v318: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 3897/299652 objects degraded (1.301%); 166/299652 objects misplaced (0.055%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:00:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:44 smithi110 ceph-mon[136032]: pgmap v318: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 3897/299652 objects degraded (1.301%); 166/299652 objects misplaced (0.055%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:00:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:00:45.362 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:45.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:45.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:45.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:45.496 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:45.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:45.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:45.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:45.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:45.656 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:45 smithi012 ceph-mon[147926]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T10:00:45.656 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:45 smithi012 ceph-mon[147926]: failed to encode map e129 with expected crc 2023-10-09T10:00:45.656 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:45 smithi012 ceph-mon[147926]: failed to encode map e129 with expected crc 2023-10-09T10:00:45.656 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:45 smithi012 ceph-mon[147926]: failed to encode map e129 with expected crc 2023-10-09T10:00:45.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:45.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:45 smithi012 ceph-mon[152263]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T10:00:45.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:45 smithi012 ceph-mon[152263]: failed to encode map e129 with expected crc 2023-10-09T10:00:45.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:45 smithi012 ceph-mon[152263]: failed to encode map e129 with expected crc 2023-10-09T10:00:45.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:45 smithi012 ceph-mon[152263]: failed to encode map e129 with expected crc 2023-10-09T10:00:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:45 smithi110 ceph-mon[136032]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T10:00:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:45 smithi110 ceph-mon[136032]: failed to encode map e129 with expected crc 2023-10-09T10:00:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:45 smithi110 ceph-mon[136032]: failed to encode map e129 with expected crc 2023-10-09T10:00:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:45 smithi110 ceph-mon[136032]: failed to encode map e129 with expected crc 2023-10-09T10:00:46.125 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:46.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:46.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:46.326 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:46.392 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:46.395 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:46.816 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:46.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:46.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[147926]: pgmap v320: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3897/293172 objects degraded (1.329%); 166/293172 objects misplaced (0.057%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:00:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[147926]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[147926]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T10:00:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[147926]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[147926]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[152263]: pgmap v320: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3897/293172 objects degraded (1.329%); 166/293172 objects misplaced (0.057%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:00:46.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[152263]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[152263]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T10:00:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[152263]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:46 smithi012 ceph-mon[152263]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:46 smithi110 ceph-mon[136032]: pgmap v320: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3897/293172 objects degraded (1.329%); 166/293172 objects misplaced (0.057%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:00:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:46 smithi110 ceph-mon[136032]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:46 smithi110 ceph-mon[136032]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T10:00:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:46 smithi110 ceph-mon[136032]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:46 smithi110 ceph-mon[136032]: failed to encode map e130 with expected crc 2023-10-09T10:00:46.963 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:47.008 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:47.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:47.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:47.112 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:47.149 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:47.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:47.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:47.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:47.203 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:47.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:47.332 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:47.376 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:47.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:47.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:47.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:47.589 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:47.748 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:47.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:47.834 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:47 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3897/293172 objects degraded (1.329%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:47.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:47 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3897/293172 objects degraded (1.329%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:47.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:47.943 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:48.030 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:48.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:48.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:48.144 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:00:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:00:47] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T10:00:48.189 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:48.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:47 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3897/293172 objects degraded (1.329%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:48.236 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:48.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:48.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:48.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:48.386 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:48.434 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:48.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:48.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:48.621 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:48.730 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:00:48.837 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:48.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:48.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:48 smithi012 ceph-mon[152263]: pgmap v322: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 3896/291876 objects degraded (1.335%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:00:48.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4026777060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:48.849 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:48 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:48.849 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:48 smithi012 ceph-mon[147926]: pgmap v322: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 3896/291876 objects degraded (1.335%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:00:48.849 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4026777060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:48.849 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:48 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:48.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:49.034 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:48 smithi110 ceph-mon[136032]: pgmap v322: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 3896/291876 objects degraded (1.335%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:00:49.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4026777060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:49.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:48 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:00:49.332 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:49.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:49.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:49.439 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:49.480 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:49.525 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:49.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:49.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:49.724 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:00:49.744 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:49.782 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:00:49.801 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:00:49.861 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:00:49.907 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:00:49.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:49.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:00:49.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[147926]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T10:00:49.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[147926]: failed to encode map e131 with expected crc 2023-10-09T10:00:49.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[147926]: failed to encode map e131 with expected crc 2023-10-09T10:00:49.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[147926]: failed to encode map e131 with expected crc 2023-10-09T10:00:49.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[147926]: pgmap v324: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 3282/287586 objects degraded (1.141%); 266/287586 objects misplaced (0.092%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T10:00:49.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2849056034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:49.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:49.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:00:49.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[152263]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T10:00:49.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[152263]: failed to encode map e131 with expected crc 2023-10-09T10:00:49.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[152263]: failed to encode map e131 with expected crc 2023-10-09T10:00:49.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[152263]: failed to encode map e131 with expected crc 2023-10-09T10:00:49.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[152263]: pgmap v324: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 3282/287586 objects degraded (1.141%); 266/287586 objects misplaced (0.092%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T10:00:49.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2849056034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:49.932 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:49.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:49.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:49.939 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:00:49.973 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:00:50.021 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:00:50.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:00:50.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:00:50.023 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:00:50.023 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:00:50.023 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:00:50.023 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:00:50.023 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:00:50.023 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:00:50.037 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:50.064 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:49 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:00:50.064 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:49 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:00:50.064 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:49 smithi110 ceph-mon[136032]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T10:00:50.064 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:49 smithi110 ceph-mon[136032]: failed to encode map e131 with expected crc 2023-10-09T10:00:50.065 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:49 smithi110 ceph-mon[136032]: failed to encode map e131 with expected crc 2023-10-09T10:00:50.065 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:49 smithi110 ceph-mon[136032]: failed to encode map e131 with expected crc 2023-10-09T10:00:50.065 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:49 smithi110 ceph-mon[136032]: pgmap v324: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 3282/287586 objects degraded (1.141%); 266/287586 objects misplaced (0.092%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T10:00:50.065 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2849056034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:50.066 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:50.108 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:50.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:50.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:50.216 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:50.246 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:50.257 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:50.258 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:50.288 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:50.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:50.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:50.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:50.483 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:50.747 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:50.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:50.761 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 6 pg_num to 26 2023-10-09T10:00:50.816 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/598246848' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:00:50.817 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/598246848' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:00:50.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:00:50.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:50.894 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:50.941 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:50.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:51.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:51.134 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:51.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/598246848' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:00:51.249 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:51.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:51.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:51.443 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:51.524 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:51.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:51.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:00:51.637 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:51.667 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:51.710 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:51.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:51.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:51.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/598246848' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:00:51.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T10:00:51.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: failed to encode map e132 with expected crc 2023-10-09T10:00:51.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: failed to encode map e132 with expected crc 2023-10-09T10:00:51.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: failed to encode map e132 with expected crc 2023-10-09T10:00:51.800 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:51.816 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:51.846 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:51.892 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:51.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:51.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:51.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/598246848' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:00:51.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T10:00:51.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: failed to encode map e132 with expected crc 2023-10-09T10:00:51.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: failed to encode map e132 with expected crc 2023-10-09T10:00:51.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: failed to encode map e132 with expected crc 2023-10-09T10:00:51.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: failed to encode map e132 with expected crc 2023-10-09T10:00:51.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: pgmap v326: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 3282/283524 objects degraded (1.158%); 266/283524 objects misplaced (0.094%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T10:00:51.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:00:51.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:51 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:00:52.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:52.109 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/598246848' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:00:52.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T10:00:52.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: failed to encode map e132 with expected crc 2023-10-09T10:00:52.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: failed to encode map e132 with expected crc 2023-10-09T10:00:52.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: failed to encode map e132 with expected crc 2023-10-09T10:00:52.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: failed to encode map e132 with expected crc 2023-10-09T10:00:52.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: pgmap v326: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 3282/283524 objects degraded (1.158%); 266/283524 objects misplaced (0.094%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T10:00:52.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:00:52.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:00:52.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: failed to encode map e132 with expected crc 2023-10-09T10:00:52.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: pgmap v326: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 3282/283524 objects degraded (1.158%); 266/283524 objects misplaced (0.094%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T10:00:52.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:00:52.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:51 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:00:52.118 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:52.196 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:52.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:52.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:52.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:52.335 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:52.425 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:52.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:52.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:52.529 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:52.551 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:52.556 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:52.588 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:52.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:52.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:52.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:00:52.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:52.745 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:52.822 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:52.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:52.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:52 smithi110 ceph-mon[136032]: failed to encode map e133 with expected crc 2023-10-09T10:00:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:00:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:52 smithi110 ceph-mon[136032]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T10:00:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:52 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3282/283524 objects degraded (1.158%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:52 smithi110 ceph-mon[136032]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T10:00:52.958 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:53.040 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:53.044 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[147926]: failed to encode map e133 with expected crc 2023-10-09T10:00:53.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:00:53.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[147926]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T10:00:53.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3282/283524 objects degraded (1.158%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:53.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[147926]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T10:00:53.045 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[152263]: failed to encode map e133 with expected crc 2023-10-09T10:00:53.045 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:00:53.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[152263]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T10:00:53.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3282/283524 objects degraded (1.158%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:53.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:52 smithi012 ceph-mon[152263]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T10:00:53.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:53.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:53.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:53.179 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:53.265 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:53.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:53.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:53.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:53.404 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:53.461 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:00:53.461 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:00:53.749 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:53.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:53.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:53.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:53.881 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:53.911 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:53.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:53.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:53.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:00:53.999 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: failed to encode map e134 with expected crc 2023-10-09T10:00:53.999 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: failed to encode map e134 with expected crc 2023-10-09T10:00:53.999 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: failed to encode map e134 with expected crc 2023-10-09T10:00:53.999 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: failed to encode map e134 with expected crc 2023-10-09T10:00:53.999 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: pgmap v329: 99 pgs: 9 unknown, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 1.44k op/s; 3282/289425 objects degraded (1.134%); 266/289425 objects misplaced (0.092%) 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: 6.7 scrub starts 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[147926]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.000 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: pgmap v329: 99 pgs: 9 unknown, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 1.44k op/s; 3282/289425 objects degraded (1.134%); 266/289425 objects misplaced (0.092%) 2023-10-09T10:00:54.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: 6.7 scrub starts 2023-10-09T10:00:54.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:53 smithi012 ceph-mon[152263]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T10:00:54.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:54.044 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:54.157 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:54.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: failed to encode map e134 with expected crc 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: pgmap v329: 99 pgs: 9 unknown, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 1.44k op/s; 3282/289425 objects degraded (1.134%); 266/289425 objects misplaced (0.092%) 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: 6.7 scrub starts 2023-10-09T10:00:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:53 smithi110 ceph-mon[136032]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T10:00:54.349 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:54.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:54.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:54.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:54.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:54.533 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:54.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:54.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:54.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:54.689 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:54.790 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:54.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:54.817 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: 6.6 scrub starts 2023-10-09T10:00:54.817 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: failed to encode map e135 with expected crc 2023-10-09T10:00:54.817 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: failed to encode map e135 with expected crc 2023-10-09T10:00:54.817 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: failed to encode map e135 with expected crc 2023-10-09T10:00:54.817 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: failed to encode map e135 with expected crc 2023-10-09T10:00:54.817 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: failed to encode map e135 with expected crc 2023-10-09T10:00:54.817 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: failed to encode map e135 with expected crc 2023-10-09T10:00:54.818 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: failed to encode map e135 with expected crc 2023-10-09T10:00:54.818 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: failed to encode map e135 with expected crc 2023-10-09T10:00:54.818 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:54 smithi110 ceph-mon[136032]: 6.7 scrub starts 2023-10-09T10:00:54.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:00:54.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:54.984 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:00:54.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:54.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:00:55.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: 6.6 scrub starts 2023-10-09T10:00:55.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[147926]: 6.7 scrub starts 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: 6.6 scrub starts 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: failed to encode map e135 with expected crc 2023-10-09T10:00:55.017 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:54 smithi012 ceph-mon[152263]: 6.7 scrub starts 2023-10-09T10:00:55.063 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:55.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:55.175 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:55.343 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:55.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:55.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:55.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:55.543 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:55.628 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:55.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:55.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:55.739 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:55.811 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:55.812 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:00:55.871 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:00:55.877 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:00:55.900 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:55.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:55.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:55.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3697705229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:55.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:55 smithi012 ceph-mon[147926]: pgmap v331: 99 pgs: 4 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 3 unknown, 1 peering; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 1.43k op/s; 2612/285681 objects degraded (0.914%); 266/285681 objects misplaced (0.093%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T10:00:55.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2614451532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:55.912 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3697705229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:55.912 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:55 smithi012 ceph-mon[152263]: pgmap v331: 99 pgs: 4 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 3 unknown, 1 peering; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 1.43k op/s; 2612/285681 objects degraded (0.914%); 266/285681 objects misplaced (0.093%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T10:00:55.912 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2614451532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:55.914 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:00:55.962 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:00:56.000 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:00:56.012 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:56.031 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:00:56.050 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:56.072 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:00:56.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:56.094 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3697705229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:00:56.095 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:55 smithi110 ceph-mon[136032]: pgmap v331: 99 pgs: 4 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 3 unknown, 1 peering; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 1.43k op/s; 2612/285681 objects degraded (0.914%); 266/285681 objects misplaced (0.093%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T10:00:56.095 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2614451532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:00:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:56.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:56.109 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:00:56.125 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:00:56.125 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:00:56.126 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:00:56.126 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:00:56.126 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:00:56.126 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:00:56.126 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:00:56.126 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:00:56.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:56.231 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:00:56.231 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:00:56.231 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T10:00:56.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T10:00:56.264 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:56.667 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:56.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:56.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:00:56.870 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:56.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:00:56.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3697705229' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:56.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[152263]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T10:00:56.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[152263]: failed to encode map e136 with expected crc 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[152263]: failed to encode map e136 with expected crc 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[152263]: failed to encode map e136 with expected crc 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[152263]: failed to encode map e136 with expected crc 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[152263]: 6.7 scrub starts 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3697705229' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[147926]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[147926]: failed to encode map e136 with expected crc 2023-10-09T10:00:56.934 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[147926]: failed to encode map e136 with expected crc 2023-10-09T10:00:56.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[147926]: failed to encode map e136 with expected crc 2023-10-09T10:00:56.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[147926]: failed to encode map e136 with expected crc 2023-10-09T10:00:56.935 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:56 smithi012 ceph-mon[147926]: 6.7 scrub starts 2023-10-09T10:00:56.942 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:00:56.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:56.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:00:57.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:57.166 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:56 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:00:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3697705229' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:00:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:56 smithi110 ceph-mon[136032]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T10:00:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:56 smithi110 ceph-mon[136032]: failed to encode map e136 with expected crc 2023-10-09T10:00:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:56 smithi110 ceph-mon[136032]: failed to encode map e136 with expected crc 2023-10-09T10:00:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:56 smithi110 ceph-mon[136032]: failed to encode map e136 with expected crc 2023-10-09T10:00:57.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:56 smithi110 ceph-mon[136032]: failed to encode map e136 with expected crc 2023-10-09T10:00:57.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:56 smithi110 ceph-mon[136032]: 6.7 scrub starts 2023-10-09T10:00:57.245 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:00:57.245 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:00:57.245 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.245 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:121887 6.18 6:1a8b34cf:::benchmark_data_smithi012_182576_object16366:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T10:00:57.245 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.233768+0000", 2023-10-09T10:00:57.245 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0087899970000000008, 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0088222860000000004, 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121887 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.246 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.233768+0000", 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.233768+0000", 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.233766+0000", 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.233773+0000", 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0380000000000002e-06 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.233785+0000", 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2102e-05 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.233793+0000", 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0309999999999998e-06 2023-10-09T10:00:57.248 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.238114+0000", 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043209850000000003 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.238889+0000", 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00077431600000000005 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.249 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.239813+0000", 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00092400900000000005 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.242097+0000", 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022845489999999999 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.250 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.34797.0:121873 6.8 e136/134)", 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.241701+0000", 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00085732899999999999, 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00097659599999999993, 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241701+0000", 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.251 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241701+0000", 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241700+0000", 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.252 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241704+0000", 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8550000000000004e-06 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241718+0000", 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4085e-05 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241726+0000", 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4120000000000004e-06 2023-10-09T10:00:57.253 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.34797.0:121877 6.8 e136/134)", 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.241732+0000", 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00082651700000000003, 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099407300000000005, 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.254 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241732+0000", 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241732+0000", 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.255 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241731+0000", 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241734+0000", 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.385e-06 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241741+0000", 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6839999999999996e-06 2023-10-09T10:00:57.256 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241746+0000", 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.369e-06 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.34797.0:121878 6.8 e136/134)", 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.241749+0000", 2023-10-09T10:00:57.257 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00080877, 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001023389, 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241749+0000", 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.258 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241749+0000", 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241749+0000", 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241751+0000", 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.877e-06 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.259 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241792+0000", 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0865000000000003e-05 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.241798+0000", 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6899999999999997e-06 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.260 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:121868 6.f 6:f9a5e5a0:::benchmark_data_smithi012_182576_object16347:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.214191+0000", 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.028367427000000001, 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028636225000000001, 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121868 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.261 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214191+0000", 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214191+0000", 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.262 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214189+0000", 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214196+0000", 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7800000000000003e-06 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214208+0000", 2023-10-09T10:00:57.263 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1789e-05 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214216+0000", 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1319999999999994e-06 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214333+0000", 2023-10-09T10:00:57.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117249 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214459+0000", 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012591999999999999 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.214582+0000", 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012307999999999999 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218284+0000", 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037017560000000001 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.221689+0000", 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034056440000000002 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.266 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:121873 6.8 6:14364a93:::benchmark_data_smithi012_182576_object16352:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.218434+0000", 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.024124621999999998, 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024482439000000002, 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121873 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.267 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218434+0000", 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218434+0000", 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218432+0000", 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218440+0000", 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7400000000000004e-06 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218454+0000", 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4039999999999999e-05 2023-10-09T10:00:57.269 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218463+0000", 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2029999999999998e-06 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.220563+0000", 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002100223 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.270 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.221185+0000", 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062168800000000001 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.221981+0000", 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00079592599999999997 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.227053+0000", 2023-10-09T10:00:57.271 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005072226 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.234202+0000", 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071492300000000003 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.242803+0000", 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0086011979999999991 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.272 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:121874 6.f 6:f2cc39a0:::benchmark_data_smithi012_182576_object16353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.218470+0000", 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.02408803, 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0245399, 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:57.273 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121874 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218470+0000", 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218470+0000", 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.274 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218470+0000", 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218472+0000", 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8650000000000001e-06 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.275 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218479+0000", 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6270000000000002e-06 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218485+0000", 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5300000000000004e-06 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218525+0000", 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0027999999999997e-05 2023-10-09T10:00:57.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218636+0000", 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011182400000000001 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218706+0000", 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9653000000000005e-05 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:00:57.277 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.221907+0000", 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032011209999999999 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.225236+0000", 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033290820000000001 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.278 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:121877 6.8 6:11f9c0d4:::benchmark_data_smithi012_182576_object16356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.218513+0000", 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.024045569999999999, 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024593771, 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121877 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218513+0000", 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218513+0000", 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218512+0000", 2023-10-09T10:00:57.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218515+0000", 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2849999999999999e-06 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218523+0000", 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6040000000000004e-06 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.218529+0000", 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0220000000000001e-06 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.227115+0000", 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085859089999999992 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.227682+0000", 2023-10-09T10:00:57.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00056667600000000001 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.228362+0000", 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00068081800000000003 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.235646+0000", 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072840159999999999 2023-10-09T10:00:57.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.236408+0000", 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000761106 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:121878 6.8 6:17dde899:::benchmark_data_smithi012_182576_object16357:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:57.225990+0000", 2023-10-09T10:00:57.284 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.016567812000000001, 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017210056000000001, 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121878 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:00:57.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.225990+0000", 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.225990+0000", 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.225989+0000", 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:00:57.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.225995+0000", 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6590000000000001e-06 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.226012+0000", 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6268000000000001e-05 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.226020+0000", 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2889999999999998e-06 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.230653+0000", 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046325020000000001 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.231268+0000", 2023-10-09T10:00:57.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000615786 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.231968+0000", 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00069954100000000003 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.235790+0000", 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003822308 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:00:57.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:57.238144+0000", 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023535779999999998 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 9 2023-10-09T10:00:57.290 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:00:57.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:57.291 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:57.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:57.318 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:57.392 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:00:57.434 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:57.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:00:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:57.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:00:57.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:57.643 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:57.908 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:57 smithi012 ceph-mon[152263]: pgmap v333: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 unknown, 1 peering; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 1.15k op/s; 2287/284616 objects degraded (0.804%); 266/284616 objects misplaced (0.093%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T10:00:57.908 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:57 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 2612/285681 objects degraded (0.914%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:57.909 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:57 smithi012 ceph-mon[147926]: pgmap v333: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 unknown, 1 peering; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 1.15k op/s; 2287/284616 objects degraded (0.804%); 266/284616 objects misplaced (0.093%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T10:00:57.910 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:57 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 2612/285681 objects degraded (0.914%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:57.988 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:57.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:57.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:58.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:58.125 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:57 smithi110 ceph-mon[136032]: pgmap v333: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 unknown, 1 peering; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 1.15k op/s; 2287/284616 objects degraded (0.804%); 266/284616 objects misplaced (0.093%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T10:00:58.125 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:57 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 2612/285681 objects degraded (0.914%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:00:58.125 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:58.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:00:58.166 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:58.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:58.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:00:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:00:57] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T10:00:58.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:00:58.362 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:58.472 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:00:58.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:58.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:00:58.582 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:00:58.614 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:58.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:00:58.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:58.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:00:58.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:00:58.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:00:58.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:58.903 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:59.081 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:00:59.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:59.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:00:59.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:00:59.195 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:59.283 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:59.437 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:00:59.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:59.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:00:59.555 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:00:59.583 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:59.623 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:00:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:00:59.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:00:59.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:00:59.787 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:00:59.696+0000 7f85a99d3700 -1 mon.a@0(leader).osd e136 definitely_dead 0 2023-10-09T10:00:59.798 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:00:59.831 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:59.831 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:00:59.947 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.4. 2023-10-09T10:01:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[147926]: pgmap v334: 99 pgs: 1 active+recovering, 7 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 2290/269814 objects degraded (0.849%); 270/269814 objects misplaced (0.100%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T10:01:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1337140683' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:01:00.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:01:00.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[147926]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T10:01:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:00.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[152263]: pgmap v334: 99 pgs: 1 active+recovering, 7 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 2290/269814 objects degraded (0.849%); 270/269814 objects misplaced (0.100%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T10:01:00.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1337140683' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:01:00.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:01:00.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:00:59 smithi012 ceph-mon[152263]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T10:01:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:59 smithi110 ceph-mon[136032]: pgmap v334: 99 pgs: 1 active+recovering, 7 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 2290/269814 objects degraded (0.849%); 270/269814 objects misplaced (0.100%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T10:01:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1337140683' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:01:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:01:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:00:59 smithi110 ceph-mon[136032]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T10:01:00.238 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:00.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:00.344 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:00.370 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:00.401 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:00.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:00.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:00.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:00.530 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:00.563 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:00.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:00.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:00.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:00.692 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:00.725 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:00.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:00.829 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:00.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:00.891 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:00.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:00.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:00.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:00 smithi012 ceph-mon[147926]: failed to encode map e137 with expected crc 2023-10-09T10:01:00.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:00 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T10:01:00.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:00 smithi012 ceph-mon[147926]: osdmap e137: 8 total, 8 up, 7 in 2023-10-09T10:01:00.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:00 smithi012 ceph-mon[152263]: failed to encode map e137 with expected crc 2023-10-09T10:01:00.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:00 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T10:01:00.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:00 smithi012 ceph-mon[152263]: osdmap e137: 8 total, 8 up, 7 in 2023-10-09T10:01:00.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:00 smithi110 ceph-mon[136032]: failed to encode map e137 with expected crc 2023-10-09T10:01:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:00 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T10:01:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:00 smithi110 ceph-mon[136032]: osdmap e137: 8 total, 8 up, 7 in 2023-10-09T10:01:00.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:01.034 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:01.273 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:01:01.273 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:01:01.274 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:01:01.274 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:01:01.274 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:01:01.274 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:01.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:01.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:01.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:01.414 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:01.642 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:01.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:01.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:01.778 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:01.864 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:01.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:01.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:01.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:01:01.970 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:02.084 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[147926]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[147926]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[147926]: osdmap e138: 8 total, 8 up, 7 in 2023-10-09T10:01:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[147926]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[147926]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[147926]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[147926]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[147926]: pgmap v337: 99 pgs: 1 active+recovering, 7 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 2290/263790 objects degraded (0.868%); 270/263790 objects misplaced (0.102%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:01:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[152263]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[152263]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[152263]: osdmap e138: 8 total, 8 up, 7 in 2023-10-09T10:01:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[152263]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[152263]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[152263]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[152263]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:01 smithi012 ceph-mon[152263]: pgmap v337: 99 pgs: 1 active+recovering, 7 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 2290/263790 objects degraded (0.868%); 270/263790 objects misplaced (0.102%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:01:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:01 smithi110 ceph-mon[136032]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:01 smithi110 ceph-mon[136032]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:01 smithi110 ceph-mon[136032]: osdmap e138: 8 total, 8 up, 7 in 2023-10-09T10:01:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:01 smithi110 ceph-mon[136032]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:01 smithi110 ceph-mon[136032]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:01 smithi110 ceph-mon[136032]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:01 smithi110 ceph-mon[136032]: failed to encode map e138 with expected crc 2023-10-09T10:01:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:01 smithi110 ceph-mon[136032]: pgmap v337: 99 pgs: 1 active+recovering, 7 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 2290/263790 objects degraded (0.868%); 270/263790 objects misplaced (0.102%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:01:02.198 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:02.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:02.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:02.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:02.380 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:02.486 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:02.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:02.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:02.526 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:01:02.547 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:01:02.569 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:01:02.589 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:02.603 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:01:02.628 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:01:02.631 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:02.659 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:01:02.686 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:02.688 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:01:02.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:02.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:02.722 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:01:02.725 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:01:02.725 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:01:02.725 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:01:02.725 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:01:02.725 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:01:02.725 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:01:02.725 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:01:02.726 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:01:02.773 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:02.791 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:02.843 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 136'7622 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: osdmap e139: 8 total, 8 up, 7 in 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: 6.f starting backfill to osd.6 from (0'0,123'6545] MIN to 136'7706 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 136'7820 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: 6.16 starting backfill to osd.6 from (0'0,0'0] MAX to 136'7619 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1772661218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 2290/263790 objects degraded (0.868%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 136'7622 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: osdmap e139: 8 total, 8 up, 7 in 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: 6.f starting backfill to osd.6 from (0'0,123'6545] MIN to 136'7706 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 136'7820 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: 6.16 starting backfill to osd.6 from (0'0,0'0] MAX to 136'7619 2023-10-09T10:01:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1772661218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:02 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 2290/263790 objects degraded (0.868%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 136'7622 2023-10-09T10:01:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: osdmap e139: 8 total, 8 up, 7 in 2023-10-09T10:01:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: 6.f starting backfill to osd.6 from (0'0,123'6545] MIN to 136'7706 2023-10-09T10:01:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 136'7820 2023-10-09T10:01:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: 6.16 starting backfill to osd.6 from (0'0,0'0] MAX to 136'7619 2023-10-09T10:01:03.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: failed to encode map e139 with expected crc 2023-10-09T10:01:03.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1772661218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:03.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:02 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 2290/263790 objects degraded (0.868%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:03.935 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:01:04.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2003375653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:03 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:03 smithi012 ceph-mon[147926]: pgmap v339: 99 pgs: 1 activating+remapped, 1 active+recovering, 5 active+recovery_wait+degraded, 92 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 145 op/s; 1650/261717 objects degraded (0.630%); 5624/261717 objects misplaced (2.149%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:01:04.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2003375653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:04.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:03 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:04.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:03 smithi012 ceph-mon[152263]: pgmap v339: 99 pgs: 1 activating+remapped, 1 active+recovering, 5 active+recovery_wait+degraded, 92 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 145 op/s; 1650/261717 objects degraded (0.630%); 5624/261717 objects misplaced (2.149%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:01:04.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2003375653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:04.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:03 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:04.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:03 smithi110 ceph-mon[136032]: pgmap v339: 99 pgs: 1 activating+remapped, 1 active+recovering, 5 active+recovery_wait+degraded, 92 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 145 op/s; 1650/261717 objects degraded (0.630%); 5624/261717 objects misplaced (2.149%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:01:04.340 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:04.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:04.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:04.480 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:05.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:01:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[147926]: osdmap e140: 8 total, 8 up, 7 in 2023-10-09T10:01:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[147926]: failed to encode map e140 with expected crc 2023-10-09T10:01:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[147926]: failed to encode map e140 with expected crc 2023-10-09T10:01:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:01:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[152263]: osdmap e140: 8 total, 8 up, 7 in 2023-10-09T10:01:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[152263]: failed to encode map e140 with expected crc 2023-10-09T10:01:05.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:04 smithi012 ceph-mon[152263]: failed to encode map e140 with expected crc 2023-10-09T10:01:05.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:04 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:05.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:04 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:01:05.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:04 smithi110 ceph-mon[136032]: osdmap e140: 8 total, 8 up, 7 in 2023-10-09T10:01:05.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:04 smithi110 ceph-mon[136032]: failed to encode map e140 with expected crc 2023-10-09T10:01:05.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:04 smithi110 ceph-mon[136032]: failed to encode map e140 with expected crc 2023-10-09T10:01:05.290 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:05.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:05.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:05.429 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:05.470 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:05.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:05.574 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:05.608 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:05.712 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 5, 6, 7, 2] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:01:05.712 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:01:05.713 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-09T10:01:05.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:01:06.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:05 smithi110 ceph-mon[136032]: failed to encode map e140 with expected crc 2023-10-09T10:01:06.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:05 smithi110 ceph-mon[136032]: failed to encode map e140 with expected crc 2023-10-09T10:01:06.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:05 smithi110 ceph-mon[136032]: pgmap v341: 99 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+degraded, 91 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 3914/260970 objects degraded (1.500%); 13807/260970 objects misplaced (5.291%); 18 MiB/s, 1 keys/s, 298 objects/s recovering 2023-10-09T10:01:06.194 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:05 smithi012 ceph-mon[147926]: failed to encode map e140 with expected crc 2023-10-09T10:01:06.194 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:05 smithi012 ceph-mon[147926]: failed to encode map e140 with expected crc 2023-10-09T10:01:06.194 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:05 smithi012 ceph-mon[147926]: pgmap v341: 99 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+degraded, 91 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 3914/260970 objects degraded (1.500%); 13807/260970 objects misplaced (5.291%); 18 MiB/s, 1 keys/s, 298 objects/s recovering 2023-10-09T10:01:06.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:05 smithi012 ceph-mon[152263]: failed to encode map e140 with expected crc 2023-10-09T10:01:06.194 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:05 smithi012 ceph-mon[152263]: failed to encode map e140 with expected crc 2023-10-09T10:01:06.195 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:05 smithi012 ceph-mon[152263]: pgmap v341: 99 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+degraded, 91 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 3914/260970 objects degraded (1.500%); 13807/260970 objects misplaced (5.291%); 18 MiB/s, 1 keys/s, 298 objects/s recovering 2023-10-09T10:01:06.452 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:06.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:06.558 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:06.589 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:06.628 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:06.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:06.732 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:06.761 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:06.981 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:07.110 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:07.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:07.110 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:06 smithi110 ceph-mon[136032]: failed to encode map e140 with expected crc 2023-10-09T10:01:07.154 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:06 smithi012 ceph-mon[147926]: failed to encode map e140 with expected crc 2023-10-09T10:01:07.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:06 smithi012 ceph-mon[152263]: failed to encode map e140 with expected crc 2023-10-09T10:01:07.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:07.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:01:07.618 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:01:07.619 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:01:07.692 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:07.799 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:07.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:07.908 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:07.945 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:07 smithi012 ceph-mon[147926]: pgmap v342: 99 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait+degraded, 92 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 3914/260925 objects degraded (1.500%); 13569/260925 objects misplaced (5.200%); 18 MiB/s, 1 keys/s, 287 objects/s recovering 2023-10-09T10:01:07.945 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:07 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3914/260970 objects degraded (1.500%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:07.945 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:01:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:01:07] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T10:01:07.955 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:07 smithi012 ceph-mon[152263]: pgmap v342: 99 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait+degraded, 92 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 3914/260925 objects degraded (1.500%); 13569/260925 objects misplaced (5.200%); 18 MiB/s, 1 keys/s, 287 objects/s recovering 2023-10-09T10:01:07.955 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:07 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3914/260970 objects degraded (1.500%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:08.042 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:08.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:01:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:08.379 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:08.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:08.379 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:07 smithi110 ceph-mon[136032]: pgmap v342: 99 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait+degraded, 92 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 3914/260925 objects degraded (1.500%); 13569/260925 objects misplaced (5.200%); 18 MiB/s, 1 keys/s, 287 objects/s recovering 2023-10-09T10:01:08.380 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:07 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3914/260970 objects degraded (1.500%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:08.480 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:08.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:01:08.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:08.551 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:08.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:08.657 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:08.777 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:08.870 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:08.958 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:08.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:09.066 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:09.095 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:09.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:09.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:09.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:09.290 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:01:09.300 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:01:09.393 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:09.714 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:09.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:09.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:09.791 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:09.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:09.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:09.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:09.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:10.009 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:10.053 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:10 smithi012 ceph-mon[147926]: pgmap v343: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 9940/259266 objects degraded (3.834%); 14994/259266 objects misplaced (5.783%); 44 MiB/s, 1 keys/s, 709 objects/s recovering 2023-10-09T10:01:10.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:10 smithi012 ceph-mon[147926]: from='client.34956 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:10.054 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:10.070 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":343,"stamp":"2023-10-09T10:01:08.983953+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5662709688,"num_objects":86422,"num_object_clones":0,"num_object_copies":259266,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":9940,"num_objects_misplaced":14994,"num_objects_unfound":0,"num_objects_dirty":86422,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":124900,"num_write_kb":6755598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9662,"num_bytes_recovered":631576488,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":190571,"ondisk_log_size":190571,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":305,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16497992,"kb_used_data":15136208,"kb_used_omap":126,"kb_used_meta":1361601,"kb_avail":639718072,"statfs":{"total":671965249536,"available":655071305728,"internally_reserved":0,"allocated":15499476992,"data_stored":15495673171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1394280059},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1,9,0,0,6],"upper_bound":8192},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-98828288,"num_objects":-1508,"num_object_clones":0,"num_object_copies":-4524,"num_objects_missing_on_primary":-605,"num_objects_missing":-605,"num_objects_degraded":7650,"num_objects_misplaced":14724,"num_objects_unfound":0,"num_objects_dirty":-1508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1508,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5679,"num_bytes_recovered":371200784,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004420"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":14,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:03.620248+0000","last_change":"2023-10-09T10:01:00.939779+0000","last_active":"2023-10-09T10:01:03.620248+0000","last_peered":"2023-10-09T10:01:03.620248+0000","last_clean":"2023-10-09T10:01:03.620248+0000","last_became_active":"2023-10-09T10:01:00.939403+0000","last_became_peered":"2023-10-09T10:01:00.939403+0000","last_unstale":"2023-10-09T10:01:03.620248+0000","last_undegraded":"2023-10-09T10:01:03.620248+0000","last_fullsized":"2023-10-09T10:01:03.620248+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:05.091492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.920866+0000","last_change":"2023-10-09T10:01:00.930519+0000","last_active":"2023-10-09T10:01:01.920866+0000","last_peered":"2023-10-09T10:01:01.920866+0000","last_clean":"2023-10-09T10:01:01.920866+0000","last_became_active":"2023-10-09T10:01:00.930236+0000","last_became_peered":"2023-10-09T10:01:00.930236+0000","last_unstale":"2023-10-09T10:01:01.920866+0000","last_undegraded":"2023-10-09T10:01:01.920866+0000","last_fullsized":"2023-10-09T10:01:01.920866+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:25.568164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.930088+0000","last_change":"2023-10-09T10:01:00.930723+0000","last_active":"2023-10-09T10:01:01.930088+0000","last_peered":"2023-10-09T10:01:01.930088+0000","last_clean":"2023-10-09T10:01:01.930088+0000","last_became_active":"2023-10-09T10:01:00.930428+0000","last_became_peered":"2023-10-09T10:01:00.930428+0000","last_unstale":"2023-10-09T10:01:01.930088+0000","last_undegraded":"2023-10-09T10:01:01.930088+0000","last_fullsized":"2023-10-09T10:01:01.930088+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:37.700582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.928761+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T10:01:01.928761+0000","last_peered":"2023-10-09T10:01:01.928761+0000","last_clean":"2023-10-09T10:01:01.928761+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T10:01:01.928761+0000","last_undegraded":"2023-10-09T10:01:01.928761+0000","last_fullsized":"2023-10-09T10:01:01.928761+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.920485+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T10:01:01.920485+0000","last_peered":"2023-10-09T10:01:01.920485+0000","last_clean":"2023-10-09T10:01:01.920485+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T10:01:01.920485+0000","last_undegraded":"2023-10-09T10:01:01.920485+0000","last_fullsized":"2023-10-09T10:01:01.920485+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.217494+0000","last_change":"2023-10-09T10:00:09.714491+0000","last_active":"2023-10-09T10:01:02.217494+0000","last_peered":"2023-10-09T10:01:02.217494+0000","last_clean":"2023-10-09T10:01:02.217494+0000","last_became_active":"2023-10-09T10:00:09.714161+0000","last_became_peered":"2023-10-09T10:00:09.714161+0000","last_unstale":"2023-10-09T10:01:02.217494+0000","last_undegraded":"2023-10-09T10:01:02.217494+0000","last_fullsized":"2023-10-09T10:01:02.217494+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"136'7717","reported_seq":7789,"reported_epoch":140,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T10:01:07.150613+0000","last_change":"2023-10-09T10:01:07.150613+0000","last_active":"2023-10-09T10:00:59.925802+0000","last_peered":"2023-10-09T10:00:59.795470+0000","last_clean":"2023-10-09T10:00:08.575217+0000","last_became_active":"2023-10-09T10:00:52.770183+0000","last_became_peered":"2023-10-09T10:00:52.770183+0000","last_unstale":"2023-10-09T10:01:07.150613+0000","last_undegraded":"2023-10-09T10:01:02.035072+0000","last_fullsized":"2023-10-09T10:01:01.922438+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180486144,"num_objects":2754,"num_object_clones":0,"num_object_copies":8262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3363,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3955,"num_write_kb":214688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"2,3","objects":-24},{"shards":"2,3,4","objects":2442},{"shards":"3,4,6","objects":336}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.928422+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T10:01:01.928422+0000","last_peered":"2023-10-09T10:01:01.928422+0000","last_clean":"2023-10-09T10:01:01.928422+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T10:01:01.928422+0000","last_undegraded":"2023-10-09T10:01:01.928422+0000","last_fullsized":"2023-10-09T10:01:01.928422+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.920512+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T10:01:01.920512+0000","last_peered":"2023-10-09T10:01:01.920512+0000","last_clean":"2023-10-09T10:01:01.920512+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T10:01:01.920512+0000","last_undegraded":"2023-10-09T10:01:01.920512+0000","last_fullsized":"2023-10-09T10:01:01.920512+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"139'7634","reported_seq":9492,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.248827+0000","last_change":"2023-10-09T10:00:53.519038+0000","last_active":"2023-10-09T10:01:02.248827+0000","last_peered":"2023-10-09T10:01:02.248827+0000","last_clean":"2023-10-09T10:01:02.248827+0000","last_became_active":"2023-10-09T10:00:52.732215+0000","last_became_peered":"2023-10-09T10:00:52.732215+0000","last_unstale":"2023-10-09T10:01:02.248827+0000","last_undegraded":"2023-10-09T10:01:02.248827+0000","last_fullsized":"2023-10-09T10:01:02.248827+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7634,"log_dups_size":0,"ondisk_log_size":7634,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180191232,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3902,"num_write_kb":212832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":613,"num_bytes_recovered":40140800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.925850+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T10:01:01.925850+0000","last_peered":"2023-10-09T10:01:01.925850+0000","last_clean":"2023-10-09T10:01:01.925850+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T10:01:01.925850+0000","last_undegraded":"2023-10-09T10:01:01.925850+0000","last_fullsized":"2023-10-09T10:01:01.925850+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.920661+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T10:01:01.920661+0000","last_peered":"2023-10-09T10:01:01.920661+0000","last_clean":"2023-10-09T10:01:01.920661+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T10:01:01.920661+0000","last_undegraded":"2023-10-09T10:01:01.920661+0000","last_fullsized":"2023-10-09T10:01:01.920661+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"139'7558","reported_seq":8889,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.298583+0000","last_change":"2023-10-09T10:00:52.687122+0000","last_active":"2023-10-09T10:01:02.298583+0000","last_peered":"2023-10-09T10:01:02.298583+0000","last_clean":"2023-10-09T10:01:02.298583+0000","last_became_active":"2023-10-09T10:00:52.676306+0000","last_became_peered":"2023-10-09T10:00:52.676306+0000","last_unstale":"2023-10-09T10:01:02.298583+0000","last_undegraded":"2023-10-09T10:01:02.298583+0000","last_fullsized":"2023-10-09T10:01:02.298583+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175472640,"num_objects":2678,"num_object_clones":0,"num_object_copies":8034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3865,"num_write_kb":209344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20414464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.301459+0000","last_change":"2023-10-09T10:01:01.301253+0000","last_active":"2023-10-09T10:01:01.301459+0000","last_peered":"2023-10-09T10:01:01.301459+0000","last_clean":"2023-10-09T10:01:01.301459+0000","last_became_active":"2023-10-09T10:01:00.933163+0000","last_became_peered":"2023-10-09T10:01:00.933163+0000","last_unstale":"2023-10-09T10:01:01.301459+0000","last_undegraded":"2023-10-09T10:01:01.301459+0000","last_fullsized":"2023-10-09T10:01:01.301459+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":112,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.930239+0000","last_change":"2023-10-09T10:00:30.790050+0000","last_active":"2023-10-09T10:01:01.930239+0000","last_peered":"2023-10-09T10:01:01.930239+0000","last_clean":"2023-10-09T10:01:01.930239+0000","last_became_active":"2023-10-09T10:00:30.789740+0000","last_became_peered":"2023-10-09T10:00:30.789740+0000","last_unstale":"2023-10-09T10:01:01.930239+0000","last_undegraded":"2023-10-09T10:01:01.930239+0000","last_fullsized":"2023-10-09T10:01:01.930239+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:36.206642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"140'7632","reported_seq":8305,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:08.524304+0000","last_change":"2023-10-09T10:01:00.721483+0000","last_active":"2023-10-09T10:01:08.524304+0000","last_peered":"2023-10-09T10:01:08.524304+0000","last_clean":"2023-10-09T10:01:08.524304+0000","last_became_active":"2023-10-09T10:00:52.779409+0000","last_became_peered":"2023-10-09T10:00:52.779409+0000","last_unstale":"2023-10-09T10:01:08.524304+0000","last_undegraded":"2023-10-09T10:01:08.524304+0000","last_fullsized":"2023-10-09T10:01:08.524304+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7632,"log_dups_size":0,"ondisk_log_size":7632,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179896320,"num_objects":2745,"num_object_clones":0,"num_object_copies":8235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3910,"num_write_kb":212960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":299,"num_bytes_recovered":19595264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":70,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.210072+0000","last_change":"2023-10-09T10:01:01.209859+0000","last_active":"2023-10-09T10:01:01.210072+0000","last_peered":"2023-10-09T10:01:01.210072+0000","last_clean":"2023-10-09T10:01:01.210072+0000","last_became_active":"2023-10-09T10:01:00.931767+0000","last_became_peered":"2023-10-09T10:01:00.931767+0000","last_unstale":"2023-10-09T10:01:01.210072+0000","last_undegraded":"2023-10-09T10:01:01.210072+0000","last_fullsized":"2023-10-09T10:01:01.210072+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":56,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.135737+0000","last_change":"2023-10-09T10:01:00.930805+0000","last_active":"2023-10-09T10:01:02.135737+0000","last_peered":"2023-10-09T10:01:02.135737+0000","last_clean":"2023-10-09T10:01:02.135737+0000","last_became_active":"2023-10-09T10:01:00.930521+0000","last_became_peered":"2023-10-09T10:01:00.930521+0000","last_unstale":"2023-10-09T10:01:02.135737+0000","last_undegraded":"2023-10-09T10:01:02.135737+0000","last_fullsized":"2023-10-09T10:01:02.135737+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.16","version":"139'7623","reported_seq":7700,"reported_epoch":139,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:01:02.703753+0000","last_change":"2023-10-09T10:01:02.703753+0000","last_active":"2023-10-09T10:01:02.703753+0000","last_peered":"2023-10-09T10:01:02.703753+0000","last_clean":"2023-10-09T10:00:59.912614+0000","last_became_active":"2023-10-09T10:01:02.185720+0000","last_became_peered":"2023-10-09T10:01:02.185720+0000","last_unstale":"2023-10-09T10:01:02.703753+0000","last_undegraded":"2023-10-09T10:01:02.703753+0000","last_fullsized":"2023-10-09T10:01:02.703753+0000","mapping_epoch":138,"log_start":"102'300","ondisk_log_start":"102'300","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7323,"log_dups_size":0,"ondisk_log_size":7323,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177209344,"num_objects":2704,"num_object_clones":0,"num_object_copies":8112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2704,"num_objects_unfound":0,"num_objects_dirty":2704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3896,"num_write_kb":211200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":2704}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":151,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.304482+0000","last_change":"2023-10-09T10:01:01.304294+0000","last_active":"2023-10-09T10:01:01.304482+0000","last_peered":"2023-10-09T10:01:01.304482+0000","last_clean":"2023-10-09T10:01:01.304482+0000","last_became_active":"2023-10-09T10:01:00.936526+0000","last_became_peered":"2023-10-09T10:01:00.936526+0000","last_unstale":"2023-10-09T10:01:01.304482+0000","last_undegraded":"2023-10-09T10:01:01.304482+0000","last_fullsized":"2023-10-09T10:01:01.304482+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.921539+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T10:01:01.921539+0000","last_peered":"2023-10-09T10:01:01.921539+0000","last_clean":"2023-10-09T10:01:01.921539+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T10:01:01.921539+0000","last_undegraded":"2023-10-09T10:01:01.921539+0000","last_fullsized":"2023-10-09T10:01:01.921539+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"139'7441","reported_seq":7509,"reported_epoch":139,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:01:02.380451+0000","last_change":"2023-10-09T10:01:02.099633+0000","last_active":"2023-10-09T10:01:02.380451+0000","last_peered":"2023-10-09T10:01:02.380451+0000","last_clean":"2023-10-09T10:00:59.881901+0000","last_became_active":"2023-10-09T10:01:02.089341+0000","last_became_peered":"2023-10-09T10:01:02.089341+0000","last_unstale":"2023-10-09T10:01:02.380451+0000","last_undegraded":"2023-10-09T10:01:01.938801+0000","last_fullsized":"2023-10-09T10:01:01.925575+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7441,"log_dups_size":0,"ondisk_log_size":7441,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171769856,"num_objects":2621,"num_object_clones":0,"num_object_copies":7863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3229,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3822,"num_write_kb":206176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2626},{"shards":"3,5","objects":-14}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.927574+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T10:01:01.927574+0000","last_peered":"2023-10-09T10:01:01.927574+0000","last_clean":"2023-10-09T10:01:01.927574+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T10:01:01.927574+0000","last_undegraded":"2023-10-09T10:01:01.927574+0000","last_fullsized":"2023-10-09T10:01:01.927574+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.926223+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T10:01:01.926223+0000","last_peered":"2023-10-09T10:01:01.926223+0000","last_clean":"2023-10-09T10:01:01.926223+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T10:01:01.926223+0000","last_undegraded":"2023-10-09T10:01:01.926223+0000","last_fullsized":"2023-10-09T10:01:01.926223+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"140'7607","reported_seq":7669,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:08.499793+0000","last_change":"2023-10-09T10:00:52.685558+0000","last_active":"2023-10-09T10:01:08.499793+0000","last_peered":"2023-10-09T10:01:08.499793+0000","last_clean":"2023-10-09T10:01:08.499793+0000","last_became_active":"2023-10-09T10:00:52.677109+0000","last_became_peered":"2023-10-09T10:00:52.677109+0000","last_unstale":"2023-10-09T10:01:08.499793+0000","last_undegraded":"2023-10-09T10:01:08.499793+0000","last_fullsized":"2023-10-09T10:01:08.499793+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7607,"log_dups_size":0,"ondisk_log_size":7607,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174489600,"num_objects":2663,"num_object_clones":0,"num_object_copies":7989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3907,"num_write_kb":210208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":17,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.136619+0000","last_change":"2023-10-09T10:01:00.927906+0000","last_active":"2023-10-09T10:01:02.136619+0000","last_peered":"2023-10-09T10:01:02.136619+0000","last_clean":"2023-10-09T10:01:02.136619+0000","last_became_active":"2023-10-09T10:01:00.927587+0000","last_became_peered":"2023-10-09T10:01:00.927587+0000","last_unstale":"2023-10-09T10:01:02.136619+0000","last_undegraded":"2023-10-09T10:01:02.136619+0000","last_fullsized":"2023-10-09T10:01:02.136619+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.928721+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T10:01:01.928721+0000","last_peered":"2023-10-09T10:01:01.928721+0000","last_clean":"2023-10-09T10:01:01.928721+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T10:01:01.928721+0000","last_undegraded":"2023-10-09T10:01:01.928721+0000","last_fullsized":"2023-10-09T10:01:01.928721+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"139'7531","reported_seq":7589,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.302084+0000","last_change":"2023-10-09T10:00:52.704990+0000","last_active":"2023-10-09T10:01:02.302084+0000","last_peered":"2023-10-09T10:01:02.302084+0000","last_clean":"2023-10-09T10:01:02.302084+0000","last_became_active":"2023-10-09T10:00:52.699362+0000","last_became_peered":"2023-10-09T10:00:52.699362+0000","last_unstale":"2023-10-09T10:01:02.302084+0000","last_undegraded":"2023-10-09T10:01:02.302084+0000","last_fullsized":"2023-10-09T10:01:02.302084+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7531,"log_dups_size":0,"ondisk_log_size":7531,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175931392,"num_objects":2685,"num_object_clones":0,"num_object_copies":8055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3846,"num_write_kb":208960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":113,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.135625+0000","last_change":"2023-10-09T09:56:13.623204+0000","last_active":"2023-10-09T10:01:02.135625+0000","last_peered":"2023-10-09T10:01:02.135625+0000","last_clean":"2023-10-09T10:01:02.135625+0000","last_became_active":"2023-10-09T09:56:13.622853+0000","last_became_peered":"2023-10-09T09:56:13.622853+0000","last_unstale":"2023-10-09T10:01:02.135625+0000","last_undegraded":"2023-10-09T10:01:02.135625+0000","last_fullsized":"2023-10-09T10:01:02.135625+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.921178+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T10:01:01.921178+0000","last_peered":"2023-10-09T10:01:01.921178+0000","last_clean":"2023-10-09T10:01:01.921178+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T10:01:01.921178+0000","last_undegraded":"2023-10-09T10:01:01.921178+0000","last_fullsized":"2023-10-09T10:01:01.921178+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"139'7796","reported_seq":7847,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.341555+0000","last_change":"2023-10-09T09:58:52.780310+0000","last_active":"2023-10-09T10:01:02.341555+0000","last_peered":"2023-10-09T10:01:02.341555+0000","last_clean":"2023-10-09T10:01:02.341555+0000","last_became_active":"2023-10-09T09:58:52.779910+0000","last_became_peered":"2023-10-09T09:58:52.779910+0000","last_unstale":"2023-10-09T10:01:02.341555+0000","last_undegraded":"2023-10-09T10:01:02.341555+0000","last_fullsized":"2023-10-09T10:01:02.341555+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7796,"log_dups_size":0,"ondisk_log_size":7796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355598336,"num_objects":5426,"num_object_clones":0,"num_object_copies":16278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7796,"num_write_kb":423104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":118,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T10:01:00.940217+0000","last_change":"2023-10-09T10:01:00.939824+0000","last_active":"2023-10-09T10:01:00.940217+0000","last_peered":"2023-10-09T10:01:00.940217+0000","last_clean":"2023-10-09T10:01:00.940217+0000","last_became_active":"2023-10-09T10:01:00.922153+0000","last_became_peered":"2023-10-09T10:01:00.922153+0000","last_unstale":"2023-10-09T10:01:00.940217+0000","last_undegraded":"2023-10-09T10:01:00.940217+0000","last_fullsized":"2023-10-09T10:01:00.940217+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":90,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.925710+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T10:01:01.925710+0000","last_peered":"2023-10-09T10:01:01.925710+0000","last_clean":"2023-10-09T10:01:01.925710+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T10:01:01.925710+0000","last_undegraded":"2023-10-09T10:01:01.925710+0000","last_fullsized":"2023-10-09T10:01:01.925710+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"139'7907","reported_seq":7960,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:04.833629+0000","last_change":"2023-10-09T09:58:52.625388+0000","last_active":"2023-10-09T10:01:04.833629+0000","last_peered":"2023-10-09T10:01:04.833629+0000","last_clean":"2023-10-09T10:01:04.833629+0000","last_became_active":"2023-10-09T09:58:52.625135+0000","last_became_peered":"2023-10-09T09:58:52.625135+0000","last_unstale":"2023-10-09T10:01:04.833629+0000","last_undegraded":"2023-10-09T10:01:04.833629+0000","last_fullsized":"2023-10-09T10:01:04.833629+0000","mapping_epoch":97,"log_start":"105'2447","ondisk_log_start":"105'2447","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":5460,"log_dups_size":0,"ondisk_log_size":5460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:28.713713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358023168,"num_objects":5463,"num_object_clones":0,"num_object_copies":16389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7907,"num_write_kb":427840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":119,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.035722+0000","last_change":"2023-10-09T10:01:01.035488+0000","last_active":"2023-10-09T10:01:01.035722+0000","last_peered":"2023-10-09T10:01:01.035722+0000","last_clean":"2023-10-09T10:01:01.035722+0000","last_became_active":"2023-10-09T10:01:00.926607+0000","last_became_peered":"2023-10-09T10:01:00.926607+0000","last_unstale":"2023-10-09T10:01:01.035722+0000","last_undegraded":"2023-10-09T10:01:01.035722+0000","last_fullsized":"2023-10-09T10:01:01.035722+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":81,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:03.620180+0000","last_change":"2023-10-09T10:01:00.937704+0000","last_active":"2023-10-09T10:01:03.620180+0000","last_peered":"2023-10-09T10:01:03.620180+0000","last_clean":"2023-10-09T10:01:03.620180+0000","last_became_active":"2023-10-09T10:01:00.937399+0000","last_became_peered":"2023-10-09T10:01:00.937399+0000","last_unstale":"2023-10-09T10:01:03.620180+0000","last_undegraded":"2023-10-09T10:01:03.620180+0000","last_fullsized":"2023-10-09T10:01:03.620180+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:06.336319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.8","version":"139'7791","reported_seq":8527,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.370698+0000","last_change":"2023-10-09T10:01:00.234694+0000","last_active":"2023-10-09T10:01:02.370698+0000","last_peered":"2023-10-09T10:01:02.370698+0000","last_clean":"2023-10-09T10:01:02.370698+0000","last_became_active":"2023-10-09T10:00:53.649276+0000","last_became_peered":"2023-10-09T10:00:53.649276+0000","last_unstale":"2023-10-09T10:01:02.370698+0000","last_undegraded":"2023-10-09T10:01:02.370698+0000","last_fullsized":"2023-10-09T10:01:02.370698+0000","mapping_epoch":134,"log_start":"102'400","ondisk_log_start":"102'400","created":97,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7391,"log_dups_size":0,"ondisk_log_size":7391,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181829632,"num_objects":2774,"num_object_clones":0,"num_object_copies":8322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3986,"num_write_kb":216352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":332,"num_bytes_recovered":21757952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":42,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.926156+0000","last_change":"2023-10-09T10:00:19.641304+0000","last_active":"2023-10-09T10:01:01.926156+0000","last_peered":"2023-10-09T10:01:01.926156+0000","last_clean":"2023-10-09T10:01:01.926156+0000","last_became_active":"2023-10-09T10:00:19.640958+0000","last_became_peered":"2023-10-09T10:00:19.640958+0000","last_unstale":"2023-10-09T10:01:01.926156+0000","last_undegraded":"2023-10-09T10:01:01.926156+0000","last_fullsized":"2023-10-09T10:01:01.926156+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":104,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.184010+0000","last_change":"2023-10-09T09:56:45.556365+0000","last_active":"2023-10-09T10:01:02.184010+0000","last_peered":"2023-10-09T10:01:02.184010+0000","last_clean":"2023-10-09T10:01:02.184010+0000","last_became_active":"2023-10-09T09:56:45.555832+0000","last_became_peered":"2023-10-09T09:56:45.555832+0000","last_unstale":"2023-10-09T10:01:02.184010+0000","last_undegraded":"2023-10-09T10:01:02.184010+0000","last_fullsized":"2023-10-09T10:01:02.184010+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"139'7596","reported_seq":9890,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:03.930661+0000","last_change":"2023-10-09T10:01:02.702863+0000","last_active":"2023-10-09T10:01:03.930661+0000","last_peered":"2023-10-09T10:01:03.930661+0000","last_clean":"2023-10-09T10:01:03.930661+0000","last_became_active":"2023-10-09T10:00:53.661339+0000","last_became_peered":"2023-10-09T10:00:53.661339+0000","last_unstale":"2023-10-09T10:01:03.930661+0000","last_undegraded":"2023-10-09T10:01:03.930661+0000","last_fullsized":"2023-10-09T10:01:03.930661+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7596,"log_dups_size":0,"ondisk_log_size":7596,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172654592,"num_objects":2634,"num_object_clones":0,"num_object_copies":7902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3898,"num_write_kb":209056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":704,"num_bytes_recovered":46170112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.925988+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T10:01:01.925988+0000","last_peered":"2023-10-09T10:01:01.925988+0000","last_clean":"2023-10-09T10:01:01.925988+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T10:01:01.925988+0000","last_undegraded":"2023-10-09T10:01:01.925988+0000","last_fullsized":"2023-10-09T10:01:01.925988+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.927901+0000","last_change":"2023-10-09T10:01:00.937849+0000","last_active":"2023-10-09T10:01:01.927901+0000","last_peered":"2023-10-09T10:01:01.927901+0000","last_clean":"2023-10-09T10:01:01.927901+0000","last_became_active":"2023-10-09T10:01:00.937536+0000","last_became_peered":"2023-10-09T10:01:00.937536+0000","last_unstale":"2023-10-09T10:01:01.927901+0000","last_undegraded":"2023-10-09T10:01:01.927901+0000","last_fullsized":"2023-10-09T10:01:01.927901+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:29.118995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"140'7889","reported_seq":13348,"reported_epoch":140,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T10:01:08.617283+0000","last_change":"2023-10-09T10:01:04.125064+0000","last_active":"2023-10-09T10:01:08.617283+0000","last_peered":"2023-10-09T10:01:08.617283+0000","last_clean":"2023-10-09T10:00:59.887126+0000","last_became_active":"2023-10-09T10:01:02.022511+0000","last_became_peered":"2023-10-09T10:01:02.022511+0000","last_unstale":"2023-10-09T10:01:08.617283+0000","last_undegraded":"2023-10-09T10:01:08.617283+0000","last_fullsized":"2023-10-09T10:01:08.617283+0000","mapping_epoch":138,"log_start":"105'2160","ondisk_log_start":"105'2160","created":97,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:58.605111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348979200,"num_objects":5325,"num_object_clones":0,"num_object_copies":15975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3315,"num_objects_unfound":0,"num_objects_dirty":5325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7889,"num_write_kb":422848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2688,"num_bytes_recovered":176160768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":5325}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":119,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.920819+0000","last_change":"2023-10-09T10:01:00.964746+0000","last_active":"2023-10-09T10:01:01.920819+0000","last_peered":"2023-10-09T10:01:01.920819+0000","last_clean":"2023-10-09T10:01:01.920819+0000","last_became_active":"2023-10-09T10:01:00.930510+0000","last_became_peered":"2023-10-09T10:01:00.930510+0000","last_unstale":"2023-10-09T10:01:01.920819+0000","last_undegraded":"2023-10-09T10:01:01.920819+0000","last_fullsized":"2023-10-09T10:01:01.920819+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:11.047472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:04.834031+0000","last_change":"2023-10-09T10:01:00.932525+0000","last_active":"2023-10-09T10:01:04.834031+0000","last_peered":"2023-10-09T10:01:04.834031+0000","last_clean":"2023-10-09T10:01:04.834031+0000","last_became_active":"2023-10-09T10:01:00.931364+0000","last_became_peered":"2023-10-09T10:01:00.931364+0000","last_unstale":"2023-10-09T10:01:04.834031+0000","last_undegraded":"2023-10-09T10:01:04.834031+0000","last_fullsized":"2023-10-09T10:01:04.834031+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:54.467044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"139'7714","reported_seq":8947,"reported_epoch":140,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:01:05.561476+0000","last_change":"2023-10-09T10:01:03.349776+0000","last_active":"2023-10-09T10:01:05.561476+0000","last_peered":"2023-10-09T10:01:05.561476+0000","last_clean":"2023-10-09T10:00:59.886279+0000","last_became_active":"2023-10-09T10:01:02.225568+0000","last_became_peered":"2023-10-09T10:01:02.225568+0000","last_unstale":"2023-10-09T10:01:05.561476+0000","last_undegraded":"2023-10-09T10:01:05.561476+0000","last_fullsized":"2023-10-09T10:01:05.561476+0000","mapping_epoch":138,"log_start":"102'200","ondisk_log_start":"102'200","created":97,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7514,"log_dups_size":0,"ondisk_log_size":7514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:45.432958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352452608,"num_objects":5378,"num_object_clones":0,"num_object_copies":16134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5378,"num_objects_unfound":0,"num_objects_dirty":5378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7714,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":614,"num_bytes_recovered":40239104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":5378}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":114,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.183878+0000","last_change":"2023-10-09T10:01:00.933271+0000","last_active":"2023-10-09T10:01:02.183878+0000","last_peered":"2023-10-09T10:01:02.183878+0000","last_clean":"2023-10-09T10:01:02.183878+0000","last_became_active":"2023-10-09T10:01:00.933011+0000","last_became_peered":"2023-10-09T10:01:00.933011+0000","last_unstale":"2023-10-09T10:01:02.183878+0000","last_undegraded":"2023-10-09T10:01:02.183878+0000","last_fullsized":"2023-10-09T10:01:02.183878+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":129,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.192404+0000","last_change":"2023-10-09T10:01:00.935684+0000","last_active":"2023-10-09T10:01:02.192404+0000","last_peered":"2023-10-09T10:01:02.192404+0000","last_clean":"2023-10-09T10:01:02.192404+0000","last_became_active":"2023-10-09T10:01:00.935290+0000","last_became_peered":"2023-10-09T10:01:00.935290+0000","last_unstale":"2023-10-09T10:01:02.192404+0000","last_undegraded":"2023-10-09T10:01:02.192404+0000","last_fullsized":"2023-10-09T10:01:02.192404+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:33.930594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"139'7801","reported_seq":7854,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:04.833960+0000","last_change":"2023-10-09T09:58:52.780096+0000","last_active":"2023-10-09T10:01:04.833960+0000","last_peered":"2023-10-09T10:01:04.833960+0000","last_clean":"2023-10-09T10:01:04.833960+0000","last_became_active":"2023-10-09T09:58:52.779754+0000","last_became_peered":"2023-10-09T09:58:52.779754+0000","last_unstale":"2023-10-09T10:01:04.833960+0000","last_undegraded":"2023-10-09T10:01:04.833960+0000","last_fullsized":"2023-10-09T10:01:04.833960+0000","mapping_epoch":97,"log_start":"105'2344","ondisk_log_start":"105'2344","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:19:28.304859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355008512,"num_objects":5417,"num_object_clones":0,"num_object_copies":16251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7801,"num_write_kb":422976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.192373+0000","last_change":"2023-10-09T10:01:00.935986+0000","last_active":"2023-10-09T10:01:02.192373+0000","last_peered":"2023-10-09T10:01:02.192373+0000","last_clean":"2023-10-09T10:01:02.192373+0000","last_became_active":"2023-10-09T10:01:00.935495+0000","last_became_peered":"2023-10-09T10:01:00.935495+0000","last_unstale":"2023-10-09T10:01:02.192373+0000","last_undegraded":"2023-10-09T10:01:02.192373+0000","last_fullsized":"2023-10-09T10:01:02.192373+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:11.501226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":56,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.185016+0000","last_change":"2023-10-09T10:01:00.930864+0000","last_active":"2023-10-09T10:01:02.185016+0000","last_peered":"2023-10-09T10:01:02.185016+0000","last_clean":"2023-10-09T10:01:02.185016+0000","last_became_active":"2023-10-09T10:01:00.930553+0000","last_became_peered":"2023-10-09T10:01:00.930553+0000","last_unstale":"2023-10-09T10:01:02.185016+0000","last_undegraded":"2023-10-09T10:01:02.185016+0000","last_fullsized":"2023-10-09T10:01:02.185016+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"139'7572","reported_seq":7624,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.348038+0000","last_change":"2023-10-09T09:58:52.626617+0000","last_active":"2023-10-09T10:01:02.348038+0000","last_peered":"2023-10-09T10:01:02.348038+0000","last_clean":"2023-10-09T10:01:02.348038+0000","last_became_active":"2023-10-09T09:58:52.626409+0000","last_became_peered":"2023-10-09T09:58:52.626409+0000","last_unstale":"2023-10-09T10:01:02.348038+0000","last_undegraded":"2023-10-09T10:01:02.348038+0000","last_fullsized":"2023-10-09T10:01:02.348038+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7572,"log_dups_size":0,"ondisk_log_size":7572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351469592,"num_objects":5364,"num_object_clones":0,"num_object_copies":16092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5364,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7572,"num_write_kb":413889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":109,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.920392+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T10:01:01.920392+0000","last_peered":"2023-10-09T10:01:01.920392+0000","last_clean":"2023-10-09T10:01:01.920392+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T10:01:01.920392+0000","last_undegraded":"2023-10-09T10:01:01.920392+0000","last_fullsized":"2023-10-09T10:01:01.920392+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.944032+0000","last_change":"2023-10-09T10:01:00.933557+0000","last_active":"2023-10-09T10:01:01.944032+0000","last_peered":"2023-10-09T10:01:01.944032+0000","last_clean":"2023-10-09T10:01:01.944032+0000","last_became_active":"2023-10-09T10:01:00.933259+0000","last_became_peered":"2023-10-09T10:01:00.933259+0000","last_unstale":"2023-10-09T10:01:01.944032+0000","last_undegraded":"2023-10-09T10:01:01.944032+0000","last_fullsized":"2023-10-09T10:01:01.944032+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:47.058899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":120,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.930965+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T10:01:01.930965+0000","last_peered":"2023-10-09T10:01:01.930965+0000","last_clean":"2023-10-09T10:01:01.930965+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T10:01:01.930965+0000","last_undegraded":"2023-10-09T10:01:01.930965+0000","last_fullsized":"2023-10-09T10:01:01.930965+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":139,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.926274+0000","last_change":"2023-10-09T10:00:30.785799+0000","last_active":"2023-10-09T10:01:01.926274+0000","last_peered":"2023-10-09T10:01:01.926274+0000","last_clean":"2023-10-09T10:01:01.926274+0000","last_became_active":"2023-10-09T10:00:30.785396+0000","last_became_peered":"2023-10-09T10:00:30.785396+0000","last_unstale":"2023-10-09T10:01:01.926274+0000","last_undegraded":"2023-10-09T10:01:01.926274+0000","last_fullsized":"2023-10-09T10:01:01.926274+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":43,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.929955+0000","last_change":"2023-10-09T10:01:00.929552+0000","last_active":"2023-10-09T10:01:01.929955+0000","last_peered":"2023-10-09T10:01:01.929955+0000","last_clean":"2023-10-09T10:01:01.929955+0000","last_became_active":"2023-10-09T10:01:00.929225+0000","last_became_peered":"2023-10-09T10:01:00.929225+0000","last_unstale":"2023-10-09T10:01:01.929955+0000","last_undegraded":"2023-10-09T10:01:01.929955+0000","last_fullsized":"2023-10-09T10:01:01.929955+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:11.162776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"139'7569","reported_seq":8901,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.363384+0000","last_change":"2023-10-09T10:00:52.654643+0000","last_active":"2023-10-09T10:01:02.363384+0000","last_peered":"2023-10-09T10:01:02.363384+0000","last_clean":"2023-10-09T10:01:02.363384+0000","last_became_active":"2023-10-09T10:00:52.648538+0000","last_became_peered":"2023-10-09T10:00:52.648538+0000","last_unstale":"2023-10-09T10:01:02.363384+0000","last_undegraded":"2023-10-09T10:01:02.363384+0000","last_fullsized":"2023-10-09T10:01:02.363384+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7569,"log_dups_size":0,"ondisk_log_size":7569,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174751744,"num_objects":2666,"num_object_clones":0,"num_object_copies":7998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3875,"num_write_kb":209344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20414464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"139'7627","reported_seq":11092,"reported_epoch":140,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T10:01:07.198506+0000","last_change":"2023-10-09T10:01:04.122858+0000","last_active":"2023-10-09T10:01:07.198506+0000","last_peered":"2023-10-09T10:01:07.198506+0000","last_clean":"2023-10-09T10:00:59.894671+0000","last_became_active":"2023-10-09T10:01:02.194954+0000","last_became_peered":"2023-10-09T10:01:02.194954+0000","last_unstale":"2023-10-09T10:01:07.198506+0000","last_undegraded":"2023-10-09T10:01:07.198506+0000","last_fullsized":"2023-10-09T10:01:07.198506+0000","mapping_epoch":138,"log_start":"102'300","ondisk_log_start":"102'300","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7327,"log_dups_size":0,"ondisk_log_size":7327,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176947200,"num_objects":2700,"num_object_clones":0,"num_object_copies":8100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1005,"num_objects_unfound":0,"num_objects_dirty":2700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3900,"num_write_kb":211200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1695,"num_bytes_recovered":111083520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":2700}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":110,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.926106+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T10:01:01.926106+0000","last_peered":"2023-10-09T10:01:01.926106+0000","last_clean":"2023-10-09T10:01:01.926106+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T10:01:01.926106+0000","last_undegraded":"2023-10-09T10:01:01.926106+0000","last_fullsized":"2023-10-09T10:01:01.926106+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":52,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.927681+0000","last_change":"2023-10-09T10:01:00.937214+0000","last_active":"2023-10-09T10:01:01.927681+0000","last_peered":"2023-10-09T10:01:01.927681+0000","last_clean":"2023-10-09T10:01:01.927681+0000","last_became_active":"2023-10-09T10:01:00.936560+0000","last_became_peered":"2023-10-09T10:01:00.936560+0000","last_unstale":"2023-10-09T10:01:01.927681+0000","last_undegraded":"2023-10-09T10:01:01.927681+0000","last_fullsized":"2023-10-09T10:01:01.927681+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:53.374219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":128,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.930896+0000","last_change":"2023-10-09T10:00:30.782106+0000","last_active":"2023-10-09T10:01:01.930896+0000","last_peered":"2023-10-09T10:01:01.930896+0000","last_clean":"2023-10-09T10:01:01.930896+0000","last_became_active":"2023-10-09T10:00:30.781623+0000","last_became_peered":"2023-10-09T10:00:30.781623+0000","last_unstale":"2023-10-09T10:01:01.930896+0000","last_undegraded":"2023-10-09T10:01:01.930896+0000","last_fullsized":"2023-10-09T10:01:01.930896+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:14.657623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"139'7534","reported_seq":7593,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.375428+0000","last_change":"2023-10-09T10:00:52.678089+0000","last_active":"2023-10-09T10:01:02.375428+0000","last_peered":"2023-10-09T10:01:02.375428+0000","last_clean":"2023-10-09T10:01:02.375428+0000","last_became_active":"2023-10-09T10:00:52.671121+0000","last_became_peered":"2023-10-09T10:00:52.671121+0000","last_unstale":"2023-10-09T10:01:02.375428+0000","last_undegraded":"2023-10-09T10:01:02.375428+0000","last_fullsized":"2023-10-09T10:01:02.375428+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7534,"log_dups_size":0,"ondisk_log_size":7534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175734784,"num_objects":2681,"num_object_clones":0,"num_object_copies":8043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3848,"num_write_kb":208960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.925341+0000","last_change":"2023-10-09T10:00:30.892640+0000","last_active":"2023-10-09T10:01:01.925341+0000","last_peered":"2023-10-09T10:01:01.925341+0000","last_clean":"2023-10-09T10:01:01.925341+0000","last_became_active":"2023-10-09T10:00:30.892119+0000","last_became_peered":"2023-10-09T10:00:30.892119+0000","last_unstale":"2023-10-09T10:01:01.925341+0000","last_undegraded":"2023-10-09T10:01:01.925341+0000","last_fullsized":"2023-10-09T10:01:01.925341+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:37:45.627291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":112,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.929755+0000","last_change":"2023-10-09T10:00:30.786721+0000","last_active":"2023-10-09T10:01:01.929755+0000","last_peered":"2023-10-09T10:01:01.929755+0000","last_clean":"2023-10-09T10:01:01.929755+0000","last_became_active":"2023-10-09T10:00:30.786375+0000","last_became_peered":"2023-10-09T10:00:30.786375+0000","last_unstale":"2023-10-09T10:01:01.929755+0000","last_undegraded":"2023-10-09T10:01:01.929755+0000","last_fullsized":"2023-10-09T10:01:01.929755+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:19:38.052691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:03.620419+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T10:01:03.620419+0000","last_peered":"2023-10-09T10:01:03.620419+0000","last_clean":"2023-10-09T10:01:03.620419+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T10:01:03.620419+0000","last_undegraded":"2023-10-09T10:01:03.620419+0000","last_fullsized":"2023-10-09T10:01:03.620419+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"139'7402","reported_seq":7604,"reported_epoch":140,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:01:04.827447+0000","last_change":"2023-10-09T10:01:02.111094+0000","last_active":"2023-10-09T10:01:04.827447+0000","last_peered":"2023-10-09T10:01:04.827447+0000","last_clean":"2023-10-09T10:00:59.913112+0000","last_became_active":"2023-10-09T10:01:02.088331+0000","last_became_peered":"2023-10-09T10:01:02.088331+0000","last_unstale":"2023-10-09T10:01:04.827447+0000","last_undegraded":"2023-10-09T10:01:01.956690+0000","last_fullsized":"2023-10-09T10:01:01.940057+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7402,"log_dups_size":0,"ondisk_log_size":7402,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174325760,"num_objects":2660,"num_object_clones":0,"num_object_copies":7980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":2592,"num_objects_unfound":0,"num_objects_dirty":2660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3783,"num_write_kb":206176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2592},{"shards":"3,5","objects":67}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":128,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.925871+0000","last_change":"2023-10-09T10:00:30.790235+0000","last_active":"2023-10-09T10:01:01.925871+0000","last_peered":"2023-10-09T10:01:01.925871+0000","last_clean":"2023-10-09T10:01:01.925871+0000","last_became_active":"2023-10-09T10:00:30.789866+0000","last_became_peered":"2023-10-09T10:00:30.789866+0000","last_unstale":"2023-10-09T10:01:01.925871+0000","last_undegraded":"2023-10-09T10:01:01.925871+0000","last_fullsized":"2023-10-09T10:01:01.925871+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":340,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T10:01:00.917035+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T10:01:00.917035+0000","last_peered":"2023-10-09T10:01:00.917035+0000","last_clean":"2023-10-09T10:01:00.917035+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T10:01:00.917035+0000","last_undegraded":"2023-10-09T10:01:00.917035+0000","last_fullsized":"2023-10-09T10:01:00.917035+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":68,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T10:01:00.912184+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T10:01:00.912184+0000","last_peered":"2023-10-09T10:01:00.912184+0000","last_clean":"2023-10-09T10:01:00.912184+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T10:01:00.912184+0000","last_undegraded":"2023-10-09T10:01:00.912184+0000","last_fullsized":"2023-10-09T10:01:00.912184+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.135676+0000","last_change":"2023-10-09T10:01:00.927995+0000","last_active":"2023-10-09T10:01:02.135676+0000","last_peered":"2023-10-09T10:01:02.135676+0000","last_clean":"2023-10-09T10:01:02.135676+0000","last_became_active":"2023-10-09T10:01:00.927592+0000","last_became_peered":"2023-10-09T10:01:00.927592+0000","last_unstale":"2023-10-09T10:01:02.135676+0000","last_undegraded":"2023-10-09T10:01:02.135676+0000","last_fullsized":"2023-10-09T10:01:02.135676+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":92,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.929471+0000","last_change":"2023-10-09T10:00:19.650114+0000","last_active":"2023-10-09T10:01:01.929471+0000","last_peered":"2023-10-09T10:01:01.929471+0000","last_clean":"2023-10-09T10:01:01.929471+0000","last_became_active":"2023-10-09T10:00:19.649776+0000","last_became_peered":"2023-10-09T10:00:19.649776+0000","last_unstale":"2023-10-09T10:01:01.929471+0000","last_undegraded":"2023-10-09T10:01:01.929471+0000","last_fullsized":"2023-10-09T10:01:01.929471+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:09:03.140113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":107,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.927753+0000","last_change":"2023-10-09T10:01:00.932106+0000","last_active":"2023-10-09T10:01:01.927753+0000","last_peered":"2023-10-09T10:01:01.927753+0000","last_clean":"2023-10-09T10:01:01.927753+0000","last_became_active":"2023-10-09T10:01:00.931851+0000","last_became_peered":"2023-10-09T10:01:00.931851+0000","last_unstale":"2023-10-09T10:01:01.927753+0000","last_undegraded":"2023-10-09T10:01:01.927753+0000","last_fullsized":"2023-10-09T10:01:01.927753+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:31.563093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":135,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.928841+0000","last_change":"2023-10-09T10:00:30.785844+0000","last_active":"2023-10-09T10:01:01.928841+0000","last_peered":"2023-10-09T10:01:01.928841+0000","last_clean":"2023-10-09T10:01:01.928841+0000","last_became_active":"2023-10-09T10:00:30.785565+0000","last_became_peered":"2023-10-09T10:00:30.785565+0000","last_unstale":"2023-10-09T10:01:01.928841+0000","last_undegraded":"2023-10-09T10:01:01.928841+0000","last_fullsized":"2023-10-09T10:01:01.928841+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:56.229142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"140'7621","reported_seq":7684,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:08.446185+0000","last_change":"2023-10-09T10:00:52.682102+0000","last_active":"2023-10-09T10:01:08.446185+0000","last_peered":"2023-10-09T10:01:08.446185+0000","last_clean":"2023-10-09T10:01:08.446185+0000","last_became_active":"2023-10-09T10:00:52.668775+0000","last_became_peered":"2023-10-09T10:00:52.668775+0000","last_unstale":"2023-10-09T10:01:08.446185+0000","last_undegraded":"2023-10-09T10:01:08.446185+0000","last_fullsized":"2023-10-09T10:01:08.446185+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7621,"log_dups_size":0,"ondisk_log_size":7621,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173572096,"num_objects":2648,"num_object_clones":0,"num_object_copies":7944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3920,"num_write_kb":210208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":117,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T10:01:00.928608+0000","last_change":"2023-10-09T10:01:00.927846+0000","last_active":"2023-10-09T10:01:00.928608+0000","last_peered":"2023-10-09T10:01:00.928608+0000","last_clean":"2023-10-09T10:01:00.928608+0000","last_became_active":"2023-10-09T10:01:00.927543+0000","last_became_peered":"2023-10-09T10:01:00.927543+0000","last_unstale":"2023-10-09T10:01:00.928608+0000","last_undegraded":"2023-10-09T10:01:00.928608+0000","last_fullsized":"2023-10-09T10:01:00.928608+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":113,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.931041+0000","last_change":"2023-10-09T10:00:30.786812+0000","last_active":"2023-10-09T10:01:01.931041+0000","last_peered":"2023-10-09T10:01:01.931041+0000","last_clean":"2023-10-09T10:01:01.931041+0000","last_became_active":"2023-10-09T10:00:30.786500+0000","last_became_peered":"2023-10-09T10:00:30.786500+0000","last_unstale":"2023-10-09T10:01:01.931041+0000","last_undegraded":"2023-10-09T10:01:01.931041+0000","last_fullsized":"2023-10-09T10:01:01.931041+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:25.060464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":84,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.927758+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T10:01:01.927758+0000","last_peered":"2023-10-09T10:01:01.927758+0000","last_clean":"2023-10-09T10:01:01.927758+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T10:01:01.927758+0000","last_undegraded":"2023-10-09T10:01:01.927758+0000","last_fullsized":"2023-10-09T10:01:01.927758+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"140'7653","reported_seq":8403,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:08.542446+0000","last_change":"2023-10-09T10:01:04.122514+0000","last_active":"2023-10-09T10:01:08.542446+0000","last_peered":"2023-10-09T10:01:08.542446+0000","last_clean":"2023-10-09T10:01:08.542446+0000","last_became_active":"2023-10-09T10:00:53.648018+0000","last_became_peered":"2023-10-09T10:00:53.648018+0000","last_unstale":"2023-10-09T10:01:08.542446+0000","last_undegraded":"2023-10-09T10:01:08.542446+0000","last_fullsized":"2023-10-09T10:01:08.542446+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7653,"log_dups_size":0,"ondisk_log_size":7653,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3931,"num_write_kb":212960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":21954560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":106,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.921114+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T10:01:01.921114+0000","last_peered":"2023-10-09T10:01:01.921114+0000","last_clean":"2023-10-09T10:01:01.921114+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T10:01:01.921114+0000","last_undegraded":"2023-10-09T10:01:01.921114+0000","last_fullsized":"2023-10-09T10:01:01.921114+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":88,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:03.620348+0000","last_change":"2023-10-09T10:01:00.933551+0000","last_active":"2023-10-09T10:01:03.620348+0000","last_peered":"2023-10-09T10:01:03.620348+0000","last_clean":"2023-10-09T10:01:03.620348+0000","last_became_active":"2023-10-09T10:01:00.933229+0000","last_became_peered":"2023-10-09T10:01:00.933229+0000","last_unstale":"2023-10-09T10:01:03.620348+0000","last_undegraded":"2023-10-09T10:01:03.620348+0000","last_fullsized":"2023-10-09T10:01:03.620348+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:52.813857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.920422+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T10:01:01.920422+0000","last_peered":"2023-10-09T10:01:01.920422+0000","last_clean":"2023-10-09T10:01:01.920422+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T10:01:01.920422+0000","last_undegraded":"2023-10-09T10:01:01.920422+0000","last_fullsized":"2023-10-09T10:01:01.920422+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"136'7702","reported_seq":7779,"reported_epoch":140,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T10:01:07.153584+0000","last_change":"2023-10-09T10:01:07.153584+0000","last_active":"2023-10-09T10:01:00.933503+0000","last_peered":"2023-10-09T10:00:59.912692+0000","last_clean":"2023-10-09T10:00:08.575217+0000","last_became_active":"2023-10-09T10:00:53.659646+0000","last_became_peered":"2023-10-09T10:00:53.659646+0000","last_unstale":"2023-10-09T10:01:07.153584+0000","last_undegraded":"2023-10-09T10:01:07.139881+0000","last_fullsized":"2023-10-09T10:01:01.923140+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7702,"log_dups_size":0,"ondisk_log_size":7702,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181469184,"num_objects":2769,"num_object_clones":0,"num_object_copies":8307,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":3347,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3940,"num_write_kb":214688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["4"],"object_location_counts":[{"shards":"2,3","objects":24},{"shards":"2,3,4","objects":2397},{"shards":"3,4,6","objects":348}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.217393+0000","last_change":"2023-10-09T10:00:30.851245+0000","last_active":"2023-10-09T10:01:02.217393+0000","last_peered":"2023-10-09T10:01:02.217393+0000","last_clean":"2023-10-09T10:01:02.217393+0000","last_became_active":"2023-10-09T10:00:30.850854+0000","last_became_peered":"2023-10-09T10:00:30.850854+0000","last_unstale":"2023-10-09T10:01:02.217393+0000","last_undegraded":"2023-10-09T10:01:02.217393+0000","last_fullsized":"2023-10-09T10:01:02.217393+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.928831+0000","last_change":"2023-10-09T10:01:00.947866+0000","last_active":"2023-10-09T10:01:01.928831+0000","last_peered":"2023-10-09T10:01:01.928831+0000","last_clean":"2023-10-09T10:01:01.928831+0000","last_became_active":"2023-10-09T10:01:00.924897+0000","last_became_peered":"2023-10-09T10:01:00.924897+0000","last_unstale":"2023-10-09T10:01:01.928831+0000","last_undegraded":"2023-10-09T10:01:01.928831+0000","last_fullsized":"2023-10-09T10:01:01.928831+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:07.277714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":15,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.926581+0000","last_change":"2023-10-09T10:01:00.927941+0000","last_active":"2023-10-09T10:01:01.926581+0000","last_peered":"2023-10-09T10:01:01.926581+0000","last_clean":"2023-10-09T10:01:01.926581+0000","last_became_active":"2023-10-09T10:01:00.927456+0000","last_became_peered":"2023-10-09T10:01:00.927456+0000","last_unstale":"2023-10-09T10:01:01.926581+0000","last_undegraded":"2023-10-09T10:01:01.926581+0000","last_fullsized":"2023-10-09T10:01:01.926581+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.3","version":"139'7648","reported_seq":9505,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:03.341055+0000","last_change":"2023-10-09T10:01:03.341055+0000","last_active":"2023-10-09T10:01:03.341055+0000","last_peered":"2023-10-09T10:01:03.341055+0000","last_clean":"2023-10-09T10:01:03.341055+0000","last_became_active":"2023-10-09T10:00:53.652818+0000","last_became_peered":"2023-10-09T10:00:53.652818+0000","last_unstale":"2023-10-09T10:01:03.341055+0000","last_undegraded":"2023-10-09T10:01:03.341055+0000","last_fullsized":"2023-10-09T10:01:03.341055+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7648,"log_dups_size":0,"ondisk_log_size":7648,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179273728,"num_objects":2735,"num_object_clones":0,"num_object_copies":8205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3915,"num_write_kb":212832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":609,"num_bytes_recovered":39944192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":95,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.927989+0000","last_change":"2023-10-09T10:01:00.926495+0000","last_active":"2023-10-09T10:01:01.927989+0000","last_peered":"2023-10-09T10:01:01.927989+0000","last_clean":"2023-10-09T10:01:01.927989+0000","last_became_active":"2023-10-09T10:01:00.925943+0000","last_became_peered":"2023-10-09T10:01:00.925943+0000","last_unstale":"2023-10-09T10:01:01.927989+0000","last_undegraded":"2023-10-09T10:01:01.927989+0000","last_fullsized":"2023-10-09T10:01:01.927989+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:13.486263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":26,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.926087+0000","last_change":"2023-10-09T10:00:30.779150+0000","last_active":"2023-10-09T10:01:01.926087+0000","last_peered":"2023-10-09T10:01:01.926087+0000","last_clean":"2023-10-09T10:01:01.926087+0000","last_became_active":"2023-10-09T10:00:30.778817+0000","last_became_peered":"2023-10-09T10:00:30.778817+0000","last_unstale":"2023-10-09T10:01:01.926087+0000","last_undegraded":"2023-10-09T10:01:01.926087+0000","last_fullsized":"2023-10-09T10:01:01.926087+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:04.893208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"139'7569","reported_seq":9900,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.381513+0000","last_change":"2023-10-09T10:00:56.208970+0000","last_active":"2023-10-09T10:01:02.381513+0000","last_peered":"2023-10-09T10:01:02.381513+0000","last_clean":"2023-10-09T10:01:02.381513+0000","last_became_active":"2023-10-09T10:00:52.760143+0000","last_became_peered":"2023-10-09T10:00:52.760143+0000","last_unstale":"2023-10-09T10:01:02.381513+0000","last_undegraded":"2023-10-09T10:01:02.381513+0000","last_fullsized":"2023-10-09T10:01:02.381513+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7569,"log_dups_size":0,"ondisk_log_size":7569,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174424064,"num_objects":2662,"num_object_clones":0,"num_object_copies":7986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3872,"num_write_kb":209056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":726,"num_bytes_recovered":47546368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":121,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.925940+0000","last_change":"2023-10-09T10:01:00.967634+0000","last_active":"2023-10-09T10:01:01.925940+0000","last_peered":"2023-10-09T10:01:01.925940+0000","last_clean":"2023-10-09T10:01:01.925940+0000","last_became_active":"2023-10-09T10:01:00.925301+0000","last_became_peered":"2023-10-09T10:01:00.925301+0000","last_unstale":"2023-10-09T10:01:01.925940+0000","last_undegraded":"2023-10-09T10:01:01.925940+0000","last_fullsized":"2023-10-09T10:01:01.925940+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":81,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T10:01:04.833629+0000","last_change":"2023-10-09T10:01:00.935856+0000","last_active":"2023-10-09T10:01:04.833629+0000","last_peered":"2023-10-09T10:01:04.833629+0000","last_clean":"2023-10-09T10:01:04.833629+0000","last_became_active":"2023-10-09T10:01:00.935517+0000","last_became_peered":"2023-10-09T10:01:00.935517+0000","last_unstale":"2023-10-09T10:01:04.833629+0000","last_undegraded":"2023-10-09T10:01:04.833629+0000","last_fullsized":"2023-10-09T10:01:04.833629+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:13.048073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"139'7794","reported_seq":8545,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.366226+0000","last_change":"2023-10-09T10:01:01.555042+0000","last_active":"2023-10-09T10:01:02.366226+0000","last_peered":"2023-10-09T10:01:02.366226+0000","last_clean":"2023-10-09T10:01:02.366226+0000","last_became_active":"2023-10-09T10:00:53.683716+0000","last_became_peered":"2023-10-09T10:00:53.683716+0000","last_unstale":"2023-10-09T10:01:02.366226+0000","last_undegraded":"2023-10-09T10:01:02.366226+0000","last_fullsized":"2023-10-09T10:01:02.366226+0000","mapping_epoch":133,"log_start":"102'400","ondisk_log_start":"102'400","created":133,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181633024,"num_objects":2772,"num_object_clones":0,"num_object_copies":8316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3990,"num_write_kb":216352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":336,"num_bytes_recovered":22020096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":106,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.926328+0000","last_change":"2023-10-09T10:01:00.942712+0000","last_active":"2023-10-09T10:01:01.926328+0000","last_peered":"2023-10-09T10:01:01.926328+0000","last_clean":"2023-10-09T10:01:01.926328+0000","last_became_active":"2023-10-09T10:01:00.924401+0000","last_became_peered":"2023-10-09T10:01:00.924401+0000","last_unstale":"2023-10-09T10:01:01.926328+0000","last_undegraded":"2023-10-09T10:01:01.926328+0000","last_fullsized":"2023-10-09T10:01:01.926328+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":80,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.038722+0000","last_change":"2023-10-09T10:01:00.934578+0000","last_active":"2023-10-09T10:01:02.038722+0000","last_peered":"2023-10-09T10:01:02.038722+0000","last_clean":"2023-10-09T10:01:02.038722+0000","last_became_active":"2023-10-09T10:01:00.934237+0000","last_became_peered":"2023-10-09T10:01:00.934237+0000","last_unstale":"2023-10-09T10:01:02.038722+0000","last_undegraded":"2023-10-09T10:01:02.038722+0000","last_fullsized":"2023-10-09T10:01:02.038722+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:36.229511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":132,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.921884+0000","last_change":"2023-10-09T10:00:30.779348+0000","last_active":"2023-10-09T10:01:01.921884+0000","last_peered":"2023-10-09T10:01:01.921884+0000","last_clean":"2023-10-09T10:01:01.921884+0000","last_became_active":"2023-10-09T10:00:30.778963+0000","last_became_peered":"2023-10-09T10:00:30.778963+0000","last_unstale":"2023-10-09T10:01:01.921884+0000","last_undegraded":"2023-10-09T10:01:01.921884+0000","last_fullsized":"2023-10-09T10:01:01.921884+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.112454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":145,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.920585+0000","last_change":"2023-10-09T10:00:30.865002+0000","last_active":"2023-10-09T10:01:01.920585+0000","last_peered":"2023-10-09T10:01:01.920585+0000","last_clean":"2023-10-09T10:01:01.920585+0000","last_became_active":"2023-10-09T10:00:30.864670+0000","last_became_peered":"2023-10-09T10:00:30.864670+0000","last_unstale":"2023-10-09T10:01:01.920585+0000","last_undegraded":"2023-10-09T10:01:01.920585+0000","last_fullsized":"2023-10-09T10:01:01.920585+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:16.562219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":135,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:01.928342+0000","last_change":"2023-10-09T10:00:44.129942+0000","last_active":"2023-10-09T10:01:01.928342+0000","last_peered":"2023-10-09T10:01:01.928342+0000","last_clean":"2023-10-09T10:01:01.928342+0000","last_became_active":"2023-10-09T10:00:30.861856+0000","last_became_peered":"2023-10-09T10:00:30.861856+0000","last_unstale":"2023-10-09T10:01:01.928342+0000","last_undegraded":"2023-10-09T10:01:01.928342+0000","last_fullsized":"2023-10-09T10:01:01.928342+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:34.634782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":56,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T10:01:02.135969+0000","last_change":"2023-10-09T10:01:00.933983+0000","last_active":"2023-10-09T10:01:02.135969+0000","last_peered":"2023-10-09T10:01:02.135969+0000","last_clean":"2023-10-09T10:01:02.135969+0000","last_became_active":"2023-10-09T10:01:00.933670+0000","last_became_peered":"2023-10-09T10:01:00.933670+0000","last_unstale":"2023-10-09T10:01:02.135969+0000","last_undegraded":"2023-10-09T10:01:02.135969+0000","last_fullsized":"2023-10-09T10:01:02.135969+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":5662113816,"num_objects":86398,"num_object_clones":0,"num_object_copies":259194,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":9940,"num_objects_misplaced":14994,"num_objects_unfound":0,"num_objects_dirty":86398,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124640,"num_write_kb":6753153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9640,"num_bytes_recovered":631570432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17270648832,"data_stored":17270636616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":190377,"ondisk_log_size":190377,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6056,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430377,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2421992,"kb_used_data":2120672,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91323160,"statfs":{"total":95995035648,"available":93514915840,"internally_reserved":0,"allocated":2171568128,"data_stored":2171095146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":0.82699999999999996,"15min":0.67100000000000004},"min":{"1min":0.251,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5519999999999996,"5min":4.5519999999999996,"15min":4.5519999999999996},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.9870000000000001,"5min":0.84299999999999997,"15min":0.64200000000000002},"min":{"1min":0.25900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":8.0259999999999998,"5min":8.0259999999999998,"15min":8.0259999999999998},"last":0.41499999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5699999999999998,"5min":1.0660000000000001,"15min":0.71799999999999997},"min":{"1min":0.27000000000000002,"5min":0.246,"15min":0.246},"max":{"1min":5.9470000000000001,"5min":5.9470000000000001,"15min":5.9470000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.113,"5min":1.0549999999999999,"15min":0.74099999999999999},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.1879999999999997,"5min":6.1879999999999997,"15min":6.1879999999999997},"last":0.35799999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":0.70199999999999996,"15min":0.63400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.806,"5min":2.806,"15min":2.806},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.1160000000000001,"5min":0.86299999999999999,"15min":0.66400000000000003},"min":{"1min":0.29999999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.7400000000000002,"5min":6.7400000000000002,"15min":6.7400000000000002},"last":0.47899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":1.1060000000000001,"15min":0.75800000000000001},"min":{"1min":0.378,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.8140000000000001,"5min":9.8140000000000001,"15min":9.8140000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.3540000000000001,"5min":1.0880000000000001,"15min":0.754},"min":{"1min":0.34699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":5.8209999999999997,"5min":5.8209999999999997,"15min":5.8209999999999997},"last":0.40300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:00:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":0.69799999999999995,"15min":0.53600000000000003},"min":{"1min":0.29099999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.1689999999999996,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.626,"15min":0.52200000000000002},"min":{"1min":0.32900000000000001,"5min":0.314,"15min":0.26500000000000001},"max":{"1min":4.1550000000000002,"5min":4.1550000000000002,"15min":4.1550000000000002},"last":0.50700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:00:36 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":0.79000000000000004,"15min":0.59299999999999997},"min":{"1min":0.31900000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.958,"5min":12.958,"15min":12.958},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.3029999999999999,"5min":0.71799999999999997,"15min":0.58499999999999996},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":6.1420000000000003,"5min":6.1420000000000003,"15min":6.1420000000000003},"last":0.49199999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:00:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":0.71399999999999997,"15min":0.59699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":5.4630000000000001,"5min":5.4630000000000001,"15min":5.4630000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.03,"5min":0.68500000000000005,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.33100000000000002},"max":{"1min":4.407,"5min":4.407,"15min":4.407},"last":0.55000000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626605,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2147292,"kb_used_data":1841876,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91597860,"statfs":{"total":95995035648,"available":93796208640,"internally_reserved":0,"allocated":1886081024,"data_stored":1885619903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":0.91300000000000003,"15min":0.67500000000000004},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":8.8650000000000002,"5min":8.8650000000000002,"15min":8.8650000000000002},"last":0.23799999999999999},{"interface":"front","average":{"1min":1.843,"5min":0.875,"15min":0.629},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.246},"max":{"1min":12.439,"5min":12.439,"15min":12.439},"last":0.29499999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:00:49 2023","interfaces":[{"interface":"back","average":{"1min":3.6669999999999998,"5min":1.292,"15min":0.79700000000000004},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":18.853000000000002,"5min":18.853000000000002,"15min":18.853000000000002},"last":0.28699999999999998},{"interface":"front","average":{"1min":2.2549999999999999,"5min":0.96899999999999997,"15min":0.67900000000000005},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":12.380000000000001,"5min":12.380000000000001,"15min":12.380000000000001},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:00:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8210000000000002,"5min":1.0229999999999999,"15min":0.70099999999999996},"min":{"1min":0.41799999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":18.800000000000001,"5min":18.800000000000001,"15min":18.800000000000001},"last":0.311},{"interface":"front","average":{"1min":2.7570000000000001,"5min":1.0780000000000001,"15min":0.75},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":12.413,"5min":12.413,"15min":12.413},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:00:54 2023","interfaces":[{"interface":"back","average":{"1min":3.0369999999999999,"5min":1.2669999999999999,"15min":0.83299999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":12.337,"5min":12.337,"15min":12.337},"last":0.65300000000000002},{"interface":"front","average":{"1min":2.8319999999999999,"5min":1.1719999999999999,"15min":0.79000000000000004},"min":{"1min":0.39600000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":7.0110000000000001,"5min":7.0110000000000001,"15min":7.0110000000000001},"last":0.66200000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:00:12 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":0.622,"15min":0.50600000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":6.6219999999999999,"5min":6.6219999999999999,"15min":6.6219999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.2849999999999999,"5min":0.66500000000000004,"15min":0.54600000000000004},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.2379999999999995,"5min":8.2379999999999995,"15min":8.2379999999999995},"last":0.32800000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:00:12 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.55900000000000005,"15min":0.51900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.8920000000000003,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.63200000000000001,"15min":0.54600000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7069999999999999,"5min":5.7069999999999999,"15min":5.7069999999999999},"last":0.37}]},{"osd":7,"last update":"Mon Oct 9 10:00:39 2023","interfaces":[{"interface":"back","average":{"1min":1.496,"5min":0.77200000000000002,"15min":0.60399999999999998},"min":{"1min":0.32400000000000001,"5min":0.315,"15min":0.315},"max":{"1min":5.5449999999999999,"5min":5.5449999999999999,"15min":5.5449999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.538,"5min":0.83199999999999996,"15min":0.66600000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.5759999999999996,"5min":5.5759999999999996,"15min":5.5759999999999996},"last":0.39900000000000002}]}]},{"osd":1,"up_from":84,"seq":360777252919,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1888624,"kb_used_data":1802920,"kb_used_omap":9,"kb_used_meta":85686,"kb_avail":91856528,"statfs":{"total":95995035648,"available":94061084672,"internally_reserved":0,"allocated":1846190080,"data_stored":1845599369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":87743465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.0609999999999999,"15min":0.58799999999999997},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.249},{"interface":"front","average":{"1min":2.601,"5min":1.04,"15min":0.61199999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.805999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:00:23 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":0.95899999999999996,"15min":0.57899999999999996},"min":{"1min":0.30099999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.6839999999999993,"5min":9.6839999999999993,"15min":9.6839999999999993},"last":0.39100000000000001},{"interface":"front","average":{"1min":3.3340000000000001,"5min":1.0249999999999999,"15min":0.60199999999999998},"min":{"1min":0.32100000000000001,"5min":0.248,"15min":0.248},"max":{"1min":12.069000000000001,"5min":12.069000000000001,"15min":12.069000000000001},"last":0.33500000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0819999999999999,"5min":1.7849999999999999,"15min":0.84699999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":25.129999999999999,"5min":25.129999999999999,"15min":25.129999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":5.1189999999999998,"5min":1.645,"15min":0.81100000000000005},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":22.847999999999999,"5min":22.847999999999999,"15min":22.847999999999999},"last":0.28100000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.899,"5min":1.3140000000000001,"15min":0.84499999999999997},"min":{"1min":0.32700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":7.6529999999999996,"5min":9.5139999999999993,"15min":9.5139999999999993},"last":0.434},{"interface":"front","average":{"1min":1.625,"5min":1.0509999999999999,"15min":0.77800000000000002},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.2779999999999996,"5min":6.2539999999999996,"15min":6.2539999999999996},"last":0.32400000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":1.256,"15min":0.83799999999999997},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":12.022,"5min":12.022,"15min":12.022},"last":0.35499999999999998},{"interface":"front","average":{"1min":3.6699999999999999,"5min":1.387,"15min":0.90100000000000002},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":19.706,"5min":19.706,"15min":19.706},"last":0.379}]},{"osd":6,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":1.2869999999999999,"15min":0.88700000000000001},"min":{"1min":0.39000000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.4760000000000009,"5min":8.4760000000000009,"15min":8.4760000000000009},"last":0.41399999999999998},{"interface":"front","average":{"1min":3.2669999999999999,"5min":1.3200000000000001,"15min":0.89000000000000001},"min":{"1min":0.45500000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.057,"5min":12.057,"15min":12.057},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":3.613,"5min":1.492,"15min":0.92500000000000004},"min":{"1min":0.40799999999999997,"5min":0.379,"15min":0.379},"max":{"1min":12.766999999999999,"5min":12.766999999999999,"15min":12.766999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.222,"15min":0.86399999999999999},"min":{"1min":0.44900000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.4290000000000003,"5min":7.4290000000000003,"15min":7.4290000000000003},"last":0.503}]}]},{"osd":0,"up_from":79,"seq":339302416445,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1445316,"kb_used_data":1373500,"kb_used_omap":28,"kb_used_meta":71779,"kb_avail":92299836,"statfs":{"total":95995035648,"available":94515032064,"internally_reserved":0,"allocated":1406464000,"data_stored":1405853924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":73502544},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":1.056,"15min":0.60199999999999998},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.191},"max":{"1min":10.202,"5min":10.202,"15min":10.202},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.25,"5min":1.042,"15min":0.61199999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.1219999999999999,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":0.40400000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.4529999999999998,"5min":0.83299999999999996,"15min":0.53900000000000003},"min":{"1min":0.23000000000000001,"5min":0.216,"15min":0.216},"max":{"1min":10.175000000000001,"5min":10.175000000000001,"15min":10.175000000000001},"last":0.378},{"interface":"front","average":{"1min":3.6930000000000001,"5min":1.107,"15min":0.65300000000000002},"min":{"1min":0.29599999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":17.920999999999999,"5min":17.920999999999999,"15min":17.920999999999999},"last":0.44}]},{"osd":3,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3730000000000002,"5min":1.206,"15min":0.65600000000000003},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.1029999999999998,"5min":9.9380000000000006,"15min":9.9380000000000006},"last":0.66200000000000003},{"interface":"front","average":{"1min":2.7810000000000001,"5min":1.373,"15min":0.751},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.0800000000000001,"5min":8.0800000000000001,"15min":8.0800000000000001},"last":0.71499999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":0.94199999999999995,"15min":0.752},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.035,"5min":10.035,"15min":10.035},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.6499999999999999,"5min":1.0860000000000001,"15min":0.81299999999999994},"min":{"1min":0.40899999999999997,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":8.6370000000000005,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":0.629}]},{"osd":5,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2349999999999999,"5min":0.999,"15min":0.76100000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":6.5010000000000003,"5min":6.5010000000000003,"15min":6.5010000000000003},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.1749999999999998,"5min":0.95699999999999996,"15min":0.76300000000000001},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":10.083,"5min":10.083,"15min":10.083},"last":0.59899999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.1060000000000001,"15min":0.82799999999999996},"min":{"1min":0.432,"5min":0.379,"15min":0.379},"max":{"1min":10.115,"5min":10.115,"15min":10.115},"last":0.5},{"interface":"front","average":{"1min":2.9689999999999999,"5min":1.143,"15min":0.84199999999999997},"min":{"1min":0.57099999999999995,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":8.7940000000000005,"5min":8.7940000000000005,"15min":8.7940000000000005},"last":0.67400000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:00:20 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.038,"15min":0.79000000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.3179999999999996,"5min":7.3179999999999996,"15min":7.3179999999999996},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.7189999999999999,"5min":1.0860000000000001,"15min":0.81699999999999995},"min":{"1min":0.46600000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":10.147,"5min":10.147,"15min":10.147},"last":0.56499999999999995}]}]},{"osd":2,"up_from":89,"seq":382252089392,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2869764,"kb_used_data":2739644,"kb_used_omap":81,"kb_used_meta":130030,"kb_avail":90875388,"statfs":{"total":95995035648,"available":93056397312,"internally_reserved":0,"allocated":2805395456,"data_stored":2804789574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":133150938},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":4.1399999999999997,"5min":1.2030000000000001,"15min":0.65700000000000003},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":56.883000000000003,"5min":56.883000000000003,"15min":56.883000000000003},"last":0.33600000000000002},{"interface":"front","average":{"1min":4.6769999999999996,"5min":1.331,"15min":0.70999999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":57.595999999999997,"5min":57.595999999999997,"15min":57.595999999999997},"last":0.42199999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.0189999999999999,"15min":0.58399999999999996},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.035,"5min":12.035,"15min":12.035},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.2330000000000001,"5min":0.82599999999999996,"15min":0.54900000000000004},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":13.217000000000001,"5min":13.217000000000001,"15min":13.217000000000001},"last":0.495}]},{"osd":3,"last update":"Mon Oct 9 10:00:53 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.635,"15min":0.82399999999999995},"min":{"1min":0.33000000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8410000000000002,"5min":58.067,"15min":58.067},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.6120000000000001,"5min":1.8360000000000001,"15min":0.88},"min":{"1min":0.36899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.4829999999999997,"5min":60.456000000000003,"15min":60.456000000000003},"last":0.51500000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":4.2789999999999999,"5min":1.405,"15min":0.871},"min":{"1min":0.39300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":57.646999999999998,"5min":57.646999999999998,"15min":57.646999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":4.5570000000000004,"5min":1.554,"15min":0.93799999999999994},"min":{"1min":0.38800000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":57.024000000000001,"5min":57.024000000000001,"15min":57.024000000000001},"last":0.442}]},{"osd":5,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":3.96,"5min":2.1339999999999999,"15min":1.1279999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":56.969999999999999,"5min":79.016000000000005,"15min":79.016000000000005},"last":0.57199999999999995},{"interface":"front","average":{"1min":4.5199999999999996,"5min":2.2330000000000001,"15min":1.1639999999999999},"min":{"1min":0.438,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":57.731999999999999,"5min":79.302999999999997,"15min":79.302999999999997},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":4.7309999999999999,"5min":1.5109999999999999,"15min":0.91300000000000003},"min":{"1min":0.46600000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":57.698,"5min":57.698,"15min":57.698},"last":0.58299999999999996},{"interface":"front","average":{"1min":4.6559999999999997,"5min":1.569,"15min":0.94499999999999995},"min":{"1min":0.42599999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":57.058,"5min":57.058,"15min":57.058},"last":0.47999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:00:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0549999999999997,"5min":1.3819999999999999,"15min":0.89700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":57.003,"5min":57.003,"15min":57.003},"last":0.60199999999999998},{"interface":"front","average":{"1min":4.4290000000000003,"5min":1.4630000000000001,"15min":0.92800000000000005},"min":{"1min":0.48999999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":57.676000000000002,"5min":57.676000000000002,"15min":57.676000000000002},"last":0.73199999999999998}]}]},{"osd":3,"up_from":94,"seq":403726925865,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3699448,"kb_used_data":3537456,"kb_used_omap":7,"kb_used_meta":161976,"kb_avail":90045704,"statfs":{"total":95995035648,"available":92206800896,"internally_reserved":0,"allocated":3622354944,"data_stored":3621749243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":165864040},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1,9,0,0,6],"upper_bound":8192},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":1.5580000000000001,"15min":1.1819999999999999},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.798999999999999,"5min":11.958,"15min":11.958},"last":1.0980000000000001},{"interface":"front","average":{"1min":3.1829999999999998,"5min":1.397,"15min":0.76600000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.704000000000001,"5min":10.704000000000001,"15min":10.704000000000001},"last":1.3180000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.3400000000000001,"15min":0.72599999999999998},"min":{"1min":0.249,"5min":0.189,"15min":0.189},"max":{"1min":9.3670000000000009,"5min":9.3670000000000009,"15min":9.3670000000000009},"last":1.351},{"interface":"front","average":{"1min":2.7109999999999999,"5min":1.3160000000000001,"15min":0.71499999999999997},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.839,"5min":10.839,"15min":10.839},"last":1.129}]},{"osd":2,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.667,"15min":1.2529999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.632999999999999,"5min":15.632999999999999,"15min":15.632999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.169,"5min":1.304,"15min":0.73899999999999999},"min":{"1min":0.29599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.4669999999999996,"5min":11.496,"15min":11.496},"last":1.163}]},{"osd":4,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.0760000000000001,"15min":0.78900000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.6479999999999997,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":1.0209999999999999},{"interface":"front","average":{"1min":2.8319999999999999,"5min":1.3320000000000001,"15min":0.90200000000000002},"min":{"1min":0.38900000000000001,"5min":0.247,"15min":0.247},"max":{"1min":7.9740000000000002,"5min":7.9740000000000002,"15min":7.9740000000000002},"last":1.284}]},{"osd":5,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":1.3680000000000001,"15min":0.91500000000000004},"min":{"1min":0.48399999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.135999999999999,"5min":10.135999999999999,"15min":10.135999999999999},"last":1.218},{"interface":"front","average":{"1min":2.6680000000000001,"5min":1.5349999999999999,"15min":1.2130000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.127000000000001,"5min":10.127000000000001,"15min":10.127000000000001},"last":0.441}]},{"osd":6,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":3.7789999999999999,"5min":1.536,"15min":0.96799999999999997},"min":{"1min":0.45800000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.846,"5min":12.846,"15min":12.846},"last":1.252},{"interface":"front","average":{"1min":2.7679999999999998,"5min":1.5009999999999999,"15min":1.2050000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":9.5050000000000008,"5min":9.5050000000000008,"15min":9.5050000000000008},"last":0.52100000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.494,"15min":1.194},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.4719999999999995,"5min":9.4719999999999995,"15min":9.4719999999999995},"last":1.984},{"interface":"front","average":{"1min":1.8129999999999999,"5min":1.258,"15min":0.85599999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":9.6039999999999992,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":1.0600000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019061,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618822833,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2025556,"kb_used_data":1720140,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91719596,"statfs":{"total":95995035648,"available":93920866304,"internally_reserved":0,"allocated":1761423360,"data_stored":1760966012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1,"15min":0.69199999999999995},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":13.112,"5min":13.112,"15min":13.112},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.4489999999999998,"5min":0.94999999999999996,"15min":0.67400000000000004},"min":{"1min":0.32700000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.8529999999999998,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":0.30399999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:00:51 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":1.226,"15min":0.76900000000000002},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.248},"max":{"1min":10.308,"5min":10.308,"15min":10.308},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.1359999999999999,"15min":0.74199999999999999},"min":{"1min":0.40500000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.970000000000001,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:00:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":1.1379999999999999,"15min":0.84399999999999997},"min":{"1min":0.30099999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.582000000000001,"5min":12.582000000000001,"15min":12.582000000000001},"last":0.26700000000000002},{"interface":"front","average":{"1min":2.3490000000000002,"5min":1.0289999999999999,"15min":0.81200000000000006},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.438000000000001,"5min":12.438000000000001,"15min":12.438000000000001},"last":0.32600000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":1.2889999999999999,"15min":0.88300000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.254,"5min":10.254,"15min":10.254},"last":0.318},{"interface":"front","average":{"1min":3.3039999999999998,"5min":1.357,"15min":0.88300000000000001},"min":{"1min":0.33300000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":12.332000000000001,"5min":12.332000000000001,"15min":12.332000000000001},"last":0.40999999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":0.81899999999999995,"15min":0.57199999999999995},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":12.304,"5min":12.304,"15min":12.304},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.8780000000000001,"5min":1.0880000000000001,"15min":0.67800000000000005},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":27.768000000000001,"5min":27.768000000000001,"15min":27.768000000000001},"last":0.378}]},{"osd":6,"last update":"Mon Oct 9 10:00:19 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":1.0429999999999999,"15min":0.68000000000000005},"min":{"1min":0.34599999999999997,"5min":0.317,"15min":0.27600000000000002},"max":{"1min":27.716000000000001,"5min":27.716000000000001,"15min":27.716000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.6819999999999999,"5min":0.82799999999999996,"15min":0.61499999999999999},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.173999999999999,"5min":10.173999999999999,"15min":10.173999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:00:38 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":0.96999999999999997,"15min":0.68799999999999994},"min":{"1min":0.36399999999999999,"5min":0.318,"15min":0.309},"max":{"1min":10.334,"5min":10.334,"15min":10.334},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.46,"5min":1.0009999999999999,"15min":0.68500000000000005},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.317},"max":{"1min":12.356,"5min":12.356,"15min":12.356},"last":0.443}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1404375040,"data_stored":1404370968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1826492416,"data_stored":1826488344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2803892224,"data_stored":2803892224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3607101440,"data_stored":3607101440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1768423424,"data_stored":1768423424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1783894016,"data_stored":1783889944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1906835456,"data_stored":1906835456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2169634816,"data_stored":2169634816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:01:10.077 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 26 2023-10-09T10:01:10.103 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:10.110 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:10.111 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:10 smithi110 ceph-mon[136032]: pgmap v343: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 9940/259266 objects degraded (3.834%); 14994/259266 objects misplaced (5.783%); 44 MiB/s, 1 keys/s, 709 objects/s recovering 2023-10-09T10:01:10.111 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:10 smithi110 ceph-mon[136032]: from='client.34956 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:10.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:10.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:10.307 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:10.397 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:10.414 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:10 smithi012 ceph-mon[152263]: pgmap v343: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 9940/259266 objects degraded (3.834%); 14994/259266 objects misplaced (5.783%); 44 MiB/s, 1 keys/s, 709 objects/s recovering 2023-10-09T10:01:10.414 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:10 smithi012 ceph-mon[152263]: from='client.34956 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:10.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:10.517 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:10.552 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:10.606 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:10.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:10.714 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:10.749 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:10.790 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:10.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:10.885 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:01:10.896 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:10.929 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:10.950 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:01:10.970 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:10.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:10.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:10.986 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:01:11.035 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:01:11.073 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:11.077 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:01:11.114 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:11.124 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:01:11.160 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:11.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:11.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:11.172 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:01:11.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/353825690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:11.208 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:01:11.231 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:01:11.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:01:11.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:01:11.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:01:11.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:01:11.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:01:11.233 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:01:11.233 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:01:11.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:11.350 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/353825690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:11.350 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/353825690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:11.364 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:12.090 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:12.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:12.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:12.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:12.285 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:12.299 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:12.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:12.400 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:128401 6.f e140/138 6:f96a4c56:::benchmark_data_smithi012_182576_object22880:head v 140'7956, mlcod=140'7954)", 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:01:12.392490+0000", 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0057643099999999999, 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0057960030000000001, 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:01:12.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392490+0000", 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392490+0000", 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:01:12.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392489+0000", 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392499+0000", 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7529999999999996e-06 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392520+0000", 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0594e-05 2023-10-09T10:01:12.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392528+0000", 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4379999999999995e-06 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392975+0000", 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00044708900000000002 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:01:12.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.393004+0000", 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8875999999999999e-05 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:128402 6.f e140/138 6:f32c2265:::benchmark_data_smithi012_182576_object22881:head v 140'7957, mlcod=140'7954)", 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:01:12.392542+0000", 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.005712267, 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058193689999999996, 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:01:12.405 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392542+0000", 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392542+0000", 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392542+0000", 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392546+0000", 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1760000000000003e-06 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392552+0000", 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6829999999999996e-06 2023-10-09T10:01:12.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.392556+0000", 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0509999999999998e-06 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.397096+0000", 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045398390000000004 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:01:12.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.397135+0000", 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9149000000000001e-05 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:128405 6.f e140/138)", 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:01:12.396975+0000", 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.001279851, 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014452009999999999, 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:01:12.409 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.396975+0000", 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.396975+0000", 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.396974+0000", 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.396982+0000", 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3510000000000002e-06 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.396998+0000", 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5943999999999998e-05 2023-10-09T10:01:12.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:12.397006+0000", 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8089999999999993e-06 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 3 2023-10-09T10:01:12.412 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:01:12.413 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:12.414 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:12 smithi012 ceph-mon[147926]: pgmap v344: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 91 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 9940/257616 objects degraded (3.858%); 14878/257616 objects misplaced (5.775%); 39 MiB/s, 1 keys/s, 631 objects/s recovering 2023-10-09T10:01:12.414 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:12 smithi012 ceph-mon[152263]: pgmap v344: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 91 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 9940/257616 objects degraded (3.858%); 14878/257616 objects misplaced (5.775%); 39 MiB/s, 1 keys/s, 631 objects/s recovering 2023-10-09T10:01:12.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:12 smithi110 ceph-mon[136032]: pgmap v344: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 91 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 9940/257616 objects degraded (3.858%); 14878/257616 objects misplaced (5.775%); 39 MiB/s, 1 keys/s, 631 objects/s recovering 2023-10-09T10:01:12.466 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:12.530 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:12.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:12.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:12.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:12.759 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:12.941 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:12.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:12.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:13.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:01:13.048 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:13.085 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:13.129 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:13.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:13.234 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:13.331 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:13 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:01:13.331 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 9940/257616 objects degraded (3.858%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:13.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:13 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:01:13.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 9940/257616 objects degraded (3.858%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:13.352 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:13.438 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:13.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:13 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:01:13.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 9940/257616 objects degraded (3.858%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:13.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:13.552 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:13.595 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:13.751 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:13.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:13.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:13.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:13.974 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:13.976 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:14.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:01:14.148 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:01:14.311 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:14.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:14.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[152263]: pgmap v345: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 6734/252072 objects degraded (2.671%); 16800/252072 objects misplaced (6.665%); 34 MiB/s, 0 keys/s, 542 objects/s recovering 2023-10-09T10:01:14.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3335994580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:14.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:14.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:14.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:14.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[147926]: pgmap v345: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 6734/252072 objects degraded (2.671%); 16800/252072 objects misplaced (6.665%); 34 MiB/s, 0 keys/s, 542 objects/s recovering 2023-10-09T10:01:14.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3335994580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:14.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:14.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:14.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:14.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:14 smithi110 ceph-mon[136032]: pgmap v345: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 6734/252072 objects degraded (2.671%); 16800/252072 objects misplaced (6.665%); 34 MiB/s, 0 keys/s, 542 objects/s recovering 2023-10-09T10:01:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3335994580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:14.525 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:14.895 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:14.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:15.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:15.117 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:15.167 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 6 pgp_num to 26 2023-10-09T10:01:15.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: osdmap e141: 8 total, 8 up, 7 in 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2473639443' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:15.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:15.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:15.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:15.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: osdmap e141: 8 total, 8 up, 7 in 2023-10-09T10:01:15.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2473639443' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:15.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:15.206 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:15.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:15.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:15.315 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:15.352 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:15.407 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:15.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:15.418 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: osdmap e141: 8 total, 8 up, 7 in 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: failed to encode map e141 with expected crc 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2473639443' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:15.419 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:15.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:15.559 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:15.627 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:15.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:15.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:15.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:15.755 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:15.837 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:15.919 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:15.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:16.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:16.135 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:16.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[152263]: pgmap v347: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 6734/246267 objects degraded (2.734%); 15860/246267 objects misplaced (6.440%); 34 MiB/s, 540 objects/s recovering 2023-10-09T10:01:16.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:01:16.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[152263]: osdmap e142: 8 total, 8 up, 7 in 2023-10-09T10:01:16.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[152263]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[152263]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[152263]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[152263]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.217 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[147926]: pgmap v347: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 6734/246267 objects degraded (2.734%); 15860/246267 objects misplaced (6.440%); 34 MiB/s, 540 objects/s recovering 2023-10-09T10:01:16.217 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:01:16.217 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[147926]: osdmap e142: 8 total, 8 up, 7 in 2023-10-09T10:01:16.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[147926]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[147926]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[147926]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:16 smithi012 ceph-mon[147926]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.235 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:16.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:16.262 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:01:16.263 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:01:16.263 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:01:16.263 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:01:16.263 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:01:16.350 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:16.394 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:16 smithi110 ceph-mon[136032]: pgmap v347: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 6734/246267 objects degraded (2.734%); 15860/246267 objects misplaced (6.440%); 34 MiB/s, 540 objects/s recovering 2023-10-09T10:01:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:16 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:01:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:16 smithi110 ceph-mon[136032]: osdmap e142: 8 total, 8 up, 7 in 2023-10-09T10:01:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:16 smithi110 ceph-mon[136032]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:16 smithi110 ceph-mon[136032]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:16 smithi110 ceph-mon[136032]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:16 smithi110 ceph-mon[136032]: failed to encode map e142 with expected crc 2023-10-09T10:01:16.449 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:16.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:16.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:16.556 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:16.641 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:17.058 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:01:17.794 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:17.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:17.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:01:17.908 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:18.067 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:01:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:01:17] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.33.4" 2023-10-09T10:01:18.084 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:18.222 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:18.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:18.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:18.351 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:18.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:18.447 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:18.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:18.514 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:18.552 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:18.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:18 smithi110 ceph-mon[136032]: pgmap v349: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 6734/244857 objects degraded (2.750%); 15860/244857 objects misplaced (6.477%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T10:01:18.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:18 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:18.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:18 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:18.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6734/244857 objects degraded (2.750%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:18.696 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:18.702 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:01:18.755 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:01:18.789 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:18.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:18.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:18.796 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:01:18.833 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:01:18.876 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:01:18.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:18 smithi012 ceph-mon[152263]: pgmap v349: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 6734/244857 objects degraded (2.750%); 15860/244857 objects misplaced (6.477%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T10:01:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:18 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:18 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6734/244857 objects degraded (2.750%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:18 smithi012 ceph-mon[147926]: pgmap v349: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 6734/244857 objects degraded (2.750%); 15860/244857 objects misplaced (6.477%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T10:01:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:18 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:18 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6734/244857 objects degraded (2.750%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:18.936 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:01:18.980 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:01:18.996 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:19.023 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:01:19.042 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:01:19.043 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:01:19.043 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:01:19.043 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:01:19.043 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:01:19.043 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:01:19.043 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:01:19.043 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:01:19.088 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:19.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:19.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:19.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:19.267 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:19.408 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:19.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:19.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:19.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:19.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3837758227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:19.641 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:19.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3837758227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:19.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3837758227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:20.067 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:20.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:20.114 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:20.177 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:20.233 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:20.283 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:20.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:20.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:20.389 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:20.425 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:20.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:20.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:20.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:20.581 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:20.619 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:20.686 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:20.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:20.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:20.751 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 5, 6, 7, 2] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:01:20.751 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:01:20.751 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:01:20.751 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-10-09T10:01:20.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T10:01:20.790 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:20.818 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:20 smithi110 ceph-mon[136032]: pgmap v350: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 6838/238131 objects degraded (2.872%); 14368/238131 objects misplaced (6.034%); 16 MiB/s, 250 objects/s recovering 2023-10-09T10:01:20.821 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:20.822 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:20 smithi012 ceph-mon[152263]: pgmap v350: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 6838/238131 objects degraded (2.872%); 14368/238131 objects misplaced (6.034%); 16 MiB/s, 250 objects/s recovering 2023-10-09T10:01:20.822 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:20 smithi012 ceph-mon[147926]: pgmap v350: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 6838/238131 objects degraded (2.872%); 14368/238131 objects misplaced (6.034%); 16 MiB/s, 250 objects/s recovering 2023-10-09T10:01:20.867 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:20.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:20.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:20.971 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:21.078 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:21.408 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:21.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:21.518 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:21.550 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:21.596 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:21.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:21.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:21.656 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:01:21.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2311269123' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:21.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:21 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:21.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:21 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:21.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:21 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:21.691 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2311269123' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:21.691 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:21 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:21.691 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:21 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:21.691 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:21 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:21.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:21.776 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:21.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2311269123' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:21.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:21 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:21.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:21 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:21.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:21 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:22.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:01:22.188 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:22.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:22.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:22.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:22.326 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:22.335 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:01:22.335 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:01:22.345 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:22.365 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:22.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:22.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:22.470 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:22.504 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:22.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:22.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:22.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:22.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: pgmap v351: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 6838/234630 objects degraded (2.914%); 14253/234630 objects misplaced (6.075%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:01:22.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:22.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:01:22.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: osdmap e143: 8 total, 8 up, 7 in 2023-10-09T10:01:22.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: failed to encode map e143 with expected crc 2023-10-09T10:01:22.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: failed to encode map e143 with expected crc 2023-10-09T10:01:22.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:01:22.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:01:22.648 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:22 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:22.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:22.739 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: pgmap v351: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 6838/234630 objects degraded (2.914%); 14253/234630 objects misplaced (6.075%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: osdmap e143: 8 total, 8 up, 7 in 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: failed to encode map e143 with expected crc 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: failed to encode map e143 with expected crc 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:01:22.801 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: pgmap v351: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 6838/234630 objects degraded (2.914%); 14253/234630 objects misplaced (6.075%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: osdmap e143: 8 total, 8 up, 7 in 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: failed to encode map e143 with expected crc 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: failed to encode map e143 with expected crc 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:01:22.805 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:22 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:22.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:01:23.054 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:23.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:23.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:23.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:23.233 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:23.311 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:23.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:23.423 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:23.487 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:23.540 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:23.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:23.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:23.560 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:23.585 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:23.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:23.655 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:23 smithi012 ceph-mon[147926]: 6.0 scrub starts 2023-10-09T10:01:23.655 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:23 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:01:23.655 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/234630 objects degraded (2.914%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:23.694 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:23.754 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:23.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:23.766 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:23 smithi110 ceph-mon[136032]: 6.0 scrub starts 2023-10-09T10:01:23.766 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:23 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:01:23.766 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/234630 objects degraded (2.914%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:23.866 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:23.919 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:23.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:23 smithi012 ceph-mon[152263]: 6.0 scrub starts 2023-10-09T10:01:23.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:23 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:01:23.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/234630 objects degraded (2.914%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:23.983 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:23.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:24.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:24.163 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:24.238 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:24.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:24.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:24.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:01:24.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:24.467 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:24.578 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:24.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:24.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:24.746 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:24 smithi110 ceph-mon[136032]: pgmap v353: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 6838/228849 objects degraded (2.988%); 13659/228849 objects misplaced (5.969%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T10:01:24.757 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:24 smithi012 ceph-mon[147926]: pgmap v353: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 6838/228849 objects degraded (2.988%); 13659/228849 objects misplaced (5.969%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T10:01:24.758 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:24 smithi012 ceph-mon[152263]: pgmap v353: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 6838/228849 objects degraded (2.988%); 13659/228849 objects misplaced (5.969%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T10:01:24.776 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:24.947 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:01:24.947 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:01:24.980 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:24.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:24.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:25.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:25.183 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:25.251 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:01:25.319 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:01:25.346 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:25.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:25.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:25.365 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:01:25.404 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:25.417 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:01:25.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:25.466 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:01:25.496 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:25.513 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:01:25.552 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:01:25.568 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:25.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:25.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:25.592 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:01:25.639 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:01:25.657 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:01:25.657 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:01:25.657 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:01:25.657 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:01:25.657 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:01:25.657 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:01:25.658 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:01:25.658 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:01:25.676 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:25.727 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:25.747 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T10:01:25.776 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:25.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:25.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:25.881 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:25.937 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1313932993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:25.937 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2835315123' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:01:25.937 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:01:25.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:25.993 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:25.997 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:26.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:26.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:26.108 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:26.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1313932993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:26.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2835315123' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:01:26.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:01:26.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1313932993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:26.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2835315123' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:01:26.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:01:26.192 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:26.450 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:26.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:26.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:26.561 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:26.622 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:26.709 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: pgmap v354: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 6838/222969 objects degraded (3.067%); 12789/222969 objects misplaced (5.736%); 13 MiB/s, 205 objects/s recovering 2023-10-09T10:01:26.709 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.783 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:26.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: pgmap v354: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 6838/222969 objects degraded (3.067%); 12789/222969 objects misplaced (5.736%); 13 MiB/s, 205 objects/s recovering 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: osdmap e144: 8 total, 8 up, 7 in 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.825 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:26 smithi110 ceph-mon[136032]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:26.931 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: pgmap v354: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 6838/222969 objects degraded (3.067%); 12789/222969 objects misplaced (5.736%); 13 MiB/s, 205 objects/s recovering 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: osdmap e144: 8 total, 8 up, 7 in 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[152263]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:01:26.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: osdmap e144: 8 total, 8 up, 7 in 2023-10-09T10:01:26.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: failed to encode map e144 with expected crc 2023-10-09T10:01:26.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:26 smithi012 ceph-mon[147926]: failed to encode map e144 with expected crc 2023-10-09T10:01:27.095 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:27.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:27.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:27.207 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:27.261 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:27.345 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:27.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:27.371 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:01:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:01:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:01:27.371 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:01:27.371 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:01:27.451 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:27.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:27.798 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:27.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:27.810 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:01:27.911 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:27.986 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:27 smithi110 ceph-mon[136032]: 6.10 scrub starts 2023-10-09T10:01:27.986 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:27 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/222969 objects degraded (3.067%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:27.987 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3843571565' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:27.987 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:27.996 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:28.066 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:28.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:28.173 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:28.180 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:01:27] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.33.4" 2023-10-09T10:01:28.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-mon[147926]: 6.10 scrub starts 2023-10-09T10:01:28.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/222969 objects degraded (3.067%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:28.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3843571565' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:28.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:28.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-mon[152263]: 6.10 scrub starts 2023-10-09T10:01:28.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/222969 objects degraded (3.067%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:28.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3843571565' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:28.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:28.216 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:28.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:01:28.276 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:28.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:28.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:28.422 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:28.479 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:28.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:28.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:28.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:28.610 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:28.629 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:28.707 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:01:28.757 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:28.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:28.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:28.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:28.882 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:28 smithi110 ceph-mon[136032]: pgmap v356: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 6838/221469 objects degraded (3.088%); 12789/221469 objects misplaced (5.775%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:01:28.882 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:28 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:01:28.882 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:28 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:28.882 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:28 smithi110 ceph-mon[136032]: osdmap e145: 8 total, 8 up, 7 in 2023-10-09T10:01:28.917 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:28.953 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:29.176 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:28 smithi012 ceph-mon[152263]: pgmap v356: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 6838/221469 objects degraded (3.088%); 12789/221469 objects misplaced (5.775%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:01:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:28 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:01:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:28 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:29.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:28 smithi012 ceph-mon[152263]: osdmap e145: 8 total, 8 up, 7 in 2023-10-09T10:01:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:28 smithi012 ceph-mon[147926]: pgmap v356: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 6838/221469 objects degraded (3.088%); 12789/221469 objects misplaced (5.775%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:01:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:28 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:01:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:28 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:28 smithi012 ceph-mon[147926]: osdmap e145: 8 total, 8 up, 7 in 2023-10-09T10:01:29.919 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:01:29.919 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:01:29.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:36581 6.e 6:78a07a32:::benchmark_data_smithi012_182576_object36580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:17.203437+0000", 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 132.711728446, 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.073460469, 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.920 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36581 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203437+0000", 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203437+0000", 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:29.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203435+0000", 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7761000000000004e-05 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203503+0000", 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6582999999999999e-05 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203530+0000", 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9340000000000003e-06 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203536+0000", 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5926000000000001e-05 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203572+0000", 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9265999999999995e-05 2023-10-09T10:01:29.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203661+0000", 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104348 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.203765+0000", 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001083606 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.204849+0000", 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028363999999999999 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.205132+0000", 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.071660868000000003 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.276793+0000", 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0739999999999998e-05 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.276854+0000", 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3216000000000001e-05 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.276897+0000", 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.073460469 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.926 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:57899 6.2 6:4ae5067f:::benchmark_data_smithi012_182576_object57898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:32.431948+0000", 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.48321728000001, 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076099255000000005, 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.927 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 57899 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.431948+0000", 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.431948+0000", 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:01:29.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.431946+0000", 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2293999999999999e-05 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.431988+0000", 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4036000000000001e-05 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432002+0000", 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8179999999999998e-06 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432009+0000", 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7142999999999999e-05 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432056+0000", 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103881 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432160+0000", 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136062 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.432296+0000", 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001651764 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.433948+0000", 2023-10-09T10:01:29.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015851750000000001 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435533+0000", 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072430576999999996 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507964+0000", 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6255999999999999e-05 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.508010+0000", 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7110999999999998e-05 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.508047+0000", 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076099255000000005 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.934 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:57909 6.e 6:77ce4db4:::benchmark_data_smithi012_182576_object57908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T10:01:29.934 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:32.435413+0000", 2023-10-09T10:01:29.934 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.479752328, 2023-10-09T10:01:29.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079633525999999996, 2023-10-09T10:01:29.934 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 57909 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435413+0000", 2023-10-09T10:01:29.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435413+0000", 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435412+0000", 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3256999999999994e-05 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435475+0000", 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1637000000000001e-05 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435486+0000", 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.037e-06 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435492+0000", 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072267670000000006 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507760+0000", 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014061400000000001 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507901+0000", 2023-10-09T10:01:29.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015924400000000001 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.508060+0000", 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036826929999999999 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.511743+0000", 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001201964 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.512945+0000", 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020054690000000002 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.514950+0000", 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3223000000000002e-05 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.515003+0000", 2023-10-09T10:01:29.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3075999999999999e-05 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.515046+0000", 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079633525999999996 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:57903 6.7 e109/97 6:e524b57e:::benchmark_data_smithi012_182576_object57902:head v 109'3509, mlcod=109'3507)", 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:32.435800+0000", 2023-10-09T10:01:29.941 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 117.479364893, 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076747323000000006, 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435800+0000", 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435800+0000", 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435800+0000", 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3377e-05 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.435823+0000", 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.071937872 2023-10-09T10:01:29.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507761+0000", 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2327000000000001e-05 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507773+0000", 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9439e-05 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507813+0000", 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5551999999999999e-05 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.507828+0000", 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046742590000000001 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.512502+0000", 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5252e-05 2023-10-09T10:01:29.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:32.512548+0000", 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076747323000000006 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96606 6.e 6:72c2866f:::benchmark_data_smithi012_182576_object96605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.045317+0000", 2023-10-09T10:01:29.946 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.869848271999999, 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072861500999999995, 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96606 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045317+0000", 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045317+0000", 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045316+0000", 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5337999999999999e-05 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045361+0000", 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0549e-05 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045371+0000", 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5650000000000001e-06 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045377+0000", 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044907799999999998 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.049868+0000", 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013155900000000001 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.049999+0000", 2023-10-09T10:01:29.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013841799999999999 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.050138+0000", 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097119619999999993 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.059850+0000", 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023379799999999999 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.060084+0000", 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053718578000000003 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.113802+0000", 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6309999999999994e-05 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.113868+0000", 2023-10-09T10:01:29.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004310019 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.118178+0000", 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072861500999999995 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96609 6.e 6:754c9d3c:::benchmark_data_smithi012_182576_object96608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:29.953 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.045383+0000", 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.869781949, 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.077528253000000005, 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96609 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045383+0000", 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045383+0000", 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045383+0000", 2023-10-09T10:01:29.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7487000000000002e-05 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045420+0000", 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4599999999999997e-06 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045428+0000", 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6099999999999999e-06 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045432+0000", 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007678514 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.053111+0000", 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000171588 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.053282+0000", 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000197095 2023-10-09T10:01:29.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.053479+0000", 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0066118660000000001 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.060091+0000", 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055322749999999997 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.065624+0000", 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052631171999999997 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.118255+0000", 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0145e-05 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.118325+0000", 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045865990000000002 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.122912+0000", 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.077528253000000005 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96611 6.e 6:7ff1b730:::benchmark_data_smithi012_182576_object96610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:29.960 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.045765+0000", 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.869400381000005, 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081713884000000001, 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96611 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045765+0000", 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045765+0000", 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045764+0000", 2023-10-09T10:01:29.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6659000000000002e-05 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045811+0000", 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3148e-05 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045824+0000", 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8950000000000003e-06 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.045830+0000", 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010609782 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.056439+0000", 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000175575 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.056615+0000", 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021842899999999999 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.056833+0000", 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032659680000000002 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.060099+0000", 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057422180000000003 2023-10-09T10:01:29.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.065842+0000", 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057150713999999998 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.122992+0000", 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6161000000000005e-05 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.123068+0000", 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044102960000000002 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.127479+0000", 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081713884000000001 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96930 6.e 6:710aa3be:::benchmark_data_smithi012_182576_object96929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.321480223999998, 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098520888000000001, 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96930 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.968 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593683+0000", 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.313e-05 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593746+0000", 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2289999999999992e-06 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593756+0000", 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0159999999999999e-06 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593761+0000", 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8661999999999998e-05 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593859+0000", 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146317 2023-10-09T10:01:29.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.594006+0000", 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000179419 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.594185+0000", 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081610680000000005 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602346+0000", 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061366659999999998 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608483+0000", 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079327541000000001 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.687810+0000", 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1600999999999995e-05 2023-10-09T10:01:29.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.687872+0000", 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043339989999999998 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692206+0000", 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098520888000000001 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96932 6.e 6:7facdbe4:::benchmark_data_smithi012_182576_object96931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.313023975999997, 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094685332999999997, 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96932 2023-10-09T10:01:29.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602140+0000", 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8144999999999999e-05 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602168+0000", 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2849999999999997e-06 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602175+0000", 2023-10-09T10:01:29.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7560000000000001e-06 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602179+0000", 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000195152 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602374+0000", 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206584 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602581+0000", 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000174067 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602755+0000", 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032482050000000001 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.606003+0000", 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024921650000000002 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608495+0000", 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083792852000000001 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692288+0000", 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6909000000000006e-05 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692355+0000", 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044717339999999998 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.696827+0000", 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094685332999999997 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96937 6.e 6:7e02d1d2:::benchmark_data_smithi012_182576_object96936:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.608811+0000", 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.306354141, 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092425560000000004, 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96937 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.982 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608811+0000", 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608811+0000", 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608809+0000", 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.863e-05 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608858+0000", 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6682000000000001e-05 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608875+0000", 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1960000000000008e-06 2023-10-09T10:01:29.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608883+0000", 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3739000000000003e-05 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608927+0000", 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017568999999999999 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.609102+0000", 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023748899999999999 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.609340+0000", 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095543239999999995 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.618894+0000", 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013873755 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.632768+0000", 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.064114811999999993 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.696883+0000", 2023-10-09T10:01:29.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4169999999999998e-05 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.696937+0000", 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042997770000000003 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.701237+0000", 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092425560000000004 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.988 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96941 6.e 6:7534db65:::benchmark_data_smithi012_182576_object96940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.618691+0000", 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.296474380000006, 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086786896000000002, 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.989 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96941 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.618691+0000", 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.618691+0000", 2023-10-09T10:01:29.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.618689+0000", 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3998000000000001e-05 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.618733+0000", 2023-10-09T10:01:29.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4504e-05 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.618748+0000", 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9899999999999997e-06 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.618753+0000", 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060053779999999996 2023-10-09T10:01:29.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624758+0000", 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016460100000000001 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624923+0000", 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000217319 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:29.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.625140+0000", 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011620081000000001 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.636760+0000", 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3186999999999999e-05 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.636773+0000", 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.064519003000000005 2023-10-09T10:01:29.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.701292+0000", 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0532000000000003e-05 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.701353+0000", 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041250030000000003 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.705478+0000", 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086786896000000002 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96942 6.e 6:75dff677:::benchmark_data_smithi012_182576_object96941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.623067+0000", 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.292098655000004, 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086698033999999993, 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:29.996 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96942 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623067+0000", 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623067+0000", 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623065+0000", 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6143000000000001e-05 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:29.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623121+0000", 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2059e-05 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623143+0000", 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4569999999999994e-06 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623152+0000", 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005442166 2023-10-09T10:01:29.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.628594+0000", 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015196799999999999 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.628746+0000", 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023095599999999999 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:30.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.628977+0000", 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083261080000000005 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637303+0000", 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036562349999999999 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.640960+0000", 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.064607132999999997 2023-10-09T10:01:30.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.705567+0000", 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2649000000000003e-05 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.705629+0000", 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041351349999999999 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.709765+0000", 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086698033999999993 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96943 6.e 6:76634981:::benchmark_data_smithi012_182576_object96942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.623160+0000", 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.292005035000003, 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.091338833999999994, 2023-10-09T10:01:30.003 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96943 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623160+0000", 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:30.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623160+0000", 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623160+0000", 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0851999999999999e-05 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:30.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623220+0000", 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0860000000000008e-06 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623230+0000", 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6619999999999996e-06 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.623234+0000", 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095923899999999993 2023-10-09T10:01:30.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.632827+0000", 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017610099999999999 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.633003+0000", 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020964700000000001 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:30.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.633212+0000", 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077569730000000003 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.640969+0000", 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00069796300000000001 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641667+0000", 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.068181437999999997 2023-10-09T10:01:30.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.709849+0000", 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8600999999999997e-05 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.709907+0000", 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045916869999999997 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714499+0000", 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.091338833999999994 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96944 6.e 6:71fd26c2:::benchmark_data_smithi012_182576_object96943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 76.290409550999996, 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094319584999999997, 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:30.010 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96944 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624753+0000", 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9678999999999994e-05 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:30.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624833+0000", 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0849000000000001e-05 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624844+0000", 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5359999999999998e-06 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624850+0000", 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012482843 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637333+0000", 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016598700000000001 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637499+0000", 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027263199999999997 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:30.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637772+0000", 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033034599999999998 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641075+0000", 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088881500000000005 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641964+0000", 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072603763000000002 2023-10-09T10:01:30.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714568+0000", 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4980000000000005e-05 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714633+0000", 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044422960000000001 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.719075+0000", 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094319584999999997 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:104594 6.e 6:73e7c7a8:::benchmark_data_smithi012_182576_object104593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:28.231569+0000", 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 61.683595895000003, 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.074807870999999998, 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:30.017 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 104594 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.231569+0000", 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.231569+0000", 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.231569+0000", 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3602e-05 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.231612+0000", 2023-10-09T10:01:30.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8759999999999994e-06 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.231621+0000", 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7000000000000002e-06 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.231625+0000", 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024419126999999999 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.256044+0000", 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015866900000000001 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.256203+0000", 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000194496 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.256397+0000", 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074750629999999997 2023-10-09T10:01:30.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.263872+0000", 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034593860000000001 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.267332+0000", 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.034275303 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.301607+0000", 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3690999999999997e-05 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.301671+0000", 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047064610000000003 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:28.306377+0000", 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.074807870999999998 2023-10-09T10:01:30.023 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:105483 6.2 6:45b0cc0b:::benchmark_data_smithi012_182576_object105482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.192524835, 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1741883319999999, 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:30.024 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105483 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:30.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5788999999999999e-05 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722676+0000", 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.089e-06 2023-10-09T10:01:30.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722684+0000", 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5950000000000004e-06 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722690+0000", 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010708354 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733399+0000", 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013741599999999999 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733536+0000", 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017242200000000001 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:01:30.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733708+0000", 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003714143 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.737423+0000", 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015821828 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.753244+0000", 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036249568000000003 2023-10-09T10:01:30.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789494+0000", 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8799999999999997e-06 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789503+0000", 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1072112519999999 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.896714+0000", 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114647 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.896829+0000", 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1741883319999999 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:105508 6.e 6:77549acc:::benchmark_data_smithi012_182576_object105507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 60.140088464999998, 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1483757859999999, 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105508 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.032 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775075+0000", 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9004000000000003e-05 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775174+0000", 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.751e-05 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775192+0000", 2023-10-09T10:01:30.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6340000000000001e-06 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775199+0000", 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4343000000000003e-05 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775274+0000", 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000152091 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775426+0000", 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019803600000000001 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775624+0000", 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057209089999999997 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.781345+0000", 2023-10-09T10:01:30.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079732469999999993 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789318+0000", 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8120000000000002e-06 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789328+0000", 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.114660743 2023-10-09T10:01:30.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.903988+0000", 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000495179 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.904484+0000", 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018860534000000002 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.923344+0000", 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010841900000000001 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.923453+0000", 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1483757859999999 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.039 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:115341 6.2 6:46f4cbb7:::benchmark_data_smithi012_182576_object9820:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:46.905154+0000", 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 43.010011173999999, 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.085739226000000002, 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 115341 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:30.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905154+0000", 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905154+0000", 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:30.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905152+0000", 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2860000000000002e-06 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905160+0000", 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5364999999999999e-05 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905175+0000", 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1319999999999999e-06 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905181+0000", 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9752000000000005e-05 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905261+0000", 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000123108 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905384+0000", 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000165761 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.905550+0000", 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037939610000000002 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.909344+0000", 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067184530000000001 2023-10-09T10:01:30.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.916062+0000", 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.074121988 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.990184+0000", 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2249000000000007e-05 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:30.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.990246+0000", 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00064690400000000001 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.990893+0000", 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.085739226000000002 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.046 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:115347 6.2 6:418849dc:::benchmark_data_smithi012_182576_object9826:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:46.910506+0000", 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 43.004659056000001, 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.080950145000000001, 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 115347 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:30.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910506+0000", 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910506+0000", 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910504+0000", 2023-10-09T10:01:30.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8359000000000002e-05 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910523+0000", 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5438999999999999e-05 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910538+0000", 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5079999999999999e-06 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910546+0000", 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3638999999999999e-05 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910599+0000", 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000143561 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910743+0000", 2023-10-09T10:01:30.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017262399999999999 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.910915+0000", 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027872109999999999 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.913703+0000", 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024013979999999999 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.916104+0000", 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.074850994000000004 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.990955+0000", 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3174000000000002e-05 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.991018+0000", 2023-10-09T10:01:30.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043816800000000001 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.991456+0000", 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.080950145000000001 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:115349 6.2 6:4518ea1d:::benchmark_data_smithi012_182576_object9828:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-10-09T10:01:30.053 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:46.917436+0000", 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 42.997728907999999, 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082217249000000006, 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 115349 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917436+0000", 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917436+0000", 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917435+0000", 2023-10-09T10:01:30.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1799999999999999e-06 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917442+0000", 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1891e-05 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917454+0000", 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6609999999999997e-06 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917460+0000", 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2332000000000001e-05 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917502+0000", 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108198 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917610+0000", 2023-10-09T10:01:30.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7523999999999997e-05 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.917708+0000", 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027081169999999999 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.920416+0000", 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014352200000000001 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.920559+0000", 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.078374519000000004 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.998934+0000", 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7607e-05 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:30.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.998982+0000", 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00067206899999999997 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:46.999654+0000", 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082217249000000006 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:30.060 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:01:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:30.061 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:29 smithi110 ceph-mon[136032]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.062 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:29 smithi110 ceph-mon[136032]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.062 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:29 smithi110 ceph-mon[136032]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.062 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:29 smithi110 ceph-mon[136032]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.062 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:30.062 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:30.062 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:29 smithi110 ceph-mon[136032]: pgmap v358: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 6838/214467 objects degraded (3.188%); 11505/214467 objects misplaced (5.364%); 16 MiB/s, 256 objects/s recovering 2023-10-09T10:01:30.062 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:30.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:30.163 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:30.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[147926]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[147926]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[147926]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[147926]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:30.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:30.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[147926]: pgmap v358: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 6838/214467 objects degraded (3.188%); 11505/214467 objects misplaced (5.364%); 16 MiB/s, 256 objects/s recovering 2023-10-09T10:01:30.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[152263]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[152263]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[152263]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[152263]: failed to encode map e145 with expected crc 2023-10-09T10:01:30.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:30.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:30.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:29 smithi012 ceph-mon[152263]: pgmap v358: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 6838/214467 objects degraded (3.188%); 11505/214467 objects misplaced (5.364%); 16 MiB/s, 256 objects/s recovering 2023-10-09T10:01:30.239 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:30.327 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:01:31.135 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:31.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:30 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:01:31.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:30 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:01:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:30 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:01:31.709 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 5, 6, 7, 2] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:01:31.710 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:01:31.710 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:01:31.710 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2023-10-09T10:01:31.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0 2023-10-09T10:01:31.872 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:31.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:31 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:01:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:31 smithi110 ceph-mon[136032]: pgmap v359: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 758 op/s; 6838/210618 objects degraded (3.247%); 11392/210618 objects misplaced (5.409%); 14 MiB/s, 222 objects/s recovering 2023-10-09T10:01:31.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:32.037 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:32.109 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:32.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:32.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:32.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:31 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:01:32.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:31 smithi012 ceph-mon[147926]: pgmap v359: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 758 op/s; 6838/210618 objects degraded (3.247%); 11392/210618 objects misplaced (5.409%); 14 MiB/s, 222 objects/s recovering 2023-10-09T10:01:32.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:31 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:01:32.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:31 smithi012 ceph-mon[152263]: pgmap v359: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 758 op/s; 6838/210618 objects degraded (3.247%); 11392/210618 objects misplaced (5.409%); 14 MiB/s, 222 objects/s recovering 2023-10-09T10:01:32.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:32.258 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:32.344 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:32.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:32.448 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:32.474 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:01:32.475 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:01:32.475 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:01:32.475 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:01:32.475 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:32.482 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:32.521 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:32.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:32.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:32.554 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:01:32.579 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:01:32.602 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:01:32.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:32.626 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:01:32.645 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:01:32.655 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:32.670 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:01:32.688 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:01:32.707 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:01:32.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:01:32.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:01:32.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:01:32.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:01:32.709 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:01:32.709 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:01:32.709 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:01:32.709 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:01:33.071 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:33.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:33.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:33.107 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:01:33.173 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:01:33.173 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[147926]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.173 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[147926]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.173 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[147926]: osdmap e146: 8 total, 8 up, 7 in 2023-10-09T10:01:33.173 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[147926]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.174 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[147926]: 6.9 scrub starts 2023-10-09T10:01:33.174 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/962594886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:33.174 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/210618 objects degraded (3.247%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:33.175 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:01:33.175 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[152263]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.175 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[152263]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.175 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[152263]: osdmap e146: 8 total, 8 up, 7 in 2023-10-09T10:01:33.175 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[152263]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.175 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[152263]: 6.9 scrub starts 2023-10-09T10:01:33.175 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/962594886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:33.176 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:32 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/210618 objects degraded (3.247%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:33.176 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:33.186 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:32 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:01:33.186 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:32 smithi110 ceph-mon[136032]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.186 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:32 smithi110 ceph-mon[136032]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:32 smithi110 ceph-mon[136032]: osdmap e146: 8 total, 8 up, 7 in 2023-10-09T10:01:33.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:32 smithi110 ceph-mon[136032]: failed to encode map e146 with expected crc 2023-10-09T10:01:33.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:32 smithi110 ceph-mon[136032]: 6.9 scrub starts 2023-10-09T10:01:33.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/962594886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:33.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:32 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/210618 objects degraded (3.247%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:33.269 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:33.390 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:33.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:33.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:33.502 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:33.547 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:33.604 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:33.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:33.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:33.715 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:33.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:01:33.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:33.826 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:33.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:33.931 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:33.941 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:33.993 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:34.122 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:34.124 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:33 smithi110 ceph-mon[136032]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.124 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:33 smithi110 ceph-mon[136032]: osdmap e147: 8 total, 8 up, 7 in 2023-10-09T10:01:34.124 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:33 smithi110 ceph-mon[136032]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.125 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:33 smithi110 ceph-mon[136032]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.125 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:33 smithi110 ceph-mon[136032]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.125 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:33 smithi110 ceph-mon[136032]: pgmap v362: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 965 op/s; 6838/204090 objects degraded (3.350%); 10663/204090 objects misplaced (5.225%); 13 MiB/s, 206 objects/s recovering 2023-10-09T10:01:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:34.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[152263]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[152263]: osdmap e147: 8 total, 8 up, 7 in 2023-10-09T10:01:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[152263]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[152263]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[152263]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[152263]: pgmap v362: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 965 op/s; 6838/204090 objects degraded (3.350%); 10663/204090 objects misplaced (5.225%); 13 MiB/s, 206 objects/s recovering 2023-10-09T10:01:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[147926]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[147926]: osdmap e147: 8 total, 8 up, 7 in 2023-10-09T10:01:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[147926]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[147926]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[147926]: failed to encode map e147 with expected crc 2023-10-09T10:01:34.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:33 smithi012 ceph-mon[147926]: pgmap v362: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 91 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 965 op/s; 6838/204090 objects degraded (3.350%); 10663/204090 objects misplaced (5.225%); 13 MiB/s, 206 objects/s recovering 2023-10-09T10:01:34.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:34.324 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:34.406 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:34.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:34.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:34.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:34.557 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:34.564 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:34.792 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:34.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:34.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:34.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:34.976 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:34.982 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.6 primary-affinity to 0 (802) 2023-10-09T10:01:35.039 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:35.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:35.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:34 smithi110 ceph-mon[136032]: 6.19 scrub starts 2023-10-09T10:01:35.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:34 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1878251193' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:01:35.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:34 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:01:35.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:35.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:34 smithi012 ceph-mon[147926]: 6.19 scrub starts 2023-10-09T10:01:35.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:34 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1878251193' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:01:35.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:34 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:01:35.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:34 smithi012 ceph-mon[152263]: 6.19 scrub starts 2023-10-09T10:01:35.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:34 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1878251193' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:01:35.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:34 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:01:35.195 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:35.266 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:35.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:35.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:35.338 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:01:35.339 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:01:35.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:35.446 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:35.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:01:35.519 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:35.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:35.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:35.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:35.710 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:35.713 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:01:35.836 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:35.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:35.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:35.941 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:35.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[147926]: failed to encode map e148 with expected crc 2023-10-09T10:01:35.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T10:01:35.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[147926]: osdmap e148: 8 total, 8 up, 7 in 2023-10-09T10:01:35.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[147926]: failed to encode map e148 with expected crc 2023-10-09T10:01:35.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[147926]: failed to encode map e148 with expected crc 2023-10-09T10:01:35.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[147926]: failed to encode map e148 with expected crc 2023-10-09T10:01:35.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[147926]: failed to encode map e148 with expected crc 2023-10-09T10:01:35.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[147926]: pgmap v364: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 881 op/s; 6838/198591 objects degraded (3.443%); 10270/198591 objects misplaced (5.171%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T10:01:36.002 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:36.071 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:36.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:36.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:36.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:36.227 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[152263]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T10:01:36.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[152263]: osdmap e148: 8 total, 8 up, 7 in 2023-10-09T10:01:36.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[152263]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[152263]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[152263]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[152263]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:35 smithi012 ceph-mon[152263]: pgmap v364: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 881 op/s; 6838/198591 objects degraded (3.443%); 10270/198591 objects misplaced (5.171%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T10:01:36.263 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:36.440 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:36.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:35 smithi110 ceph-mon[136032]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T10:01:36.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:35 smithi110 ceph-mon[136032]: osdmap e148: 8 total, 8 up, 7 in 2023-10-09T10:01:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:35 smithi110 ceph-mon[136032]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:35 smithi110 ceph-mon[136032]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:35 smithi110 ceph-mon[136032]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:35 smithi110 ceph-mon[136032]: failed to encode map e148 with expected crc 2023-10-09T10:01:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:35 smithi110 ceph-mon[136032]: pgmap v364: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 881 op/s; 6838/198591 objects degraded (3.443%); 10270/198591 objects misplaced (5.171%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T10:01:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:36.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:36.449 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:36.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:36.618 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:36.646 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:36.717 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:36.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:36.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:36.906 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:37.035 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:01:37.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[147926]: osdmap e149: 8 total, 8 up, 7 in 2023-10-09T10:01:37.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[147926]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[147926]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[147926]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[147926]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[147926]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2542641928' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[152263]: osdmap e149: 8 total, 8 up, 7 in 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[152263]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[152263]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[152263]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[152263]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[152263]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2542641928' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:37.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:37.301 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:36 smithi110 ceph-mon[136032]: osdmap e149: 8 total, 8 up, 7 in 2023-10-09T10:01:37.301 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:36 smithi110 ceph-mon[136032]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.301 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:36 smithi110 ceph-mon[136032]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.301 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:36 smithi110 ceph-mon[136032]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.301 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:36 smithi110 ceph-mon[136032]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.301 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:36 smithi110 ceph-mon[136032]: failed to encode map e149 with expected crc 2023-10-09T10:01:37.302 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2542641928' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:37.302 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:01:37.878 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:01:37.879 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:01:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:01:37] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.33.4" 2023-10-09T10:01:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[152263]: failed to encode map e150 with expected crc 2023-10-09T10:01:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:01:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[152263]: osdmap e150: 8 total, 8 up, 7 in 2023-10-09T10:01:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[152263]: failed to encode map e150 with expected crc 2023-10-09T10:01:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[152263]: pgmap v367: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 6838/197355 objects degraded (3.465%); 10270/197355 objects misplaced (5.204%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/198591 objects degraded (3.443%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:38.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:38.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[147926]: failed to encode map e150 with expected crc 2023-10-09T10:01:38.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:01:38.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[147926]: osdmap e150: 8 total, 8 up, 7 in 2023-10-09T10:01:38.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[147926]: failed to encode map e150 with expected crc 2023-10-09T10:01:38.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[147926]: pgmap v367: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 6838/197355 objects degraded (3.465%); 10270/197355 objects misplaced (5.204%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:37 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/198591 objects degraded (3.443%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:38.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:37 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:38.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:37 smithi110 ceph-mon[136032]: failed to encode map e150 with expected crc 2023-10-09T10:01:38.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:01:38.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:37 smithi110 ceph-mon[136032]: osdmap e150: 8 total, 8 up, 7 in 2023-10-09T10:01:38.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:37 smithi110 ceph-mon[136032]: failed to encode map e150 with expected crc 2023-10-09T10:01:38.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:37 smithi110 ceph-mon[136032]: pgmap v367: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 6838/197355 objects degraded (3.465%); 10270/197355 objects misplaced (5.204%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:38.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:37 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/198591 objects degraded (3.443%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:38.735 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:01:38.872 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:38.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:38.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:38.982 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:39.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:39.077 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:39.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:39.185 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:39.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:39.285 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:39.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:39.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:39.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:39.453 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:39.514 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:39.779 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:39.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:39.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:39.964 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:40.041 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:40.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:40.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:40.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:40.327 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:40 smithi012 ceph-mon[152263]: pgmap v368: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 6838/190758 objects degraded (3.585%); 9550/190758 objects misplaced (5.006%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T10:01:40.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:40 smithi012 ceph-mon[147926]: pgmap v368: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 6838/190758 objects degraded (3.585%); 9550/190758 objects misplaced (5.006%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T10:01:40.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:40 smithi110 ceph-mon[136032]: pgmap v368: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 6838/190758 objects degraded (3.585%); 9550/190758 objects misplaced (5.006%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T10:01:40.516 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 5, 6, 7, 2] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:01:40.516 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:01:40.516 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-09T10:01:40.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:01:40.753 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:40.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:40.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:40.858 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:40.902 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:40.948 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:40.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:40.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:41.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:41.139 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:41.223 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:41.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:41.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:41.374 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:41.591 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:01:41.624 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:01:41.660 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:01:41.679 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:41.708 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:01:41.737 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:01:41.767 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:01:41.781 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:41.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:41.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:41.800 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:01:41.839 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:01:41.840 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:01:41.840 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:01:41.840 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:01:41.841 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:01:41.841 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:01:41.841 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:01:41.841 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:01:41.841 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:01:41.865 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:01:41.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:41.948 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:42.131 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:42 smithi012 ceph-mon[147926]: pgmap v369: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 6838/186699 objects degraded (3.663%); 9437/186699 objects misplaced (5.055%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:01:42.131 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/574549972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:42.131 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:42 smithi012 ceph-mon[152263]: pgmap v369: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 6838/186699 objects degraded (3.663%); 9437/186699 objects misplaced (5.055%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:01:42.132 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/574549972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:42.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:42.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:42.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:42.254 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:42.301 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:42.344 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:42 smithi110 ceph-mon[136032]: pgmap v369: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 6838/186699 objects degraded (3.663%); 9437/186699 objects misplaced (5.055%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:01:42.345 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/574549972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:42.369 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:42.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:42.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:42.482 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:42.518 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:42.620 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:42.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:42.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:42.776 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:42.841 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:42.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:42.904 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:01:42.904 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:01:42.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.904 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:152796 6.f e150/138 6:fab17f30:::benchmark_data_smithi012_182576_object47275:head v 150'9507, mlcod=150'9504)", 2023-10-09T10:01:42.904 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:01:42.900547+0000", 2023-10-09T10:01:42.904 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0036124870000000002, 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003641236, 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.900547+0000", 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.900547+0000", 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.900546+0000", 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9309999999999997e-06 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.900555+0000", 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5671999999999998e-05 2023-10-09T10:01:42.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.900571+0000", 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3320000000000003e-06 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.900578+0000", 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2967999999999996e-05 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.900661+0000", 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.595e-05 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.900697+0000", 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014994100000000001 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:152804 6.a e150/97 6:56dcf14b:::benchmark_data_smithi012_182576_object47283:head v 150'9580, mlcod=150'9578)", 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:01:42.903058+0000", 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0011008249999999999, 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012007910000000001, 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.903058+0000", 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:42.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.903058+0000", 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.903058+0000", 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9849999999999995e-06 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.903067+0000", 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1354e-05 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.903078+0000", 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.19e-06 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.903084+0000", 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5720999999999998e-05 2023-10-09T10:01:42.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.903130+0000", 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0248e-05 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:42.903150+0000", 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1666e-05 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:42.912 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:42.913 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:42.913 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:01:42.913 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 2 2023-10-09T10:01:42.913 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:01:42.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:42.999 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:43.037 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/186699 objects degraded (3.663%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:43.292 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:43.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:43.352 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:01:43.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:43.421 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/186699 objects degraded (3.663%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:43.421 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/186699 objects degraded (3.663%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:43.445 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:43.501 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:43.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:43.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:43.606 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:43.661 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:43.728 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:43.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:43.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:43.831 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:01:43.912 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:43.916 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:01:44.071 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:44.126 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:44.195 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:44.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:44.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:44.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:44.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:44.570 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:01:44.662 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:44.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:44.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:44.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:44 smithi110 ceph-mon[136032]: pgmap v370: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 6838/180141 objects degraded (3.796%); 8658/180141 objects misplaced (4.806%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:01:44.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:44.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:44.871 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:44 smithi012 ceph-mon[147926]: pgmap v370: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 6838/180141 objects degraded (3.796%); 8658/180141 objects misplaced (4.806%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:01:44.871 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:44.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:44.872 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:44 smithi012 ceph-mon[152263]: pgmap v370: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 6838/180141 objects degraded (3.796%); 8658/180141 objects misplaced (4.806%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:01:44.872 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:01:44.873 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:44.879 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:44.889 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":370,"stamp":"2023-10-09T10:01:43.015044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3934197688,"num_objects":60047,"num_object_clones":0,"num_object_copies":180141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6838,"num_objects_misplaced":8658,"num_objects_unfound":0,"num_objects_dirty":60047,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":151275,"num_write_kb":6755598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15306,"num_bytes_recovered":1001396136,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":212246,"ondisk_log_size":212246,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12188808,"kb_used_data":10794832,"kb_used_omap":126,"kb_used_meta":1393601,"kb_avail":644027256,"statfs":{"total":671965249536,"available":659483910144,"internally_reserved":0,"allocated":11053907968,"data_stored":11049989111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1427048059},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-403046400,"num_objects":-6150,"num_object_clones":0,"num_object_copies":-18450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1612,"num_objects_unfound":0,"num_objects_dirty":-6150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6150,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":751,"num_bytes_recovered":49217536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004119"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":24,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980628+0000","last_change":"2023-10-09T10:01:00.939779+0000","last_active":"2023-10-09T10:01:35.980628+0000","last_peered":"2023-10-09T10:01:35.980628+0000","last_clean":"2023-10-09T10:01:35.980628+0000","last_became_active":"2023-10-09T10:01:00.939403+0000","last_became_peered":"2023-10-09T10:01:00.939403+0000","last_unstale":"2023-10-09T10:01:35.980628+0000","last_undegraded":"2023-10-09T10:01:35.980628+0000","last_fullsized":"2023-10-09T10:01:35.980628+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:05.091492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":127,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980031+0000","last_change":"2023-10-09T10:01:00.930519+0000","last_active":"2023-10-09T10:01:35.980031+0000","last_peered":"2023-10-09T10:01:35.980031+0000","last_clean":"2023-10-09T10:01:35.980031+0000","last_became_active":"2023-10-09T10:01:00.930236+0000","last_became_peered":"2023-10-09T10:01:00.930236+0000","last_unstale":"2023-10-09T10:01:35.980031+0000","last_undegraded":"2023-10-09T10:01:35.980031+0000","last_fullsized":"2023-10-09T10:01:35.980031+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:25.568164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980937+0000","last_change":"2023-10-09T10:01:00.930723+0000","last_active":"2023-10-09T10:01:35.980937+0000","last_peered":"2023-10-09T10:01:35.980937+0000","last_clean":"2023-10-09T10:01:35.980937+0000","last_became_active":"2023-10-09T10:01:00.930428+0000","last_became_peered":"2023-10-09T10:01:00.930428+0000","last_unstale":"2023-10-09T10:01:35.980937+0000","last_undegraded":"2023-10-09T10:01:35.980937+0000","last_fullsized":"2023-10-09T10:01:35.980937+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:28:37.700582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985381+0000","last_change":"2023-10-09T09:57:49.622656+0000","last_active":"2023-10-09T10:01:35.985381+0000","last_peered":"2023-10-09T10:01:35.985381+0000","last_clean":"2023-10-09T10:01:35.985381+0000","last_became_active":"2023-10-09T09:57:49.622342+0000","last_became_peered":"2023-10-09T09:57:49.622342+0000","last_unstale":"2023-10-09T10:01:35.985381+0000","last_undegraded":"2023-10-09T10:01:35.985381+0000","last_fullsized":"2023-10-09T10:01:35.985381+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:44.295133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":134,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.979864+0000","last_change":"2023-10-09T09:57:49.624475+0000","last_active":"2023-10-09T10:01:35.979864+0000","last_peered":"2023-10-09T10:01:35.979864+0000","last_clean":"2023-10-09T10:01:35.979864+0000","last_became_active":"2023-10-09T09:57:49.623486+0000","last_became_peered":"2023-10-09T09:57:49.623486+0000","last_unstale":"2023-10-09T10:01:35.979864+0000","last_undegraded":"2023-10-09T10:01:35.979864+0000","last_fullsized":"2023-10-09T10:01:35.979864+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.319474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":103,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:36.022138+0000","last_change":"2023-10-09T10:01:36.021940+0000","last_active":"2023-10-09T10:01:36.022138+0000","last_peered":"2023-10-09T10:01:36.022138+0000","last_clean":"2023-10-09T10:01:36.022138+0000","last_became_active":"2023-10-09T10:01:36.021602+0000","last_became_peered":"2023-10-09T10:01:36.021602+0000","last_unstale":"2023-10-09T10:01:36.022138+0000","last_undegraded":"2023-10-09T10:01:36.022138+0000","last_fullsized":"2023-10-09T10:01:36.022138+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"150'8597","reported_seq":8687,"reported_epoch":150,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:01:42.256082+0000","last_change":"2023-10-09T10:01:07.543051+0000","last_active":"2023-10-09T10:01:42.256082+0000","last_peered":"2023-10-09T10:01:42.256082+0000","last_clean":"2023-10-09T10:00:08.575217+0000","last_became_active":"2023-10-09T10:01:07.211454+0000","last_became_peered":"2023-10-09T10:01:07.211454+0000","last_unstale":"2023-10-09T10:01:42.256082+0000","last_undegraded":"2023-10-09T10:01:07.537865+0000","last_fullsized":"2023-10-09T10:01:01.922438+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8597,"log_dups_size":0,"ondisk_log_size":8597,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122814464,"num_objects":1874,"num_object_clones":0,"num_object_copies":5622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3363,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4835,"num_write_kb":214688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"2,3","objects":-904},{"shards":"2,3,4","objects":1562},{"shards":"3,4,6","objects":336}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":108,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985806+0000","last_change":"2023-10-09T09:57:49.626421+0000","last_active":"2023-10-09T10:01:35.985806+0000","last_peered":"2023-10-09T10:01:35.985806+0000","last_clean":"2023-10-09T10:01:35.985806+0000","last_became_active":"2023-10-09T09:57:49.626017+0000","last_became_peered":"2023-10-09T09:57:49.626017+0000","last_unstale":"2023-10-09T10:01:35.985806+0000","last_undegraded":"2023-10-09T10:01:35.985806+0000","last_fullsized":"2023-10-09T10:01:35.985806+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:16.523567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":118,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.979925+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T10:01:35.979925+0000","last_peered":"2023-10-09T10:01:35.979925+0000","last_clean":"2023-10-09T10:01:35.979925+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T10:01:35.979925+0000","last_undegraded":"2023-10-09T10:01:35.979925+0000","last_fullsized":"2023-10-09T10:01:35.979925+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"150'8448","reported_seq":10317,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:39.850048+0000","last_change":"2023-10-09T10:00:53.519038+0000","last_active":"2023-10-09T10:01:39.850048+0000","last_peered":"2023-10-09T10:01:39.850048+0000","last_clean":"2023-10-09T10:01:39.850048+0000","last_became_active":"2023-10-09T10:00:52.732215+0000","last_became_peered":"2023-10-09T10:00:52.732215+0000","last_unstale":"2023-10-09T10:01:39.850048+0000","last_undegraded":"2023-10-09T10:01:39.850048+0000","last_fullsized":"2023-10-09T10:01:39.850048+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8448,"log_dups_size":0,"ondisk_log_size":8448,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126844928,"num_objects":1936,"num_object_clones":0,"num_object_copies":5808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4716,"num_write_kb":212832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":613,"num_bytes_recovered":40140800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985409+0000","last_change":"2023-10-09T09:56:45.554057+0000","last_active":"2023-10-09T10:01:35.985409+0000","last_peered":"2023-10-09T10:01:35.985409+0000","last_clean":"2023-10-09T10:01:35.985409+0000","last_became_active":"2023-10-09T09:56:45.553720+0000","last_became_peered":"2023-10-09T09:56:45.553720+0000","last_unstale":"2023-10-09T10:01:35.985409+0000","last_undegraded":"2023-10-09T10:01:35.985409+0000","last_fullsized":"2023-10-09T10:01:35.985409+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.979772+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T10:01:35.979772+0000","last_peered":"2023-10-09T10:01:35.979772+0000","last_clean":"2023-10-09T10:01:35.979772+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T10:01:35.979772+0000","last_undegraded":"2023-10-09T10:01:35.979772+0000","last_fullsized":"2023-10-09T10:01:35.979772+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"150'8395","reported_seq":9737,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:41.626540+0000","last_change":"2023-10-09T10:01:26.147159+0000","last_active":"2023-10-09T10:01:41.626540+0000","last_peered":"2023-10-09T10:01:41.626540+0000","last_clean":"2023-10-09T10:01:41.626540+0000","last_became_active":"2023-10-09T10:00:52.676306+0000","last_became_peered":"2023-10-09T10:00:52.676306+0000","last_unstale":"2023-10-09T10:01:41.626540+0000","last_undegraded":"2023-10-09T10:01:41.626540+0000","last_fullsized":"2023-10-09T10:01:41.626540+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8395,"log_dups_size":0,"ondisk_log_size":8395,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120619008,"num_objects":1841,"num_object_clones":0,"num_object_copies":5523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4702,"num_write_kb":209344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20414464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":133,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.987039+0000","last_change":"2023-10-09T10:01:01.301253+0000","last_active":"2023-10-09T10:01:35.987039+0000","last_peered":"2023-10-09T10:01:35.987039+0000","last_clean":"2023-10-09T10:01:35.987039+0000","last_became_active":"2023-10-09T10:01:00.933163+0000","last_became_peered":"2023-10-09T10:01:00.933163+0000","last_unstale":"2023-10-09T10:01:35.987039+0000","last_undegraded":"2023-10-09T10:01:35.987039+0000","last_fullsized":"2023-10-09T10:01:35.987039+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980765+0000","last_change":"2023-10-09T10:00:30.790050+0000","last_active":"2023-10-09T10:01:35.980765+0000","last_peered":"2023-10-09T10:01:35.980765+0000","last_clean":"2023-10-09T10:01:35.980765+0000","last_became_active":"2023-10-09T10:00:30.789740+0000","last_became_peered":"2023-10-09T10:00:30.789740+0000","last_unstale":"2023-10-09T10:01:35.980765+0000","last_undegraded":"2023-10-09T10:01:35.980765+0000","last_fullsized":"2023-10-09T10:01:35.980765+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:36.206642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"150'8413","reported_seq":9096,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:38.593672+0000","last_change":"2023-10-09T10:01:00.721483+0000","last_active":"2023-10-09T10:01:38.593672+0000","last_peered":"2023-10-09T10:01:38.593672+0000","last_clean":"2023-10-09T10:01:38.593672+0000","last_became_active":"2023-10-09T10:00:52.779409+0000","last_became_peered":"2023-10-09T10:00:52.779409+0000","last_unstale":"2023-10-09T10:01:38.593672+0000","last_undegraded":"2023-10-09T10:01:38.593672+0000","last_fullsized":"2023-10-09T10:01:38.593672+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8413,"log_dups_size":0,"ondisk_log_size":8413,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712704,"num_objects":1964,"num_object_clones":0,"num_object_copies":5892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4691,"num_write_kb":212960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":299,"num_bytes_recovered":19595264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":81,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.986159+0000","last_change":"2023-10-09T10:01:01.209859+0000","last_active":"2023-10-09T10:01:35.986159+0000","last_peered":"2023-10-09T10:01:35.986159+0000","last_clean":"2023-10-09T10:01:35.986159+0000","last_became_active":"2023-10-09T10:01:00.931767+0000","last_became_peered":"2023-10-09T10:01:00.931767+0000","last_unstale":"2023-10-09T10:01:35.986159+0000","last_undegraded":"2023-10-09T10:01:35.986159+0000","last_fullsized":"2023-10-09T10:01:35.986159+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:37.009956+0000","last_change":"2023-10-09T10:01:36.019525+0000","last_active":"2023-10-09T10:01:37.009956+0000","last_peered":"2023-10-09T10:01:37.009956+0000","last_clean":"2023-10-09T10:01:37.009956+0000","last_became_active":"2023-10-09T10:01:36.019150+0000","last_became_peered":"2023-10-09T10:01:36.019150+0000","last_unstale":"2023-10-09T10:01:37.009956+0000","last_undegraded":"2023-10-09T10:01:37.009956+0000","last_fullsized":"2023-10-09T10:01:37.009956+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:09:49.743729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.16","version":"150'8461","reported_seq":8548,"reported_epoch":150,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:01:42.259203+0000","last_change":"2023-10-09T10:01:02.703753+0000","last_active":"2023-10-09T10:01:42.259203+0000","last_peered":"2023-10-09T10:01:42.259203+0000","last_clean":"2023-10-09T10:00:59.912614+0000","last_became_active":"2023-10-09T10:01:02.185720+0000","last_became_peered":"2023-10-09T10:01:02.185720+0000","last_unstale":"2023-10-09T10:01:42.259203+0000","last_undegraded":"2023-10-09T10:01:42.259203+0000","last_fullsized":"2023-10-09T10:01:42.259203+0000","mapping_epoch":138,"log_start":"102'300","ondisk_log_start":"102'300","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8161,"log_dups_size":0,"ondisk_log_size":8161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122290176,"num_objects":1866,"num_object_clones":0,"num_object_copies":5598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1866,"num_objects_unfound":0,"num_objects_dirty":1866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4734,"num_write_kb":211200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1866}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":162,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.986250+0000","last_change":"2023-10-09T10:01:01.304294+0000","last_active":"2023-10-09T10:01:35.986250+0000","last_peered":"2023-10-09T10:01:35.986250+0000","last_clean":"2023-10-09T10:01:35.986250+0000","last_became_active":"2023-10-09T10:01:00.936526+0000","last_became_peered":"2023-10-09T10:01:00.936526+0000","last_unstale":"2023-10-09T10:01:35.986250+0000","last_undegraded":"2023-10-09T10:01:35.986250+0000","last_fullsized":"2023-10-09T10:01:35.986250+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":118,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.983528+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T10:01:35.983528+0000","last_peered":"2023-10-09T10:01:35.983528+0000","last_clean":"2023-10-09T10:01:35.983528+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T10:01:35.983528+0000","last_undegraded":"2023-10-09T10:01:35.983528+0000","last_fullsized":"2023-10-09T10:01:35.983528+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"150'8210","reported_seq":8288,"reported_epoch":150,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:01:39.832128+0000","last_change":"2023-10-09T10:01:02.099633+0000","last_active":"2023-10-09T10:01:39.832128+0000","last_peered":"2023-10-09T10:01:39.832128+0000","last_clean":"2023-10-09T10:00:59.881901+0000","last_became_active":"2023-10-09T10:01:02.089341+0000","last_became_peered":"2023-10-09T10:01:02.089341+0000","last_unstale":"2023-10-09T10:01:39.832128+0000","last_undegraded":"2023-10-09T10:01:01.938801+0000","last_fullsized":"2023-10-09T10:01:01.925575+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8210,"log_dups_size":0,"ondisk_log_size":8210,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121372672,"num_objects":1852,"num_object_clones":0,"num_object_copies":5556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3229,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4591,"num_write_kb":206176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1857},{"shards":"3,5","objects":-783}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":108,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.984934+0000","last_change":"2023-10-09T09:57:49.624030+0000","last_active":"2023-10-09T10:01:35.984934+0000","last_peered":"2023-10-09T10:01:35.984934+0000","last_clean":"2023-10-09T10:01:35.984934+0000","last_became_active":"2023-10-09T09:57:49.623732+0000","last_became_peered":"2023-10-09T09:57:49.623732+0000","last_unstale":"2023-10-09T10:01:35.984934+0000","last_undegraded":"2023-10-09T10:01:35.984934+0000","last_fullsized":"2023-10-09T10:01:35.984934+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:46.345923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":115,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985291+0000","last_change":"2023-10-09T09:57:49.627770+0000","last_active":"2023-10-09T10:01:35.985291+0000","last_peered":"2023-10-09T10:01:35.985291+0000","last_clean":"2023-10-09T10:01:35.985291+0000","last_became_active":"2023-10-09T09:57:49.627262+0000","last_became_peered":"2023-10-09T09:57:49.627262+0000","last_unstale":"2023-10-09T10:01:35.985291+0000","last_undegraded":"2023-10-09T10:01:35.985291+0000","last_fullsized":"2023-10-09T10:01:35.985291+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"150'8355","reported_seq":8427,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:38.616602+0000","last_change":"2023-10-09T10:00:52.685558+0000","last_active":"2023-10-09T10:01:38.616602+0000","last_peered":"2023-10-09T10:01:38.616602+0000","last_clean":"2023-10-09T10:01:38.616602+0000","last_became_active":"2023-10-09T10:00:52.677109+0000","last_became_peered":"2023-10-09T10:00:52.677109+0000","last_unstale":"2023-10-09T10:01:38.616602+0000","last_undegraded":"2023-10-09T10:01:38.616602+0000","last_fullsized":"2023-10-09T10:01:38.616602+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8355,"log_dups_size":0,"ondisk_log_size":8355,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125468672,"num_objects":1915,"num_object_clones":0,"num_object_copies":5745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4655,"num_write_kb":210208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":19,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:36.714862+0000","last_change":"2023-10-09T10:01:36.017955+0000","last_active":"2023-10-09T10:01:36.714862+0000","last_peered":"2023-10-09T10:01:36.714862+0000","last_clean":"2023-10-09T10:01:36.714862+0000","last_became_active":"2023-10-09T10:01:36.017665+0000","last_became_peered":"2023-10-09T10:01:36.017665+0000","last_unstale":"2023-10-09T10:01:36.714862+0000","last_undegraded":"2023-10-09T10:01:36.714862+0000","last_fullsized":"2023-10-09T10:01:36.714862+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:17.939180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985301+0000","last_change":"2023-10-09T09:57:49.622372+0000","last_active":"2023-10-09T10:01:35.985301+0000","last_peered":"2023-10-09T10:01:35.985301+0000","last_clean":"2023-10-09T10:01:35.985301+0000","last_became_active":"2023-10-09T09:57:49.621883+0000","last_became_peered":"2023-10-09T09:57:49.621883+0000","last_unstale":"2023-10-09T10:01:35.985301+0000","last_undegraded":"2023-10-09T10:01:35.985301+0000","last_fullsized":"2023-10-09T10:01:35.985301+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:22.304026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"150'8361","reported_seq":8430,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:39.848766+0000","last_change":"2023-10-09T10:00:52.704990+0000","last_active":"2023-10-09T10:01:39.848766+0000","last_peered":"2023-10-09T10:01:39.848766+0000","last_clean":"2023-10-09T10:01:39.848766+0000","last_became_active":"2023-10-09T10:00:52.699362+0000","last_became_peered":"2023-10-09T10:00:52.699362+0000","last_unstale":"2023-10-09T10:01:39.848766+0000","last_undegraded":"2023-10-09T10:01:39.848766+0000","last_fullsized":"2023-10-09T10:01:39.848766+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8361,"log_dups_size":0,"ondisk_log_size":8361,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121536512,"num_objects":1855,"num_object_clones":0,"num_object_copies":5565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4676,"num_write_kb":208960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":24,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:36.714831+0000","last_change":"2023-10-09T10:01:36.024732+0000","last_active":"2023-10-09T10:01:36.714831+0000","last_peered":"2023-10-09T10:01:36.714831+0000","last_clean":"2023-10-09T10:01:36.714831+0000","last_became_active":"2023-10-09T10:01:36.024405+0000","last_became_peered":"2023-10-09T10:01:36.024405+0000","last_unstale":"2023-10-09T10:01:36.714831+0000","last_undegraded":"2023-10-09T10:01:36.714831+0000","last_fullsized":"2023-10-09T10:01:36.714831+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:02.136940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":118,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980570+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T10:01:35.980570+0000","last_peered":"2023-10-09T10:01:35.980570+0000","last_clean":"2023-10-09T10:01:35.980570+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T10:01:35.980570+0000","last_undegraded":"2023-10-09T10:01:35.980570+0000","last_fullsized":"2023-10-09T10:01:35.980570+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"150'9380","reported_seq":9441,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:39.847834+0000","last_change":"2023-10-09T09:58:52.780310+0000","last_active":"2023-10-09T10:01:39.847834+0000","last_peered":"2023-10-09T10:01:39.847834+0000","last_clean":"2023-10-09T10:01:39.847834+0000","last_became_active":"2023-10-09T09:58:52.779910+0000","last_became_peered":"2023-10-09T09:58:52.779910+0000","last_unstale":"2023-10-09T10:01:39.847834+0000","last_undegraded":"2023-10-09T10:01:39.847834+0000","last_fullsized":"2023-10-09T10:01:39.847834+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9380,"log_dups_size":0,"ondisk_log_size":9380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251789312,"num_objects":3842,"num_object_clones":0,"num_object_copies":11526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9380,"num_write_kb":423104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":129,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.987490+0000","last_change":"2023-10-09T10:01:00.939824+0000","last_active":"2023-10-09T10:01:35.987490+0000","last_peered":"2023-10-09T10:01:35.987490+0000","last_clean":"2023-10-09T10:01:35.987490+0000","last_became_active":"2023-10-09T10:01:00.922153+0000","last_became_peered":"2023-10-09T10:01:00.922153+0000","last_unstale":"2023-10-09T10:01:35.987490+0000","last_undegraded":"2023-10-09T10:01:35.987490+0000","last_fullsized":"2023-10-09T10:01:35.987490+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":100,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.984900+0000","last_change":"2023-10-09T09:48:27.155176+0000","last_active":"2023-10-09T10:01:35.984900+0000","last_peered":"2023-10-09T10:01:35.984900+0000","last_clean":"2023-10-09T10:01:35.984900+0000","last_became_active":"2023-10-09T09:48:27.154819+0000","last_became_peered":"2023-10-09T09:48:27.154819+0000","last_unstale":"2023-10-09T10:01:35.984900+0000","last_undegraded":"2023-10-09T10:01:35.984900+0000","last_fullsized":"2023-10-09T10:01:35.984900+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"150'9704","reported_seq":9767,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:42.308109+0000","last_change":"2023-10-09T09:58:52.625388+0000","last_active":"2023-10-09T10:01:42.308109+0000","last_peered":"2023-10-09T10:01:42.308109+0000","last_clean":"2023-10-09T10:01:42.308109+0000","last_became_active":"2023-10-09T09:58:52.625135+0000","last_became_peered":"2023-10-09T09:58:52.625135+0000","last_unstale":"2023-10-09T10:01:42.308109+0000","last_undegraded":"2023-10-09T10:01:42.308109+0000","last_fullsized":"2023-10-09T10:01:42.308109+0000","mapping_epoch":97,"log_start":"110'3947","ondisk_log_start":"110'3947","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:28.713713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240254976,"num_objects":3666,"num_object_clones":0,"num_object_copies":10998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9704,"num_write_kb":427840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":130,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.986445+0000","last_change":"2023-10-09T10:01:01.035488+0000","last_active":"2023-10-09T10:01:35.986445+0000","last_peered":"2023-10-09T10:01:35.986445+0000","last_clean":"2023-10-09T10:01:35.986445+0000","last_became_active":"2023-10-09T10:01:00.926607+0000","last_became_peered":"2023-10-09T10:01:00.926607+0000","last_unstale":"2023-10-09T10:01:35.986445+0000","last_undegraded":"2023-10-09T10:01:35.986445+0000","last_fullsized":"2023-10-09T10:01:35.986445+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":91,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980849+0000","last_change":"2023-10-09T10:01:00.937704+0000","last_active":"2023-10-09T10:01:35.980849+0000","last_peered":"2023-10-09T10:01:35.980849+0000","last_clean":"2023-10-09T10:01:35.980849+0000","last_became_active":"2023-10-09T10:01:00.937399+0000","last_became_peered":"2023-10-09T10:01:00.937399+0000","last_unstale":"2023-10-09T10:01:35.980849+0000","last_undegraded":"2023-10-09T10:01:35.980849+0000","last_fullsized":"2023-10-09T10:01:35.980849+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:06.336319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.8","version":"150'8671","reported_seq":9417,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:42.214126+0000","last_change":"2023-10-09T10:01:00.234694+0000","last_active":"2023-10-09T10:01:42.214126+0000","last_peered":"2023-10-09T10:01:42.214126+0000","last_clean":"2023-10-09T10:01:42.214126+0000","last_became_active":"2023-10-09T10:00:53.649276+0000","last_became_peered":"2023-10-09T10:00:53.649276+0000","last_unstale":"2023-10-09T10:01:42.214126+0000","last_undegraded":"2023-10-09T10:01:42.214126+0000","last_fullsized":"2023-10-09T10:01:42.214126+0000","mapping_epoch":134,"log_start":"102'400","ondisk_log_start":"102'400","created":97,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8271,"log_dups_size":0,"ondisk_log_size":8271,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124157952,"num_objects":1894,"num_object_clones":0,"num_object_copies":5682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4866,"num_write_kb":216352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":332,"num_bytes_recovered":21757952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":52,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985205+0000","last_change":"2023-10-09T10:00:19.641304+0000","last_active":"2023-10-09T10:01:35.985205+0000","last_peered":"2023-10-09T10:01:35.985205+0000","last_clean":"2023-10-09T10:01:35.985205+0000","last_became_active":"2023-10-09T10:00:19.640958+0000","last_became_peered":"2023-10-09T10:00:19.640958+0000","last_unstale":"2023-10-09T10:01:35.985205+0000","last_undegraded":"2023-10-09T10:01:35.985205+0000","last_fullsized":"2023-10-09T10:01:35.985205+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":54,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:37.008373+0000","last_change":"2023-10-09T10:01:36.025782+0000","last_active":"2023-10-09T10:01:37.008373+0000","last_peered":"2023-10-09T10:01:37.008373+0000","last_clean":"2023-10-09T10:01:37.008373+0000","last_became_active":"2023-10-09T10:01:36.025196+0000","last_became_peered":"2023-10-09T10:01:36.025196+0000","last_unstale":"2023-10-09T10:01:37.008373+0000","last_undegraded":"2023-10-09T10:01:37.008373+0000","last_fullsized":"2023-10-09T10:01:37.008373+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:04.486165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"150'8485","reported_seq":10790,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:41.502296+0000","last_change":"2023-10-09T10:01:32.227124+0000","last_active":"2023-10-09T10:01:41.502296+0000","last_peered":"2023-10-09T10:01:41.502296+0000","last_clean":"2023-10-09T10:01:41.502296+0000","last_became_active":"2023-10-09T10:00:53.661339+0000","last_became_peered":"2023-10-09T10:00:53.661339+0000","last_unstale":"2023-10-09T10:01:41.502296+0000","last_undegraded":"2023-10-09T10:01:41.502296+0000","last_fullsized":"2023-10-09T10:01:41.502296+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8485,"log_dups_size":0,"ondisk_log_size":8485,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114393088,"num_objects":1745,"num_object_clones":0,"num_object_copies":5235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4787,"num_write_kb":209056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":704,"num_bytes_recovered":46170112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985177+0000","last_change":"2023-10-09T09:57:49.627539+0000","last_active":"2023-10-09T10:01:35.985177+0000","last_peered":"2023-10-09T10:01:35.985177+0000","last_clean":"2023-10-09T10:01:35.985177+0000","last_became_active":"2023-10-09T09:57:49.627203+0000","last_became_peered":"2023-10-09T09:57:49.627203+0000","last_unstale":"2023-10-09T10:01:35.985177+0000","last_undegraded":"2023-10-09T10:01:35.985177+0000","last_fullsized":"2023-10-09T10:01:35.985177+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":118,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.984984+0000","last_change":"2023-10-09T10:01:00.937849+0000","last_active":"2023-10-09T10:01:35.984984+0000","last_peered":"2023-10-09T10:01:35.984984+0000","last_clean":"2023-10-09T10:01:35.984984+0000","last_became_active":"2023-10-09T10:01:00.937536+0000","last_became_peered":"2023-10-09T10:01:00.937536+0000","last_unstale":"2023-10-09T10:01:35.984984+0000","last_undegraded":"2023-10-09T10:01:35.984984+0000","last_fullsized":"2023-10-09T10:01:35.984984+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:29.118995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"150'9367","reported_seq":20827,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:38.603549+0000","last_change":"2023-10-09T10:01:32.951000+0000","last_active":"2023-10-09T10:01:38.603549+0000","last_peered":"2023-10-09T10:01:38.603549+0000","last_clean":"2023-10-09T10:01:38.603549+0000","last_became_active":"2023-10-09T10:01:32.935658+0000","last_became_peered":"2023-10-09T10:01:32.935658+0000","last_unstale":"2023-10-09T10:01:38.603549+0000","last_undegraded":"2023-10-09T10:01:38.603549+0000","last_fullsized":"2023-10-09T10:01:38.603549+0000","mapping_epoch":146,"log_start":"109'3460","ondisk_log_start":"109'3460","created":97,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":5907,"log_dups_size":0,"ondisk_log_size":5907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:26.305928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252116992,"num_objects":3847,"num_object_clones":0,"num_object_copies":11541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9367,"num_write_kb":422848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5677,"num_bytes_recovered":372047872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":129,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.981808+0000","last_change":"2023-10-09T10:01:00.964746+0000","last_active":"2023-10-09T10:01:35.981808+0000","last_peered":"2023-10-09T10:01:35.981808+0000","last_clean":"2023-10-09T10:01:35.981808+0000","last_became_active":"2023-10-09T10:01:00.930510+0000","last_became_peered":"2023-10-09T10:01:00.930510+0000","last_unstale":"2023-10-09T10:01:35.981808+0000","last_undegraded":"2023-10-09T10:01:35.981808+0000","last_fullsized":"2023-10-09T10:01:35.981808+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:11.047472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":76,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.983746+0000","last_change":"2023-10-09T10:01:00.932525+0000","last_active":"2023-10-09T10:01:35.983746+0000","last_peered":"2023-10-09T10:01:35.983746+0000","last_clean":"2023-10-09T10:01:35.983746+0000","last_became_active":"2023-10-09T10:01:00.931364+0000","last_became_peered":"2023-10-09T10:01:00.931364+0000","last_unstale":"2023-10-09T10:01:35.983746+0000","last_undegraded":"2023-10-09T10:01:35.983746+0000","last_fullsized":"2023-10-09T10:01:35.983746+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:54.467044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"150'9471","reported_seq":10966,"reported_epoch":150,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:01:42.191732+0000","last_change":"2023-10-09T10:01:07.551328+0000","last_active":"2023-10-09T10:01:42.191732+0000","last_peered":"2023-10-09T10:01:42.191732+0000","last_clean":"2023-10-09T10:00:59.886279+0000","last_became_active":"2023-10-09T10:01:02.225568+0000","last_became_peered":"2023-10-09T10:01:02.225568+0000","last_unstale":"2023-10-09T10:01:42.191732+0000","last_undegraded":"2023-10-09T10:01:42.191732+0000","last_fullsized":"2023-10-09T10:01:42.191732+0000","mapping_epoch":138,"log_start":"103'600","ondisk_log_start":"103'600","created":97,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8871,"log_dups_size":0,"ondisk_log_size":8871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:45.432958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3538,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9471,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":737,"num_bytes_recovered":48300032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":3621}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:37.009609+0000","last_change":"2023-10-09T10:01:36.025986+0000","last_active":"2023-10-09T10:01:37.009609+0000","last_peered":"2023-10-09T10:01:37.009609+0000","last_clean":"2023-10-09T10:01:37.009609+0000","last_became_active":"2023-10-09T10:01:36.025650+0000","last_became_peered":"2023-10-09T10:01:36.025650+0000","last_unstale":"2023-10-09T10:01:37.009609+0000","last_undegraded":"2023-10-09T10:01:37.009609+0000","last_fullsized":"2023-10-09T10:01:37.009609+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:57.872953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":139,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.991459+0000","last_change":"2023-10-09T10:01:00.935684+0000","last_active":"2023-10-09T10:01:35.991459+0000","last_peered":"2023-10-09T10:01:35.991459+0000","last_clean":"2023-10-09T10:01:35.991459+0000","last_became_active":"2023-10-09T10:01:00.935290+0000","last_became_peered":"2023-10-09T10:01:00.935290+0000","last_unstale":"2023-10-09T10:01:35.991459+0000","last_undegraded":"2023-10-09T10:01:35.991459+0000","last_fullsized":"2023-10-09T10:01:35.991459+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:33.930594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"150'9532","reported_seq":9595,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:42.275880+0000","last_change":"2023-10-09T09:58:52.780096+0000","last_active":"2023-10-09T10:01:42.275880+0000","last_peered":"2023-10-09T10:01:42.275880+0000","last_clean":"2023-10-09T10:01:42.275880+0000","last_became_active":"2023-10-09T09:58:52.779754+0000","last_became_peered":"2023-10-09T09:58:52.779754+0000","last_unstale":"2023-10-09T10:01:42.275880+0000","last_undegraded":"2023-10-09T10:01:42.275880+0000","last_fullsized":"2023-10-09T10:01:42.275880+0000","mapping_epoch":97,"log_start":"111'3844","ondisk_log_start":"111'3844","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:19:28.304859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241565696,"num_objects":3686,"num_object_clones":0,"num_object_copies":11058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9532,"num_write_kb":422976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":125,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.992185+0000","last_change":"2023-10-09T10:01:00.935986+0000","last_active":"2023-10-09T10:01:35.992185+0000","last_peered":"2023-10-09T10:01:35.992185+0000","last_clean":"2023-10-09T10:01:35.992185+0000","last_became_active":"2023-10-09T10:01:00.935495+0000","last_became_peered":"2023-10-09T10:01:00.935495+0000","last_unstale":"2023-10-09T10:01:35.992185+0000","last_undegraded":"2023-10-09T10:01:35.992185+0000","last_fullsized":"2023-10-09T10:01:35.992185+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:11.501226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:37.012720+0000","last_change":"2023-10-09T10:01:36.018926+0000","last_active":"2023-10-09T10:01:37.012720+0000","last_peered":"2023-10-09T10:01:37.012720+0000","last_clean":"2023-10-09T10:01:37.012720+0000","last_became_active":"2023-10-09T10:01:36.018560+0000","last_became_peered":"2023-10-09T10:01:36.018560+0000","last_unstale":"2023-10-09T10:01:37.012720+0000","last_undegraded":"2023-10-09T10:01:37.012720+0000","last_fullsized":"2023-10-09T10:01:37.012720+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:31.179164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.d","version":"150'9105","reported_seq":9167,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:39.830628+0000","last_change":"2023-10-09T09:58:52.626617+0000","last_active":"2023-10-09T10:01:39.830628+0000","last_peered":"2023-10-09T10:01:39.830628+0000","last_clean":"2023-10-09T10:01:39.830628+0000","last_became_active":"2023-10-09T09:58:52.626409+0000","last_became_peered":"2023-10-09T09:58:52.626409+0000","last_unstale":"2023-10-09T10:01:39.830628+0000","last_undegraded":"2023-10-09T10:01:39.830628+0000","last_fullsized":"2023-10-09T10:01:39.830628+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9105,"log_dups_size":0,"ondisk_log_size":9105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251002904,"num_objects":3831,"num_object_clones":0,"num_object_copies":11493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3831,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9105,"num_write_kb":413889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":119,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.979714+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T10:01:35.979714+0000","last_peered":"2023-10-09T10:01:35.979714+0000","last_clean":"2023-10-09T10:01:35.979714+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T10:01:35.979714+0000","last_undegraded":"2023-10-09T10:01:35.979714+0000","last_fullsized":"2023-10-09T10:01:35.979714+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985520+0000","last_change":"2023-10-09T10:01:00.933557+0000","last_active":"2023-10-09T10:01:35.985520+0000","last_peered":"2023-10-09T10:01:35.985520+0000","last_clean":"2023-10-09T10:01:35.985520+0000","last_became_active":"2023-10-09T10:01:00.933259+0000","last_became_peered":"2023-10-09T10:01:00.933259+0000","last_unstale":"2023-10-09T10:01:35.985520+0000","last_undegraded":"2023-10-09T10:01:35.985520+0000","last_fullsized":"2023-10-09T10:01:35.985520+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:47.058899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":130,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.982847+0000","last_change":"2023-10-09T09:56:45.550709+0000","last_active":"2023-10-09T10:01:35.982847+0000","last_peered":"2023-10-09T10:01:35.982847+0000","last_clean":"2023-10-09T10:01:35.982847+0000","last_became_active":"2023-10-09T09:56:45.550022+0000","last_became_peered":"2023-10-09T09:56:45.550022+0000","last_unstale":"2023-10-09T10:01:35.982847+0000","last_undegraded":"2023-10-09T10:01:35.982847+0000","last_fullsized":"2023-10-09T10:01:35.982847+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:32.314353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":149,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985357+0000","last_change":"2023-10-09T10:00:30.785799+0000","last_active":"2023-10-09T10:01:35.985357+0000","last_peered":"2023-10-09T10:01:35.985357+0000","last_clean":"2023-10-09T10:01:35.985357+0000","last_became_active":"2023-10-09T10:00:30.785396+0000","last_became_peered":"2023-10-09T10:00:30.785396+0000","last_unstale":"2023-10-09T10:01:35.985357+0000","last_undegraded":"2023-10-09T10:01:35.985357+0000","last_fullsized":"2023-10-09T10:01:35.985357+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":53,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980552+0000","last_change":"2023-10-09T10:01:00.929552+0000","last_active":"2023-10-09T10:01:35.980552+0000","last_peered":"2023-10-09T10:01:35.980552+0000","last_clean":"2023-10-09T10:01:35.980552+0000","last_became_active":"2023-10-09T10:01:00.929225+0000","last_became_peered":"2023-10-09T10:01:00.929225+0000","last_unstale":"2023-10-09T10:01:35.980552+0000","last_undegraded":"2023-10-09T10:01:35.980552+0000","last_fullsized":"2023-10-09T10:01:35.980552+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:11.162776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"150'8389","reported_seq":9732,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:41.713554+0000","last_change":"2023-10-09T10:01:22.075553+0000","last_active":"2023-10-09T10:01:41.713554+0000","last_peered":"2023-10-09T10:01:41.713554+0000","last_clean":"2023-10-09T10:01:41.713554+0000","last_became_active":"2023-10-09T10:00:52.648538+0000","last_became_peered":"2023-10-09T10:00:52.648538+0000","last_unstale":"2023-10-09T10:01:41.713554+0000","last_undegraded":"2023-10-09T10:01:41.713554+0000","last_fullsized":"2023-10-09T10:01:41.713554+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8389,"log_dups_size":0,"ondisk_log_size":8389,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121012224,"num_objects":1846,"num_object_clones":0,"num_object_copies":5538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4695,"num_write_kb":209344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20414464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"150'8480","reported_seq":11985,"reported_epoch":150,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:01:42.290440+0000","last_change":"2023-10-09T10:01:07.222699+0000","last_active":"2023-10-09T10:01:42.290440+0000","last_peered":"2023-10-09T10:01:42.290440+0000","last_clean":"2023-10-09T10:00:59.894671+0000","last_became_active":"2023-10-09T10:01:02.194954+0000","last_became_peered":"2023-10-09T10:01:02.194954+0000","last_unstale":"2023-10-09T10:01:42.290440+0000","last_undegraded":"2023-10-09T10:01:42.290440+0000","last_fullsized":"2023-10-09T10:01:42.290440+0000","mapping_epoch":138,"log_start":"102'300","ondisk_log_start":"102'300","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8180,"log_dups_size":0,"ondisk_log_size":8180,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121044992,"num_objects":1847,"num_object_clones":0,"num_object_copies":5541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":679,"num_objects_unfound":0,"num_objects_dirty":1847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4753,"num_write_kb":211200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1709,"num_bytes_recovered":112001024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1847}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":120,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985146+0000","last_change":"2023-10-09T09:56:45.553337+0000","last_active":"2023-10-09T10:01:35.985146+0000","last_peered":"2023-10-09T10:01:35.985146+0000","last_clean":"2023-10-09T10:01:35.985146+0000","last_became_active":"2023-10-09T09:56:45.552986+0000","last_became_peered":"2023-10-09T09:56:45.552986+0000","last_unstale":"2023-10-09T10:01:35.985146+0000","last_undegraded":"2023-10-09T10:01:35.985146+0000","last_fullsized":"2023-10-09T10:01:35.985146+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":62,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.984758+0000","last_change":"2023-10-09T10:01:00.937214+0000","last_active":"2023-10-09T10:01:35.984758+0000","last_peered":"2023-10-09T10:01:35.984758+0000","last_clean":"2023-10-09T10:01:35.984758+0000","last_became_active":"2023-10-09T10:01:00.936560+0000","last_became_peered":"2023-10-09T10:01:00.936560+0000","last_unstale":"2023-10-09T10:01:35.984758+0000","last_undegraded":"2023-10-09T10:01:35.984758+0000","last_fullsized":"2023-10-09T10:01:35.984758+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:53.374219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":138,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.982777+0000","last_change":"2023-10-09T10:00:30.782106+0000","last_active":"2023-10-09T10:01:35.982777+0000","last_peered":"2023-10-09T10:01:35.982777+0000","last_clean":"2023-10-09T10:01:35.982777+0000","last_became_active":"2023-10-09T10:00:30.781623+0000","last_became_peered":"2023-10-09T10:00:30.781623+0000","last_unstale":"2023-10-09T10:01:35.982777+0000","last_undegraded":"2023-10-09T10:01:35.982777+0000","last_fullsized":"2023-10-09T10:01:35.982777+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:14.657623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"150'8345","reported_seq":8423,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:39.764752+0000","last_change":"2023-10-09T10:01:31.634910+0000","last_active":"2023-10-09T10:01:39.764752+0000","last_peered":"2023-10-09T10:01:39.764752+0000","last_clean":"2023-10-09T10:01:39.764752+0000","last_became_active":"2023-10-09T10:00:52.671121+0000","last_became_peered":"2023-10-09T10:00:52.671121+0000","last_unstale":"2023-10-09T10:01:39.764752+0000","last_undegraded":"2023-10-09T10:01:39.764752+0000","last_fullsized":"2023-10-09T10:01:39.764752+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8345,"log_dups_size":0,"ondisk_log_size":8345,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122585088,"num_objects":1870,"num_object_clones":0,"num_object_copies":5610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4659,"num_write_kb":208960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.992108+0000","last_change":"2023-10-09T10:00:30.892640+0000","last_active":"2023-10-09T10:01:35.992108+0000","last_peered":"2023-10-09T10:01:35.992108+0000","last_clean":"2023-10-09T10:01:35.992108+0000","last_became_active":"2023-10-09T10:00:30.892119+0000","last_became_peered":"2023-10-09T10:00:30.892119+0000","last_unstale":"2023-10-09T10:01:35.992108+0000","last_undegraded":"2023-10-09T10:01:35.992108+0000","last_fullsized":"2023-10-09T10:01:35.992108+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:37:45.627291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":122,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.982089+0000","last_change":"2023-10-09T10:00:30.786721+0000","last_active":"2023-10-09T10:01:35.982089+0000","last_peered":"2023-10-09T10:01:35.982089+0000","last_clean":"2023-10-09T10:01:35.982089+0000","last_became_active":"2023-10-09T10:00:30.786375+0000","last_became_peered":"2023-10-09T10:00:30.786375+0000","last_unstale":"2023-10-09T10:01:35.982089+0000","last_undegraded":"2023-10-09T10:01:35.982089+0000","last_fullsized":"2023-10-09T10:01:35.982089+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:19:38.052691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":125,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985445+0000","last_change":"2023-10-09T09:56:45.547436+0000","last_active":"2023-10-09T10:01:35.985445+0000","last_peered":"2023-10-09T10:01:35.985445+0000","last_clean":"2023-10-09T10:01:35.985445+0000","last_became_active":"2023-10-09T09:56:45.547107+0000","last_became_peered":"2023-10-09T09:56:45.547107+0000","last_unstale":"2023-10-09T10:01:35.985445+0000","last_undegraded":"2023-10-09T10:01:35.985445+0000","last_fullsized":"2023-10-09T10:01:35.985445+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:23.604988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"150'8166","reported_seq":9982,"reported_epoch":150,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:01:39.832154+0000","last_change":"2023-10-09T10:01:02.111094+0000","last_active":"2023-10-09T10:01:39.832154+0000","last_peered":"2023-10-09T10:01:39.832154+0000","last_clean":"2023-10-09T10:00:59.913112+0000","last_became_active":"2023-10-09T10:01:02.088331+0000","last_became_peered":"2023-10-09T10:01:02.088331+0000","last_unstale":"2023-10-09T10:01:39.832154+0000","last_undegraded":"2023-10-09T10:01:01.956690+0000","last_fullsized":"2023-10-09T10:01:01.940057+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8166,"log_dups_size":0,"ondisk_log_size":8166,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124256256,"num_objects":1896,"num_object_clones":0,"num_object_copies":5688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1790,"num_objects_unfound":0,"num_objects_dirty":1896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4547,"num_write_kb":206176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1790},{"shards":"3,5","objects":105}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":138,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.984974+0000","last_change":"2023-10-09T10:00:30.790235+0000","last_active":"2023-10-09T10:01:35.984974+0000","last_peered":"2023-10-09T10:01:35.984974+0000","last_clean":"2023-10-09T10:01:35.984974+0000","last_became_active":"2023-10-09T10:00:30.789866+0000","last_became_peered":"2023-10-09T10:00:30.789866+0000","last_unstale":"2023-10-09T10:01:35.984974+0000","last_undegraded":"2023-10-09T10:01:35.984974+0000","last_fullsized":"2023-10-09T10:01:35.984974+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":351,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985974+0000","last_change":"2023-10-09T09:56:13.614446+0000","last_active":"2023-10-09T10:01:35.985974+0000","last_peered":"2023-10-09T10:01:35.985974+0000","last_clean":"2023-10-09T10:01:35.985974+0000","last_became_active":"2023-10-09T09:56:13.613994+0000","last_became_peered":"2023-10-09T09:56:13.613994+0000","last_unstale":"2023-10-09T10:01:35.985974+0000","last_undegraded":"2023-10-09T10:01:35.985974+0000","last_fullsized":"2023-10-09T10:01:35.985974+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985719+0000","last_change":"2023-10-09T09:57:49.622586+0000","last_active":"2023-10-09T10:01:35.985719+0000","last_peered":"2023-10-09T10:01:35.985719+0000","last_clean":"2023-10-09T10:01:35.985719+0000","last_became_active":"2023-10-09T09:57:49.622279+0000","last_became_peered":"2023-10-09T09:57:49.622279+0000","last_unstale":"2023-10-09T10:01:35.985719+0000","last_undegraded":"2023-10-09T10:01:35.985719+0000","last_fullsized":"2023-10-09T10:01:35.985719+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:39:20.490011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":100,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:36.022065+0000","last_change":"2023-10-09T10:01:36.019638+0000","last_active":"2023-10-09T10:01:36.022065+0000","last_peered":"2023-10-09T10:01:36.022065+0000","last_clean":"2023-10-09T10:01:36.022065+0000","last_became_active":"2023-10-09T10:01:36.019310+0000","last_became_peered":"2023-10-09T10:01:36.019310+0000","last_unstale":"2023-10-09T10:01:36.022065+0000","last_undegraded":"2023-10-09T10:01:36.022065+0000","last_fullsized":"2023-10-09T10:01:36.022065+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":102,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980162+0000","last_change":"2023-10-09T10:00:19.650114+0000","last_active":"2023-10-09T10:01:35.980162+0000","last_peered":"2023-10-09T10:01:35.980162+0000","last_clean":"2023-10-09T10:01:35.980162+0000","last_became_active":"2023-10-09T10:00:19.649776+0000","last_became_peered":"2023-10-09T10:00:19.649776+0000","last_unstale":"2023-10-09T10:01:35.980162+0000","last_undegraded":"2023-10-09T10:01:35.980162+0000","last_fullsized":"2023-10-09T10:01:35.980162+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:09:03.140113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":117,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.986116+0000","last_change":"2023-10-09T10:01:00.932106+0000","last_active":"2023-10-09T10:01:35.986116+0000","last_peered":"2023-10-09T10:01:35.986116+0000","last_clean":"2023-10-09T10:01:35.986116+0000","last_became_active":"2023-10-09T10:01:00.931851+0000","last_became_peered":"2023-10-09T10:01:00.931851+0000","last_unstale":"2023-10-09T10:01:35.986116+0000","last_undegraded":"2023-10-09T10:01:35.986116+0000","last_fullsized":"2023-10-09T10:01:35.986116+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:31.563093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":145,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985552+0000","last_change":"2023-10-09T10:00:30.785844+0000","last_active":"2023-10-09T10:01:35.985552+0000","last_peered":"2023-10-09T10:01:35.985552+0000","last_clean":"2023-10-09T10:01:35.985552+0000","last_became_active":"2023-10-09T10:00:30.785565+0000","last_became_peered":"2023-10-09T10:00:30.785565+0000","last_unstale":"2023-10-09T10:01:35.985552+0000","last_undegraded":"2023-10-09T10:01:35.985552+0000","last_fullsized":"2023-10-09T10:01:35.985552+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:56.229142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"150'8366","reported_seq":8439,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:38.577736+0000","last_change":"2023-10-09T10:00:52.682102+0000","last_active":"2023-10-09T10:01:38.577736+0000","last_peered":"2023-10-09T10:01:38.577736+0000","last_clean":"2023-10-09T10:01:38.577736+0000","last_became_active":"2023-10-09T10:00:52.668775+0000","last_became_peered":"2023-10-09T10:00:52.668775+0000","last_unstale":"2023-10-09T10:01:38.577736+0000","last_undegraded":"2023-10-09T10:01:38.577736+0000","last_fullsized":"2023-10-09T10:01:38.577736+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8366,"log_dups_size":0,"ondisk_log_size":8366,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124747776,"num_objects":1903,"num_object_clones":0,"num_object_copies":5709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4665,"num_write_kb":210208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":128,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.986956+0000","last_change":"2023-10-09T10:01:00.927846+0000","last_active":"2023-10-09T10:01:35.986956+0000","last_peered":"2023-10-09T10:01:35.986956+0000","last_clean":"2023-10-09T10:01:35.986956+0000","last_became_active":"2023-10-09T10:01:00.927543+0000","last_became_peered":"2023-10-09T10:01:00.927543+0000","last_unstale":"2023-10-09T10:01:35.986956+0000","last_undegraded":"2023-10-09T10:01:35.986956+0000","last_fullsized":"2023-10-09T10:01:35.986956+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.982927+0000","last_change":"2023-10-09T10:00:30.786812+0000","last_active":"2023-10-09T10:01:35.982927+0000","last_peered":"2023-10-09T10:01:35.982927+0000","last_clean":"2023-10-09T10:01:35.982927+0000","last_became_active":"2023-10-09T10:00:30.786500+0000","last_became_peered":"2023-10-09T10:00:30.786500+0000","last_unstale":"2023-10-09T10:01:35.982927+0000","last_undegraded":"2023-10-09T10:01:35.982927+0000","last_fullsized":"2023-10-09T10:01:35.982927+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:25.060464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":94,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.984838+0000","last_change":"2023-10-09T09:56:13.621559+0000","last_active":"2023-10-09T10:01:35.984838+0000","last_peered":"2023-10-09T10:01:35.984838+0000","last_clean":"2023-10-09T10:01:35.984838+0000","last_became_active":"2023-10-09T09:56:13.614597+0000","last_became_peered":"2023-10-09T09:56:13.614597+0000","last_unstale":"2023-10-09T10:01:35.984838+0000","last_undegraded":"2023-10-09T10:01:35.984838+0000","last_fullsized":"2023-10-09T10:01:35.984838+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:33.239162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"150'8438","reported_seq":9198,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:38.593715+0000","last_change":"2023-10-09T10:01:04.122514+0000","last_active":"2023-10-09T10:01:38.593715+0000","last_peered":"2023-10-09T10:01:38.593715+0000","last_clean":"2023-10-09T10:01:38.593715+0000","last_became_active":"2023-10-09T10:00:53.648018+0000","last_became_peered":"2023-10-09T10:00:53.648018+0000","last_unstale":"2023-10-09T10:01:38.593715+0000","last_undegraded":"2023-10-09T10:01:38.593715+0000","last_fullsized":"2023-10-09T10:01:38.593715+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8438,"log_dups_size":0,"ondisk_log_size":8438,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127074304,"num_objects":1939,"num_object_clones":0,"num_object_copies":5817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4716,"num_write_kb":212960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":21954560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.980498+0000","last_change":"2023-10-09T09:57:49.624047+0000","last_active":"2023-10-09T10:01:35.980498+0000","last_peered":"2023-10-09T10:01:35.980498+0000","last_clean":"2023-10-09T10:01:35.980498+0000","last_became_active":"2023-10-09T09:57:49.620878+0000","last_became_peered":"2023-10-09T09:57:49.620878+0000","last_unstale":"2023-10-09T10:01:35.980498+0000","last_undegraded":"2023-10-09T10:01:35.980498+0000","last_fullsized":"2023-10-09T10:01:35.980498+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:31.319360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.981922+0000","last_change":"2023-10-09T10:01:00.933551+0000","last_active":"2023-10-09T10:01:35.981922+0000","last_peered":"2023-10-09T10:01:35.981922+0000","last_clean":"2023-10-09T10:01:35.981922+0000","last_became_active":"2023-10-09T10:01:00.933229+0000","last_became_peered":"2023-10-09T10:01:00.933229+0000","last_unstale":"2023-10-09T10:01:35.981922+0000","last_undegraded":"2023-10-09T10:01:35.981922+0000","last_fullsized":"2023-10-09T10:01:35.981922+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:52.813857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":113,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.979797+0000","last_change":"2023-10-09T09:57:49.624797+0000","last_active":"2023-10-09T10:01:35.979797+0000","last_peered":"2023-10-09T10:01:35.979797+0000","last_clean":"2023-10-09T10:01:35.979797+0000","last_became_active":"2023-10-09T09:57:49.624210+0000","last_became_peered":"2023-10-09T09:57:49.624210+0000","last_unstale":"2023-10-09T10:01:35.979797+0000","last_undegraded":"2023-10-09T10:01:35.979797+0000","last_fullsized":"2023-10-09T10:01:35.979797+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:01.885760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"150'8607","reported_seq":12133,"reported_epoch":150,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:01:42.299929+0000","last_change":"2023-10-09T10:01:07.557994+0000","last_active":"2023-10-09T10:01:42.299929+0000","last_peered":"2023-10-09T10:01:42.299929+0000","last_clean":"2023-10-09T10:00:08.575217+0000","last_became_active":"2023-10-09T10:01:07.227487+0000","last_became_peered":"2023-10-09T10:01:07.227487+0000","last_unstale":"2023-10-09T10:01:42.299929+0000","last_undegraded":"2023-10-09T10:01:07.139881+0000","last_fullsized":"2023-10-09T10:01:01.923140+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8607,"log_dups_size":0,"ondisk_log_size":8607,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122159104,"num_objects":1864,"num_object_clones":0,"num_object_copies":5592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":245,"num_objects_misplaced":785,"num_objects_unfound":0,"num_objects_dirty":1864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4845,"num_write_kb":214688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1717,"num_bytes_recovered":112394240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3","objects":834},{"shards":"2,3,4","objects":785}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":116,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:37.026713+0000","last_change":"2023-10-09T10:01:36.024079+0000","last_active":"2023-10-09T10:01:37.026713+0000","last_peered":"2023-10-09T10:01:37.026713+0000","last_clean":"2023-10-09T10:01:37.026713+0000","last_became_active":"2023-10-09T10:01:36.023515+0000","last_became_peered":"2023-10-09T10:01:36.023515+0000","last_unstale":"2023-10-09T10:01:37.026713+0000","last_undegraded":"2023-10-09T10:01:37.026713+0000","last_fullsized":"2023-10-09T10:01:37.026713+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:36.977968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":125,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985473+0000","last_change":"2023-10-09T10:01:00.947866+0000","last_active":"2023-10-09T10:01:35.985473+0000","last_peered":"2023-10-09T10:01:35.985473+0000","last_clean":"2023-10-09T10:01:35.985473+0000","last_became_active":"2023-10-09T10:01:00.924897+0000","last_became_peered":"2023-10-09T10:01:00.924897+0000","last_unstale":"2023-10-09T10:01:35.985473+0000","last_undegraded":"2023-10-09T10:01:35.985473+0000","last_fullsized":"2023-10-09T10:01:35.985473+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:07.277714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.984870+0000","last_change":"2023-10-09T10:01:00.927941+0000","last_active":"2023-10-09T10:01:35.984870+0000","last_peered":"2023-10-09T10:01:35.984870+0000","last_clean":"2023-10-09T10:01:35.984870+0000","last_became_active":"2023-10-09T10:01:00.927456+0000","last_became_peered":"2023-10-09T10:01:00.927456+0000","last_unstale":"2023-10-09T10:01:35.984870+0000","last_undegraded":"2023-10-09T10:01:35.984870+0000","last_fullsized":"2023-10-09T10:01:35.984870+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.3","version":"150'8449","reported_seq":10317,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:39.859808+0000","last_change":"2023-10-09T10:01:03.341055+0000","last_active":"2023-10-09T10:01:39.859808+0000","last_peered":"2023-10-09T10:01:39.859808+0000","last_clean":"2023-10-09T10:01:39.859808+0000","last_became_active":"2023-10-09T10:00:53.652818+0000","last_became_peered":"2023-10-09T10:00:53.652818+0000","last_unstale":"2023-10-09T10:01:39.859808+0000","last_undegraded":"2023-10-09T10:01:39.859808+0000","last_fullsized":"2023-10-09T10:01:39.859808+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8449,"log_dups_size":0,"ondisk_log_size":8449,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126779392,"num_objects":1934,"num_object_clones":0,"num_object_copies":5802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4716,"num_write_kb":212832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":609,"num_bytes_recovered":39944192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":106,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985070+0000","last_change":"2023-10-09T10:01:00.926495+0000","last_active":"2023-10-09T10:01:35.985070+0000","last_peered":"2023-10-09T10:01:35.985070+0000","last_clean":"2023-10-09T10:01:35.985070+0000","last_became_active":"2023-10-09T10:01:00.925943+0000","last_became_peered":"2023-10-09T10:01:00.925943+0000","last_unstale":"2023-10-09T10:01:35.985070+0000","last_undegraded":"2023-10-09T10:01:35.985070+0000","last_fullsized":"2023-10-09T10:01:35.985070+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:13.486263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":36,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.991558+0000","last_change":"2023-10-09T10:00:30.779150+0000","last_active":"2023-10-09T10:01:35.991558+0000","last_peered":"2023-10-09T10:01:35.991558+0000","last_clean":"2023-10-09T10:01:35.991558+0000","last_became_active":"2023-10-09T10:00:30.778817+0000","last_became_peered":"2023-10-09T10:00:30.778817+0000","last_unstale":"2023-10-09T10:01:35.991558+0000","last_undegraded":"2023-10-09T10:01:35.991558+0000","last_fullsized":"2023-10-09T10:01:35.991558+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:04.893208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"150'8402","reported_seq":10744,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:41.544627+0000","last_change":"2023-10-09T10:01:34.061561+0000","last_active":"2023-10-09T10:01:41.544627+0000","last_peered":"2023-10-09T10:01:41.544627+0000","last_clean":"2023-10-09T10:01:41.544627+0000","last_became_active":"2023-10-09T10:00:52.760143+0000","last_became_peered":"2023-10-09T10:00:52.760143+0000","last_unstale":"2023-10-09T10:01:41.544627+0000","last_undegraded":"2023-10-09T10:01:41.544627+0000","last_fullsized":"2023-10-09T10:01:41.544627+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8402,"log_dups_size":0,"ondisk_log_size":8402,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119832576,"num_objects":1829,"num_object_clones":0,"num_object_copies":5487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4705,"num_write_kb":209056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":726,"num_bytes_recovered":47546368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985305+0000","last_change":"2023-10-09T10:01:00.967634+0000","last_active":"2023-10-09T10:01:35.985305+0000","last_peered":"2023-10-09T10:01:35.985305+0000","last_clean":"2023-10-09T10:01:35.985305+0000","last_became_active":"2023-10-09T10:01:00.925301+0000","last_became_peered":"2023-10-09T10:01:00.925301+0000","last_unstale":"2023-10-09T10:01:35.985305+0000","last_undegraded":"2023-10-09T10:01:35.985305+0000","last_fullsized":"2023-10-09T10:01:35.985305+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":91,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.981679+0000","last_change":"2023-10-09T10:01:00.935856+0000","last_active":"2023-10-09T10:01:35.981679+0000","last_peered":"2023-10-09T10:01:35.981679+0000","last_clean":"2023-10-09T10:01:35.981679+0000","last_became_active":"2023-10-09T10:01:00.935517+0000","last_became_peered":"2023-10-09T10:01:00.935517+0000","last_unstale":"2023-10-09T10:01:35.981679+0000","last_undegraded":"2023-10-09T10:01:35.981679+0000","last_fullsized":"2023-10-09T10:01:35.981679+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:13.048073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"150'8706","reported_seq":9467,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:42.194713+0000","last_change":"2023-10-09T10:01:01.555042+0000","last_active":"2023-10-09T10:01:42.194713+0000","last_peered":"2023-10-09T10:01:42.194713+0000","last_clean":"2023-10-09T10:01:42.194713+0000","last_became_active":"2023-10-09T10:00:53.683716+0000","last_became_peered":"2023-10-09T10:00:53.683716+0000","last_unstale":"2023-10-09T10:01:42.194713+0000","last_undegraded":"2023-10-09T10:01:42.194713+0000","last_fullsized":"2023-10-09T10:01:42.194713+0000","mapping_epoch":133,"log_start":"102'400","ondisk_log_start":"102'400","created":133,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8306,"log_dups_size":0,"ondisk_log_size":8306,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121864192,"num_objects":1860,"num_object_clones":0,"num_object_copies":5580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4902,"num_write_kb":216352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":336,"num_bytes_recovered":22020096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985432+0000","last_change":"2023-10-09T10:01:00.942712+0000","last_active":"2023-10-09T10:01:35.985432+0000","last_peered":"2023-10-09T10:01:35.985432+0000","last_clean":"2023-10-09T10:01:35.985432+0000","last_became_active":"2023-10-09T10:01:00.924401+0000","last_became_peered":"2023-10-09T10:01:00.924401+0000","last_unstale":"2023-10-09T10:01:35.985432+0000","last_undegraded":"2023-10-09T10:01:35.985432+0000","last_fullsized":"2023-10-09T10:01:35.985432+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985886+0000","last_change":"2023-10-09T10:01:00.934578+0000","last_active":"2023-10-09T10:01:35.985886+0000","last_peered":"2023-10-09T10:01:35.985886+0000","last_clean":"2023-10-09T10:01:35.985886+0000","last_became_active":"2023-10-09T10:01:00.934237+0000","last_became_peered":"2023-10-09T10:01:00.934237+0000","last_unstale":"2023-10-09T10:01:35.985886+0000","last_undegraded":"2023-10-09T10:01:35.985886+0000","last_fullsized":"2023-10-09T10:01:35.985886+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:36.229511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":142,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.981074+0000","last_change":"2023-10-09T10:00:30.779348+0000","last_active":"2023-10-09T10:01:35.981074+0000","last_peered":"2023-10-09T10:01:35.981074+0000","last_clean":"2023-10-09T10:01:35.981074+0000","last_became_active":"2023-10-09T10:00:30.778963+0000","last_became_peered":"2023-10-09T10:00:30.778963+0000","last_unstale":"2023-10-09T10:01:35.981074+0000","last_undegraded":"2023-10-09T10:01:35.981074+0000","last_fullsized":"2023-10-09T10:01:35.981074+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.112454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":155,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.979723+0000","last_change":"2023-10-09T10:00:30.865002+0000","last_active":"2023-10-09T10:01:35.979723+0000","last_peered":"2023-10-09T10:01:35.979723+0000","last_clean":"2023-10-09T10:01:35.979723+0000","last_became_active":"2023-10-09T10:00:30.864670+0000","last_became_peered":"2023-10-09T10:00:30.864670+0000","last_unstale":"2023-10-09T10:01:35.979723+0000","last_undegraded":"2023-10-09T10:01:35.979723+0000","last_fullsized":"2023-10-09T10:01:35.979723+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:16.562219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":145,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:01:35.985720+0000","last_change":"2023-10-09T10:00:44.129942+0000","last_active":"2023-10-09T10:01:35.985720+0000","last_peered":"2023-10-09T10:01:35.985720+0000","last_clean":"2023-10-09T10:01:35.985720+0000","last_became_active":"2023-10-09T10:00:30.861856+0000","last_became_peered":"2023-10-09T10:00:30.861856+0000","last_unstale":"2023-10-09T10:01:35.985720+0000","last_undegraded":"2023-10-09T10:01:35.985720+0000","last_fullsized":"2023-10-09T10:01:35.985720+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:34.634782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T10:01:37.008322+0000","last_change":"2023-10-09T10:01:36.019526+0000","last_active":"2023-10-09T10:01:37.008322+0000","last_peered":"2023-10-09T10:01:37.008322+0000","last_clean":"2023-10-09T10:01:37.008322+0000","last_became_active":"2023-10-09T10:01:36.019172+0000","last_became_peered":"2023-10-09T10:01:36.019172+0000","last_unstale":"2023-10-09T10:01:37.008322+0000","last_undegraded":"2023-10-09T10:01:37.008322+0000","last_fullsized":"2023-10-09T10:01:37.008322+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:30.046178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":3933601816,"num_objects":60023,"num_object_clones":0,"num_object_copies":180069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6838,"num_objects_misplaced":8658,"num_objects_unfound":0,"num_objects_dirty":60023,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":151015,"num_write_kb":6753153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15284,"num_bytes_recovered":1001390080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12448641024,"data_stored":12448628808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":212052,"ondisk_log_size":212052,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6056,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430384,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1768256,"kb_used_data":1462840,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91976896,"statfs":{"total":95995035648,"available":94184341504,"internally_reserved":0,"allocated":1497948160,"data_stored":1497473847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:01:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.0980000000000001,"15min":0.76200000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.3179999999999996,"5min":4.5519999999999996,"15min":4.5519999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.044,"15min":0.70899999999999996},"min":{"1min":0.28199999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.0990000000000002,"5min":8.0259999999999998,"15min":8.0259999999999998},"last":0.47599999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5699999999999998,"5min":1.0660000000000001,"15min":0.71799999999999997},"min":{"1min":0.27000000000000002,"5min":0.246,"15min":0.246},"max":{"1min":5.9470000000000001,"5min":5.9470000000000001,"15min":5.9470000000000001},"last":1.1459999999999999},{"interface":"front","average":{"1min":2.113,"5min":1.0549999999999999,"15min":0.74099999999999999},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.1879999999999997,"5min":6.1879999999999997,"15min":6.1879999999999997},"last":1.3049999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:01:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":1.0600000000000001,"15min":0.753},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":1.337},{"interface":"front","average":{"1min":2.5169999999999999,"5min":1.2529999999999999,"15min":0.79400000000000004},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.7669999999999995,"5min":9.7669999999999995,"15min":9.7669999999999995},"last":0.70599999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":1.1060000000000001,"15min":0.75800000000000001},"min":{"1min":0.378,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.8140000000000001,"5min":9.8140000000000001,"15min":9.8140000000000001},"last":3.3109999999999999},{"interface":"front","average":{"1min":2.3540000000000001,"5min":1.0880000000000001,"15min":0.754},"min":{"1min":0.34699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":5.8209999999999997,"5min":5.8209999999999997,"15min":5.8209999999999997},"last":6.5110000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:01:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.75700000000000001,"15min":0.55800000000000005},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.024,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.755,"5min":0.67400000000000004,"15min":0.54100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.26500000000000001},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":0.36899999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:01:36 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.93700000000000006,"15min":0.64400000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9009999999999998,"5min":12.958,"15min":12.958},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.532,"5min":0.91600000000000004,"15min":0.65200000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.6120000000000001,"5min":6.6120000000000001,"15min":6.6120000000000001},"last":0.57799999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:01:36 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.77900000000000003,"15min":0.621},"min":{"1min":0.35199999999999998,"5min":0.34300000000000003,"15min":0.32900000000000001},"max":{"1min":4.1340000000000003,"5min":5.4630000000000001,"15min":5.4630000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.751,"15min":0.60999999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33100000000000002},"max":{"1min":4.2469999999999999,"5min":4.407,"15min":4.407},"last":0.54100000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626612,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1729588,"kb_used_data":1428268,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92015564,"statfs":{"total":95995035648,"available":94223937536,"internally_reserved":0,"allocated":1462546432,"data_stored":1462083980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:01:21 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.052,"15min":0.72099999999999997},"min":{"1min":0.23799999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.609,"5min":8.8650000000000002,"15min":8.8650000000000002},"last":0.23599999999999999},{"interface":"front","average":{"1min":1.29,"5min":1.0309999999999999,"15min":0.68100000000000005},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.3499999999999996,"5min":12.439,"15min":12.439},"last":2.464}]},{"osd":1,"last update":"Mon Oct 9 10:00:49 2023","interfaces":[{"interface":"back","average":{"1min":3.6669999999999998,"5min":1.292,"15min":0.79700000000000004},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":18.853000000000002,"5min":18.853000000000002,"15min":18.853000000000002},"last":5.734},{"interface":"front","average":{"1min":2.2549999999999999,"5min":0.96899999999999997,"15min":0.67900000000000005},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":12.380000000000001,"5min":12.380000000000001,"15min":12.380000000000001},"last":0.57199999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:01:21 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":1.395,"15min":0.82499999999999996},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.83,"5min":18.800000000000001,"15min":18.800000000000001},"last":1.681},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.304,"15min":0.82599999999999996},"min":{"1min":0.308,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0960000000000001,"5min":12.413,"15min":12.413},"last":0.52400000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:00:54 2023","interfaces":[{"interface":"back","average":{"1min":3.0369999999999999,"5min":1.2669999999999999,"15min":0.83299999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":12.337,"5min":12.337,"15min":12.337},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.8319999999999999,"5min":1.1719999999999999,"15min":0.79000000000000004},"min":{"1min":0.39600000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":7.0110000000000001,"5min":7.0110000000000001,"15min":7.0110000000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:01:16 2023","interfaces":[{"interface":"back","average":{"1min":3.4430000000000001,"5min":1.214,"15min":0.70699999999999996},"min":{"1min":0.27600000000000002,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":23.446000000000002,"5min":23.446000000000002,"15min":23.446000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.6220000000000001,"5min":0.88400000000000001,"15min":0.622},"min":{"1min":0.308,"5min":0.247,"15min":0.247},"max":{"1min":6.984,"5min":8.2379999999999995,"15min":8.2379999999999995},"last":0.71999999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:01:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":0.81100000000000005,"15min":0.60399999999999998},"min":{"1min":0.32500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":14.09,"5min":14.09,"15min":14.09},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.137,"5min":0.75,"15min":0.58899999999999997},"min":{"1min":0.35299999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.3540000000000001,"5min":6.3540000000000001,"15min":6.3540000000000001},"last":0.38400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.82999999999999996,"15min":0.626},"min":{"1min":0.33900000000000002,"5min":0.315,"15min":0.315},"max":{"1min":5.9089999999999998,"5min":5.9089999999999998,"15min":5.9089999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.89000000000000001,"15min":0.68799999999999994},"min":{"1min":0.39900000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0860000000000003,"5min":6.0860000000000003,"15min":6.0860000000000003},"last":0.46100000000000002}]}]},{"osd":1,"up_from":84,"seq":360777252925,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1347748,"kb_used_data":1262044,"kb_used_omap":9,"kb_used_meta":85686,"kb_avail":92397404,"statfs":{"total":95995035648,"available":94614941696,"internally_reserved":0,"allocated":1292333056,"data_stored":1291707146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":87743465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.0609999999999999,"15min":0.58799999999999997},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":1.2589999999999999},{"interface":"front","average":{"1min":2.601,"5min":1.04,"15min":0.61199999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.805999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.67000000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:01:26 2023","interfaces":[{"interface":"back","average":{"1min":3.669,"5min":1.615,"15min":0.79700000000000004},"min":{"1min":0.33700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":18.109000000000002,"5min":18.109000000000002,"15min":18.109000000000002},"last":0.376},{"interface":"front","average":{"1min":2.794,"5min":1.506,"15min":0.76200000000000001},"min":{"1min":0.313,"5min":0.248,"15min":0.248},"max":{"1min":9.4760000000000009,"5min":12.069000000000001,"15min":12.069000000000001},"last":0.45600000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0819999999999999,"5min":1.7849999999999999,"15min":0.84699999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":25.129999999999999,"5min":25.129999999999999,"15min":25.129999999999999},"last":2.2469999999999999},{"interface":"front","average":{"1min":5.1189999999999998,"5min":1.645,"15min":0.81100000000000005},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":22.847999999999999,"5min":22.847999999999999,"15min":22.847999999999999},"last":2.1080000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.899,"5min":1.3140000000000001,"15min":0.84499999999999997},"min":{"1min":0.32700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":7.6529999999999996,"5min":9.5139999999999993,"15min":9.5139999999999993},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.625,"5min":1.0509999999999999,"15min":0.77800000000000002},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.2779999999999996,"5min":6.2539999999999996,"15min":6.2539999999999996},"last":1.232}]},{"osd":5,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":1.256,"15min":0.83799999999999997},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":12.022,"5min":12.022,"15min":12.022},"last":0.41399999999999998},{"interface":"front","average":{"1min":3.6699999999999999,"5min":1.387,"15min":0.90100000000000002},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":19.706,"5min":19.706,"15min":19.706},"last":2.0699999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":1.2869999999999999,"15min":0.88700000000000001},"min":{"1min":0.39000000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.4760000000000009,"5min":8.4760000000000009,"15min":8.4760000000000009},"last":1.536},{"interface":"front","average":{"1min":3.2669999999999999,"5min":1.3200000000000001,"15min":0.89000000000000001},"min":{"1min":0.45500000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.057,"5min":12.057,"15min":12.057},"last":1.548}]},{"osd":7,"last update":"Mon Oct 9 10:00:58 2023","interfaces":[{"interface":"back","average":{"1min":3.613,"5min":1.492,"15min":0.92500000000000004},"min":{"1min":0.40799999999999997,"5min":0.379,"15min":0.379},"max":{"1min":12.766999999999999,"5min":12.766999999999999,"15min":12.766999999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.222,"15min":0.86399999999999999},"min":{"1min":0.44900000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.4290000000000003,"5min":7.4290000000000003,"15min":7.4290000000000003},"last":1.6299999999999999}]}]},{"osd":0,"up_from":79,"seq":339302416452,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1015720,"kb_used_data":952032,"kb_used_omap":28,"kb_used_meta":63587,"kb_avail":92729432,"statfs":{"total":95995035648,"available":94954938368,"internally_reserved":0,"allocated":974880768,"data_stored":974250429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":65113936},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":1.056,"15min":0.60199999999999998},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.191},"max":{"1min":10.202,"5min":10.202,"15min":10.202},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.25,"5min":1.042,"15min":0.61199999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.1219999999999999,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":1.2969999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:01:21 2023","interfaces":[{"interface":"back","average":{"1min":3.6099999999999999,"5min":1.476,"15min":0.754},"min":{"1min":0.224,"5min":0.216,"15min":0.216},"max":{"1min":11.787000000000001,"5min":11.787000000000001,"15min":11.787000000000001},"last":2.2789999999999999},{"interface":"front","average":{"1min":2.3079999999999998,"5min":1.4830000000000001,"15min":0.77800000000000002},"min":{"1min":0.28399999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.2930000000000001,"5min":17.920999999999999,"15min":17.920999999999999},"last":2.3839999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3730000000000002,"5min":1.206,"15min":0.65600000000000003},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.1029999999999998,"5min":9.9380000000000006,"15min":9.9380000000000006},"last":4.6689999999999996},{"interface":"front","average":{"1min":2.7810000000000001,"5min":1.373,"15min":0.751},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.0800000000000001,"5min":8.0800000000000001,"15min":8.0800000000000001},"last":2.8410000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:01:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":1.2190000000000001,"15min":0.84499999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":11.587999999999999,"5min":11.587999999999999,"15min":11.587999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.268,"15min":0.874},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":5.2530000000000001,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":1.339}]},{"osd":5,"last update":"Mon Oct 9 10:01:21 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.1950000000000001,"15min":0.82699999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.4160000000000004,"5min":7.4160000000000004,"15min":7.4160000000000004},"last":9.0329999999999995},{"interface":"front","average":{"1min":2.427,"5min":1.3089999999999999,"15min":0.88},"min":{"1min":0.47699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":11.667,"5min":11.667,"15min":11.667},"last":5.4329999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:01:21 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":1.5109999999999999,"15min":0.96299999999999997},"min":{"1min":0.437,"5min":0.379,"15min":0.379},"max":{"1min":11.699,"5min":11.699,"15min":11.699},"last":5.4829999999999997},{"interface":"front","average":{"1min":2.4449999999999998,"5min":1.494,"15min":0.95899999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":14.536,"5min":14.536,"15min":14.536},"last":3.9049999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:01:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":1.28,"15min":0.871},"min":{"1min":0.45000000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.2910000000000004,"5min":7.3179999999999996,"15min":7.3179999999999996},"last":26.978000000000002},{"interface":"front","average":{"1min":2.5790000000000002,"5min":1.4650000000000001,"15min":0.94299999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":11.757,"5min":11.757,"15min":11.757},"last":26.768000000000001}]}]},{"osd":2,"up_from":89,"seq":382252089399,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2014000,"kb_used_data":1869864,"kb_used_omap":81,"kb_used_meta":143982,"kb_avail":91731152,"statfs":{"total":95995035648,"available":93932699648,"internally_reserved":0,"allocated":1914740736,"data_stored":1914106961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":147437786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:01:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.494,"15min":0.754},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":6.8680000000000003,"5min":56.883000000000003,"15min":56.883000000000003},"last":1.661},{"interface":"front","average":{"1min":2.8159999999999998,"5min":1.8140000000000001,"15min":0.871},"min":{"1min":0.313,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":12.278,"5min":57.595999999999997,"15min":57.595999999999997},"last":6.8730000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:01:23 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.3939999999999999,"15min":0.70899999999999996},"min":{"1min":0.248,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.3559999999999999,"5min":12.035,"15min":12.035},"last":1.0449999999999999},{"interface":"front","average":{"1min":2.7810000000000001,"5min":1.3,"15min":0.70699999999999996},"min":{"1min":0.29399999999999998,"5min":0.224,"15min":0.224},"max":{"1min":15.936999999999999,"5min":15.936999999999999,"15min":15.936999999999999},"last":11.369999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:00:53 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.635,"15min":0.82399999999999995},"min":{"1min":0.33000000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8410000000000002,"5min":58.067,"15min":58.067},"last":1.3799999999999999},{"interface":"front","average":{"1min":2.6120000000000001,"5min":1.8360000000000001,"15min":0.88},"min":{"1min":0.36899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.4829999999999997,"5min":60.456000000000003,"15min":60.456000000000003},"last":7.2610000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:01:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":1.619,"15min":0.94199999999999995},"min":{"1min":0.371,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.090999999999999,"5min":57.646999999999998,"15min":57.646999999999998},"last":4.3140000000000001},{"interface":"front","average":{"1min":2.286,"5min":1.885,"15min":1.048},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.2880000000000003,"5min":57.024000000000001,"15min":57.024000000000001},"last":0.83299999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:01:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":2.431,"15min":1.2270000000000001},"min":{"1min":0.34000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.4779999999999998,"5min":79.016000000000005,"15min":79.016000000000005},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.9159999999999999,"5min":2.4910000000000001,"15min":1.25},"min":{"1min":0.44800000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.138999999999999,"5min":79.302999999999997,"15min":79.302999999999997},"last":4.3789999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:01:23 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.7809999999999999,"15min":1.0029999999999999},"min":{"1min":0.46300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.4189999999999996,"5min":57.698,"15min":57.698},"last":4.3520000000000003},{"interface":"front","average":{"1min":2.375,"5min":1.917,"15min":1.0609999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":8.3290000000000006,"5min":57.058,"15min":57.058},"last":0.86799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:01:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":1.591,"15min":0.96699999999999997},"min":{"1min":0.39200000000000002,"5min":0.375,"15min":0.375},"max":{"1min":4.1609999999999996,"5min":57.003,"15min":57.003},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.649,"15min":0.98999999999999999},"min":{"1min":0.51400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9100000000000001,"5min":57.676000000000002,"15min":57.676000000000002},"last":4.4020000000000001}]}]},{"osd":3,"up_from":94,"seq":403726925872,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2533164,"kb_used_data":2348964,"kb_used_omap":7,"kb_used_meta":184120,"kb_avail":91211988,"statfs":{"total":95995035648,"available":93401075712,"internally_reserved":0,"allocated":2405339136,"data_stored":2404705747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":188539496},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":1.5580000000000001,"15min":1.1819999999999999},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.798999999999999,"5min":11.958,"15min":11.958},"last":3.2570000000000001},{"interface":"front","average":{"1min":3.1829999999999998,"5min":1.397,"15min":0.76600000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.704000000000001,"5min":10.704000000000001,"15min":10.704000000000001},"last":3.1230000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.3400000000000001,"15min":0.72599999999999998},"min":{"1min":0.249,"5min":0.189,"15min":0.189},"max":{"1min":9.3670000000000009,"5min":9.3670000000000009,"15min":9.3670000000000009},"last":2.7050000000000001},{"interface":"front","average":{"1min":2.7109999999999999,"5min":1.3160000000000001,"15min":0.71499999999999997},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.839,"5min":10.839,"15min":10.839},"last":2.8450000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.667,"15min":1.2529999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.632999999999999,"5min":15.632999999999999,"15min":15.632999999999999},"last":4.2759999999999998},{"interface":"front","average":{"1min":2.169,"5min":1.304,"15min":0.73899999999999999},"min":{"1min":0.29599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.4669999999999996,"5min":11.496,"15min":11.496},"last":2.8199999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.0760000000000001,"15min":0.78900000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.6479999999999997,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":2.7490000000000001},{"interface":"front","average":{"1min":2.8319999999999999,"5min":1.3320000000000001,"15min":0.90200000000000002},"min":{"1min":0.38900000000000001,"5min":0.247,"15min":0.247},"max":{"1min":7.9740000000000002,"5min":7.9740000000000002,"15min":7.9740000000000002},"last":0.38600000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":1.3680000000000001,"15min":0.91500000000000004},"min":{"1min":0.48399999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.135999999999999,"5min":10.135999999999999,"15min":10.135999999999999},"last":0.314},{"interface":"front","average":{"1min":2.6680000000000001,"5min":1.5349999999999999,"15min":1.2130000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.127000000000001,"5min":10.127000000000001,"15min":10.127000000000001},"last":0.55600000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":3.7789999999999999,"5min":1.536,"15min":0.96799999999999997},"min":{"1min":0.45800000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.846,"5min":12.846,"15min":12.846},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.7679999999999998,"5min":1.5009999999999999,"15min":1.2050000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":9.5050000000000008,"5min":9.5050000000000008,"15min":9.5050000000000008},"last":0.46100000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.494,"15min":1.194},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.4719999999999995,"5min":9.4719999999999995,"15min":9.4719999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.8129999999999999,"5min":1.258,"15min":0.85599999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":9.6039999999999992,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":2.7890000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019068,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618822840,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1780332,"kb_used_data":1470820,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91964820,"statfs":{"total":95995035648,"available":94171975680,"internally_reserved":0,"allocated":1506119680,"data_stored":1505661001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:01:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":1.1399999999999999,"15min":0.73799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":5.242,"5min":13.112,"15min":13.112},"last":15.773999999999999},{"interface":"front","average":{"1min":2.0710000000000002,"5min":1.256,"15min":0.77600000000000002},"min":{"1min":0.29799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":12.259,"5min":12.259,"15min":12.259},"last":15.699}]},{"osd":1,"last update":"Mon Oct 9 10:00:51 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":1.226,"15min":0.76900000000000002},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.248},"max":{"1min":10.308,"5min":10.308,"15min":10.308},"last":15.689},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.1359999999999999,"15min":0.74199999999999999},"min":{"1min":0.40500000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.970000000000001,"5min":12.970000000000001,"15min":12.970000000000001},"last":15.896000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":1.292,"15min":0.89500000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3280000000000003,"5min":12.582000000000001,"15min":12.582000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.3370000000000002,"5min":1.3560000000000001,"15min":0.92100000000000004},"min":{"1min":0.32600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.9260000000000002,"5min":12.438000000000001,"15min":12.438000000000001},"last":0.307}]},{"osd":3,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":1.2889999999999999,"15min":0.88300000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.254,"5min":10.254,"15min":10.254},"last":39.345999999999997},{"interface":"front","average":{"1min":3.3039999999999998,"5min":1.357,"15min":0.88300000000000001},"min":{"1min":0.33300000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":12.332000000000001,"5min":12.332000000000001,"15min":12.332000000000001},"last":38.906999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:00:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":0.81899999999999995,"15min":0.57199999999999995},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":12.304,"5min":12.304,"15min":12.304},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.8780000000000001,"5min":1.0880000000000001,"15min":0.67800000000000005},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":27.768000000000001,"5min":27.768000000000001,"15min":27.768000000000001},"last":0.33100000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:01:20 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.1100000000000001,"15min":0.70599999999999996},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":6.157,"5min":27.716000000000001,"15min":27.716000000000001},"last":0.37},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.79400000000000004,"15min":0.61299999999999999},"min":{"1min":0.313,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.88500000000000001,"5min":10.173999999999999,"15min":10.173999999999999},"last":0.42099999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:01:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.96999999999999997,"15min":0.69599999999999995},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.309},"max":{"1min":3.2509999999999999,"5min":10.334,"15min":10.334},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.1180000000000001,"15min":0.73099999999999998},"min":{"1min":0.34200000000000003,"5min":0.33200000000000002,"15min":0.317},"max":{"1min":9.6430000000000007,"5min":12.356,"15min":12.356},"last":0.35799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":964366336,"data_stored":964362264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1272385536,"data_stored":1272381464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1910636544,"data_stored":1910636544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2403074048,"data_stored":2403074048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1467088896,"data_stored":1467088896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1492324352,"data_stored":1492320280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1448083456,"data_stored":1448083456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1490681856,"data_stored":1490681856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:01:44.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 26 2023-10-09T10:01:44.948 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:44.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:44.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:45.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:45.107 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:45.156 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:45.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:45.261 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:45.308 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:01:45.308 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:01:45.308 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:01:45.308 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:01:45.308 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:01:45.316 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:45.415 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:45.697 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:45.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:45.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:45.718 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:01:45.739 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:01:45.795 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:45 smithi110 ceph-mon[136032]: from='client.44954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:45.795 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/125058985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:45.795 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:45 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:45.805 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:45.870 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:45.951 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:45.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:45.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:46.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:46.124 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:45 smithi012 ceph-mon[152263]: from='client.44954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:46.124 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/125058985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:46.124 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:45 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:46.126 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:45 smithi012 ceph-mon[147926]: from='client.44954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:01:46.126 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/125058985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:46.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:45 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:01:46.129 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:46.551 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:46.622 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:46.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:46.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:46.729 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:46.796 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[147926]: pgmap v371: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 913 op/s; 6838/173805 objects degraded (3.934%); 8542/173805 objects misplaced (4.915%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[147926]: osdmap e151: 8 total, 8 up, 7 in 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[147926]: failed to encode map e151 with expected crc 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[147926]: failed to encode map e151 with expected crc 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[147926]: failed to encode map e151 with expected crc 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[147926]: failed to encode map e151 with expected crc 2023-10-09T10:01:46.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[152263]: pgmap v371: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 913 op/s; 6838/173805 objects degraded (3.934%); 8542/173805 objects misplaced (4.915%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[152263]: osdmap e151: 8 total, 8 up, 7 in 2023-10-09T10:01:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[152263]: failed to encode map e151 with expected crc 2023-10-09T10:01:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[152263]: failed to encode map e151 with expected crc 2023-10-09T10:01:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[152263]: failed to encode map e151 with expected crc 2023-10-09T10:01:46.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:46 smithi012 ceph-mon[152263]: failed to encode map e151 with expected crc 2023-10-09T10:01:47.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:46 smithi110 ceph-mon[136032]: pgmap v371: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 913 op/s; 6838/173805 objects degraded (3.934%); 8542/173805 objects misplaced (4.915%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:47.027 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:46 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:47.027 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:46 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:01:47.027 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:46 smithi110 ceph-mon[136032]: osdmap e151: 8 total, 8 up, 7 in 2023-10-09T10:01:47.027 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:46 smithi110 ceph-mon[136032]: failed to encode map e151 with expected crc 2023-10-09T10:01:47.027 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:46 smithi110 ceph-mon[136032]: failed to encode map e151 with expected crc 2023-10-09T10:01:47.027 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:46 smithi110 ceph-mon[136032]: failed to encode map e151 with expected crc 2023-10-09T10:01:47.027 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:46 smithi110 ceph-mon[136032]: failed to encode map e151 with expected crc 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13899 6.c e103/97 6:39dbd6a1:::benchmark_data_smithi012_182576_object13898:head v 103'862, mlcod=103'860)", 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 162.424414617, 2023-10-09T10:01:47.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109510709, 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:01:47.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545772+0000", 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4119999999999999e-05 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545806+0000", 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4049000000000001e-05 2023-10-09T10:01:47.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545820+0000", 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1470000000000004e-06 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545827+0000", 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5004000000000002e-05 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545902+0000", 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3674999999999999e-05 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545926+0000", 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109217777 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655143+0000", 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014009299999999999 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655284+0000", 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109510709 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13895 6.3 e103/97 6:cb4c122f:::benchmark_data_smithi012_182576_object13894:head v 103'855, mlcod=103'853)", 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 162.42388584400001, 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108952985, 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:01:47.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546301+0000", 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2656000000000001e-05 2023-10-09T10:01:47.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546324+0000", 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6180000000000001e-05 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546340+0000", 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9619999999999998e-06 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546346+0000", 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5816000000000003e-05 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546391+0000", 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0018e-05 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546421+0000", 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108744009 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655165+0000", 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9089e-05 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655255+0000", 2023-10-09T10:01:47.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108952985 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13902 6.5 e103/97 6:a6447830:::benchmark_data_smithi012_182576_object13901:head v 103'874, mlcod=103'872)", 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 162.42382748099999, 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110437017, 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:47.985 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:47.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0539000000000001e-05 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546380+0000", 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8390000000000001e-06 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546387+0000", 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2400000000000001e-06 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546391+0000", 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1564999999999998e-05 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546443+0000", 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1464999999999999e-05 2023-10-09T10:01:47.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546464+0000", 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110295141 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656759+0000", 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7586999999999998e-05 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656797+0000", 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110437017 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13914 6.a e103/97 6:527234fb:::benchmark_data_smithi012_182576_object13913:head v 103'853, mlcod=103'851)", 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 162.41906258399999, 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.105616578, 2023-10-09T10:01:47.990 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:01:47.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551124+0000", 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4997000000000001e-05 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551149+0000", 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104127211 2023-10-09T10:01:47.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655276+0000", 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4699999999999999e-06 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655283+0000", 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9079000000000003e-05 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655322+0000", 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1195000000000002e-05 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655353+0000", 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013534339999999999 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656706+0000", 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5194000000000002e-05 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656741+0000", 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.105616578 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:47.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13926 6.1 e103/97 6:8de8341f:::benchmark_data_smithi012_182576_object13925:head v 103'823, mlcod=103'821)", 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 162.40636058699999, 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092614976000000002, 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:47.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563825+0000", 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014807399999999999 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:47.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563974+0000", 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.091111844999999997 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655085+0000", 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0784000000000002e-05 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655116+0000", 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017427 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655290+0000", 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4281999999999999e-05 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655315+0000", 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010930600000000001 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656408+0000", 2023-10-09T10:01:47.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4076999999999999e-05 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656442+0000", 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092614976000000002 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13929 6.c e103/97 6:39e98511:::benchmark_data_smithi012_182576_object13928:head v 103'863, mlcod=103'860)", 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:01:48.000 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 162.40620284799999, 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092504991999999994, 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563984+0000", 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2965000000000004e-05 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564047+0000", 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.091099323999999995 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655146+0000", 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0924000000000002e-05 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655167+0000", 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016929299999999999 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655336+0000", 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0599999999999999e-05 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655357+0000", 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011074889999999999 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656464+0000", 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5162999999999999e-05 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656490+0000", 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092504991999999994 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96028 6.f e118/117 6:f25f2574:::benchmark_data_smithi012_182576_object96027:head v 118'5972, mlcod=0'0)", 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 95.810838837999995, 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041325972000000002, 2023-10-09T10:01:48.005 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159348+0000", 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0521e-05 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159388+0000", 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9960000000000005e-06 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159398+0000", 2023-10-09T10:01:48.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.981e-06 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159404+0000", 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0812e-05 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159465+0000", 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2714999999999999e-05 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159488+0000", 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041139283999999998 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.200627+0000", 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7336000000000002e-05 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.200675+0000", 2023-10-09T10:01:48.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041325972000000002 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96906 6.8 e119/117 6:1349a12f:::benchmark_data_smithi012_182576_object96905:head v 119'6178, mlcod=0'0)", 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 94.406541040999997, 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043713813999999997, 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.010 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4641e-05 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563671+0000", 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3489999999999997e-06 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563677+0000", 2023-10-09T10:01:48.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0620000000000002e-06 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563681+0000", 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024566689999999999 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.566138+0000", 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8526000000000001e-05 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.566166+0000", 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041156198999999997 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.607322+0000", 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7769999999999999e-05 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.607360+0000", 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043713813999999997 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96908 6.8 e119/117 6:16fa2f2f:::benchmark_data_smithi012_182576_object96907:head v 119'6179, mlcod=0'0)", 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 94.406495871999994, 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052663933000000003, 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.015 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563691+0000", 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3872000000000001e-05 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563715+0000", 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0090000000000004e-06 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563721+0000", 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9999999999999998e-06 2023-10-09T10:01:48.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563725+0000", 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010034111 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.573759+0000", 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.565e-05 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.573785+0000", 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042522310000000001 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616307+0000", 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8340000000000001e-05 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616356+0000", 2023-10-09T10:01:48.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052663933000000003 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96912 6.8 e119/117 6:11471bcc:::benchmark_data_smithi012_182576_object96911:head v 119'6180, mlcod=0'0)", 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 94.399319715000004, 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04550949, 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570867+0000", 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5475000000000002e-05 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570903+0000", 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1657999999999999e-05 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570914+0000", 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8560000000000001e-06 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570921+0000", 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027926853000000001 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.598848+0000", 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8767000000000001e-05 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.598877+0000", 2023-10-09T10:01:48.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017483862999999999 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616361+0000", 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6659000000000001e-05 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616377+0000", 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04550949 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.024 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96916 6.8 e119/117 6:11cbd0a6:::benchmark_data_smithi012_182576_object96915:head v 119'6181, mlcod=0'0)", 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 94.374795785000003, 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042488638000000002, 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595390+0000", 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6316e-05 2023-10-09T10:01:48.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595416+0000", 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.643e-05 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595433+0000", 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0700000000000007e-06 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595441+0000", 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013293442000000001 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608734+0000", 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4462000000000003e-05 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608769+0000", 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029077441999999998 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637846+0000", 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3998999999999998e-05 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637880+0000", 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042488638000000002 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96919 6.8 e119/117 6:16cabaa4:::benchmark_data_smithi012_182576_object96918:head v 119'6182, mlcod=0'0)", 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 94.374738808999993, 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042453923999999997, 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:01:48.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595448+0000", 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9534e-05 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595468+0000", 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1179999999999997e-06 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595474+0000", 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2780000000000004e-06 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595478+0000", 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020955610999999999 2023-10-09T10:01:48.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616434+0000", 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5219000000000003e-05 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616469+0000", 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021415979000000002 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637885+0000", 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7509000000000001e-05 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637903+0000", 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042453923999999997 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:98780 6.2 e119/117 6:46aa6df3:::benchmark_data_smithi012_182576_object98779:head v 119'6253, mlcod=0'0)", 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.292535822999994, 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053554971999999999, 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677650+0000", 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0443e-05 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677681+0000", 2023-10-09T10:01:48.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8221999999999999e-05 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677699+0000", 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1400000000000002e-06 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677706+0000", 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070735980000000004 2023-10-09T10:01:48.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.684780+0000", 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4288000000000001e-05 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.684814+0000", 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046346799000000001 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731161+0000", 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5666e-05 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731207+0000", 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053554971999999999 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:98784 6.2 e119/117 6:44e3e91a:::benchmark_data_smithi012_182576_object98783:head v 119'6254, mlcod=0'0)", 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.291179967000005, 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052221194999999998, 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:01:48.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6214e-05 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679033+0000", 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6022999999999999e-05 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679049+0000", 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7379999999999997e-06 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679056+0000", 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013390713 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.692446+0000", 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8688e-05 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.692475+0000", 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038735865000000001 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731211+0000", 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.783e-05 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731229+0000", 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052221194999999998 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103547 6.8 e122/121 6:1259b0ac:::benchmark_data_smithi012_182576_object103546:head v 122'6626, mlcod=0'0)", 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.487065254000001, 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040879891000000002, 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483121+0000", 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105214 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483226+0000", 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9423999999999999e-05 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483246+0000", 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4619999999999999e-06 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483253+0000", 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038037510000000002 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.487057+0000", 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.73e-05 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.487084+0000", 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036875568999999997 2023-10-09T10:01:48.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.523960+0000", 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2120000000000003e-05 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.524002+0000", 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040879891000000002 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103568 6.f e122/117 6:f1ce72d6:::benchmark_data_smithi012_182576_object103567:head v 122'6429, mlcod=0'0)", 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.450398088, 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047394631, 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:01:48.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7826e-05 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519827+0000", 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.409e-06 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519833+0000", 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1130000000000003e-06 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519837+0000", 2023-10-09T10:01:48.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.008335238 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.528173+0000", 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7257e-05 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.528200+0000", 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038942002000000003 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.567142+0000", 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2160999999999998e-05 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.567184+0000", 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047394631 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103834 6.2 e122/117 6:481c5bb1:::benchmark_data_smithi012_182576_object103833:head v 122'6597, mlcod=0'0)", 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 81.016686023999995, 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040357787999999999, 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:01:48.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953500+0000", 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8389999999999997e-05 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953538+0000", 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6200000000000001e-05 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953554+0000", 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.303e-06 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953563+0000", 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011028718999999999 2023-10-09T10:01:48.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.964591+0000", 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2540000000000001e-05 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.964614+0000", 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029196083000000001 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.993810+0000", 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9212999999999998e-05 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.993859+0000", 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040357787999999999 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:104205 6.f e122/117 6:f3405025:::benchmark_data_smithi012_182576_object104204:head v 122'6474, mlcod=0'0)", 2023-10-09T10:01:48.059 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 80.390956387000003, 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048731662000000002, 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1229999999999997e-05 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579262+0000", 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1770000000000001e-05 2023-10-09T10:01:48.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579274+0000", 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8019999999999999e-06 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579278+0000", 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079426949999999996 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.587221+0000", 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8532000000000001e-05 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.587250+0000", 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040668007999999999 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.627918+0000", 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5040999999999998e-05 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.627963+0000", 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048731662000000002 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.064 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:105353 6.2 e123/117 6:4730719a:::benchmark_data_smithi012_182576_object105352:head v 123'6695, mlcod=0'0)", 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 78.449606549999999, 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041308386000000002, 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1875e-05 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520602+0000", 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4280000000000001e-06 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520609+0000", 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0799999999999999e-06 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520613+0000", 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078419579999999996 2023-10-09T10:01:48.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.528455+0000", 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0639999999999998e-05 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.528486+0000", 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033364444 2023-10-09T10:01:48.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.561850+0000", 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9289999999999998e-05 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.561889+0000", 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041308386000000002 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.069 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(6.2 144/138 [PushOp(6:4011f362:::benchmark_data_smithi012_182576_object40319:head, version: 105'2578, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:4011f362:::benchmark_data_smithi012_182576_object40319:head@105'2578, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 21.003712434000001, 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048028721000000003, 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966474+0000", 2023-10-09T10:01:48.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8900000000000002e-05 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966533+0000", 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7698000000000001e-05 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966561+0000", 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2663000000000001e-05 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966573+0000", 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.039901351000000002 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.006475+0000", 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4449999999999996e-06 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.006483+0000", 2023-10-09T10:01:48.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080206500000000007 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.014504+0000", 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048028721000000003 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:01:48.074 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:01:48.075 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:48.075 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:47 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/173805 objects degraded (3.934%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:48.075 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:47 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/173805 objects degraded (3.934%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:48.076 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:01:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:01:47] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T10:01:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:48.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:48.077 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:47 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/173805 objects degraded (3.934%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:48.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:48.280 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:48.350 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:48.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:48.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:48.439 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:01:48.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:48.472 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:48.546 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:48.867 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:48.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:48.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:48.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:48.985 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:48 smithi012 ceph-mon[152263]: pgmap v373: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 6838/172284 objects degraded (3.969%); 8542/172284 objects misplaced (4.958%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:01:48.986 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:48 smithi012 ceph-mon[147926]: pgmap v373: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 6838/172284 objects degraded (3.969%); 8542/172284 objects misplaced (4.958%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:01:49.015 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:48 smithi110 ceph-mon[136032]: pgmap v373: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 6838/172284 objects degraded (3.969%); 8542/172284 objects misplaced (4.958%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:01:49.023 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:49.092 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:49.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:49.199 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:49.226 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:49.264 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:49.335 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:49.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:49.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:49.441 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:49.485 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:49.554 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:49.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:49.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:49.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:49.748 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/33923803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:49.769 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:49.770 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:01:49.790 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:01:49.829 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:49.832 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:01:49.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:49.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:49.867 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:01:49.910 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:01:49.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/33923803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:49.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/33923803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:01:49.942 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:49.951 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:01:49.982 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:49.992 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:01:50.043 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:01:50.045 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:50.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:50.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:50.062 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:01:50.062 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:01:50.063 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:01:50.063 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:01:50.063 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:01:50.063 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:01:50.063 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:01:50.063 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:01:50.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:50.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:01:50.215 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:50.272 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:50.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:50.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:50.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:50.488 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:50.577 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:50.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:50.610 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:01:50.610 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:01:50.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:50.757 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:50.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:50 smithi012 ceph-mon[147926]: pgmap v374: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 6838/164997 objects degraded (4.144%); 7803/164997 objects misplaced (4.729%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T10:01:50.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3081047243' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:50.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:50 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:50.773 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:50 smithi012 ceph-mon[152263]: pgmap v374: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 6838/164997 objects degraded (4.144%); 7803/164997 objects misplaced (4.729%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T10:01:50.773 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3081047243' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:50.773 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:50 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:50.799 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 6 pgp_num to 26 2023-10-09T10:01:50.919 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:50 smithi110 ceph-mon[136032]: pgmap v374: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 6838/164997 objects degraded (4.144%); 7803/164997 objects misplaced (4.729%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T10:01:50.919 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3081047243' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:50.919 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:50 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:01:50.940 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:50.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:51.030 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:01:51.047 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:51.127 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:51.168 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:51.200 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:51.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:51.307 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:51.384 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:51.473 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:51.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:51.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:51.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:01:51.656 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:51.729 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:01:51.732 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:51.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:51.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:01:51.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:51.920 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:51.995 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:52.002 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:01:52.003 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: osdmap e152: 8 total, 8 up, 7 in 2023-10-09T10:01:52.003 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.003 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.003 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.003 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.003 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.003 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.003 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[152263]: pgmap v376: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 6838/161067 objects degraded (4.245%); 7686/161067 objects misplaced (4.772%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:01:52.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:52.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:52.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:01:52.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: osdmap e152: 8 total, 8 up, 7 in 2023-10-09T10:01:52.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:51 smithi012 ceph-mon[147926]: pgmap v376: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 6838/161067 objects degraded (4.245%); 7686/161067 objects misplaced (4.772%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: osdmap e152: 8 total, 8 up, 7 in 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: failed to encode map e152 with expected crc 2023-10-09T10:01:52.033 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:51 smithi110 ceph-mon[136032]: pgmap v376: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 6838/161067 objects degraded (4.245%); 7686/161067 objects misplaced (4.772%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:01:52.105 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:52.156 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:52.216 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:52.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:52.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:52.393 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:52.478 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:52.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:52.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:52.550 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:01:52.596 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:52.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:52.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:52.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:52.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:52.871 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:52 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/161067 objects degraded (4.245%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:52.871 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:01:52.921 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:52.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:52 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/161067 objects degraded (4.245%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:52.934 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:52 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/161067 objects degraded (4.245%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:53.024 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:01:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:53.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:01:53.126 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:01:53.127 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:01:53.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:53.182 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:53.350 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:53.450 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:53.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:53.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:53.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:01:53.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:53.647 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:53.815 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2820796360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:53.815 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:53 smithi110 ceph-mon[136032]: pgmap v377: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 6838/154575 objects degraded (4.424%); 6877/154575 objects misplaced (4.449%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:53.838 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:01:53.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2820796360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:53.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:53 smithi012 ceph-mon[147926]: pgmap v377: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 6838/154575 objects degraded (4.424%); 6877/154575 objects misplaced (4.449%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:53.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2820796360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:01:53.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:53 smithi012 ceph-mon[152263]: pgmap v377: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 6838/154575 objects degraded (4.424%); 6877/154575 objects misplaced (4.449%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:01:54.337 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:01:54.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:54 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:54.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:54 smithi110 ceph-mon[136032]: failed to encode map e153 with expected crc 2023-10-09T10:01:54.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:54 smithi110 ceph-mon[136032]: failed to encode map e153 with expected crc 2023-10-09T10:01:54.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:54 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2820796360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:01:54.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:54 smithi110 ceph-mon[136032]: osdmap e153: 8 total, 8 up, 7 in 2023-10-09T10:01:54.872 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:01:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:54.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:01:54.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:55.050 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:55.176 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:55.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[152263]: failed to encode map e153 with expected crc 2023-10-09T10:01:55.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[152263]: failed to encode map e153 with expected crc 2023-10-09T10:01:55.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2820796360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:01:55.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[152263]: osdmap e153: 8 total, 8 up, 7 in 2023-10-09T10:01:55.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:01:55.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[147926]: failed to encode map e153 with expected crc 2023-10-09T10:01:55.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[147926]: failed to encode map e153 with expected crc 2023-10-09T10:01:55.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2820796360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:01:55.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:54 smithi012 ceph-mon[147926]: osdmap e153: 8 total, 8 up, 7 in 2023-10-09T10:01:55.660 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:01:55.660 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:01:55.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:55 smithi110 ceph-mon[136032]: pgmap v379: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 982 op/s; 6838/148701 objects degraded (4.598%); 6763/148701 objects misplaced (4.548%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T10:01:56.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:01:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:55 smithi012 ceph-mon[147926]: pgmap v379: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 982 op/s; 6838/148701 objects degraded (4.598%); 6763/148701 objects misplaced (4.548%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T10:01:56.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:55 smithi012 ceph-mon[152263]: pgmap v379: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 982 op/s; 6838/148701 objects degraded (4.598%); 6763/148701 objects misplaced (4.548%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T10:01:56.267 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 5, 6, 7, 2] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:01:56.267 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:01:56.267 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 3, 5, 6, 7, 2] 2023-10-09T10:01:56.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T10:01:57.176 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:57.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:57.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:57.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:57.339 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:57.390 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:57.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:57.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:57.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:01:57.587 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:57.656 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:01:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:57.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:01:57.779 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:01:57.832 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:57.891 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:01:57.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:57.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:01:57.995 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:01:58.043 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:01:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:01:57] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T10:01:58.052 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:58.106 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:01:58.111 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:58 smithi110 ceph-mon[136032]: pgmap v380: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 6838/147126 objects degraded (4.648%); 6763/147126 objects misplaced (4.597%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T10:01:58.111 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/147126 objects degraded (4.648%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:58.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:01:58.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:58.261 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:58.276 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:58.308 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:58.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:58.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:58.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:58.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:58 smithi012 ceph-mon[147926]: pgmap v380: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 6838/147126 objects degraded (4.648%); 6763/147126 objects misplaced (4.597%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T10:01:58.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/147126 objects degraded (4.648%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:58.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:58 smithi012 ceph-mon[152263]: pgmap v380: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 6838/147126 objects degraded (4.648%); 6763/147126 objects misplaced (4.597%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T10:01:58.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/147126 objects degraded (4.648%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:01:58.470 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:01:58.497 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:58.635 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:58.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:58.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:58.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:01:58.789 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:58.838 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:01:58.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:58.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:01:58.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:01:59.050 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:01:59.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:01:59.209 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:01:59.254 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:01:59.304 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:01:59.362 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:01:59.401 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:01:59.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:59 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:01:59.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:59 smithi012 ceph-mon[147926]: 6.0 scrub starts 2023-10-09T10:01:59.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:01:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:59.403 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:59 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:01:59.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:59 smithi012 ceph-mon[152263]: 6.0 scrub starts 2023-10-09T10:01:59.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:01:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:59.440 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:01:59.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:59 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:01:59.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:59 smithi110 ceph-mon[136032]: 6.0 scrub starts 2023-10-09T10:01:59.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:01:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:01:59.457 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:01:59.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:01:59.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:01:59.482 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:01:59.528 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:01:59.547 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:01:59.547 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:01:59.547 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:01:59.547 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:01:59.548 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:01:59.548 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:01:59.548 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:01:59.548 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:01:59.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:01:59.648 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:00.030 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:00.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:00.144 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:00.182 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:00.230 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:00.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:00.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:00.338 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:00.383 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:00.389 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:00 smithi012 ceph-mon[147926]: pgmap v381: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 6838/139665 objects degraded (4.896%); 6192/139665 objects misplaced (4.433%); 4.5 MiB/s, 104 objects/s recovering 2023-10-09T10:02:00.389 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:00 smithi012 ceph-mon[147926]: 6.10 scrub starts 2023-10-09T10:02:00.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/810317873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:00.390 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:00 smithi012 ceph-mon[152263]: pgmap v381: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 6838/139665 objects degraded (4.896%); 6192/139665 objects misplaced (4.433%); 4.5 MiB/s, 104 objects/s recovering 2023-10-09T10:02:00.390 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:00 smithi012 ceph-mon[152263]: 6.10 scrub starts 2023-10-09T10:02:00.390 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/810317873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:00.434 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:00.438 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:00 smithi110 ceph-mon[136032]: pgmap v381: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 6838/139665 objects degraded (4.896%); 6192/139665 objects misplaced (4.433%); 4.5 MiB/s, 104 objects/s recovering 2023-10-09T10:02:00.439 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:00 smithi110 ceph-mon[136032]: 6.10 scrub starts 2023-10-09T10:02:00.439 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/810317873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:00.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:00.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:00.540 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:00.591 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:00.635 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:00.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:00.710 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:00.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:00.865 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:01.001 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:01.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:01.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:01.110 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:01.130 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:02:01.131 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:02:01.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.131 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:168088 6.0 6:016c442c:::benchmark_data_smithi012_182576_object62567:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T10:02:01.131 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:01.105593+0000", 2023-10-09T10:02:01.131 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.024267165, 2023-10-09T10:02:01.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024298061999999999, 2023-10-09T10:02:01.131 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:01.131 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 168088 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.105593+0000", 2023-10-09T10:02:01.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.105593+0000", 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.105591+0000", 2023-10-09T10:02:01.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.105599+0000", 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.602e-06 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.105619+0000", 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9111e-05 2023-10-09T10:02:01.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.105628+0000", 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8409999999999995e-06 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.106027+0000", 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000398828 2023-10-09T10:02:01.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.106200+0000", 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017276300000000001 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.106696+0000", 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00049647299999999999 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.120245+0000", 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013548927000000001 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.120267+0000", 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1688e-05 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:01.137 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:168094 6.19 6:9f2d208a:::benchmark_data_smithi012_182576_object62573:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:01.112626+0000", 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.017233595000000001, 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017372533999999999, 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:01.138 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 168094 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.112626+0000", 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.112626+0000", 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.112625+0000", 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.112634+0000", 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1849999999999993e-06 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.112654+0000", 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9998999999999999e-05 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.112664+0000", 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0138e-05 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.112718+0000", 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4227000000000001e-05 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.112847+0000", 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128863 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.113285+0000", 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043845599999999999 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.120894+0000", 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076082479999999997 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.121497+0000", 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00060309700000000003 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:01.144 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:168106 6.9 6:95ad4879:::benchmark_data_smithi012_182576_object62585:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:01.129299+0000", 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00056126600000000004, 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000800521, 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:01.145 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 168106 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129299+0000", 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129299+0000", 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:01.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129297+0000", 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129306+0000", 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5079999999999994e-06 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:01.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129345+0000", 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8549e-05 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129355+0000", 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0152e-05 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:01.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129408+0000", 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3226999999999997e-05 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129528+0000", 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120333 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:01.129990+0000", 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00046173500000000001 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 3 2023-10-09T10:02:01.151 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:02:01.152 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:01.201 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:01.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:01.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:01.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:01.307 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:02:01.162+0000 7f85a99d3700 -1 mon.a@0(leader).osd e153 definitely_dead 0 2023-10-09T10:02:01.386 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:01.609 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:02:01.624 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:01.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:01.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:01.730 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:01.771 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:01.822 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:01.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:01.926 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:01.969 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:02.015 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:02.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:02.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:02.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:02.172 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.0. 2023-10-09T10:02:02.212 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:02.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:02:02.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:02:02.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[147926]: pgmap v382: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 2.8 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 6838/135729 objects degraded (5.038%); 6075/135729 objects misplaced (4.476%); 4.5 MiB/s, 97 objects/s recovering 2023-10-09T10:02:02.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[147926]: 6.9 scrub starts 2023-10-09T10:02:02.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1112229686' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:02:02.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[147926]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:02:02.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:02:02.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[152263]: pgmap v382: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 2.8 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 6838/135729 objects degraded (5.038%); 6075/135729 objects misplaced (4.476%); 4.5 MiB/s, 97 objects/s recovering 2023-10-09T10:02:02.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[152263]: 6.9 scrub starts 2023-10-09T10:02:02.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1112229686' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:02:02.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:02 smithi012 ceph-mon[152263]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:02:02.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:02 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:02:02.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:02 smithi110 ceph-mon[136032]: pgmap v382: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 2.8 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 6838/135729 objects degraded (5.038%); 6075/135729 objects misplaced (4.476%); 4.5 MiB/s, 97 objects/s recovering 2023-10-09T10:02:02.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:02 smithi110 ceph-mon[136032]: 6.9 scrub starts 2023-10-09T10:02:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1112229686' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:02:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:02 smithi110 ceph-mon[136032]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:02:02.608 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:02.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:02.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:02.761 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:02.796 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:02.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:02.884 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:02.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:02.926 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:02.959 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:02.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:02.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:03.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:03.090 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:03.122 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:03.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:03.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: 6.19 scrub starts 2023-10-09T10:02:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1112229686' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:02:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: osdmap e154: 8 total, 8 up, 6 in 2023-10-09T10:02:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6838/135729 objects degraded (5.038%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: osdmap e155: 8 total, 8 up, 6 in 2023-10-09T10:02:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: failed to encode map e155 with expected crc 2023-10-09T10:02:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/777492861' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:03 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:03.227 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:03.255 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:03.390 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:03.393 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:03.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:03.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: 6.19 scrub starts 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1112229686' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: osdmap e154: 8 total, 8 up, 6 in 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6838/135729 objects degraded (5.038%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: osdmap e155: 8 total, 8 up, 6 in 2023-10-09T10:02:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: failed to encode map e155 with expected crc 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/777492861' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: 6.19 scrub starts 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1112229686' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: osdmap e154: 8 total, 8 up, 6 in 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: failed to encode map e154 with expected crc 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6838/135729 objects degraded (5.038%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: osdmap e155: 8 total, 8 up, 6 in 2023-10-09T10:02:03.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: failed to encode map e155 with expected crc 2023-10-09T10:02:03.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/777492861' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:03.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:03 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:03.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:03.527 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:03.618 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:03.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:03.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:03.700 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:02:03.722 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:03.753 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:03.792 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:03.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:03.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:03.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:03.976 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:04.051 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:04.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:04.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: 6.4 scrub starts 2023-10-09T10:02:04.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: pgmap v385: 99 pgs: 4 unknown, 11 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 77 active+clean; 2.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 794 op/s; 6650/129615 objects degraded (5.131%); 5667/129615 objects misplaced (4.372%); 2.0 MiB/s, 130 objects/s recovering 2023-10-09T10:02:04.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:02:04.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:04.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: osdmap e156: 8 total, 8 up, 6 in 2023-10-09T10:02:04.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:04 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:02:04.163 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:04.203 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:04.264 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:04.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:04.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:04.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: 6.4 scrub starts 2023-10-09T10:02:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: pgmap v385: 99 pgs: 4 unknown, 11 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 77 active+clean; 2.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 794 op/s; 6650/129615 objects degraded (5.131%); 5667/129615 objects misplaced (4.372%); 2.0 MiB/s, 130 objects/s recovering 2023-10-09T10:02:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:02:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: osdmap e156: 8 total, 8 up, 6 in 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: failed to encode map e155 with expected crc 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: 6.4 scrub starts 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: pgmap v385: 99 pgs: 4 unknown, 11 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 77 active+clean; 2.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 794 op/s; 6650/129615 objects degraded (5.131%); 5667/129615 objects misplaced (4.372%); 2.0 MiB/s, 130 objects/s recovering 2023-10-09T10:02:04.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:02:04.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:04.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: osdmap e156: 8 total, 8 up, 6 in 2023-10-09T10:02:04.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:04 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:02:04.465 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:04.873 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:04.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:04.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:04.983 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:05.090 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:05.271 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[147926]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.271 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[147926]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.271 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[147926]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.271 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[147926]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 153'10260 2023-10-09T10:02:05.272 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[147926]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 153'10588 2023-10-09T10:02:05.272 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[147926]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[152263]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[152263]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[152263]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[152263]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 153'10260 2023-10-09T10:02:05.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[152263]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 153'10588 2023-10-09T10:02:05.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:05 smithi012 ceph-mon[152263]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.289 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:05.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:05.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:05.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:05 smithi110 ceph-mon[136032]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:05 smithi110 ceph-mon[136032]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:05 smithi110 ceph-mon[136032]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:05 smithi110 ceph-mon[136032]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 153'10260 2023-10-09T10:02:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:05 smithi110 ceph-mon[136032]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 153'10588 2023-10-09T10:02:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:05 smithi110 ceph-mon[136032]: failed to encode map e156 with expected crc 2023-10-09T10:02:05.514 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:05.569 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:02:05.602 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:05.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:05.612 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:02:05.612 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:02:05.623 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:02:05.653 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:02:05.704 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:02:05.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:05.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:02:05.788 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:02:05.806 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:05.845 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:02:05.883 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:02:05.905 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:02:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:02:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:02:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:02:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:02:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:02:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:02:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:02:05.935 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:05.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:05.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:05.982 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:02:06.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:06.160 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:06.310 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:06.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:06 smithi012 ceph-mon[147926]: pgmap v387: 99 pgs: 1 active+clean+scrubbing, 4 unknown, 11 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 2.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 6590/125487 objects degraded (5.252%); 10794/125487 objects misplaced (8.602%); 2.3 MiB/s, 125 objects/s recovering 2023-10-09T10:02:06.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2442522359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:06.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:06 smithi012 ceph-mon[152263]: pgmap v387: 99 pgs: 1 active+clean+scrubbing, 4 unknown, 11 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 2.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 6590/125487 objects degraded (5.252%); 10794/125487 objects misplaced (8.602%); 2.3 MiB/s, 125 objects/s recovering 2023-10-09T10:02:06.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2442522359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:06.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:06.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:06.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:06.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:06 smithi110 ceph-mon[136032]: pgmap v387: 99 pgs: 1 active+clean+scrubbing, 4 unknown, 11 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 2.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 6590/125487 objects degraded (5.252%); 10794/125487 objects misplaced (8.602%); 2.3 MiB/s, 125 objects/s recovering 2023-10-09T10:02:06.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2442522359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:06.528 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:06.602 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:06.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:06.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:06.812 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:06.885 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:06.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:06.931 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:06.994 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:07.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:02:07.035 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:07.089 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:07.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:07.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:07.198 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:07.240 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:07.283 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:07.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:07.388 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:07.421 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:07.481 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:07.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:07.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:07.585 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:07.637 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:07.688 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:07.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:07.771 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 5, 6, 7, 2] out_osds: [4, 0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:02:07.771 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:02:07.771 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T10:02:07.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T10:02:07.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:07.931 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:07.996 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:08.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:08.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:08.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:02:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:02:07] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T10:02:08.185 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:08.194 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:08.264 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:08.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:08.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:08.411 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:08.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:02:08.458 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:08.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:08.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:08.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:08 smithi110 ceph-mon[136032]: pgmap v388: 99 pgs: 1 active+clean+scrubbing, 4 unknown, 11 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 2.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 6590/125487 objects degraded (5.252%); 10794/125487 objects misplaced (8.602%); 1.1 MiB/s, 106 objects/s recovering 2023-10-09T10:02:08.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6590/125487 objects degraded (5.252%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:08.493 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:08 smithi012 ceph-mon[147926]: pgmap v388: 99 pgs: 1 active+clean+scrubbing, 4 unknown, 11 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 2.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 6590/125487 objects degraded (5.252%); 10794/125487 objects misplaced (8.602%); 1.1 MiB/s, 106 objects/s recovering 2023-10-09T10:02:08.493 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6590/125487 objects degraded (5.252%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:08.494 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:08 smithi012 ceph-mon[152263]: pgmap v388: 99 pgs: 1 active+clean+scrubbing, 4 unknown, 11 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 73 active+clean; 2.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 6590/125487 objects degraded (5.252%); 10794/125487 objects misplaced (8.602%); 1.1 MiB/s, 106 objects/s recovering 2023-10-09T10:02:08.494 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6590/125487 objects degraded (5.252%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:08.565 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:08.600 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:08.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:08.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:08.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:08.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:08.856 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:09.284 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:02:09.284 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:02:09.619 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:09.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:09.631 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:09 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:02:09.633 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:09 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:02:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:09 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:02:09.732 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:09.774 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:09.831 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:09.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:09.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:09.968 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:10.010 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:10.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:10.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:10.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:10.194 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:10.255 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:10 smithi012 ceph-mon[147926]: pgmap v389: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 17744/119919 objects degraded (14.797%); 11610/119919 objects misplaced (9.682%); 3.2 MiB/s, 136 objects/s recovering 2023-10-09T10:02:10.255 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:10 smithi012 ceph-mon[152263]: pgmap v389: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 17744/119919 objects degraded (14.797%); 11610/119919 objects misplaced (9.682%); 3.2 MiB/s, 136 objects/s recovering 2023-10-09T10:02:10.511 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:10.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:10.616 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:10.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:10 smithi110 ceph-mon[136032]: pgmap v389: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 17744/119919 objects degraded (14.797%); 11610/119919 objects misplaced (9.682%); 3.2 MiB/s, 136 objects/s recovering 2023-10-09T10:02:10.667 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:10.725 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:10.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:10.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:10.892 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:11.027 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:11.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:11.134 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:11.201 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:12.402 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:12.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:12.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:12.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:02:12.521 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:12.526 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:12 smithi012 ceph-mon[147926]: pgmap v390: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 17744/116271 objects degraded (15.261%); 11015/116271 objects misplaced (9.474%); 3.5 MiB/s, 63 objects/s recovering 2023-10-09T10:02:12.526 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:12 smithi012 ceph-mon[152263]: pgmap v390: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 17744/116271 objects degraded (15.261%); 11015/116271 objects misplaced (9.474%); 3.5 MiB/s, 63 objects/s recovering 2023-10-09T10:02:12.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:12 smithi110 ceph-mon[136032]: pgmap v390: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 17744/116271 objects degraded (15.261%); 11015/116271 objects misplaced (9.474%); 3.5 MiB/s, 63 objects/s recovering 2023-10-09T10:02:12.558 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:12.609 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:12.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:12.650 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:12.717 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:12.749 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:12.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:12.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:12.842 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:12.910 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:12.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:12.984 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:12.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:13.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:13.186 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:13.494 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:13.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17744/116271 objects degraded (15.261%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:13.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17744/116271 objects degraded (15.261%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:13.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:13.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:13.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:13.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17744/116271 objects degraded (15.261%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:13.636 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:13.680 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:13.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:13.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:13.882 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:14.218 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:14.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:14.263 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:14.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:14.359 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:14.400 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:14.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:14.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:14.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:14.563 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:14.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:14.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:14.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:14.720 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:14.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:14.838 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:14.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:14.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:14.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[152263]: pgmap v391: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 648 op/s; 17809/110166 objects degraded (16.166%); 10304/110166 objects misplaced (9.353%); 4.3 MiB/s, 76 objects/s recovering 2023-10-09T10:02:14.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:14.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:14.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/101034470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:02:14.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:02:14.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[147926]: pgmap v391: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 648 op/s; 17809/110166 objects degraded (16.166%); 10304/110166 objects misplaced (9.353%); 4.3 MiB/s, 76 objects/s recovering 2023-10-09T10:02:14.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:14.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:14.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/101034470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:02:14.864 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:02:14.941 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:14 smithi110 ceph-mon[136032]: pgmap v391: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 648 op/s; 17809/110166 objects degraded (16.166%); 10304/110166 objects misplaced (9.353%); 4.3 MiB/s, 76 objects/s recovering 2023-10-09T10:02:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/101034470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:02:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:02:14.949 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:02:14.977 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:15.025 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:15.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:15.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:15.239 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:15.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:02:15.363 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:02:15.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:02:15.432 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:02:15.476 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:02:15.517 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:02:15.532 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:15.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:15.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:15.553 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:02:15.601 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:02:15.633 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:02:15.633 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:02:15.633 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:02:15.633 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:02:15.633 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:02:15.633 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:02:15.633 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:02:15.633 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:02:15.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:15.684 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:15.738 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:15.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:15.845 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:15.884 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:15.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:15.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:02:15.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[147926]: osdmap e157: 8 total, 8 up, 6 in 2023-10-09T10:02:15.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[147926]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[147926]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[147926]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[147926]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2638817394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:15.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:15.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:02:15.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[152263]: osdmap e157: 8 total, 8 up, 6 in 2023-10-09T10:02:15.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[152263]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[152263]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[152263]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[152263]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.906 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2638817394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:15 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:02:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:15 smithi110 ceph-mon[136032]: osdmap e157: 8 total, 8 up, 6 in 2023-10-09T10:02:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:15 smithi110 ceph-mon[136032]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:15 smithi110 ceph-mon[136032]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:15 smithi110 ceph-mon[136032]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:15 smithi110 ceph-mon[136032]: failed to encode map e157 with expected crc 2023-10-09T10:02:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2638817394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:15.944 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:15.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:15.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:16.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:16.149 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:02:16.035+0000 7f85a99d3700 -1 mon.a@0(leader).osd e157 definitely_dead 0 2023-10-09T10:02:16.151 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:16.238 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:16.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:16.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:16.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:16.449 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:16.526 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:16.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:16.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:16.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:16.700 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:178796 6.e 6:7e1a53d2:::benchmark_data_smithi012_182576_object73275:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:16.709540+0000", 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.022387648999999999, 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022418964, 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:16.735 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178796 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709540+0000", 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709540+0000", 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709539+0000", 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:16.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709543+0000", 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2140000000000001e-06 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709551+0000", 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3729999999999993e-06 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709557+0000", 2023-10-09T10:02:16.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7400000000000001e-06 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709659+0000", 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010251099999999999 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709787+0000", 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012745899999999999 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709914+0000", 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012762200000000001 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.716552+0000", 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066372829999999999 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:02:16.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.717146+0000", 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00059429999999999997 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.731954+0000", 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014808597999999999 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:178797 6.e 6:7e0b4842:::benchmark_data_smithi012_182576_object73276:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T10:02:16.741 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:16.709574+0000", 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.022353496, 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022489180000000001, 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178797 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.742 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709574+0000", 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709574+0000", 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709573+0000", 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709576+0000", 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0199999999999999e-06 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709584+0000", 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3119999999999996e-06 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.709590+0000", 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8370000000000002e-06 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.712938+0000", 2023-10-09T10:02:16.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033481299999999999 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.713056+0000", 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118468 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.713174+0000", 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011786 2023-10-09T10:02:16.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.716559+0000", 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033845469999999999 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.717222+0000", 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00066297199999999995 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:16.747 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:178800 6.1 6:81a17e02:::benchmark_data_smithi012_182576_object73279:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:16.720568+0000", 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.011359757999999999, 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011604119, 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178800 2023-10-09T10:02:16.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720568+0000", 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720568+0000", 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720566+0000", 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720574+0000", 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4999999999999999e-06 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720596+0000", 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1178999999999998e-05 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720605+0000", 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6490000000000003e-06 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.723526+0000", 2023-10-09T10:02:16.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002920897 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.723644+0000", 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118175 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.724107+0000", 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000462151 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:16.752 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:178803 6.e 6:7740a466:::benchmark_data_smithi012_182576_object73282:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:16.720621+0000", 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.011306171, 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011709951999999999, 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:16.753 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178803 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720621+0000", 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720621+0000", 2023-10-09T10:02:16.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720621+0000", 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720624+0000", 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3979999999999999e-06 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720631+0000", 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6830000000000002e-06 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720636+0000", 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.541e-06 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720717+0000", 2023-10-09T10:02:16.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1007000000000006e-05 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720869+0000", 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015191200000000001 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720997+0000", 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012768999999999999 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.725411+0000", 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044136000000000002 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:178804 6.11 6:88ea3bf5:::benchmark_data_smithi012_182576_object73283:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:16.720655+0000", 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.011272611, 2023-10-09T10:02:16.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011787662000000001, 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178804 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:16.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720655+0000", 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720655+0000", 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720654+0000", 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720658+0000", 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2509999999999999e-06 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720665+0000", 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7989999999999994e-06 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:16.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.720671+0000", 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9020000000000004e-06 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.727010+0000", 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0063387390000000004 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.727138+0000", 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012815999999999999 2023-10-09T10:02:16.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:02:16.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.727614+0000", 2023-10-09T10:02:16.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00047612499999999999 2023-10-09T10:02:16.764 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.764 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:16.764 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:178798 6.b e157/97 6:d28f246f:::benchmark_data_smithi012_182576_object73277:head v 157'11432, mlcod=157'11430)", 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:16.728573+0000", 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0033541669999999999, 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039524060000000003, 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728573+0000", 2023-10-09T10:02:16.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728573+0000", 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728572+0000", 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728581+0000", 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8759999999999994e-06 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728593+0000", 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2145e-05 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:16.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728605+0000", 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1847e-05 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728661+0000", 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6169999999999999e-05 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728686+0000", 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5386000000000001e-05 2023-10-09T10:02:16.768 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:178805 6.b e157/97 6:d019365e:::benchmark_data_smithi012_182576_object73284:head v 157'11433, mlcod=157'11430)", 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:16.728626+0000", 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0033009670000000001, 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039693259999999996, 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:16.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728626+0000", 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728626+0000", 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728626+0000", 2023-10-09T10:02:16.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728630+0000", 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0409999999999999e-06 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728636+0000", 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8270000000000003e-06 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.728641+0000", 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4989999999999996e-06 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.731237+0000", 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002596377 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:16.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.731255+0000", 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8142e-05 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:178808 6.e 6:750f0b38:::benchmark_data_smithi012_182576_object73287:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:16.730192+0000", 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.001735917, 2023-10-09T10:02:16.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024724830000000001, 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178808 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:16.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.730192+0000", 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.730192+0000", 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.730190+0000", 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.730198+0000", 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3049999999999993e-06 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.730213+0000", 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5387e-05 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:16.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.730224+0000", 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0856000000000001e-05 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.732468+0000", 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022435290000000002 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:16.732595+0000", 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127463 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.777 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:16.778 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.778 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:16.778 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:02:16.778 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 8 2023-10-09T10:02:16.778 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:02:16.789 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:16.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:16.897 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:16.943 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:16.975 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.4. 2023-10-09T10:02:16.999 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:16 smithi012 ceph-mon[152263]: pgmap v393: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.1 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 708 op/s; 17809/104217 objects degraded (17.088%); 9712/104217 objects misplaced (9.319%); 4.4 MiB/s, 76 objects/s recovering 2023-10-09T10:02:16.999 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:16 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/714212386' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:02:16.999 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:16 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:02:17.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:16 smithi012 ceph-mon[147926]: pgmap v393: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.1 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 708 op/s; 17809/104217 objects degraded (17.088%); 9712/104217 objects misplaced (9.319%); 4.4 MiB/s, 76 objects/s recovering 2023-10-09T10:02:17.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:16 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/714212386' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:02:17.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:16 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:02:17.001 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:17.003 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:16 smithi110 ceph-mon[136032]: pgmap v393: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.1 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 708 op/s; 17809/104217 objects degraded (17.088%); 9712/104217 objects misplaced (9.319%); 4.4 MiB/s, 76 objects/s recovering 2023-10-09T10:02:17.003 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:16 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/714212386' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:02:17.003 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:16 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:02:17.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:17.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:17.105 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:17.131 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:17.163 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:17.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:17.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:17.292 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:17.325 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:17.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:17.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:17.459 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:17.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:02:17.915 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:02:17] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[147926]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[147926]: osdmap e158: 8 total, 8 up, 7 in 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[147926]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[147926]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[147926]: pgmap v395: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.1 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 653 op/s; 17809/104217 objects degraded (17.088%); 9712/104217 objects misplaced (9.319%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17809/104217 objects degraded (17.088%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[147926]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T10:02:18.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[152263]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T10:02:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[152263]: osdmap e158: 8 total, 8 up, 7 in 2023-10-09T10:02:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[152263]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[152263]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[152263]: pgmap v395: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.1 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 653 op/s; 17809/104217 objects degraded (17.088%); 9712/104217 objects misplaced (9.319%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T10:02:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17809/104217 objects degraded (17.088%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:17 smithi012 ceph-mon[152263]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T10:02:18.282 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:02:18.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:17 smithi110 ceph-mon[136032]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:17 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T10:02:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:17 smithi110 ceph-mon[136032]: osdmap e158: 8 total, 8 up, 7 in 2023-10-09T10:02:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:17 smithi110 ceph-mon[136032]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:17 smithi110 ceph-mon[136032]: failed to encode map e158 with expected crc 2023-10-09T10:02:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:17 smithi110 ceph-mon[136032]: pgmap v395: 99 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 86 active+clean; 2.1 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 653 op/s; 17809/104217 objects degraded (17.088%); 9712/104217 objects misplaced (9.319%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T10:02:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:17 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17809/104217 objects degraded (17.088%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:02:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:17 smithi110 ceph-mon[136032]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T10:02:18.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:02:18.896 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:19.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:19.161 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T10:02:19.251 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.254 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.255 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.255 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.255 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.255 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T10:02:19.255 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[147926]: 6.e starting backfill to osd.4 from (107'3060,145'9039] MIN to 157'11195 2023-10-09T10:02:19.255 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.255 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T10:02:19.256 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:18 smithi012 ceph-mon[152263]: 6.e starting backfill to osd.4 from (107'3060,145'9039] MIN to 157'11195 2023-10-09T10:02:19.257 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:19.261 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:19.327 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:19.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.539 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.539 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.539 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.539 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: failed to encode map e159 with expected crc 2023-10-09T10:02:19.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T10:02:19.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:18 smithi110 ceph-mon[136032]: 6.e starting backfill to osd.4 from (107'3060,145'9039] MIN to 157'11195 2023-10-09T10:02:20.234 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:20.241 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:20.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:20.243 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[147926]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.243 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[147926]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.243 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[147926]: 6.2 starting backfill to osd.4 from (0'0,136'7702] MIN to 157'9437 2023-10-09T10:02:20.243 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[147926]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.243 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[147926]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.243 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[147926]: pgmap v398: 99 pgs: 4 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 10220/97959 objects degraded (10.433%); 7167/97959 objects misplaced (7.316%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:02:20.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:20.244 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[152263]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.245 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[152263]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.245 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[152263]: 6.2 starting backfill to osd.4 from (0'0,136'7702] MIN to 157'9437 2023-10-09T10:02:20.245 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[152263]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.245 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[152263]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.245 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:19 smithi012 ceph-mon[152263]: pgmap v398: 99 pgs: 4 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 10220/97959 objects degraded (10.433%); 7167/97959 objects misplaced (7.316%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:02:20.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:20.386 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:19 smithi110 ceph-mon[136032]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.387 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:19 smithi110 ceph-mon[136032]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.387 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:19 smithi110 ceph-mon[136032]: 6.2 starting backfill to osd.4 from (0'0,136'7702] MIN to 157'9437 2023-10-09T10:02:20.387 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:19 smithi110 ceph-mon[136032]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.387 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:19 smithi110 ceph-mon[136032]: failed to encode map e160 with expected crc 2023-10-09T10:02:20.387 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:19 smithi110 ceph-mon[136032]: pgmap v398: 99 pgs: 4 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 10220/97959 objects degraded (10.433%); 7167/97959 objects misplaced (7.316%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:02:20.389 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:20.433 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:20.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:20.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:20.524 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:20.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:20.617 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:20.858 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:20.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:20.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:21.049 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:21.864 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:21.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:21.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:22.040 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:22 smithi012 ceph-mon[147926]: pgmap v399: 99 pgs: 4 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 10355/95547 objects degraded (10.838%); 5822/95547 objects misplaced (6.093%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T10:02:22.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:22 smithi012 ceph-mon[152263]: pgmap v399: 99 pgs: 4 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 10355/95547 objects degraded (10.838%); 5822/95547 objects misplaced (6.093%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T10:02:22.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:22 smithi110 ceph-mon[136032]: pgmap v399: 99 pgs: 4 remapped+peering, 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 10355/95547 objects degraded (10.838%); 5822/95547 objects misplaced (6.093%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T10:02:22.581 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:22.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:22.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:22.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:22.794 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:23.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:23 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:02:23.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 10355/95547 objects degraded (10.838%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:23.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:23 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:02:23.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 10355/95547 objects degraded (10.838%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:23.410 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:23.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:23.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:23 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:02:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 10355/95547 objects degraded (10.838%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:23.517 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:23.556 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:23.608 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:23.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:23.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:23.717 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:23.766 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:23.817 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:23.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:23.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:24.020 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:24.112 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:24.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:24.233 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:24.252 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 5, 6, 7, 2, 4] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:02:24.252 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:02:24.252 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T10:02:24.252 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T10:02:24.285 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:24.330 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:24.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:24.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:24.575 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:24.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:24 smithi012 ceph-mon[147926]: pgmap v400: 99 pgs: 1 remapped+peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 12069/90831 objects degraded (13.287%); 6149/90831 objects misplaced (6.770%); 11 MiB/s, 1 keys/s, 169 objects/s recovering 2023-10-09T10:02:24.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:24 smithi012 ceph-mon[152263]: pgmap v400: 99 pgs: 1 remapped+peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 12069/90831 objects degraded (13.287%); 6149/90831 objects misplaced (6.770%); 11 MiB/s, 1 keys/s, 169 objects/s recovering 2023-10-09T10:02:24.945 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:02:24.945 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:02:24.946 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:02:24.946 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:02:24.946 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:02:24.946 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:24.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:24.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:25.043 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:02:25.051 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:25.084 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:24 smithi110 ceph-mon[136032]: pgmap v400: 99 pgs: 1 remapped+peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 12069/90831 objects degraded (13.287%); 6149/90831 objects misplaced (6.770%); 11 MiB/s, 1 keys/s, 169 objects/s recovering 2023-10-09T10:02:25.086 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:02:25.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:25.124 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:02:25.130 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:25.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:25.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:25.176 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:02:25.223 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:02:25.234 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:25.262 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:02:25.292 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:25.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:02:25.333 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:02:25.345 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:25.347 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:02:25.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:02:25.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:02:25.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:02:25.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:02:25.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:02:25.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:02:25.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:02:25.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:25.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:25.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:25.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:02:25.488 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:25.499 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:25.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:25.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:25.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:25.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:25.662 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:25.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:25.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:25.810 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:25.812 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:02:25.875 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:25.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3192277268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:25.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3887352011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:26.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3192277268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:26.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3887352011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:26.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3192277268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:26.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3887352011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:26.134 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:26.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:26.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:26.237 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:26.330 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:26.456 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:26.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:26.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:26.562 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:26.600 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:26.660 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:26.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:26.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:26.791 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[152263]: pgmap v401: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 12069/85152 objects degraded (14.173%); 7453/85152 objects misplaced (8.753%); 8.0 MiB/s, 1 keys/s, 129 objects/s recovering 2023-10-09T10:02:26.791 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:26.791 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[152263]: failed to encode map e161 with expected crc 2023-10-09T10:02:26.792 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3192277268' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:26.792 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[152263]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T10:02:26.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[147926]: pgmap v401: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 12069/85152 objects degraded (14.173%); 7453/85152 objects misplaced (8.753%); 8.0 MiB/s, 1 keys/s, 129 objects/s recovering 2023-10-09T10:02:26.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:26.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[147926]: failed to encode map e161 with expected crc 2023-10-09T10:02:26.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3192277268' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:26.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:26 smithi012 ceph-mon[147926]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T10:02:26.830 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:26.907 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:26.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:26 smithi110 ceph-mon[136032]: pgmap v401: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 12069/85152 objects degraded (14.173%); 7453/85152 objects misplaced (8.753%); 8.0 MiB/s, 1 keys/s, 129 objects/s recovering 2023-10-09T10:02:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:26 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:26 smithi110 ceph-mon[136032]: failed to encode map e161 with expected crc 2023-10-09T10:02:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3192277268' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:26 smithi110 ceph-mon[136032]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T10:02:26.945 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:27.021 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:27.060 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:27.110 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:27.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:27.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:27.216 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:27.257 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:27.360 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:27.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:27.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:27.508 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:27.600 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:27.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:27.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:27.706 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:27.838 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:28.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:02:28.085+0000 7f85a99d3700 -1 mon.a@0(leader).osd e161 definitely_dead 0 2023-10-09T10:02:28.089 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:02:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:02:27] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T10:02:28.134 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:02:28.212 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:28.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:28.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:28.320 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:28.367 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:28.409 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:28.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:28.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:28.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:28.570 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:02:28.601 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:28.926 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.0. 2023-10-09T10:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[152263]: pgmap v403: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 12069/85152 objects degraded (14.173%); 7453/85152 objects misplaced (8.753%); 5.5 MiB/s, 1 keys/s, 88 objects/s recovering 2023-10-09T10:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 12069/85152 objects degraded (14.173%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3265968770' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:02:28.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[147926]: pgmap v403: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 12069/85152 objects degraded (14.173%); 7453/85152 objects misplaced (8.753%); 5.5 MiB/s, 1 keys/s, 88 objects/s recovering 2023-10-09T10:02:28.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 12069/85152 objects degraded (14.173%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:28.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:28.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:28.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3265968770' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:02:29.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:28 smithi110 ceph-mon[136032]: pgmap v403: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 12069/85152 objects degraded (14.173%); 7453/85152 objects misplaced (8.753%); 5.5 MiB/s, 1 keys/s, 88 objects/s recovering 2023-10-09T10:02:29.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 12069/85152 objects degraded (14.173%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:29.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:28 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:28 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3265968770' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:02:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[147926]: failed to encode map e162 with expected crc 2023-10-09T10:02:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3265968770' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:02:30.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[147926]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T10:02:30.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:30.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:30.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[147926]: pgmap v405: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 12069/77922 objects degraded (15.489%); 6725/77922 objects misplaced (8.630%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:02:30.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[152263]: failed to encode map e162 with expected crc 2023-10-09T10:02:30.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3265968770' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:02:30.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[152263]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T10:02:30.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:30.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:30.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:29 smithi012 ceph-mon[152263]: pgmap v405: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 12069/77922 objects degraded (15.489%); 6725/77922 objects misplaced (8.630%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:02:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:29 smithi110 ceph-mon[136032]: failed to encode map e162 with expected crc 2023-10-09T10:02:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3265968770' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:02:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:29 smithi110 ceph-mon[136032]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T10:02:30.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:30.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:30.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:29 smithi110 ceph-mon[136032]: pgmap v405: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 12069/77922 objects degraded (15.489%); 6725/77922 objects misplaced (8.630%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:02:30.249 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:30.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:30.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:30.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:30.448 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:30.936 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:30.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:30.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:31.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:31.143 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[152263]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.143 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[152263]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T10:02:31.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[152263]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[152263]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[152263]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[152263]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[152263]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[152263]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.144 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[147926]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.144 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[147926]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T10:02:31.145 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[147926]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.145 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[147926]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.145 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[147926]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.145 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[147926]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.145 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[147926]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.145 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:30 smithi012 ceph-mon[147926]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.146 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:30 smithi110 ceph-mon[136032]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:30 smithi110 ceph-mon[136032]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T10:02:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:30 smithi110 ceph-mon[136032]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:30 smithi110 ceph-mon[136032]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:30 smithi110 ceph-mon[136032]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:30 smithi110 ceph-mon[136032]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:30 smithi110 ceph-mon[136032]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:30 smithi110 ceph-mon[136032]: failed to encode map e163 with expected crc 2023-10-09T10:02:31.316 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T10:02:31.334 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:31.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:31.445 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:31.470 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:31.475 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:31.525 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:31.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:31.627 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:31.660 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:31.706 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:31.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:31.792 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:31.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:31.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:31.848 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:02:31.848 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:02:31.891 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:31.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:31.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:31.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:32.080 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:32.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[147926]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T10:02:32.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[147926]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[147926]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[147926]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[147926]: pgmap v408: 99 pgs: 2 peering, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 12047/75216 objects degraded (16.017%); 3288/75216 objects misplaced (4.371%); 5.5 MiB/s, 91 objects/s recovering 2023-10-09T10:02:32.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[152263]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T10:02:32.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[152263]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[152263]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[152263]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:31 smithi012 ceph-mon[152263]: pgmap v408: 99 pgs: 2 peering, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 12047/75216 objects degraded (16.017%); 3288/75216 objects misplaced (4.371%); 5.5 MiB/s, 91 objects/s recovering 2023-10-09T10:02:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:31 smithi110 ceph-mon[136032]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T10:02:32.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:31 smithi110 ceph-mon[136032]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:31 smithi110 ceph-mon[136032]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:31 smithi110 ceph-mon[136032]: failed to encode map e164 with expected crc 2023-10-09T10:02:32.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:31 smithi110 ceph-mon[136032]: pgmap v408: 99 pgs: 2 peering, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 12047/75216 objects degraded (16.017%); 3288/75216 objects misplaced (4.371%); 5.5 MiB/s, 91 objects/s recovering 2023-10-09T10:02:32.432 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:32.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:32.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:32.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:02:32.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:32.682 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:32.831 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:32.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:32.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:33.046 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:33.148 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:32 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 12047/75216 objects degraded (16.017%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:33.148 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:32 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 12047/75216 objects degraded (16.017%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:33.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:32 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 12047/75216 objects degraded (16.017%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:33.444 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:33.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:33.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:33.555 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:33.593 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:33.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:33.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:33.722 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:33.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:33.796 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:33.844 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:33.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:33.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:33.948 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:33.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:34.024 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:34.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:34.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:34.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:34.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:33 smithi110 ceph-mon[136032]: pgmap v409: 99 pgs: 2 active+recovering+undersized+remapped, 2 peering, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 4915/70761 objects degraded (6.946%); 2377/70761 objects misplaced (3.359%); 6.2 MiB/s, 168 objects/s recovering 2023-10-09T10:02:34.172 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:34.224 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:34.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:34.335 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:34.402 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:33 smithi012 ceph-mon[147926]: pgmap v409: 99 pgs: 2 active+recovering+undersized+remapped, 2 peering, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 4915/70761 objects degraded (6.946%); 2377/70761 objects misplaced (3.359%); 6.2 MiB/s, 168 objects/s recovering 2023-10-09T10:02:34.403 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:33 smithi012 ceph-mon[152263]: pgmap v409: 99 pgs: 2 active+recovering+undersized+remapped, 2 peering, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 4915/70761 objects degraded (6.946%); 2377/70761 objects misplaced (3.359%); 6.2 MiB/s, 168 objects/s recovering 2023-10-09T10:02:34.420 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:36.317 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 5, 6, 7, 2, 4, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:02:36.317 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:02:36.317 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [1, 3, 5, 6, 7, 2, 4, 0] 2023-10-09T10:02:36.318 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T10:02:36.370 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:36 smithi012 ceph-mon[147926]: pgmap v410: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 4917/65055 objects degraded (7.558%); 2178/65055 objects misplaced (3.348%); 1.4 MiB/s, 89 objects/s recovering 2023-10-09T10:02:36.370 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1493678834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:36.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:36 smithi012 ceph-mon[152263]: pgmap v410: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 4917/65055 objects degraded (7.558%); 2178/65055 objects misplaced (3.348%); 1.4 MiB/s, 89 objects/s recovering 2023-10-09T10:02:36.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1493678834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:36.412 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:02:36.412 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:02:36.412 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:02:36.412 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:02:36.412 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:02:36.412 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:02:36.412 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:02:36.412 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:02:36.413 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:02:36.413 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:02:36.413 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:02:36.413 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:02:36.413 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:02:36.413 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:02:36.413 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:02:36.413 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:02:36.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:36 smithi110 ceph-mon[136032]: pgmap v410: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 4917/65055 objects degraded (7.558%); 2178/65055 objects misplaced (3.348%); 1.4 MiB/s, 89 objects/s recovering 2023-10-09T10:02:36.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1493678834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:37.156 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:37.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:37.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:37.297 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:37.350 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:37.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:37.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:37 smithi012 ceph-mon[147926]: failed to encode map e165 with expected crc 2023-10-09T10:02:37.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:37 smithi012 ceph-mon[147926]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T10:02:37.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:37 smithi012 ceph-mon[147926]: failed to encode map e165 with expected crc 2023-10-09T10:02:37.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:37 smithi012 ceph-mon[152263]: failed to encode map e165 with expected crc 2023-10-09T10:02:37.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:37 smithi012 ceph-mon[152263]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T10:02:37.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:37 smithi012 ceph-mon[152263]: failed to encode map e165 with expected crc 2023-10-09T10:02:37.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:37 smithi110 ceph-mon[136032]: failed to encode map e165 with expected crc 2023-10-09T10:02:37.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:37 smithi110 ceph-mon[136032]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T10:02:37.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:37 smithi110 ceph-mon[136032]: failed to encode map e165 with expected crc 2023-10-09T10:02:37.454 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:37.484 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:37.520 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:37.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:37.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:37.661 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:37.917 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:02:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:02:37] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.33.4" 2023-10-09T10:02:38.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: pgmap v412: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 4917/64788 objects degraded (7.589%); 2178/64788 objects misplaced (3.362%); 1.2 MiB/s, 123 objects/s recovering 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: failed to encode map e166 with expected crc 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: failed to encode map e166 with expected crc 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4917/64788 objects degraded (7.589%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: pgmap v412: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 4917/64788 objects degraded (7.589%); 2178/64788 objects misplaced (3.362%); 1.2 MiB/s, 123 objects/s recovering 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: failed to encode map e166 with expected crc 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: failed to encode map e166 with expected crc 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4917/64788 objects degraded (7.589%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:02:38.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:38 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.254 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:38.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:38.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:38.365 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:38.406 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: pgmap v412: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 4917/64788 objects degraded (7.589%); 2178/64788 objects misplaced (3.362%); 1.2 MiB/s, 123 objects/s recovering 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: failed to encode map e166 with expected crc 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: failed to encode map e166 with expected crc 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4917/64788 objects degraded (7.589%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:02:38.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:02:38.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:38 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:38.453 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:38.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:38.559 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:38.595 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:38.643 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:38.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:38.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:38.847 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:38.944 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:38.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:38.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:39.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:39.128 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:39.197 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:39.210 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:39.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:39.227 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:39 smithi012 ceph-mon[147926]: failed to encode map e166 with expected crc 2023-10-09T10:02:39.227 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:39 smithi012 ceph-mon[147926]: failed to encode map e166 with expected crc 2023-10-09T10:02:39.227 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1675964346' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:02:39.227 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:39 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:02:39.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:39 smithi012 ceph-mon[152263]: failed to encode map e166 with expected crc 2023-10-09T10:02:39.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:39 smithi012 ceph-mon[152263]: failed to encode map e166 with expected crc 2023-10-09T10:02:39.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1675964346' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:02:39.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:39 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:02:39.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:39.239 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:02:39.329 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:39.368 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:39.418 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:39.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:39.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:39.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:39 smithi110 ceph-mon[136032]: failed to encode map e166 with expected crc 2023-10-09T10:02:39.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:39 smithi110 ceph-mon[136032]: failed to encode map e166 with expected crc 2023-10-09T10:02:39.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1675964346' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:02:39.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:39 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:02:39.523 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:39.559 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:39.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:39.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:39.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:39.720 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:39.753 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:39.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:39.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:39.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:39.908 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:40.025 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:40.191 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:40 smithi110 ceph-mon[136032]: pgmap v414: 99 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 4324/57627 objects degraded (7.503%); 1862/57627 objects misplaced (3.231%); 1.1 MiB/s, 206 objects/s recovering 2023-10-09T10:02:40.191 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:40 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:40.191 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:02:40.191 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:40 smithi110 ceph-mon[136032]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T10:02:40.191 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:40 smithi110 ceph-mon[136032]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.191 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:40 smithi110 ceph-mon[136032]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.191 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:40 smithi110 ceph-mon[136032]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.191 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:40 smithi110 ceph-mon[136032]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.240 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:40.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:40.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:40.394 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:02:40.394 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:02:40.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.394 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:194328 6.18 6:1f7e081c:::benchmark_data_smithi012_182576_object88807:head [delete] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:40.387501+0000", 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0060848400000000002, 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0061186820000000003, 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 194328 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:40.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387501+0000", 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387501+0000", 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:40.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387500+0000", 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387507+0000", 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7259999999999996e-06 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387524+0000", 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7683999999999999e-05 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387533+0000", 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4479999999999994e-06 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387570+0000", 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7344e-05 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:40.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387729+0000", 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015891599999999999 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388285+0000", 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00055598300000000002 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:194333 6.18 6:181f9a09:::benchmark_data_smithi012_182576_object88812:head [delete] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T10:02:40.399 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:40.387545+0000", 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.006041345, 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006159355, 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 194333 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:40.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387545+0000", 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387545+0000", 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387544+0000", 2023-10-09T10:02:40.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387548+0000", 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3639999999999999e-06 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387554+0000", 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7109999999999998e-06 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387561+0000", 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4060000000000002e-06 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391075+0000", 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003514138 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391184+0000", 2023-10-09T10:02:40.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010882000000000001 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391657+0000", 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000473437 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:194334 6.2 6:40de05ac:::benchmark_data_smithi012_182576_object88813:head [delete] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T10:02:40.404 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:40.387571+0000", 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0060154910000000004, 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062089240000000002, 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 194334 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:40.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387571+0000", 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387571+0000", 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387570+0000", 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:02:40.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387573+0000", 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.52e-06 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387579+0000", 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3389999999999998e-06 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387584+0000", 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6149999999999999e-06 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387700+0000", 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115906 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.387824+0000", 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000124027 2023-10-09T10:02:40.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4,6", 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388284+0000", 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00046003999999999999 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:194329 6.e e167/159 6:744c33a5:::benchmark_data_smithi012_182576_object88808:head v 167'12105, mlcod=167'12103)", 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:40.388250+0000", 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.005336548, 2023-10-09T10:02:40.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056037969999999998, 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388250+0000", 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388250+0000", 2023-10-09T10:02:40.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388249+0000", 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388258+0000", 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.397e-06 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388270+0000", 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1637000000000001e-05 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388277+0000", 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7619999999999995e-06 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388395+0000", 2023-10-09T10:02:40.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117209 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.388422+0000", 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7504e-05 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:194337 6.1 e167/134 6:86f7fd24:::benchmark_data_smithi012_182576_object88816:head v 167'9861, mlcod=167'9859)", 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:40.391468+0000", 2023-10-09T10:02:40.413 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0021183949999999999, 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024464230000000001, 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391468+0000", 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:02:40.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391468+0000", 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391467+0000", 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391473+0000", 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3069999999999999e-06 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391484+0000", 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0271e-05 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391491+0000", 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1709999999999996e-06 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391542+0000", 2023-10-09T10:02:40.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1631000000000001e-05 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:40.391562+0000", 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9925999999999999e-05 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 5 2023-10-09T10:02:40.417 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:02:40.455 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[147926]: pgmap v414: 99 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 4324/57627 objects degraded (7.503%); 1862/57627 objects misplaced (3.231%); 1.1 MiB/s, 206 objects/s recovering 2023-10-09T10:02:40.455 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:40.455 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:02:40.455 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[147926]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T10:02:40.455 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[147926]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.455 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[147926]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.455 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[147926]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.455 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[147926]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[152263]: pgmap v414: 99 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 4324/57627 objects degraded (7.503%); 1862/57627 objects misplaced (3.231%); 1.1 MiB/s, 206 objects/s recovering 2023-10-09T10:02:40.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:02:40.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:02:40.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[152263]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T10:02:40.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[152263]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[152263]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[152263]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.456 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:40 smithi012 ceph-mon[152263]: failed to encode map e167 with expected crc 2023-10-09T10:02:40.481 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:40.783 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:40.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:40.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:02:40.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:02:40.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:41.078 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:41.265 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:41.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:41.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:41.374 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:41.412 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:41.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:41 smithi012 ceph-mon[147926]: 6.4 scrub starts 2023-10-09T10:02:41.437 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:41 smithi012 ceph-mon[152263]: 6.4 scrub starts 2023-10-09T10:02:41.463 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:41.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:41.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:41.571 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:41.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:41 smithi110 ceph-mon[136032]: 6.4 scrub starts 2023-10-09T10:02:41.619 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:41.667 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:41.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:41.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:41.786 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:41.839 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:41.851 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:41.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:02:41.960 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:41.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:41.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:42.061 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:02:41.773+0000 7f85a99d3700 -1 mon.a@0(leader).osd e167 definitely_dead 0 2023-10-09T10:02:42.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:42.227 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:42.289 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.5. 2023-10-09T10:02:42.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:42 smithi012 ceph-mon[147926]: pgmap v416: 99 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 4324/54501 objects degraded (7.934%); 1862/54501 objects misplaced (3.416%); 0 B/s, 186 objects/s recovering 2023-10-09T10:02:42.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3104093577' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:02:42.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:02:42.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:42 smithi012 ceph-mon[147926]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T10:02:42.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:42 smithi012 ceph-mon[152263]: pgmap v416: 99 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 4324/54501 objects degraded (7.934%); 1862/54501 objects misplaced (3.416%); 0 B/s, 186 objects/s recovering 2023-10-09T10:02:42.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3104093577' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:02:42.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:02:42.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:42 smithi012 ceph-mon[152263]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T10:02:42.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:42 smithi110 ceph-mon[136032]: pgmap v416: 99 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 4324/54501 objects degraded (7.934%); 1862/54501 objects misplaced (3.416%); 0 B/s, 186 objects/s recovering 2023-10-09T10:02:42.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3104093577' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:02:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:02:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:42 smithi110 ceph-mon[136032]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T10:02:43.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T10:02:43.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T10:02:43.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4324/54501 objects degraded (7.934%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[147926]: failed to encode map e169 with expected crc 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4324/54501 objects degraded (7.934%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T10:02:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:43 smithi012 ceph-mon[152263]: failed to encode map e169 with expected crc 2023-10-09T10:02:43.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T10:02:43.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T10:02:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: failed to encode map e168 with expected crc 2023-10-09T10:02:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4324/54501 objects degraded (7.934%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:43.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T10:02:43.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:43 smithi110 ceph-mon[136032]: failed to encode map e169 with expected crc 2023-10-09T10:02:44.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:02:44.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: pgmap v419: 99 pgs: 1 active+recovering+undersized+remapped, 6 peering, 1 active+undersized+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 877 op/s; 4324/49248 objects degraded (8.780%); 1742/49248 objects misplaced (3.537%); 0 B/s, 308 objects/s recovering 2023-10-09T10:02:44.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: failed to encode map e170 with expected crc 2023-10-09T10:02:44.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T10:02:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: 6.14 starting backfill to osd.6 from (0'0,0'0] MAX to 167'9783 2023-10-09T10:02:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: 6.d starting backfill to osd.2 from (111'3860,161'11401] MIN to 167'11971 2023-10-09T10:02:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: 6.4 starting backfill to osd.6 from (0'0,0'0] MAX to 167'9821 2023-10-09T10:02:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: 6.c starting backfill to osd.7 from (0'0,0'0] MAX to 167'12263 2023-10-09T10:02:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:02:44.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:44.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:44.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:02:44.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: pgmap v419: 99 pgs: 1 active+recovering+undersized+remapped, 6 peering, 1 active+undersized+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 877 op/s; 4324/49248 objects degraded (8.780%); 1742/49248 objects misplaced (3.537%); 0 B/s, 308 objects/s recovering 2023-10-09T10:02:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: failed to encode map e170 with expected crc 2023-10-09T10:02:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T10:02:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: 6.14 starting backfill to osd.6 from (0'0,0'0] MAX to 167'9783 2023-10-09T10:02:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: 6.d starting backfill to osd.2 from (111'3860,161'11401] MIN to 167'11971 2023-10-09T10:02:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: 6.4 starting backfill to osd.6 from (0'0,0'0] MAX to 167'9821 2023-10-09T10:02:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: 6.c starting backfill to osd.7 from (0'0,0'0] MAX to 167'12263 2023-10-09T10:02:44.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:02:44.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:44.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:44.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:02:44.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: pgmap v419: 99 pgs: 1 active+recovering+undersized+remapped, 6 peering, 1 active+undersized+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 877 op/s; 4324/49248 objects degraded (8.780%); 1742/49248 objects misplaced (3.537%); 0 B/s, 308 objects/s recovering 2023-10-09T10:02:44.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: failed to encode map e170 with expected crc 2023-10-09T10:02:44.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T10:02:44.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: 6.14 starting backfill to osd.6 from (0'0,0'0] MAX to 167'9783 2023-10-09T10:02:44.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: 6.d starting backfill to osd.2 from (111'3860,161'11401] MIN to 167'11971 2023-10-09T10:02:44.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: 6.4 starting backfill to osd.6 from (0'0,0'0] MAX to 167'9821 2023-10-09T10:02:44.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: 6.c starting backfill to osd.7 from (0'0,0'0] MAX to 167'12263 2023-10-09T10:02:44.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:02:44.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:02:44.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:02:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:45 smithi012 ceph-mon[147926]: failed to encode map e170 with expected crc 2023-10-09T10:02:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:45 smithi012 ceph-mon[147926]: failed to encode map e170 with expected crc 2023-10-09T10:02:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:45 smithi012 ceph-mon[147926]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T10:02:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:45 smithi012 ceph-mon[147926]: failed to encode map e171 with expected crc 2023-10-09T10:02:45.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:45 smithi012 ceph-mon[152263]: failed to encode map e170 with expected crc 2023-10-09T10:02:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:45 smithi012 ceph-mon[152263]: failed to encode map e170 with expected crc 2023-10-09T10:02:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:45 smithi012 ceph-mon[152263]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T10:02:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:45 smithi012 ceph-mon[152263]: failed to encode map e171 with expected crc 2023-10-09T10:02:45.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:45 smithi110 ceph-mon[136032]: failed to encode map e170 with expected crc 2023-10-09T10:02:45.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:45 smithi110 ceph-mon[136032]: failed to encode map e170 with expected crc 2023-10-09T10:02:45.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:45 smithi110 ceph-mon[136032]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T10:02:45.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:45 smithi110 ceph-mon[136032]: failed to encode map e171 with expected crc 2023-10-09T10:02:45.699 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:45.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:45.806 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:45.853 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:45.900 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:45.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:46.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:46.105 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:46.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:46 smithi012 ceph-mon[147926]: failed to encode map e171 with expected crc 2023-10-09T10:02:46.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:46 smithi012 ceph-mon[147926]: failed to encode map e171 with expected crc 2023-10-09T10:02:46.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:46 smithi012 ceph-mon[147926]: pgmap v422: 99 pgs: 1 active+remapped+backfilling, 6 peering, 1 active+undersized+remapped, 2 active+recovery_wait, 4 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 966 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 4322/46401 objects degraded (9.314%); 2913/46401 objects misplaced (6.278%); 4.8 MiB/s, 326 objects/s recovering 2023-10-09T10:02:46.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:46 smithi012 ceph-mon[152263]: failed to encode map e171 with expected crc 2023-10-09T10:02:46.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:46 smithi012 ceph-mon[152263]: failed to encode map e171 with expected crc 2023-10-09T10:02:46.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:46 smithi012 ceph-mon[152263]: pgmap v422: 99 pgs: 1 active+remapped+backfilling, 6 peering, 1 active+undersized+remapped, 2 active+recovery_wait, 4 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 966 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 4322/46401 objects degraded (9.314%); 2913/46401 objects misplaced (6.278%); 4.8 MiB/s, 326 objects/s recovering 2023-10-09T10:02:46.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:46 smithi110 ceph-mon[136032]: failed to encode map e171 with expected crc 2023-10-09T10:02:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:46 smithi110 ceph-mon[136032]: failed to encode map e171 with expected crc 2023-10-09T10:02:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:46 smithi110 ceph-mon[136032]: pgmap v422: 99 pgs: 1 active+remapped+backfilling, 6 peering, 1 active+undersized+remapped, 2 active+recovery_wait, 4 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 966 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 4322/46401 objects degraded (9.314%); 2913/46401 objects misplaced (6.278%); 4.8 MiB/s, 326 objects/s recovering 2023-10-09T10:02:47.470 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:47.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:47.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:47.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:47.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[147926]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T10:02:47.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[147926]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[147926]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[147926]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[147926]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[147926]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[147926]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[147926]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[152263]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T10:02:47.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[152263]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[152263]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[152263]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[152263]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[152263]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[152263]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-mon[152263]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.656 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:47 smithi110 ceph-mon[136032]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T10:02:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:47 smithi110 ceph-mon[136032]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:47 smithi110 ceph-mon[136032]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:47 smithi110 ceph-mon[136032]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:47 smithi110 ceph-mon[136032]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:47 smithi110 ceph-mon[136032]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:47 smithi110 ceph-mon[136032]: failed to encode map e172 with expected crc 2023-10-09T10:02:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:47 smithi110 ceph-mon[136032]: failed to encode map e172 with expected crc 2023-10-09T10:02:48.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:02:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:02:47] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.33.4" 2023-10-09T10:02:48.205 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:48.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:48.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:48.340 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:48.381 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:48.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:48.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:48.471 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:48.485 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:48.518 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:48.563 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:48.568 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:48 smithi110 ceph-mon[136032]: pgmap v424: 99 pgs: 2 active+remapped+backfilling, 7 peering, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 940 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 3790/45180 objects degraded (8.389%); 3606/45180 objects misplaced (7.981%); 6.5 MiB/s, 212 objects/s recovering 2023-10-09T10:02:48.568 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:48 smithi110 ceph-mon[136032]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T10:02:48.568 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:48 smithi110 ceph-mon[136032]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.568 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:48 smithi110 ceph-mon[136032]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.569 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:48 smithi110 ceph-mon[136032]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.569 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:48 smithi110 ceph-mon[136032]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.569 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3790/45180 objects degraded (8.389%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:48.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:48.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[147926]: pgmap v424: 99 pgs: 2 active+remapped+backfilling, 7 peering, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 940 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 3790/45180 objects degraded (8.389%); 3606/45180 objects misplaced (7.981%); 6.5 MiB/s, 212 objects/s recovering 2023-10-09T10:02:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[147926]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T10:02:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[147926]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[147926]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[147926]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[147926]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3790/45180 objects degraded (8.389%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[152263]: pgmap v424: 99 pgs: 2 active+remapped+backfilling, 7 peering, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 940 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 3790/45180 objects degraded (8.389%); 3606/45180 objects misplaced (7.981%); 6.5 MiB/s, 212 objects/s recovering 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[152263]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[152263]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[152263]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[152263]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[152263]: failed to encode map e173 with expected crc 2023-10-09T10:02:48.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3790/45180 objects degraded (8.389%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:48.734 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:49.097 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:49.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:49.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:49.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:49.239 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[147926]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[147926]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[147926]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T10:02:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[147926]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[147926]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[152263]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[152263]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[152263]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T10:02:49.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[152263]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:49 smithi012 ceph-mon[152263]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.439 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:49.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:49.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:49.557 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:49.572 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:02:49.588 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:49 smithi110 ceph-mon[136032]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.588 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:49 smithi110 ceph-mon[136032]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.588 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:49 smithi110 ceph-mon[136032]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T10:02:49.588 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:49 smithi110 ceph-mon[136032]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.588 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:49 smithi110 ceph-mon[136032]: failed to encode map e174 with expected crc 2023-10-09T10:02:49.593 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:49.615 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:02:49.636 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:49.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:49.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:49.664 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:02:49.705 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:02:49.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:49.748 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:02:49.780 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:49.805 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:02:49.825 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:49.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:49.843 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:02:49.876 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:02:49.882 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:02:49.882 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:02:49.883 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:02:49.883 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:02:49.883 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:02:49.883 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:02:49.883 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:02:49.883 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:02:49.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:50.057 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:50.067 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:02:50.067 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:02:50.160 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:50.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:50.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:50.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:50.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:02:50.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: pgmap v427: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 88 active+clean; 875 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 4626/42054 objects degraded (11.000%); 4710/42054 objects misplaced (11.200%); 2.6 MiB/s, 279 objects/s recovering 2023-10-09T10:02:50.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:02:50.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1342136732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: pgmap v427: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 88 active+clean; 875 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 4626/42054 objects degraded (11.000%); 4710/42054 objects misplaced (11.200%); 2.6 MiB/s, 279 objects/s recovering 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T10:02:50.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.315 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.315 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1342136732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:50.327 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:50.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:02:50.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:02:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: pgmap v427: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 88 active+clean; 875 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 404 op/s; 4626/42054 objects degraded (11.000%); 4710/42054 objects misplaced (11.200%); 2.6 MiB/s, 279 objects/s recovering 2023-10-09T10:02:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:02:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T10:02:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: failed to encode map e175 with expected crc 2023-10-09T10:02:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1342136732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:02:50.748 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 2, 4, 0] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:02:50.748 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:02:50.748 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T10:02:50.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:02:50.855 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:50.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:50.968 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:50.994 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:51.027 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:51.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:51.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:51.130 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:51.156 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:51.189 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:51.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:51.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:51.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:51.336 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:51.401 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:51.409 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[147926]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T10:02:51.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[147926]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[147926]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[147926]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[147926]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[147926]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:51.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:51.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[152263]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T10:02:51.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[152263]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[152263]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[152263]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[152263]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:51 smithi012 ceph-mon[152263]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:51.513 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:51.557 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:51.626 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:51.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:51 smithi110 ceph-mon[136032]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T10:02:51.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:51 smithi110 ceph-mon[136032]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:51 smithi110 ceph-mon[136032]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:51 smithi110 ceph-mon[136032]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:51 smithi110 ceph-mon[136032]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:51 smithi110 ceph-mon[136032]: failed to encode map e176 with expected crc 2023-10-09T10:02:51.736 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:51.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:51.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:51.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:51.922 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:51.997 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:52.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:52.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:52.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:52.142 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:52.182 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:52.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:52.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:52.286 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:52.315 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:52.353 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:52.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:52.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:52.367 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:02:52.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: pgmap v430: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 89 active+clean; 803 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 4626/38574 objects degraded (11.993%); 3509/38574 objects misplaced (9.097%); 14 MiB/s, 358 objects/s recovering 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: 6.19 scrub starts 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4074966351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:52 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:52.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:52.549 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:52.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: pgmap v430: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 89 active+clean; 803 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 4626/38574 objects degraded (11.993%); 3509/38574 objects misplaced (9.097%); 14 MiB/s, 358 objects/s recovering 2023-10-09T10:02:52.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: 6.19 scrub starts 2023-10-09T10:02:52.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T10:02:52.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4074966351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: pgmap v430: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 89 active+clean; 803 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 4626/38574 objects degraded (11.993%); 3509/38574 objects misplaced (9.097%); 14 MiB/s, 358 objects/s recovering 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: 6.19 scrub starts 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: failed to encode map e177 with expected crc 2023-10-09T10:02:52.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4074966351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:52.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:52 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:02:52.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:02:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:02:53.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[147926]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:53.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[147926]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T10:02:53.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[147926]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[147926]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4626/38574 objects degraded (11.993%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:02:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[152263]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:53.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[152263]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T10:02:53.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[152263]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[152263]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:53 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4626/38574 objects degraded (11.993%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:53.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:53 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:02:53.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:53 smithi110 ceph-mon[136032]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:53 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:02:53.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:53 smithi110 ceph-mon[136032]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T10:02:53.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:53 smithi110 ceph-mon[136032]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:53 smithi110 ceph-mon[136032]: failed to encode map e178 with expected crc 2023-10-09T10:02:53.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:53 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4626/38574 objects degraded (11.993%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:54.065 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:54.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:54.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:54.261 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:54.293 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:54.336 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:54.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:54.441 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:54.533 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:54.905 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:54 smithi012 ceph-mon[147926]: pgmap v433: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 716 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 4694/34401 objects degraded (13.645%); 2515/34401 objects misplaced (7.311%); 24 MiB/s, 527 objects/s recovering 2023-10-09T10:02:54.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:54 smithi012 ceph-mon[152263]: pgmap v433: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 716 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 4694/34401 objects degraded (13.645%); 2515/34401 objects misplaced (7.311%); 24 MiB/s, 527 objects/s recovering 2023-10-09T10:02:54.906 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:54.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:55.021 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:02:55.093 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:54 smithi110 ceph-mon[136032]: pgmap v433: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 716 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 4694/34401 objects degraded (13.645%); 2515/34401 objects misplaced (7.311%); 24 MiB/s, 527 objects/s recovering 2023-10-09T10:02:56.096 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:02:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:56.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:02:56.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:56.255 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:56.344 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:56.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:56.449 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:56.475 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:56.507 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:56.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:56.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:56.610 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:02:56.636 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:56.671 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:02:56.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:56.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:02:56.712 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:56.748 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:02:56.748 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:02:56.754 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[147926]: pgmap v434: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 641 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 4694/30825 objects degraded (15.228%); 1735/30825 objects misplaced (5.629%); 14 MiB/s, 324 objects/s recovering 2023-10-09T10:02:56.754 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[147926]: 6.0 scrub starts 2023-10-09T10:02:56.755 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[147926]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T10:02:56.755 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[147926]: failed to encode map e179 with expected crc 2023-10-09T10:02:56.755 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[147926]: failed to encode map e179 with expected crc 2023-10-09T10:02:56.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:56.800 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:56.835 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:56.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:56.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:56.939 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:56.977 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:57.018 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:57.021 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:56 smithi110 ceph-mon[136032]: pgmap v434: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 641 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 4694/30825 objects degraded (15.228%); 1735/30825 objects misplaced (5.629%); 14 MiB/s, 324 objects/s recovering 2023-10-09T10:02:57.021 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:56 smithi110 ceph-mon[136032]: 6.0 scrub starts 2023-10-09T10:02:57.021 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:56 smithi110 ceph-mon[136032]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T10:02:57.021 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:56 smithi110 ceph-mon[136032]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.021 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:56 smithi110 ceph-mon[136032]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.021 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:56 smithi110 ceph-mon[136032]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.022 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:56 smithi110 ceph-mon[136032]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:57.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:57.123 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:57.163 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:57.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[147926]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[147926]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[152263]: pgmap v434: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 641 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 4694/30825 objects degraded (15.228%); 1735/30825 objects misplaced (5.629%); 14 MiB/s, 324 objects/s recovering 2023-10-09T10:02:57.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[152263]: 6.0 scrub starts 2023-10-09T10:02:57.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[152263]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T10:02:57.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[152263]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[152263]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[152263]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:56 smithi012 ceph-mon[152263]: failed to encode map e179 with expected crc 2023-10-09T10:02:57.207 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:57.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:57.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:57.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:57.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:02:57.466 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:57.636 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:57.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:57.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:57.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:02:57.849 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[147926]: 6.10 deep-scrub starts 2023-10-09T10:02:57.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[147926]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[147926]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T10:02:57.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[147926]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[147926]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[147926]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.851 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[147926]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.851 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4694/30825 objects degraded (15.228%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:57.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[152263]: 6.10 deep-scrub starts 2023-10-09T10:02:57.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[152263]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.852 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[152263]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T10:02:57.852 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[152263]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.852 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[152263]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.852 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[152263]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.852 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[152263]: failed to encode map e180 with expected crc 2023-10-09T10:02:57.852 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4694/30825 objects degraded (15.228%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:57.853 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:57.968 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:02:57.985 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:02:57.990 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:02:57.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:57.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:02:58.100 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:58.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:02:58.152 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:02:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:02:57] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.33.4" 2023-10-09T10:02:58.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:57 smithi110 ceph-mon[136032]: 6.10 deep-scrub starts 2023-10-09T10:02:58.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:57 smithi110 ceph-mon[136032]: failed to encode map e180 with expected crc 2023-10-09T10:02:58.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:57 smithi110 ceph-mon[136032]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T10:02:58.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:57 smithi110 ceph-mon[136032]: failed to encode map e180 with expected crc 2023-10-09T10:02:58.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:57 smithi110 ceph-mon[136032]: failed to encode map e180 with expected crc 2023-10-09T10:02:58.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:57 smithi110 ceph-mon[136032]: failed to encode map e180 with expected crc 2023-10-09T10:02:58.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:57 smithi110 ceph-mon[136032]: failed to encode map e180 with expected crc 2023-10-09T10:02:58.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:57 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4694/30825 objects degraded (15.228%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:02:58.214 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:58.417 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:02:58.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:58.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:02:58.484 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":437,"stamp":"2023-10-09T10:02:57.056774+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":628430776,"num_objects":9605,"num_object_clones":0,"num_object_copies":28815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4694,"num_objects_misplaced":1735,"num_objects_unfound":0,"num_objects_dirty":9605,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":201717,"num_write_kb":6755598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26654,"num_bytes_recovered":1413226376,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":228520,"ondisk_log_size":228520,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3866044,"kb_used_data":2304324,"kb_used_omap":126,"kb_used_meta":1560897,"kb_avail":652350020,"statfs":{"total":671965249536,"available":668006420480,"internally_reserved":0,"allocated":2359627776,"data_stored":2355302206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1598359163},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-213188608,"num_objects":-3253,"num_object_clones":0,"num_object_copies":-9759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":68,"num_objects_misplaced":-1774,"num_objects_unfound":0,"num_objects_dirty":-3253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3253,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1867,"num_bytes_recovered":82903064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.725991"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348274+0000","last_change":"2023-10-09T10:02:17.733173+0000","last_active":"2023-10-09T10:02:51.348274+0000","last_peered":"2023-10-09T10:02:51.348274+0000","last_clean":"2023-10-09T10:02:51.348274+0000","last_became_active":"2023-10-09T10:02:17.732100+0000","last_became_peered":"2023-10-09T10:02:17.732100+0000","last_unstale":"2023-10-09T10:02:51.348274+0000","last_undegraded":"2023-10-09T10:02:51.348274+0000","last_fullsized":"2023-10-09T10:02:51.348274+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":165,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.334063+0000","last_change":"2023-10-09T10:02:17.718080+0000","last_active":"2023-10-09T10:02:51.334063+0000","last_peered":"2023-10-09T10:02:51.334063+0000","last_clean":"2023-10-09T10:02:51.334063+0000","last_became_active":"2023-10-09T10:02:17.717746+0000","last_became_peered":"2023-10-09T10:02:17.717746+0000","last_unstale":"2023-10-09T10:02:51.334063+0000","last_undegraded":"2023-10-09T10:02:51.334063+0000","last_fullsized":"2023-10-09T10:02:51.334063+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:59.375104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":151,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.340774+0000","last_change":"2023-10-09T10:02:17.740018+0000","last_active":"2023-10-09T10:02:51.340774+0000","last_peered":"2023-10-09T10:02:51.340774+0000","last_clean":"2023-10-09T10:02:51.340774+0000","last_became_active":"2023-10-09T10:02:17.739691+0000","last_became_peered":"2023-10-09T10:02:17.739691+0000","last_unstale":"2023-10-09T10:02:51.340774+0000","last_undegraded":"2023-10-09T10:02:51.340774+0000","last_fullsized":"2023-10-09T10:02:51.340774+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.836600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":28,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.841090+0000","last_change":"2023-10-09T10:02:29.904188+0000","last_active":"2023-10-09T10:02:55.841090+0000","last_peered":"2023-10-09T10:02:55.841090+0000","last_clean":"2023-10-09T10:02:55.841090+0000","last_became_active":"2023-10-09T10:02:29.899658+0000","last_became_peered":"2023-10-09T10:02:29.899658+0000","last_unstale":"2023-10-09T10:02:55.841090+0000","last_undegraded":"2023-10-09T10:02:55.841090+0000","last_fullsized":"2023-10-09T10:02:55.841090+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:35.455601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":181,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.331756+0000","last_change":"2023-10-09T10:02:29.885613+0000","last_active":"2023-10-09T10:02:51.331756+0000","last_peered":"2023-10-09T10:02:51.331756+0000","last_clean":"2023-10-09T10:02:51.331756+0000","last_became_active":"2023-10-09T10:02:29.885185+0000","last_became_peered":"2023-10-09T10:02:29.885185+0000","last_unstale":"2023-10-09T10:02:51.331756+0000","last_undegraded":"2023-10-09T10:02:51.331756+0000","last_fullsized":"2023-10-09T10:02:51.331756+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:01:44.101600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":114,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.340602+0000","last_change":"2023-10-09T10:02:42.727164+0000","last_active":"2023-10-09T10:02:51.340602+0000","last_peered":"2023-10-09T10:02:51.340602+0000","last_clean":"2023-10-09T10:02:51.340602+0000","last_became_active":"2023-10-09T10:02:42.726647+0000","last_became_peered":"2023-10-09T10:02:42.726647+0000","last_unstale":"2023-10-09T10:02:51.340602+0000","last_undegraded":"2023-10-09T10:02:51.340602+0000","last_fullsized":"2023-10-09T10:02:51.340602+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:24.610528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.12","version":"178'10205","reported_seq":14550,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.728752+0000","last_change":"2023-10-09T10:02:44.760639+0000","last_active":"2023-10-09T10:02:54.728752+0000","last_peered":"2023-10-09T10:02:54.728752+0000","last_clean":"2023-10-09T10:02:54.728752+0000","last_became_active":"2023-10-09T10:02:44.723961+0000","last_became_peered":"2023-10-09T10:02:44.723961+0000","last_unstale":"2023-10-09T10:02:54.728752+0000","last_undegraded":"2023-10-09T10:02:54.728752+0000","last_fullsized":"2023-10-09T10:02:54.728752+0000","mapping_epoch":170,"log_start":"104'1880","ondisk_log_start":"104'1880","created":133,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8325,"log_dups_size":0,"ondisk_log_size":8325,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17432576,"num_objects":266,"num_object_clones":0,"num_object_copies":798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6443,"num_write_kb":214688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2101,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":37,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.761630+0000","last_change":"2023-10-09T10:02:42.732361+0000","last_active":"2023-10-09T10:02:55.761630+0000","last_peered":"2023-10-09T10:02:55.761630+0000","last_clean":"2023-10-09T10:02:55.761630+0000","last_became_active":"2023-10-09T10:02:42.732042+0000","last_became_peered":"2023-10-09T10:02:42.732042+0000","last_unstale":"2023-10-09T10:02:55.761630+0000","last_undegraded":"2023-10-09T10:02:55.761630+0000","last_fullsized":"2023-10-09T10:02:55.761630+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:31.630503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.331836+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T10:02:51.331836+0000","last_peered":"2023-10-09T10:02:51.331836+0000","last_clean":"2023-10-09T10:02:51.331836+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T10:02:51.331836+0000","last_undegraded":"2023-10-09T10:02:51.331836+0000","last_fullsized":"2023-10-09T10:02:51.331836+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"178'10107","reported_seq":12004,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.808906+0000","last_change":"2023-10-09T10:00:53.519038+0000","last_active":"2023-10-09T10:02:54.808906+0000","last_peered":"2023-10-09T10:02:54.808906+0000","last_clean":"2023-10-09T10:02:54.808906+0000","last_became_active":"2023-10-09T10:00:52.732215+0000","last_became_peered":"2023-10-09T10:00:52.732215+0000","last_unstale":"2023-10-09T10:02:54.808906+0000","last_undegraded":"2023-10-09T10:02:54.808906+0000","last_fullsized":"2023-10-09T10:02:54.808906+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":10107,"log_dups_size":0,"ondisk_log_size":10107,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18120704,"num_objects":277,"num_object_clones":0,"num_object_copies":831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6375,"num_write_kb":212832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":613,"num_bytes_recovered":40140800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":64,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.341235+0000","last_change":"2023-10-09T10:02:42.728038+0000","last_active":"2023-10-09T10:02:51.341235+0000","last_peered":"2023-10-09T10:02:51.341235+0000","last_clean":"2023-10-09T10:02:51.341235+0000","last_became_active":"2023-10-09T10:02:42.727751+0000","last_became_peered":"2023-10-09T10:02:42.727751+0000","last_unstale":"2023-10-09T10:02:51.341235+0000","last_undegraded":"2023-10-09T10:02:51.341235+0000","last_fullsized":"2023-10-09T10:02:51.341235+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:24.983171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":108,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.332283+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T10:02:51.332283+0000","last_peered":"2023-10-09T10:02:51.332283+0000","last_clean":"2023-10-09T10:02:51.332283+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T10:02:51.332283+0000","last_undegraded":"2023-10-09T10:02:51.332283+0000","last_fullsized":"2023-10-09T10:02:51.332283+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"179'9979","reported_seq":12629,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:56.384692+0000","last_change":"2023-10-09T10:02:56.321450+0000","last_active":"2023-10-09T10:02:56.384692+0000","last_peered":"2023-10-09T10:02:56.384692+0000","last_clean":"2023-10-09T10:02:56.384692+0000","last_became_active":"2023-10-09T10:02:47.331852+0000","last_became_peered":"2023-10-09T10:02:47.331852+0000","last_unstale":"2023-10-09T10:02:56.384692+0000","last_undegraded":"2023-10-09T10:02:56.384692+0000","last_fullsized":"2023-10-09T10:02:56.384692+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9979,"log_dups_size":0,"ondisk_log_size":9979,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16809984,"num_objects":257,"num_object_clones":0,"num_object_copies":771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6286,"num_write_kb":209344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":934,"num_bytes_recovered":26312704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":191,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.446733+0000","last_change":"2023-10-09T10:02:18.222658+0000","last_active":"2023-10-09T10:02:51.446733+0000","last_peered":"2023-10-09T10:02:51.446733+0000","last_clean":"2023-10-09T10:02:51.446733+0000","last_became_active":"2023-10-09T10:02:17.734984+0000","last_became_peered":"2023-10-09T10:02:17.734984+0000","last_unstale":"2023-10-09T10:02:51.446733+0000","last_undegraded":"2023-10-09T10:02:51.446733+0000","last_fullsized":"2023-10-09T10:02:51.446733+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":151,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339393+0000","last_change":"2023-10-09T10:00:30.790050+0000","last_active":"2023-10-09T10:02:51.339393+0000","last_peered":"2023-10-09T10:02:51.339393+0000","last_clean":"2023-10-09T10:02:51.339393+0000","last_became_active":"2023-10-09T10:00:30.789740+0000","last_became_peered":"2023-10-09T10:00:30.789740+0000","last_unstale":"2023-10-09T10:02:51.339393+0000","last_undegraded":"2023-10-09T10:02:51.339393+0000","last_fullsized":"2023-10-09T10:02:51.339393+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:36.206642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"178'10089","reported_seq":10800,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.048346+0000","last_change":"2023-10-09T10:01:00.721483+0000","last_active":"2023-10-09T10:02:54.048346+0000","last_peered":"2023-10-09T10:02:54.048346+0000","last_clean":"2023-10-09T10:02:54.048346+0000","last_became_active":"2023-10-09T10:00:52.779409+0000","last_became_peered":"2023-10-09T10:00:52.779409+0000","last_unstale":"2023-10-09T10:02:54.048346+0000","last_undegraded":"2023-10-09T10:02:54.048346+0000","last_fullsized":"2023-10-09T10:02:54.048346+0000","mapping_epoch":133,"log_start":"103'695","ondisk_log_start":"103'695","created":133,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9394,"log_dups_size":0,"ondisk_log_size":9394,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18874368,"num_objects":288,"num_object_clones":0,"num_object_copies":864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6367,"num_write_kb":212960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":299,"num_bytes_recovered":19595264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":39,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348498+0000","last_change":"2023-10-09T10:02:18.088005+0000","last_active":"2023-10-09T10:02:51.348498+0000","last_peered":"2023-10-09T10:02:51.348498+0000","last_clean":"2023-10-09T10:02:51.348498+0000","last_became_active":"2023-10-09T10:02:17.722407+0000","last_became_peered":"2023-10-09T10:02:17.722407+0000","last_unstale":"2023-10-09T10:02:51.348498+0000","last_undegraded":"2023-10-09T10:02:51.348498+0000","last_fullsized":"2023-10-09T10:02:51.348498+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348301+0000","last_change":"2023-10-09T10:02:17.731192+0000","last_active":"2023-10-09T10:02:51.348301+0000","last_peered":"2023-10-09T10:02:51.348301+0000","last_clean":"2023-10-09T10:02:51.348301+0000","last_became_active":"2023-10-09T10:02:17.730757+0000","last_became_peered":"2023-10-09T10:02:17.730757+0000","last_unstale":"2023-10-09T10:02:51.348301+0000","last_undegraded":"2023-10-09T10:02:51.348301+0000","last_fullsized":"2023-10-09T10:02:51.348301+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"178'9974","reported_seq":10100,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:52.366225+0000","last_change":"2023-10-09T10:02:18.759849+0000","last_active":"2023-10-09T10:02:52.366225+0000","last_peered":"2023-10-09T10:02:52.366225+0000","last_clean":"2023-10-09T10:02:52.366225+0000","last_became_active":"2023-10-09T10:02:18.733707+0000","last_became_peered":"2023-10-09T10:02:18.733707+0000","last_unstale":"2023-10-09T10:02:52.366225+0000","last_undegraded":"2023-10-09T10:02:52.366225+0000","last_fullsized":"2023-10-09T10:02:52.366225+0000","mapping_epoch":159,"log_start":"104'1516","ondisk_log_start":"104'1516","created":133,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8458,"log_dups_size":0,"ondisk_log_size":8458,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23134208,"num_objects":353,"num_object_clones":0,"num_object_copies":1059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6247,"num_write_kb":211200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":205,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.446153+0000","last_change":"2023-10-09T10:02:17.760140+0000","last_active":"2023-10-09T10:02:51.446153+0000","last_peered":"2023-10-09T10:02:51.446153+0000","last_clean":"2023-10-09T10:02:51.446153+0000","last_became_active":"2023-10-09T10:02:17.724899+0000","last_became_peered":"2023-10-09T10:02:17.724899+0000","last_unstale":"2023-10-09T10:02:51.446153+0000","last_undegraded":"2023-10-09T10:02:51.446153+0000","last_fullsized":"2023-10-09T10:02:51.446153+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.333174+0000","last_change":"2023-10-09T09:57:49.622141+0000","last_active":"2023-10-09T10:02:51.333174+0000","last_peered":"2023-10-09T10:02:51.333174+0000","last_clean":"2023-10-09T10:02:51.333174+0000","last_became_active":"2023-10-09T09:57:49.621815+0000","last_became_peered":"2023-10-09T09:57:49.621815+0000","last_unstale":"2023-10-09T10:02:51.333174+0000","last_undegraded":"2023-10-09T10:02:51.333174+0000","last_fullsized":"2023-10-09T10:02:51.333174+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:10.215337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"178'9744","reported_seq":10558,"reported_epoch":178,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:02:52.430049+0000","last_change":"2023-10-09T10:02:44.002420+0000","last_active":"2023-10-09T10:02:52.430049+0000","last_peered":"2023-10-09T10:02:52.430049+0000","last_clean":"2023-10-09T10:00:59.881901+0000","last_became_active":"2023-10-09T10:02:43.969276+0000","last_became_peered":"2023-10-09T10:02:43.969276+0000","last_unstale":"2023-10-09T10:02:52.430049+0000","last_undegraded":"2023-10-09T10:02:43.906927+0000","last_fullsized":"2023-10-09T10:02:43.813271+0000","mapping_epoch":169,"log_start":"103'924","ondisk_log_start":"103'924","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8820,"log_dups_size":0,"ondisk_log_size":8820,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20840448,"num_objects":318,"num_object_clones":0,"num_object_copies":954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4554,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6125,"num_write_kb":206176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":345,"num_bytes_recovered":21168128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4","objects":-168},{"shards":"3,4,5","objects":323}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":28,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.841259+0000","last_change":"2023-10-09T10:02:29.904731+0000","last_active":"2023-10-09T10:02:55.841259+0000","last_peered":"2023-10-09T10:02:55.841259+0000","last_clean":"2023-10-09T10:02:55.841259+0000","last_became_active":"2023-10-09T10:02:29.904290+0000","last_became_peered":"2023-10-09T10:02:29.904290+0000","last_unstale":"2023-10-09T10:02:55.841259+0000","last_undegraded":"2023-10-09T10:02:55.841259+0000","last_fullsized":"2023-10-09T10:02:55.841259+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:00.376061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.446810+0000","last_change":"2023-10-09T10:02:42.716581+0000","last_active":"2023-10-09T10:02:51.446810+0000","last_peered":"2023-10-09T10:02:51.446810+0000","last_clean":"2023-10-09T10:02:51.446810+0000","last_became_active":"2023-10-09T10:02:42.716329+0000","last_became_peered":"2023-10-09T10:02:42.716329+0000","last_unstale":"2023-10-09T10:02:51.446810+0000","last_undegraded":"2023-10-09T10:02:51.446810+0000","last_fullsized":"2023-10-09T10:02:51.446810+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"178'9957","reported_seq":10864,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.115067+0000","last_change":"2023-10-09T10:02:49.344575+0000","last_active":"2023-10-09T10:02:54.115067+0000","last_peered":"2023-10-09T10:02:54.115067+0000","last_clean":"2023-10-09T10:02:54.115067+0000","last_became_active":"2023-10-09T10:02:49.322950+0000","last_became_peered":"2023-10-09T10:02:49.322950+0000","last_unstale":"2023-10-09T10:02:54.115067+0000","last_undegraded":"2023-10-09T10:02:54.115067+0000","last_fullsized":"2023-10-09T10:02:54.115067+0000","mapping_epoch":174,"log_start":"103'540","ondisk_log_start":"103'540","created":133,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9417,"log_dups_size":0,"ondisk_log_size":9417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20480000,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6257,"num_write_kb":210208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":392,"num_bytes_recovered":25690112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348281+0000","last_change":"2023-10-09T10:02:17.730673+0000","last_active":"2023-10-09T10:02:51.348281+0000","last_peered":"2023-10-09T10:02:51.348281+0000","last_clean":"2023-10-09T10:02:51.348281+0000","last_became_active":"2023-10-09T10:02:17.729848+0000","last_became_peered":"2023-10-09T10:02:17.729848+0000","last_unstale":"2023-10-09T10:02:51.348281+0000","last_undegraded":"2023-10-09T10:02:51.348281+0000","last_fullsized":"2023-10-09T10:02:51.348281+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":28,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.840929+0000","last_change":"2023-10-09T10:02:29.900023+0000","last_active":"2023-10-09T10:02:55.840929+0000","last_peered":"2023-10-09T10:02:55.840929+0000","last_clean":"2023-10-09T10:02:55.840929+0000","last_became_active":"2023-10-09T10:02:29.899702+0000","last_became_peered":"2023-10-09T10:02:29.899702+0000","last_unstale":"2023-10-09T10:02:55.840929+0000","last_undegraded":"2023-10-09T10:02:55.840929+0000","last_fullsized":"2023-10-09T10:02:55.840929+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:39.396593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"178'9965","reported_seq":10062,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.830684+0000","last_change":"2023-10-09T10:00:52.704990+0000","last_active":"2023-10-09T10:02:54.830684+0000","last_peered":"2023-10-09T10:02:54.830684+0000","last_clean":"2023-10-09T10:02:54.830684+0000","last_became_active":"2023-10-09T10:00:52.699362+0000","last_became_peered":"2023-10-09T10:00:52.699362+0000","last_unstale":"2023-10-09T10:02:54.830684+0000","last_undegraded":"2023-10-09T10:02:54.830684+0000","last_fullsized":"2023-10-09T10:02:54.830684+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9965,"log_dups_size":0,"ondisk_log_size":9965,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16416768,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6280,"num_write_kb":208960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":47,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.761490+0000","last_change":"2023-10-09T10:02:42.734786+0000","last_active":"2023-10-09T10:02:55.761490+0000","last_peered":"2023-10-09T10:02:55.761490+0000","last_clean":"2023-10-09T10:02:55.761490+0000","last_became_active":"2023-10-09T10:02:42.734398+0000","last_became_peered":"2023-10-09T10:02:42.734398+0000","last_unstale":"2023-10-09T10:02:55.761490+0000","last_undegraded":"2023-10-09T10:02:55.761490+0000","last_fullsized":"2023-10-09T10:02:55.761490+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:46:26.098529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.335916+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T10:02:51.335916+0000","last_peered":"2023-10-09T10:02:51.335916+0000","last_clean":"2023-10-09T10:02:51.335916+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T10:02:51.335916+0000","last_undegraded":"2023-10-09T10:02:51.335916+0000","last_fullsized":"2023-10-09T10:02:51.335916+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"179'12759","reported_seq":14506,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:56.408507+0000","last_change":"2023-10-09T10:02:51.390252+0000","last_active":"2023-10-09T10:02:56.408507+0000","last_peered":"2023-10-09T10:02:56.408507+0000","last_clean":"2023-10-09T10:02:56.408507+0000","last_became_active":"2023-10-09T10:02:51.341983+0000","last_became_peered":"2023-10-09T10:02:51.341983+0000","last_unstale":"2023-10-09T10:02:56.408507+0000","last_undegraded":"2023-10-09T10:02:56.408507+0000","last_fullsized":"2023-10-09T10:02:56.408507+0000","mapping_epoch":176,"log_start":"112'4688","ondisk_log_start":"112'4688","created":97,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8071,"log_dups_size":0,"ondisk_log_size":8071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:00.020383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30343168,"num_objects":463,"num_object_clones":0,"num_object_copies":1389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12759,"num_write_kb":423104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":806,"num_bytes_recovered":52822016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":196,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.446802+0000","last_change":"2023-10-09T10:02:29.908231+0000","last_active":"2023-10-09T10:02:51.446802+0000","last_peered":"2023-10-09T10:02:51.446802+0000","last_clean":"2023-10-09T10:02:51.446802+0000","last_became_active":"2023-10-09T10:02:29.880421+0000","last_became_peered":"2023-10-09T10:02:29.880421+0000","last_unstale":"2023-10-09T10:02:51.446802+0000","last_undegraded":"2023-10-09T10:02:51.446802+0000","last_fullsized":"2023-10-09T10:02:51.446802+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":22,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.446147+0000","last_change":"2023-10-09T10:02:42.715191+0000","last_active":"2023-10-09T10:02:51.446147+0000","last_peered":"2023-10-09T10:02:51.446147+0000","last_clean":"2023-10-09T10:02:51.446147+0000","last_became_active":"2023-10-09T10:02:42.714849+0000","last_became_peered":"2023-10-09T10:02:42.714849+0000","last_unstale":"2023-10-09T10:02:51.446147+0000","last_undegraded":"2023-10-09T10:02:51.446147+0000","last_fullsized":"2023-10-09T10:02:51.446147+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"178'12712","reported_seq":12803,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:52.410219+0000","last_change":"2023-10-09T09:58:52.625388+0000","last_active":"2023-10-09T10:02:52.410219+0000","last_peered":"2023-10-09T10:02:52.410219+0000","last_clean":"2023-10-09T10:02:52.410219+0000","last_became_active":"2023-10-09T09:58:52.625135+0000","last_became_peered":"2023-10-09T09:58:52.625135+0000","last_unstale":"2023-10-09T10:02:52.410219+0000","last_undegraded":"2023-10-09T10:02:52.410219+0000","last_fullsized":"2023-10-09T10:02:52.410219+0000","mapping_epoch":97,"log_start":"128'7092","ondisk_log_start":"128'7092","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":5620,"log_dups_size":0,"ondisk_log_size":5620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:28.713713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43122688,"num_objects":658,"num_object_clones":0,"num_object_copies":1974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12712,"num_write_kb":427840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":184,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.446361+0000","last_change":"2023-10-09T10:02:18.135878+0000","last_active":"2023-10-09T10:02:51.446361+0000","last_peered":"2023-10-09T10:02:51.446361+0000","last_clean":"2023-10-09T10:02:51.446361+0000","last_became_active":"2023-10-09T10:02:17.734932+0000","last_became_peered":"2023-10-09T10:02:17.734932+0000","last_unstale":"2023-10-09T10:02:51.446361+0000","last_undegraded":"2023-10-09T10:02:51.446361+0000","last_fullsized":"2023-10-09T10:02:51.446361+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348272+0000","last_change":"2023-10-09T10:02:17.743834+0000","last_active":"2023-10-09T10:02:51.348272+0000","last_peered":"2023-10-09T10:02:51.348272+0000","last_clean":"2023-10-09T10:02:51.348272+0000","last_became_active":"2023-10-09T10:02:17.743066+0000","last_became_peered":"2023-10-09T10:02:17.743066+0000","last_unstale":"2023-10-09T10:02:51.348272+0000","last_undegraded":"2023-10-09T10:02:51.348272+0000","last_fullsized":"2023-10-09T10:02:51.348272+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"177'10177","reported_seq":10950,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:52.259589+0000","last_change":"2023-10-09T10:01:00.234694+0000","last_active":"2023-10-09T10:02:52.259589+0000","last_peered":"2023-10-09T10:02:52.259589+0000","last_clean":"2023-10-09T10:02:52.259589+0000","last_became_active":"2023-10-09T10:00:53.649276+0000","last_became_peered":"2023-10-09T10:00:53.649276+0000","last_unstale":"2023-10-09T10:02:52.259589+0000","last_undegraded":"2023-10-09T10:02:52.259589+0000","last_fullsized":"2023-10-09T10:02:52.259589+0000","mapping_epoch":134,"log_start":"102'400","ondisk_log_start":"102'400","created":97,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9777,"log_dups_size":0,"ondisk_log_size":9777,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25460736,"num_objects":388,"num_object_clones":0,"num_object_copies":1164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6372,"num_write_kb":216352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":332,"num_bytes_recovered":21757952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":24,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.333876+0000","last_change":"2023-10-09T10:02:42.774736+0000","last_active":"2023-10-09T10:02:51.333876+0000","last_peered":"2023-10-09T10:02:51.333876+0000","last_clean":"2023-10-09T10:02:51.333876+0000","last_became_active":"2023-10-09T10:02:42.715518+0000","last_became_peered":"2023-10-09T10:02:42.715518+0000","last_unstale":"2023-10-09T10:02:51.333876+0000","last_undegraded":"2023-10-09T10:02:51.333876+0000","last_fullsized":"2023-10-09T10:02:51.333876+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:14.590462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":82,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339551+0000","last_change":"2023-10-09T10:01:36.025782+0000","last_active":"2023-10-09T10:02:51.339551+0000","last_peered":"2023-10-09T10:02:51.339551+0000","last_clean":"2023-10-09T10:02:51.339551+0000","last_became_active":"2023-10-09T10:01:36.025196+0000","last_became_peered":"2023-10-09T10:01:36.025196+0000","last_unstale":"2023-10-09T10:02:51.339551+0000","last_undegraded":"2023-10-09T10:02:51.339551+0000","last_fullsized":"2023-10-09T10:02:51.339551+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:04.486165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"179'10027","reported_seq":14546,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:56.449397+0000","last_change":"2023-10-09T10:02:50.371578+0000","last_active":"2023-10-09T10:02:56.449397+0000","last_peered":"2023-10-09T10:02:56.449397+0000","last_clean":"2023-10-09T10:02:56.449397+0000","last_became_active":"2023-10-09T10:02:50.346036+0000","last_became_peered":"2023-10-09T10:02:50.346036+0000","last_unstale":"2023-10-09T10:02:56.449397+0000","last_undegraded":"2023-10-09T10:02:56.449397+0000","last_fullsized":"2023-10-09T10:02:56.449397+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":10027,"log_dups_size":0,"ondisk_log_size":10027,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13336576,"num_objects":203,"num_object_clones":0,"num_object_copies":609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6329,"num_write_kb":209056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1779,"num_bytes_recovered":83394560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.333496+0000","last_change":"2023-10-09T10:02:42.729859+0000","last_active":"2023-10-09T10:02:51.333496+0000","last_peered":"2023-10-09T10:02:51.333496+0000","last_clean":"2023-10-09T10:02:51.333496+0000","last_became_active":"2023-10-09T10:02:42.729583+0000","last_became_peered":"2023-10-09T10:02:42.729583+0000","last_unstale":"2023-10-09T10:02:51.333496+0000","last_undegraded":"2023-10-09T10:02:51.333496+0000","last_fullsized":"2023-10-09T10:02:51.333496+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:22.749051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":29,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.841127+0000","last_change":"2023-10-09T10:02:29.889376+0000","last_active":"2023-10-09T10:02:55.841127+0000","last_peered":"2023-10-09T10:02:55.841127+0000","last_clean":"2023-10-09T10:02:55.841127+0000","last_became_active":"2023-10-09T10:02:29.888420+0000","last_became_peered":"2023-10-09T10:02:29.888420+0000","last_unstale":"2023-10-09T10:02:55.841127+0000","last_undegraded":"2023-10-09T10:02:55.841127+0000","last_fullsized":"2023-10-09T10:02:55.841127+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:16.976540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"178'12616","reported_seq":24713,"reported_epoch":178,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:02:54.168854+0000","last_change":"2023-10-09T10:02:44.175198+0000","last_active":"2023-10-09T10:02:54.168854+0000","last_peered":"2023-10-09T10:02:54.168854+0000","last_clean":"2023-10-09T10:02:16.942633+0000","last_became_active":"2023-10-09T10:02:18.812469+0000","last_became_peered":"2023-10-09T10:02:18.812469+0000","last_unstale":"2023-10-09T10:02:54.168854+0000","last_undegraded":"2023-10-09T10:02:54.168854+0000","last_fullsized":"2023-10-09T10:02:54.168854+0000","mapping_epoch":159,"log_start":"119'6207","ondisk_log_start":"119'6207","created":97,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6409,"log_dups_size":0,"ondisk_log_size":6409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:02:08.210356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39190528,"num_objects":598,"num_object_clones":0,"num_object_copies":1794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":286,"num_objects_unfound":0,"num_objects_dirty":598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12616,"num_write_kb":422848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5974,"num_bytes_recovered":391446528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[1,2,5],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2,5","objects":598}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":197,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.333045+0000","last_change":"2023-10-09T10:02:29.946556+0000","last_active":"2023-10-09T10:02:51.333045+0000","last_peered":"2023-10-09T10:02:51.333045+0000","last_clean":"2023-10-09T10:02:51.333045+0000","last_became_active":"2023-10-09T10:02:29.884952+0000","last_became_peered":"2023-10-09T10:02:29.884952+0000","last_unstale":"2023-10-09T10:02:51.333045+0000","last_undegraded":"2023-10-09T10:02:51.333045+0000","last_fullsized":"2023-10-09T10:02:51.333045+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:32.438007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348399+0000","last_change":"2023-10-09T10:02:17.743630+0000","last_active":"2023-10-09T10:02:51.348399+0000","last_peered":"2023-10-09T10:02:51.348399+0000","last_clean":"2023-10-09T10:02:51.348399+0000","last_became_active":"2023-10-09T10:02:17.742997+0000","last_became_peered":"2023-10-09T10:02:17.742997+0000","last_unstale":"2023-10-09T10:02:51.348399+0000","last_undegraded":"2023-10-09T10:02:51.348399+0000","last_fullsized":"2023-10-09T10:02:51.348399+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"177'12440","reported_seq":13991,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:52.243464+0000","last_change":"2023-10-09T10:02:43.064851+0000","last_active":"2023-10-09T10:02:52.243464+0000","last_peered":"2023-10-09T10:02:52.243464+0000","last_clean":"2023-10-09T10:02:52.243464+0000","last_became_active":"2023-10-09T10:02:17.796226+0000","last_became_peered":"2023-10-09T10:02:17.796226+0000","last_unstale":"2023-10-09T10:02:52.243464+0000","last_undegraded":"2023-10-09T10:02:52.243464+0000","last_fullsized":"2023-10-09T10:02:52.243464+0000","mapping_epoch":158,"log_start":"112'4630","ondisk_log_start":"112'4630","created":97,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":7810,"log_dups_size":0,"ondisk_log_size":7810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:56.118300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42729472,"num_objects":652,"num_object_clones":0,"num_object_copies":1956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12440,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":743,"num_bytes_recovered":48627712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348392+0000","last_change":"2023-10-09T10:02:17.733313+0000","last_active":"2023-10-09T10:02:51.348392+0000","last_peered":"2023-10-09T10:02:51.348392+0000","last_clean":"2023-10-09T10:02:51.348392+0000","last_became_active":"2023-10-09T10:02:17.732658+0000","last_became_peered":"2023-10-09T10:02:17.732658+0000","last_unstale":"2023-10-09T10:02:51.348392+0000","last_undegraded":"2023-10-09T10:02:51.348392+0000","last_fullsized":"2023-10-09T10:02:51.348392+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348576+0000","last_change":"2023-10-09T10:02:17.731627+0000","last_active":"2023-10-09T10:02:51.348576+0000","last_peered":"2023-10-09T10:02:51.348576+0000","last_clean":"2023-10-09T10:02:51.348576+0000","last_became_active":"2023-10-09T10:02:17.731306+0000","last_became_peered":"2023-10-09T10:02:17.731306+0000","last_unstale":"2023-10-09T10:02:51.348576+0000","last_undegraded":"2023-10-09T10:02:51.348576+0000","last_fullsized":"2023-10-09T10:02:51.348576+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"178'12554","reported_seq":12713,"reported_epoch":178,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:02:52.428765+0000","last_change":"2023-10-09T10:02:44.010541+0000","last_active":"2023-10-09T10:02:52.428765+0000","last_peered":"2023-10-09T10:02:52.428765+0000","last_clean":"2023-10-09T10:02:42.219591+0000","last_became_active":"2023-10-09T10:02:43.790257+0000","last_became_peered":"2023-10-09T10:02:43.790257+0000","last_unstale":"2023-10-09T10:02:52.428765+0000","last_undegraded":"2023-10-09T10:02:52.428765+0000","last_fullsized":"2023-10-09T10:02:52.428765+0000","mapping_epoch":169,"log_start":"128'6890","ondisk_log_start":"128'6890","created":97,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":5664,"log_dups_size":0,"ondisk_log_size":5664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:29.731160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43515904,"num_objects":664,"num_object_clones":0,"num_object_copies":1992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":650,"num_objects_unfound":0,"num_objects_dirty":664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12554,"num_write_kb":422976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1835008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,5],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":664}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348113+0000","last_change":"2023-10-09T10:02:17.730841+0000","last_active":"2023-10-09T10:02:51.348113+0000","last_peered":"2023-10-09T10:02:51.348113+0000","last_clean":"2023-10-09T10:02:51.348113+0000","last_became_active":"2023-10-09T10:02:17.730221+0000","last_became_peered":"2023-10-09T10:02:17.730221+0000","last_unstale":"2023-10-09T10:02:51.348113+0000","last_undegraded":"2023-10-09T10:02:51.348113+0000","last_fullsized":"2023-10-09T10:02:51.348113+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348667+0000","last_change":"2023-10-09T10:02:17.732379+0000","last_active":"2023-10-09T10:02:51.348667+0000","last_peered":"2023-10-09T10:02:51.348667+0000","last_clean":"2023-10-09T10:02:51.348667+0000","last_became_active":"2023-10-09T10:02:17.729410+0000","last_became_peered":"2023-10-09T10:02:17.729410+0000","last_unstale":"2023-10-09T10:02:51.348667+0000","last_undegraded":"2023-10-09T10:02:51.348667+0000","last_fullsized":"2023-10-09T10:02:51.348667+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"178'12315","reported_seq":13650,"reported_epoch":178,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T10:02:54.176347+0000","last_change":"2023-10-09T10:02:49.946955+0000","last_active":"2023-10-09T10:02:54.176347+0000","last_peered":"2023-10-09T10:02:54.176347+0000","last_clean":"2023-10-09T10:02:42.174326+0000","last_became_active":"2023-10-09T10:02:43.916386+0000","last_became_peered":"2023-10-09T10:02:43.916386+0000","last_unstale":"2023-10-09T10:02:54.176347+0000","last_undegraded":"2023-10-09T10:02:54.176347+0000","last_fullsized":"2023-10-09T10:02:54.176347+0000","mapping_epoch":169,"log_start":"119'5921","ondisk_log_start":"119'5921","created":97,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:53.058542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40632344,"num_objects":621,"num_object_clones":0,"num_object_copies":1863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":94,"num_objects_unfound":0,"num_objects_dirty":621,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12315,"num_write_kb":413889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":607,"num_bytes_recovered":39714840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,5],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1,5","objects":621}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":148,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.331530+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T10:02:51.331530+0000","last_peered":"2023-10-09T10:02:51.331530+0000","last_clean":"2023-10-09T10:02:51.331530+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T10:02:51.331530+0000","last_undegraded":"2023-10-09T10:02:51.331530+0000","last_fullsized":"2023-10-09T10:02:51.331530+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":151,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.340415+0000","last_change":"2023-10-09T10:02:17.726854+0000","last_active":"2023-10-09T10:02:51.340415+0000","last_peered":"2023-10-09T10:02:51.340415+0000","last_clean":"2023-10-09T10:02:51.340415+0000","last_became_active":"2023-10-09T10:02:17.726570+0000","last_became_peered":"2023-10-09T10:02:17.726570+0000","last_unstale":"2023-10-09T10:02:51.340415+0000","last_undegraded":"2023-10-09T10:02:51.340415+0000","last_fullsized":"2023-10-09T10:02:51.340415+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:41.295112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":176,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339779+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:02:51.339779+0000","last_peered":"2023-10-09T10:02:51.339779+0000","last_clean":"2023-10-09T10:02:51.339779+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:02:51.339779+0000","last_undegraded":"2023-10-09T10:02:51.339779+0000","last_fullsized":"2023-10-09T10:02:51.339779+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.334200+0000","last_change":"2023-10-09T10:02:42.728439+0000","last_active":"2023-10-09T10:02:51.334200+0000","last_peered":"2023-10-09T10:02:51.334200+0000","last_clean":"2023-10-09T10:02:51.334200+0000","last_became_active":"2023-10-09T10:02:42.728157+0000","last_became_peered":"2023-10-09T10:02:42.728157+0000","last_unstale":"2023-10-09T10:02:51.334200+0000","last_undegraded":"2023-10-09T10:02:51.334200+0000","last_fullsized":"2023-10-09T10:02:51.334200+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:40.510084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":109,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339994+0000","last_change":"2023-10-09T10:02:42.724058+0000","last_active":"2023-10-09T10:02:51.339994+0000","last_peered":"2023-10-09T10:02:51.339994+0000","last_clean":"2023-10-09T10:02:51.339994+0000","last_became_active":"2023-10-09T10:02:42.723718+0000","last_became_peered":"2023-10-09T10:02:42.723718+0000","last_unstale":"2023-10-09T10:02:51.339994+0000","last_undegraded":"2023-10-09T10:02:51.339994+0000","last_fullsized":"2023-10-09T10:02:51.339994+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:20.608141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"179'10017","reported_seq":13931,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:02:56.752722+0000","last_change":"2023-10-09T10:02:55.350914+0000","last_active":"2023-10-09T10:02:56.752722+0000","last_peered":"2023-10-09T10:02:56.752722+0000","last_clean":"2023-10-09T10:02:56.752722+0000","last_became_active":"2023-10-09T10:02:43.717515+0000","last_became_peered":"2023-10-09T10:02:43.717515+0000","last_unstale":"2023-10-09T10:02:56.752722+0000","last_undegraded":"2023-10-09T10:02:56.752722+0000","last_fullsized":"2023-10-09T10:02:56.752722+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":10017,"log_dups_size":0,"ondisk_log_size":10017,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14319616,"num_objects":218,"num_object_clones":0,"num_object_copies":654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6323,"num_write_kb":209344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1568,"num_bytes_recovered":65306624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"178'10026","reported_seq":13568,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:52.432349+0000","last_change":"2023-10-09T10:02:18.759836+0000","last_active":"2023-10-09T10:02:52.432349+0000","last_peered":"2023-10-09T10:02:52.432349+0000","last_clean":"2023-10-09T10:02:52.432349+0000","last_became_active":"2023-10-09T10:02:18.731094+0000","last_became_peered":"2023-10-09T10:02:18.731094+0000","last_unstale":"2023-10-09T10:02:52.432349+0000","last_undegraded":"2023-10-09T10:02:52.432349+0000","last_fullsized":"2023-10-09T10:02:52.432349+0000","mapping_epoch":159,"log_start":"104'1819","ondisk_log_start":"104'1819","created":97,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8207,"log_dups_size":0,"ondisk_log_size":8207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19726336,"num_objects":301,"num_object_clones":0,"num_object_copies":903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6299,"num_write_kb":211200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1709,"num_bytes_recovered":112001024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":68,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339477+0000","last_change":"2023-10-09T10:02:42.724064+0000","last_active":"2023-10-09T10:02:51.339477+0000","last_peered":"2023-10-09T10:02:51.339477+0000","last_clean":"2023-10-09T10:02:51.339477+0000","last_became_active":"2023-10-09T10:02:42.723722+0000","last_became_peered":"2023-10-09T10:02:42.723722+0000","last_unstale":"2023-10-09T10:02:51.339477+0000","last_undegraded":"2023-10-09T10:02:51.339477+0000","last_fullsized":"2023-10-09T10:02:51.339477+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:15.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":38,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348336+0000","last_change":"2023-10-09T10:02:29.891638+0000","last_active":"2023-10-09T10:02:51.348336+0000","last_peered":"2023-10-09T10:02:51.348336+0000","last_clean":"2023-10-09T10:02:51.348336+0000","last_became_active":"2023-10-09T10:02:29.891265+0000","last_became_peered":"2023-10-09T10:02:29.891265+0000","last_unstale":"2023-10-09T10:02:51.348336+0000","last_undegraded":"2023-10-09T10:02:51.348336+0000","last_fullsized":"2023-10-09T10:02:51.348336+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":166,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339722+0000","last_change":"2023-10-09T10:00:30.782106+0000","last_active":"2023-10-09T10:02:51.339722+0000","last_peered":"2023-10-09T10:02:51.339722+0000","last_clean":"2023-10-09T10:02:51.339722+0000","last_became_active":"2023-10-09T10:00:30.781623+0000","last_became_peered":"2023-10-09T10:00:30.781623+0000","last_unstale":"2023-10-09T10:02:51.339722+0000","last_undegraded":"2023-10-09T10:02:51.339722+0000","last_fullsized":"2023-10-09T10:02:51.339722+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:14.657623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"178'9946","reported_seq":10060,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.869358+0000","last_change":"2023-10-09T10:02:48.916676+0000","last_active":"2023-10-09T10:02:54.869358+0000","last_peered":"2023-10-09T10:02:54.869358+0000","last_clean":"2023-10-09T10:02:54.869358+0000","last_became_active":"2023-10-09T10:00:52.671121+0000","last_became_peered":"2023-10-09T10:00:52.671121+0000","last_unstale":"2023-10-09T10:02:54.869358+0000","last_undegraded":"2023-10-09T10:02:54.869358+0000","last_fullsized":"2023-10-09T10:02:54.869358+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9946,"log_dups_size":0,"ondisk_log_size":9946,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17661952,"num_objects":269,"num_object_clones":0,"num_object_copies":807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6260,"num_write_kb":208960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":82,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T10:02:50.341458+0000","last_change":"2023-10-09T10:02:29.898739+0000","last_active":"2023-10-09T10:02:50.341458+0000","last_peered":"2023-10-09T10:02:50.341458+0000","last_clean":"2023-10-09T10:02:50.341458+0000","last_became_active":"2023-10-09T10:02:29.898403+0000","last_became_peered":"2023-10-09T10:02:29.898403+0000","last_unstale":"2023-10-09T10:02:50.341458+0000","last_undegraded":"2023-10-09T10:02:50.341458+0000","last_fullsized":"2023-10-09T10:02:50.341458+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:33.101431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":151,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339403+0000","last_change":"2023-10-09T10:00:30.786721+0000","last_active":"2023-10-09T10:02:51.339403+0000","last_peered":"2023-10-09T10:02:51.339403+0000","last_clean":"2023-10-09T10:02:51.339403+0000","last_became_active":"2023-10-09T10:00:30.786375+0000","last_became_peered":"2023-10-09T10:00:30.786375+0000","last_unstale":"2023-10-09T10:02:51.339403+0000","last_undegraded":"2023-10-09T10:02:51.339403+0000","last_fullsized":"2023-10-09T10:02:51.339403+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:19:38.052691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":180,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.340234+0000","last_change":"2023-10-09T10:02:42.727456+0000","last_active":"2023-10-09T10:02:51.340234+0000","last_peered":"2023-10-09T10:02:51.340234+0000","last_clean":"2023-10-09T10:02:51.340234+0000","last_became_active":"2023-10-09T10:02:42.727114+0000","last_became_peered":"2023-10-09T10:02:42.727114+0000","last_unstale":"2023-10-09T10:02:51.340234+0000","last_undegraded":"2023-10-09T10:02:51.340234+0000","last_fullsized":"2023-10-09T10:02:51.340234+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:25.782090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"178'9742","reported_seq":13966,"reported_epoch":178,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:02:52.469697+0000","last_change":"2023-10-09T10:02:44.028435+0000","last_active":"2023-10-09T10:02:52.469697+0000","last_peered":"2023-10-09T10:02:52.469697+0000","last_clean":"2023-10-09T10:00:59.913112+0000","last_became_active":"2023-10-09T10:02:43.938040+0000","last_became_peered":"2023-10-09T10:02:43.938040+0000","last_unstale":"2023-10-09T10:02:52.469697+0000","last_undegraded":"2023-10-09T10:02:43.812496+0000","last_fullsized":"2023-10-09T10:02:43.715886+0000","mapping_epoch":169,"log_start":"103'927","ondisk_log_start":"103'927","created":97,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8815,"log_dups_size":0,"ondisk_log_size":8815,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20971520,"num_objects":320,"num_object_clones":0,"num_object_copies":960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":140,"num_objects_misplaced":315,"num_objects_unfound":0,"num_objects_dirty":320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6123,"num_write_kb":206176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2045,"num_bytes_recovered":91291648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4","objects":-135},{"shards":"3,4,5","objects":315}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":124,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T10:02:50.341250+0000","last_change":"2023-10-09T10:02:42.722779+0000","last_active":"2023-10-09T10:02:50.341250+0000","last_peered":"2023-10-09T10:02:50.341250+0000","last_clean":"2023-10-09T10:02:50.341250+0000","last_became_active":"2023-10-09T10:02:42.721266+0000","last_became_peered":"2023-10-09T10:02:42.721266+0000","last_unstale":"2023-10-09T10:02:50.341250+0000","last_undegraded":"2023-10-09T10:02:50.341250+0000","last_fullsized":"2023-10-09T10:02:50.341250+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:39.323310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":425,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.445964+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:02:51.445964+0000","last_peered":"2023-10-09T10:02:51.445964+0000","last_clean":"2023-10-09T10:02:51.445964+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:02:51.445964+0000","last_undegraded":"2023-10-09T10:02:51.445964+0000","last_fullsized":"2023-10-09T10:02:51.445964+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.332724+0000","last_change":"2023-10-09T10:02:42.718619+0000","last_active":"2023-10-09T10:02:51.332724+0000","last_peered":"2023-10-09T10:02:51.332724+0000","last_clean":"2023-10-09T10:02:51.332724+0000","last_became_active":"2023-10-09T10:02:42.718310+0000","last_became_peered":"2023-10-09T10:02:42.718310+0000","last_unstale":"2023-10-09T10:02:51.332724+0000","last_undegraded":"2023-10-09T10:02:51.332724+0000","last_fullsized":"2023-10-09T10:02:51.332724+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:23.362057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348659+0000","last_change":"2023-10-09T10:02:17.734226+0000","last_active":"2023-10-09T10:02:51.348659+0000","last_peered":"2023-10-09T10:02:51.348659+0000","last_clean":"2023-10-09T10:02:51.348659+0000","last_became_active":"2023-10-09T10:02:17.733854+0000","last_became_peered":"2023-10-09T10:02:17.733854+0000","last_unstale":"2023-10-09T10:02:51.348659+0000","last_undegraded":"2023-10-09T10:02:51.348659+0000","last_fullsized":"2023-10-09T10:02:51.348659+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":149,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339570+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:02:51.339570+0000","last_peered":"2023-10-09T10:02:51.339570+0000","last_clean":"2023-10-09T10:02:51.339570+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:02:51.339570+0000","last_undegraded":"2023-10-09T10:02:51.339570+0000","last_fullsized":"2023-10-09T10:02:51.339570+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":28,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.761671+0000","last_change":"2023-10-09T10:02:29.953999+0000","last_active":"2023-10-09T10:02:55.761671+0000","last_peered":"2023-10-09T10:02:55.761671+0000","last_clean":"2023-10-09T10:02:55.761671+0000","last_became_active":"2023-10-09T10:02:29.953244+0000","last_became_peered":"2023-10-09T10:02:29.953244+0000","last_unstale":"2023-10-09T10:02:55.761671+0000","last_undegraded":"2023-10-09T10:02:55.761671+0000","last_fullsized":"2023-10-09T10:02:55.761671+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:05.612343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":37,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.841019+0000","last_change":"2023-10-09T10:02:42.730361+0000","last_active":"2023-10-09T10:02:55.841019+0000","last_peered":"2023-10-09T10:02:55.841019+0000","last_clean":"2023-10-09T10:02:55.841019+0000","last_became_active":"2023-10-09T10:02:42.726162+0000","last_became_peered":"2023-10-09T10:02:42.726162+0000","last_unstale":"2023-10-09T10:02:55.841019+0000","last_undegraded":"2023-10-09T10:02:55.841019+0000","last_fullsized":"2023-10-09T10:02:55.841019+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:22.829402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"178'10008","reported_seq":11054,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.120418+0000","last_change":"2023-10-09T10:02:48.341804+0000","last_active":"2023-10-09T10:02:54.120418+0000","last_peered":"2023-10-09T10:02:54.120418+0000","last_clean":"2023-10-09T10:02:54.120418+0000","last_became_active":"2023-10-09T10:02:48.317391+0000","last_became_peered":"2023-10-09T10:02:48.317391+0000","last_unstale":"2023-10-09T10:02:54.120418+0000","last_undegraded":"2023-10-09T10:02:54.120418+0000","last_fullsized":"2023-10-09T10:02:54.120418+0000","mapping_epoch":173,"log_start":"103'731","ondisk_log_start":"103'731","created":97,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9277,"log_dups_size":0,"ondisk_log_size":9277,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17137664,"num_objects":261,"num_object_clones":0,"num_object_copies":783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6307,"num_write_kb":210208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":183,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.446683+0000","last_change":"2023-10-09T10:02:29.882693+0000","last_active":"2023-10-09T10:02:51.446683+0000","last_peered":"2023-10-09T10:02:51.446683+0000","last_clean":"2023-10-09T10:02:51.446683+0000","last_became_active":"2023-10-09T10:02:29.882300+0000","last_became_peered":"2023-10-09T10:02:29.882300+0000","last_unstale":"2023-10-09T10:02:51.446683+0000","last_undegraded":"2023-10-09T10:02:51.446683+0000","last_fullsized":"2023-10-09T10:02:51.446683+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":160,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.339808+0000","last_change":"2023-10-09T10:02:42.726633+0000","last_active":"2023-10-09T10:02:51.339808+0000","last_peered":"2023-10-09T10:02:51.339808+0000","last_clean":"2023-10-09T10:02:51.339808+0000","last_became_active":"2023-10-09T10:02:42.726287+0000","last_became_peered":"2023-10-09T10:02:42.726287+0000","last_unstale":"2023-10-09T10:02:51.339808+0000","last_undegraded":"2023-10-09T10:02:51.339808+0000","last_fullsized":"2023-10-09T10:02:51.339808+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:15.897069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.840995+0000","last_change":"2023-10-09T10:02:42.726504+0000","last_active":"2023-10-09T10:02:55.840995+0000","last_peered":"2023-10-09T10:02:55.840995+0000","last_clean":"2023-10-09T10:02:55.840995+0000","last_became_active":"2023-10-09T10:02:42.726170+0000","last_became_peered":"2023-10-09T10:02:42.726170+0000","last_unstale":"2023-10-09T10:02:55.840995+0000","last_undegraded":"2023-10-09T10:02:55.840995+0000","last_fullsized":"2023-10-09T10:02:55.840995+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.725163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"178'10074","reported_seq":10862,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.120265+0000","last_change":"2023-10-09T10:01:04.122514+0000","last_active":"2023-10-09T10:02:54.120265+0000","last_peered":"2023-10-09T10:02:54.120265+0000","last_clean":"2023-10-09T10:02:54.120265+0000","last_became_active":"2023-10-09T10:00:53.648018+0000","last_became_peered":"2023-10-09T10:00:53.648018+0000","last_unstale":"2023-10-09T10:02:54.120265+0000","last_undegraded":"2023-10-09T10:02:54.120265+0000","last_fullsized":"2023-10-09T10:02:54.120265+0000","mapping_epoch":134,"log_start":"103'829","ondisk_log_start":"103'829","created":97,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9245,"log_dups_size":0,"ondisk_log_size":9245,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19857408,"num_objects":303,"num_object_clones":0,"num_object_copies":909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6352,"num_write_kb":212960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":21954560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":171,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.335846+0000","last_change":"2023-10-09T10:02:29.972836+0000","last_active":"2023-10-09T10:02:51.335846+0000","last_peered":"2023-10-09T10:02:51.335846+0000","last_clean":"2023-10-09T10:02:51.335846+0000","last_became_active":"2023-10-09T10:02:29.888134+0000","last_became_peered":"2023-10-09T10:02:29.888134+0000","last_unstale":"2023-10-09T10:02:51.335846+0000","last_undegraded":"2023-10-09T10:02:51.335846+0000","last_fullsized":"2023-10-09T10:02:51.335846+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:22.244154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":135,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.341159+0000","last_change":"2023-10-09T10:02:17.723233+0000","last_active":"2023-10-09T10:02:51.341159+0000","last_peered":"2023-10-09T10:02:51.341159+0000","last_clean":"2023-10-09T10:02:51.341159+0000","last_became_active":"2023-10-09T10:02:17.722944+0000","last_became_peered":"2023-10-09T10:02:17.722944+0000","last_unstale":"2023-10-09T10:02:51.341159+0000","last_undegraded":"2023-10-09T10:02:51.341159+0000","last_fullsized":"2023-10-09T10:02:51.341159+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:56.963212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":160,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.331669+0000","last_change":"2023-10-09T10:02:29.886514+0000","last_active":"2023-10-09T10:02:51.331669+0000","last_peered":"2023-10-09T10:02:51.331669+0000","last_clean":"2023-10-09T10:02:51.331669+0000","last_became_active":"2023-10-09T10:02:29.885706+0000","last_became_peered":"2023-10-09T10:02:29.885706+0000","last_unstale":"2023-10-09T10:02:51.331669+0000","last_undegraded":"2023-10-09T10:02:51.331669+0000","last_fullsized":"2023-10-09T10:02:51.331669+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:28:06.195775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"177'10081","reported_seq":16911,"reported_epoch":177,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:02:52.256205+0000","last_change":"2023-10-09T10:02:18.816969+0000","last_active":"2023-10-09T10:02:52.256205+0000","last_peered":"2023-10-09T10:02:52.256205+0000","last_clean":"2023-10-09T10:02:16.876602+0000","last_became_active":"2023-10-09T10:02:18.790602+0000","last_became_peered":"2023-10-09T10:02:18.790602+0000","last_unstale":"2023-10-09T10:02:52.256205+0000","last_undegraded":"2023-10-09T10:02:52.256205+0000","last_fullsized":"2023-10-09T10:02:52.256205+0000","mapping_epoch":159,"log_start":"103'1330","ondisk_log_start":"103'1330","created":97,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":8751,"log_dups_size":0,"ondisk_log_size":8751,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25559040,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":390,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6319,"num_write_kb":214688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3347,"num_bytes_recovered":163840000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[2,3,6],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":390}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":143,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T10:02:50.341823+0000","last_change":"2023-10-09T10:01:36.024079+0000","last_active":"2023-10-09T10:02:50.341823+0000","last_peered":"2023-10-09T10:02:50.341823+0000","last_clean":"2023-10-09T10:02:50.341823+0000","last_became_active":"2023-10-09T10:01:36.023515+0000","last_became_peered":"2023-10-09T10:01:36.023515+0000","last_unstale":"2023-10-09T10:02:50.341823+0000","last_undegraded":"2023-10-09T10:02:50.341823+0000","last_fullsized":"2023-10-09T10:02:50.341823+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:36.977968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":33,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.841172+0000","last_change":"2023-10-09T10:02:29.922084+0000","last_active":"2023-10-09T10:02:55.841172+0000","last_peered":"2023-10-09T10:02:55.841172+0000","last_clean":"2023-10-09T10:02:55.841172+0000","last_became_active":"2023-10-09T10:02:29.886231+0000","last_became_peered":"2023-10-09T10:02:29.886231+0000","last_unstale":"2023-10-09T10:02:55.841172+0000","last_undegraded":"2023-10-09T10:02:55.841172+0000","last_fullsized":"2023-10-09T10:02:55.841172+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:02:16.587891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":38,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348180+0000","last_change":"2023-10-09T10:02:42.737085+0000","last_active":"2023-10-09T10:02:51.348180+0000","last_peered":"2023-10-09T10:02:51.348180+0000","last_clean":"2023-10-09T10:02:51.348180+0000","last_became_active":"2023-10-09T10:02:42.736715+0000","last_became_peered":"2023-10-09T10:02:42.736715+0000","last_unstale":"2023-10-09T10:02:51.348180+0000","last_undegraded":"2023-10-09T10:02:51.348180+0000","last_fullsized":"2023-10-09T10:02:51.348180+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"178'10066","reported_seq":11962,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:02:54.811810+0000","last_change":"2023-10-09T10:01:03.341055+0000","last_active":"2023-10-09T10:02:54.811810+0000","last_peered":"2023-10-09T10:02:54.811810+0000","last_clean":"2023-10-09T10:02:54.811810+0000","last_became_active":"2023-10-09T10:00:53.652818+0000","last_became_peered":"2023-10-09T10:00:53.652818+0000","last_unstale":"2023-10-09T10:02:54.811810+0000","last_undegraded":"2023-10-09T10:02:54.811810+0000","last_fullsized":"2023-10-09T10:02:54.811810+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":97,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20807680,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6333,"num_write_kb":212832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":609,"num_bytes_recovered":39944192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":28,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.840839+0000","last_change":"2023-10-09T10:02:29.889587+0000","last_active":"2023-10-09T10:02:55.840839+0000","last_peered":"2023-10-09T10:02:55.840839+0000","last_clean":"2023-10-09T10:02:55.840839+0000","last_became_active":"2023-10-09T10:02:29.888869+0000","last_became_peered":"2023-10-09T10:02:29.888869+0000","last_unstale":"2023-10-09T10:02:55.840839+0000","last_undegraded":"2023-10-09T10:02:55.840839+0000","last_fullsized":"2023-10-09T10:02:55.840839+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:22.220539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":63,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T10:02:50.341428+0000","last_change":"2023-10-09T10:00:30.779150+0000","last_active":"2023-10-09T10:02:50.341428+0000","last_peered":"2023-10-09T10:02:50.341428+0000","last_clean":"2023-10-09T10:02:50.341428+0000","last_became_active":"2023-10-09T10:00:30.778817+0000","last_became_peered":"2023-10-09T10:00:30.778817+0000","last_unstale":"2023-10-09T10:02:50.341428+0000","last_undegraded":"2023-10-09T10:02:50.341428+0000","last_fullsized":"2023-10-09T10:02:50.341428+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:04.893208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"179'9942","reported_seq":13369,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:56.330847+0000","last_change":"2023-10-09T10:02:51.337535+0000","last_active":"2023-10-09T10:02:56.330847+0000","last_peered":"2023-10-09T10:02:56.330847+0000","last_clean":"2023-10-09T10:02:56.330847+0000","last_became_active":"2023-10-09T10:02:37.639374+0000","last_became_peered":"2023-10-09T10:02:37.639374+0000","last_unstale":"2023-10-09T10:02:56.330847+0000","last_undegraded":"2023-10-09T10:02:56.330847+0000","last_fullsized":"2023-10-09T10:02:56.330847+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":133,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9942,"log_dups_size":0,"ondisk_log_size":9942,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18907136,"num_objects":289,"num_object_clones":0,"num_object_copies":867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6245,"num_write_kb":209056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1238,"num_bytes_recovered":48005120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":165,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348332+0000","last_change":"2023-10-09T10:02:42.768970+0000","last_active":"2023-10-09T10:02:51.348332+0000","last_peered":"2023-10-09T10:02:51.348332+0000","last_clean":"2023-10-09T10:02:51.348332+0000","last_became_active":"2023-10-09T10:02:42.722677+0000","last_became_peered":"2023-10-09T10:02:42.722677+0000","last_unstale":"2023-10-09T10:02:51.348332+0000","last_undegraded":"2023-10-09T10:02:51.348332+0000","last_fullsized":"2023-10-09T10:02:51.348332+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348421+0000","last_change":"2023-10-09T10:02:17.743821+0000","last_active":"2023-10-09T10:02:51.348421+0000","last_peered":"2023-10-09T10:02:51.348421+0000","last_clean":"2023-10-09T10:02:51.348421+0000","last_became_active":"2023-10-09T10:02:17.743215+0000","last_became_peered":"2023-10-09T10:02:17.743215+0000","last_unstale":"2023-10-09T10:02:51.348421+0000","last_undegraded":"2023-10-09T10:02:51.348421+0000","last_fullsized":"2023-10-09T10:02:51.348421+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"177'10223","reported_seq":11011,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:52.226412+0000","last_change":"2023-10-09T10:01:01.555042+0000","last_active":"2023-10-09T10:02:52.226412+0000","last_peered":"2023-10-09T10:02:52.226412+0000","last_clean":"2023-10-09T10:02:52.226412+0000","last_became_active":"2023-10-09T10:00:53.683716+0000","last_became_peered":"2023-10-09T10:00:53.683716+0000","last_unstale":"2023-10-09T10:02:52.226412+0000","last_undegraded":"2023-10-09T10:02:52.226412+0000","last_fullsized":"2023-10-09T10:02:52.226412+0000","mapping_epoch":133,"log_start":"102'400","ondisk_log_start":"102'400","created":133,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:58:51.600849+0000","last_clean_scrub_stamp":"2023-10-09T09:58:51.600849+0000","objects_scrubbed":0,"log_size":9823,"log_dups_size":0,"ondisk_log_size":9823,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22446080,"num_objects":343,"num_object_clones":0,"num_object_copies":1029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6419,"num_write_kb":216352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":336,"num_bytes_recovered":22020096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":150,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348583+0000","last_change":"2023-10-09T10:02:42.784706+0000","last_active":"2023-10-09T10:02:51.348583+0000","last_peered":"2023-10-09T10:02:51.348583+0000","last_clean":"2023-10-09T10:02:51.348583+0000","last_became_active":"2023-10-09T10:02:42.722605+0000","last_became_peered":"2023-10-09T10:02:42.722605+0000","last_unstale":"2023-10-09T10:02:51.348583+0000","last_undegraded":"2023-10-09T10:02:51.348583+0000","last_fullsized":"2023-10-09T10:02:51.348583+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":38,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348359+0000","last_change":"2023-10-09T10:02:29.890059+0000","last_active":"2023-10-09T10:02:51.348359+0000","last_peered":"2023-10-09T10:02:51.348359+0000","last_clean":"2023-10-09T10:02:51.348359+0000","last_became_active":"2023-10-09T10:02:29.889678+0000","last_became_peered":"2023-10-09T10:02:29.889678+0000","last_unstale":"2023-10-09T10:02:51.348359+0000","last_undegraded":"2023-10-09T10:02:51.348359+0000","last_fullsized":"2023-10-09T10:02:51.348359+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":171,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.333596+0000","last_change":"2023-10-09T10:00:30.779348+0000","last_active":"2023-10-09T10:02:51.333596+0000","last_peered":"2023-10-09T10:02:51.333596+0000","last_clean":"2023-10-09T10:02:51.333596+0000","last_became_active":"2023-10-09T10:00:30.778963+0000","last_became_peered":"2023-10-09T10:00:30.778963+0000","last_unstale":"2023-10-09T10:02:51.333596+0000","last_undegraded":"2023-10-09T10:02:51.333596+0000","last_fullsized":"2023-10-09T10:02:51.333596+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.112454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":202,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.332208+0000","last_change":"2023-10-09T10:02:29.914116+0000","last_active":"2023-10-09T10:02:51.332208+0000","last_peered":"2023-10-09T10:02:51.332208+0000","last_clean":"2023-10-09T10:02:51.332208+0000","last_became_active":"2023-10-09T10:02:29.913629+0000","last_became_peered":"2023-10-09T10:02:29.913629+0000","last_unstale":"2023-10-09T10:02:51.332208+0000","last_undegraded":"2023-10-09T10:02:51.332208+0000","last_fullsized":"2023-10-09T10:02:51.332208+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:48.302616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":49,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:02:55.761718+0000","last_change":"2023-10-09T10:02:42.783054+0000","last_active":"2023-10-09T10:02:55.761718+0000","last_peered":"2023-10-09T10:02:55.761718+0000","last_clean":"2023-10-09T10:02:55.761718+0000","last_became_active":"2023-10-09T10:02:42.730487+0000","last_became_peered":"2023-10-09T10:02:42.730487+0000","last_unstale":"2023-10-09T10:02:55.761718+0000","last_undegraded":"2023-10-09T10:02:55.761718+0000","last_fullsized":"2023-10-09T10:02:55.761718+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:55.853794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":29,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:02:51.348111+0000","last_change":"2023-10-09T10:02:17.720763+0000","last_active":"2023-10-09T10:02:51.348111+0000","last_peered":"2023-10-09T10:02:51.348111+0000","last_clean":"2023-10-09T10:02:51.348111+0000","last_became_active":"2023-10-09T10:02:17.720442+0000","last_became_peered":"2023-10-09T10:02:17.720442+0000","last_unstale":"2023-10-09T10:02:51.348111+0000","last_undegraded":"2023-10-09T10:02:51.348111+0000","last_fullsized":"2023-10-09T10:02:51.348111+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":627834904,"num_objects":9581,"num_object_clones":0,"num_object_copies":28743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4694,"num_objects_misplaced":1735,"num_objects_unfound":0,"num_objects_dirty":9581,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":201457,"num_write_kb":6753153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26601,"num_bytes_recovered":1412038680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2508865536,"data_stored":2508849248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":228326,"ondisk_log_size":228326,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6960,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430399,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":530800,"kb_used_data":233576,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93214352,"statfs":{"total":95995035648,"available":95451496448,"internally_reserved":0,"allocated":239181824,"data_stored":238688975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:02:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.431,"15min":0.873},"min":{"1min":0.223,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":6.7779999999999996,"5min":6.7779999999999996,"15min":6.7779999999999996},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.399,"5min":1.2150000000000001,"15min":0.76600000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.6399999999999997,"5min":8.0259999999999998,"15min":8.0259999999999998},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.3089999999999999,"15min":0.79900000000000004},"min":{"1min":0.27500000000000002,"5min":0.246,"15min":0.246},"max":{"1min":12.853999999999999,"5min":12.853999999999999,"15min":12.853999999999999},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.3069999999999999,"15min":0.82499999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.996,"5min":12.996,"15min":12.996},"last":0.36399999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:02:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":1.379,"15min":0.85899999999999999},"min":{"1min":0.35499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":16.302,"5min":16.302,"15min":16.302},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.4870000000000001,"5min":1.6379999999999999,"15min":0.92200000000000004},"min":{"1min":0.377,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.295,"5min":13.295,"15min":13.295},"last":0.69899999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:01:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":1.415,"15min":0.86099999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.715999999999999,"5min":14.715999999999999,"15min":14.715999999999999},"last":5.5330000000000004},{"interface":"front","average":{"1min":3.117,"5min":1.5940000000000001,"15min":0.92300000000000004},"min":{"1min":0.37,"5min":0.312,"15min":0.312},"max":{"1min":14.538,"5min":14.538,"15min":14.538},"last":0.91600000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.82499999999999996,"15min":0.58199999999999996},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.129,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.75,"5min":0.72499999999999998,"15min":0.56000000000000005},"min":{"1min":0.33200000000000002,"5min":0.28899999999999998,"15min":0.26500000000000001},"max":{"1min":3.593,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":0.32800000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":1.1120000000000001,"15min":0.70399999999999996},"min":{"1min":0.317,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.0510000000000002,"5min":12.958,"15min":12.958},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0109999999999999,"15min":0.68600000000000005},"min":{"1min":0.36299999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.7770000000000001,"5min":6.6120000000000001,"15min":6.6120000000000001},"last":0.34399999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":0.91200000000000003,"15min":0.66700000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.32900000000000001},"max":{"1min":3.9870000000000001,"5min":5.4630000000000001,"15min":5.4630000000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.0529999999999999,"5min":0.84499999999999997,"15min":0.64500000000000002},"min":{"1min":0.373,"5min":0.34000000000000002,"15min":0.33100000000000002},"max":{"1min":3.5369999999999999,"5min":4.407,"15min":4.407},"last":0.45100000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626627,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":578212,"kb_used_data":272796,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93166940,"statfs":{"total":95995035648,"available":95402946560,"internally_reserved":0,"allocated":279343104,"data_stored":278862636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:02:26 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.272,"15min":0.79400000000000004},"min":{"1min":0.23599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.6429999999999998,"5min":8.8650000000000002,"15min":8.8650000000000002},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.2629999999999999,"15min":0.75800000000000001},"min":{"1min":0.27400000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.1399999999999997,"5min":12.439,"15min":12.439},"last":0.56399999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:02:52 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.784,"15min":0.96099999999999997},"min":{"1min":0.29399999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.5999999999999996,"5min":18.853000000000002,"15min":18.853000000000002},"last":1.792},{"interface":"front","average":{"1min":2.0670000000000002,"5min":1.5269999999999999,"15min":0.86499999999999999},"min":{"1min":0.35199999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.2069999999999999,"5min":12.380000000000001,"15min":12.380000000000001},"last":2.0089999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:02:26 2023","interfaces":[{"interface":"back","average":{"1min":2.5230000000000001,"5min":1.792,"15min":0.95699999999999996},"min":{"1min":0.27700000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.688000000000001,"5min":18.800000000000001,"15min":18.800000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.6200000000000001,"5min":1.71,"15min":0.96099999999999997},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6829999999999998,"5min":12.413,"15min":12.413},"last":2.5}]},{"osd":3,"last update":"Mon Oct 9 10:01:59 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":1.4990000000000001,"15min":0.91000000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.9740000000000002,"5min":12.337,"15min":12.337},"last":2.2709999999999999},{"interface":"front","average":{"1min":1.6950000000000001,"5min":1.391,"15min":0.86299999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.0530000000000008,"5min":8.0530000000000008,"15min":8.0530000000000008},"last":2.2970000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:02:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.347,"15min":0.753},"min":{"1min":0.28499999999999998,"5min":0.23599999999999999,"15min":0.23300000000000001},"max":{"1min":5.468,"5min":23.446000000000002,"15min":23.446000000000002},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.97999999999999998,"15min":0.65300000000000002},"min":{"1min":0.29299999999999998,"5min":0.247,"15min":0.247},"max":{"1min":3.5259999999999998,"5min":8.2379999999999995,"15min":8.2379999999999995},"last":0.42599999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:02:18 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.90000000000000002,"15min":0.63400000000000001},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.5840000000000001,"5min":14.09,"15min":14.09},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.1870000000000001,"5min":0.88600000000000001,"15min":0.63500000000000001},"min":{"1min":0.308,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.0579999999999998,"5min":6.3540000000000001,"15min":6.3540000000000001},"last":0.48499999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.92000000000000004,"15min":0.65800000000000003},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.315},"max":{"1min":3.4060000000000001,"5min":5.9089999999999998,"15min":5.9089999999999998},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.046,"5min":0.97399999999999998,"15min":0.71899999999999997},"min":{"1min":0.41299999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.4409999999999998,"5min":6.0860000000000003,"15min":6.0860000000000003},"last":0.51000000000000001}]}]},{"osd":1,"up_from":84,"seq":360777252940,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310348,"kb_used_data":206596,"kb_used_omap":9,"kb_used_meta":103734,"kb_avail":93434804,"statfs":{"total":95995035648,"available":95677239296,"internally_reserved":0,"allocated":211554304,"data_stored":210858104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":106224617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:01:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":1.5600000000000001,"15min":0.754},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":16.324999999999999,"5min":16.324999999999999,"15min":16.324999999999999},"last":6.7960000000000003},{"interface":"front","average":{"1min":3.1920000000000002,"5min":1.599,"15min":0.79800000000000004},"min":{"1min":0.26400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.573,"5min":15.573,"15min":15.573},"last":0.34499999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2000000000000002,"5min":1.9770000000000001,"15min":0.91800000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.131,"5min":18.109000000000002,"15min":18.109000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.6899999999999999,"5min":1.9650000000000001,"15min":0.91500000000000004},"min":{"1min":0.312,"5min":0.248,"15min":0.248},"max":{"1min":11.382999999999999,"5min":12.069000000000001,"15min":12.069000000000001},"last":1.1759999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:01:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3929999999999998,"5min":2.3879999999999999,"15min":1.048},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.608000000000001,"5min":25.129999999999999,"15min":25.129999999999999},"last":0.627},{"interface":"front","average":{"1min":3.0510000000000002,"5min":2.1760000000000002,"15min":0.98799999999999999},"min":{"1min":0.28100000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.725,"5min":22.847999999999999,"15min":22.847999999999999},"last":1.331}]},{"osd":4,"last update":"Mon Oct 9 10:01:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4780000000000002,"5min":1.6870000000000001,"15min":0.96899999999999997},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":11.311999999999999,"5min":11.311999999999999,"15min":11.311999999999999},"last":1.286},{"interface":"front","average":{"1min":2.4089999999999998,"5min":1.405,"15min":0.89600000000000002},"min":{"1min":0.32400000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":16.053000000000001,"5min":16.053000000000001,"15min":16.053000000000001},"last":2.7719999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:01:59 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":1.6819999999999999,"15min":0.97999999999999998},"min":{"1min":0.35499999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":15.531000000000001,"5min":15.531000000000001,"15min":15.531000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.4209999999999998,"5min":1.7390000000000001,"15min":1.0189999999999999},"min":{"1min":0.375,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.26,"5min":19.706,"15min":19.706},"last":1.2470000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:01:59 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":1.6060000000000001,"15min":0.99399999999999999},"min":{"1min":0.41399999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":16.193999999999999,"5min":16.193999999999999,"15min":16.193999999999999},"last":2.8399999999999999},{"interface":"front","average":{"1min":3.0960000000000001,"5min":1.804,"15min":1.052},"min":{"1min":0.48799999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":15.481999999999999,"5min":15.481999999999999,"15min":15.481999999999999},"last":0.54600000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:01:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":1.8180000000000001,"15min":1.034},"min":{"1min":0.44700000000000001,"5min":0.379,"15min":0.379},"max":{"1min":11.348000000000001,"5min":12.766999999999999,"15min":12.766999999999999},"last":1.3080000000000001},{"interface":"front","average":{"1min":2.5630000000000002,"5min":1.597,"15min":0.98899999999999999},"min":{"1min":0.47999999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.100999999999999,"5min":16.100999999999999,"15min":16.100999999999999},"last":2.8119999999999998}]}]},{"osd":0,"up_from":79,"seq":339302416467,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":218920,"kb_used_data":129056,"kb_used_omap":28,"kb_used_meta":89827,"kb_avail":93526232,"statfs":{"total":95995035648,"available":95770861568,"internally_reserved":0,"allocated":132153344,"data_stored":131436886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":91983696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:01:56 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":1.4470000000000001,"15min":0.73299999999999998},"min":{"1min":0.23100000000000001,"5min":0.191,"15min":0.191},"max":{"1min":11.73,"5min":11.73,"15min":11.73},"last":0.97199999999999998},{"interface":"front","average":{"1min":2.0910000000000002,"5min":1.3799999999999999,"15min":0.72499999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.6150000000000002,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":1.0069999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:02:21 2023","interfaces":[{"interface":"back","average":{"1min":3.7109999999999999,"5min":2.1400000000000001,"15min":0.97499999999999998},"min":{"1min":0.25,"5min":0.216,"15min":0.216},"max":{"1min":38.575000000000003,"5min":38.575000000000003,"15min":38.575000000000003},"last":2.899},{"interface":"front","average":{"1min":4.657,"5min":2.3290000000000002,"15min":1.0600000000000001},"min":{"1min":0.27200000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":38.332000000000001,"5min":38.332000000000001,"15min":38.332000000000001},"last":3.3820000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:01:56 2023","interfaces":[{"interface":"back","average":{"1min":2.903,"5min":1.71,"15min":0.82399999999999995},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.587999999999999,"5min":13.587999999999999,"15min":13.587999999999999},"last":3.6850000000000001},{"interface":"front","average":{"1min":3.194,"5min":1.9239999999999999,"15min":0.93400000000000005},"min":{"1min":0.377,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.211,"5min":13.211,"15min":13.211},"last":7.7110000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":1.3859999999999999,"15min":0.90000000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2249999999999996,"5min":11.587999999999999,"15min":11.587999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.6000000000000001,"5min":1.6519999999999999,"15min":1.002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":14.653,"5min":14.653,"15min":14.653},"last":0.56699999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:02:21 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.5509999999999999,"15min":0.94499999999999995},"min":{"1min":0.41199999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.0329999999999995,"5min":9.0329999999999995,"15min":9.0329999999999995},"last":0.627},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.5,"15min":0.94399999999999995},"min":{"1min":0.34699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":5.4329999999999998,"5min":11.667,"15min":11.667},"last":0.44}]},{"osd":6,"last update":"Mon Oct 9 10:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":1.744,"15min":1.04},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.5860000000000003,"5min":11.699,"15min":11.699},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.8340000000000001,"5min":1.9219999999999999,"15min":1.1020000000000001},"min":{"1min":0.41299999999999998,"5min":0.379,"15min":0.379},"max":{"1min":15.108000000000001,"5min":15.108000000000001,"15min":15.108000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:02:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":1.694,"15min":1.0089999999999999},"min":{"1min":0.35999999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":26.978000000000002,"5min":26.978000000000002,"15min":26.978000000000002},"last":0.65000000000000002},{"interface":"front","average":{"1min":3.302,"5min":1.9890000000000001,"15min":1.1180000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":26.768000000000001,"5min":26.768000000000001,"15min":26.768000000000001},"last":0.53800000000000003}]}]},{"osd":2,"up_from":89,"seq":382252089413,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":653660,"kb_used_data":431828,"kb_used_omap":81,"kb_used_meta":221550,"kb_avail":93091492,"statfs":{"total":95995035648,"available":95325687808,"internally_reserved":0,"allocated":442191872,"data_stored":441473263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":226867418},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:02:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.78,"15min":0.84899999999999998},"min":{"1min":0.20899999999999999,"5min":0.187,"15min":0.187},"max":{"1min":7.8109999999999999,"5min":56.883000000000003,"15min":56.883000000000003},"last":4.444},{"interface":"front","average":{"1min":3.0550000000000002,"5min":2.3450000000000002,"15min":1.048},"min":{"1min":0.307,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.601000000000001,"5min":57.595999999999997,"15min":57.595999999999997},"last":4.4889999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:02:24 2023","interfaces":[{"interface":"back","average":{"1min":3.4889999999999999,"5min":2.0179999999999998,"15min":0.91700000000000004},"min":{"1min":0.36399999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.782,"5min":17.782,"15min":17.782},"last":8.2210000000000001},{"interface":"front","average":{"1min":2.8929999999999998,"5min":1.7969999999999999,"15min":0.873},"min":{"1min":0.255,"5min":0.224,"15min":0.224},"max":{"1min":11.369999999999999,"5min":15.936999999999999,"15min":15.936999999999999},"last":1.1819999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:01:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":2.0139999999999998,"15min":0.94999999999999996},"min":{"1min":0.29999999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.2550000000000008,"5min":58.067,"15min":58.067},"last":1.21},{"interface":"front","average":{"1min":2.8159999999999998,"5min":2.3180000000000001,"15min":1.0409999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":16.195,"5min":60.456000000000003,"15min":60.456000000000003},"last":4.3040000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:02:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":2.0270000000000001,"15min":1.0780000000000001},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.7619999999999996,"5min":57.646999999999998,"15min":57.646999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":3.5489999999999999,"5min":2.4689999999999999,"15min":1.2430000000000001},"min":{"1min":0.45300000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":17.678000000000001,"5min":57.024000000000001,"15min":57.024000000000001},"last":8.1910000000000007}]},{"osd":5,"last update":"Mon Oct 9 10:02:24 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":2.5459999999999998,"15min":1.2649999999999999},"min":{"1min":0.33800000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.5179999999999998,"5min":79.016000000000005,"15min":79.016000000000005},"last":1.099},{"interface":"front","average":{"1min":2.9319999999999999,"5min":2.9510000000000001,"15min":1.403},"min":{"1min":0.433,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.8399999999999999,"5min":79.302999999999997,"15min":79.302999999999997},"last":0.42499999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:02:24 2023","interfaces":[{"interface":"back","average":{"1min":2.806,"5min":2.2200000000000002,"15min":1.149},"min":{"1min":0.39100000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.6040000000000001,"5min":57.698,"15min":57.698},"last":0.51600000000000001},{"interface":"front","average":{"1min":4.0990000000000002,"5min":2.6099999999999999,"15min":1.292},"min":{"1min":0.495,"5min":0.378,"15min":0.378},"max":{"1min":17.751999999999999,"5min":57.058,"15min":57.058},"last":8.1419999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:02:24 2023","interfaces":[{"interface":"back","average":{"1min":3.819,"5min":2.2240000000000002,"15min":1.1779999999999999},"min":{"1min":0.42199999999999999,"5min":0.375,"15min":0.375},"max":{"1min":43.715000000000003,"5min":57.003,"15min":57.003},"last":1.1459999999999999},{"interface":"front","average":{"1min":5.8499999999999996,"5min":2.6869999999999998,"15min":1.3360000000000001},"min":{"1min":0.50800000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":43.840000000000003,"5min":57.676000000000002,"15min":57.676000000000002},"last":0.55800000000000005}]}]},{"osd":3,"up_from":94,"seq":403726925886,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":701104,"kb_used_data":454760,"kb_used_omap":7,"kb_used_meta":245944,"kb_avail":93044048,"statfs":{"total":95995035648,"available":95277105152,"internally_reserved":0,"allocated":465674240,"data_stored":464938525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":251847272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:02:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0819999999999999,"5min":2.3969999999999998,"15min":1.462},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.925000000000001,"5min":11.958,"15min":11.958},"last":2.7160000000000002},{"interface":"front","average":{"1min":3.0630000000000002,"5min":2.698,"15min":1.2},"min":{"1min":0.31,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.289,"5min":21.285,"15min":21.285},"last":3.323}]},{"osd":1,"last update":"Mon Oct 9 10:02:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7999999999999998,"5min":2.5049999999999999,"15min":1.1140000000000001},"min":{"1min":0.218,"5min":0.189,"15min":0.189},"max":{"1min":12.894,"5min":21.341999999999999,"15min":21.341999999999999},"last":0.23999999999999999},{"interface":"front","average":{"1min":4.4589999999999996,"5min":2.7650000000000001,"15min":1.198},"min":{"1min":0.38,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":15.59,"5min":15.59,"15min":15.59},"last":4.2619999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:02:51 2023","interfaces":[{"interface":"back","average":{"1min":3.5449999999999999,"5min":2.8050000000000002,"15min":1.6319999999999999},"min":{"1min":0.42799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.593,"5min":15.632999999999999,"15min":15.632999999999999},"last":1.181},{"interface":"front","average":{"1min":3.286,"5min":2.5950000000000002,"15min":1.169},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":11.879,"5min":11.879,"15min":11.879},"last":2.4369999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:02:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":1.5880000000000001,"15min":0.95999999999999996},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.976000000000001,"5min":11.976000000000001,"15min":11.976000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.206,"5min":1.867,"15min":1.081},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.247},"max":{"1min":3.4660000000000002,"5min":21.061,"15min":21.061},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:02:51 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.9810000000000001,"15min":1.119},"min":{"1min":0.34699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.0289999999999999,"5min":21.015000000000001,"15min":21.015000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.754,"5min":2.1120000000000001,"15min":1.4059999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.9920000000000009,"5min":10.127000000000001,"15min":10.127000000000001},"last":1.2370000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:02:51 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.4860000000000002,"15min":1.2849999999999999},"min":{"1min":0.40999999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.246,"5min":21.317,"15min":21.317},"last":0.40999999999999998},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.387,"15min":1.5},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":11.125,"5min":12.106999999999999,"15min":12.106999999999999},"last":1.264}]},{"osd":7,"last update":"Mon Oct 9 10:02:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7120000000000002,"5min":2.1619999999999999,"15min":1.4159999999999999},"min":{"1min":0.39500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":9.0210000000000008,"5min":9.4719999999999995,"15min":9.4719999999999995},"last":1.21},{"interface":"front","average":{"1min":2.0710000000000002,"5min":1.865,"15min":1.0589999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.951000000000001,"5min":11.951000000000001,"15min":11.951000000000001},"last":0.51600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019083,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":873000,"kb_used_data":575712,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92872152,"statfs":{"total":95995035648,"available":95101083648,"internally_reserved":0,"allocated":589529088,"data_stored":589043817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:02:25 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.1359999999999999,"15min":0.72199999999999998},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.2430000000000003,"5min":7.7939999999999996,"15min":7.7939999999999996},"last":4.0810000000000004},{"interface":"front","average":{"1min":2.0569999999999999,"5min":1.379,"15min":0.84099999999999997},"min":{"1min":0.29199999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.4409999999999998,"5min":9.0500000000000007,"15min":9.0500000000000007},"last":0.311}]},{"osd":1,"last update":"Mon Oct 9 10:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.2549999999999999,"15min":0.76500000000000001},"min":{"1min":0.28599999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.6669999999999998,"5min":6.1210000000000004,"15min":6.1210000000000004},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.6819999999999999,"5min":1.5780000000000001,"15min":0.90600000000000003},"min":{"1min":0.33200000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.3099999999999996,"5min":8.9849999999999994,"15min":8.9849999999999994},"last":0.60899999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:02:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":1.363,"15min":0.80500000000000005},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.3710000000000004,"5min":9.4640000000000004,"15min":9.4640000000000004},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.8799999999999999,"5min":1.462,"15min":0.86199999999999999},"min":{"1min":0.34999999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":11.999000000000001,"5min":11.999000000000001,"15min":11.999000000000001},"last":3.5739999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:02:54 2023","interfaces":[{"interface":"back","average":{"1min":3.048,"5min":2.6699999999999999,"15min":1.238},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":19.555,"5min":67.156999999999996,"15min":67.156999999999996},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.9950000000000001,"5min":2.2040000000000002,"15min":1.097},"min":{"1min":0.34999999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.4240000000000004,"5min":70.238,"15min":70.238},"last":1.214}]},{"osd":5,"last update":"Mon Oct 9 10:02:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":0.90100000000000002,"15min":0.59499999999999997},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.970000000000001,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.4379999999999999,"5min":0.97299999999999998,"15min":0.64300000000000002},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":9.3109999999999999,"5min":14.773,"15min":14.773},"last":0.30099999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:02:25 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":0.73499999999999999,"15min":0.56699999999999995},"min":{"1min":0.29599999999999999,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":9.4079999999999995,"5min":9.4079999999999995,"15min":9.4079999999999995},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.143,"5min":0.90700000000000003,"15min":0.64100000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.8239999999999998,"5min":9.0109999999999992,"15min":9.0109999999999992},"last":0.42099999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.0289999999999999,"15min":0.70199999999999996},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.7400000000000002,"5min":14.81,"15min":14.81},"last":0.433},{"interface":"front","average":{"1min":1.0569999999999999,"5min":0.66100000000000003,"15min":0.59999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32200000000000001,"15min":0.316},"max":{"1min":4.0350000000000001,"5min":4.0350000000000001,"15min":4.0350000000000001},"last":0.372}]}]},{"osd":5,"up_from":0,"seq":154618822855,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":129044480,"data_stored":129040408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":195170304,"data_stored":195166232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":433917952,"data_stored":433913880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":457900032,"data_stored":457900032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":575668224,"data_stored":575668224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":244125696,"data_stored":244121624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":253624320,"data_stored":253624320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":219414528,"data_stored":219414528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:02:58.492 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:02:58.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-10-09T10:02:58.528 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:58.557 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:58.595 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:58.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:58.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:58.700 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:02:58.728 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:58.767 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:02:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:58.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:02:58.872 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:02:58.891 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:58.938 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:02:58.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:58 smithi110 ceph-mon[136032]: pgmap v437: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 599 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 4694/28815 objects degraded (16.290%); 1735/28815 objects misplaced (6.021%); 14 MiB/s, 326 objects/s recovering 2023-10-09T10:02:58.999 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:59.011 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:02:59.051 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:02:59.122 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:02:59.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:02:59.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:58 smithi012 ceph-mon[147926]: pgmap v437: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 599 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 4694/28815 objects degraded (16.290%); 1735/28815 objects misplaced (6.021%); 14 MiB/s, 326 objects/s recovering 2023-10-09T10:02:59.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:58 smithi012 ceph-mon[152263]: pgmap v437: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 599 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 4694/28815 objects degraded (16.290%); 1735/28815 objects misplaced (6.021%); 14 MiB/s, 326 objects/s recovering 2023-10-09T10:02:59.200 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:02:59.242 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:02:59.244 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:02:59.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:59.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:02:59.277 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:02:59.297 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:02:59.297 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:02:59.297 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:02:59.298 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:02:59.298 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:02:59.298 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:02:59.298 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:02:59.298 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:02:59.300 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:02:59.352 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:59.384 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:59.424 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:59.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:59.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:59.537 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:02:59.568 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:59.607 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:02:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:59.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:02:59.711 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:02:59.742 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:02:59.789 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:02:59.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:02:59.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:02:59.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:02:59.962 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:00.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[152263]: from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:03:00.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1615096483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:00.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:00.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:00.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:03:00.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[147926]: from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:03:00.100 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1615096483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:00.100 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:00.100 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:00.100 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:02:59 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:03:00.182 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:59 smithi110 ceph-mon[136032]: from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:03:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1615096483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:02:59 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:03:00.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:00.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:00.299 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:00.329 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:00.375 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:00.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:00.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:00.565 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:00.812 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:00 smithi012 ceph-mon[147926]: pgmap v438: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 510 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 4790/24510 objects degraded (19.543%); 1295/24510 objects misplaced (5.284%); 5.6 MiB/s, 213 objects/s recovering 2023-10-09T10:03:00.812 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:00 smithi012 ceph-mon[147926]: 6.5 deep-scrub starts 2023-10-09T10:03:00.812 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:00 smithi012 ceph-mon[147926]: 6.18 scrub starts 2023-10-09T10:03:00.991 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:00.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:01.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:00 smithi012 ceph-mon[152263]: pgmap v438: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 510 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 4790/24510 objects degraded (19.543%); 1295/24510 objects misplaced (5.284%); 5.6 MiB/s, 213 objects/s recovering 2023-10-09T10:03:01.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:00 smithi012 ceph-mon[152263]: 6.5 deep-scrub starts 2023-10-09T10:03:01.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:00 smithi012 ceph-mon[152263]: 6.18 scrub starts 2023-10-09T10:03:01.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:01.102 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:01.122 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:00 smithi110 ceph-mon[136032]: pgmap v438: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 93 active+clean; 510 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 4790/24510 objects degraded (19.543%); 1295/24510 objects misplaced (5.284%); 5.6 MiB/s, 213 objects/s recovering 2023-10-09T10:03:01.122 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:00 smithi110 ceph-mon[136032]: 6.5 deep-scrub starts 2023-10-09T10:03:01.122 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:00 smithi110 ceph-mon[136032]: 6.18 scrub starts 2023-10-09T10:03:01.125 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:01.164 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:01.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:01.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:01.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:01.359 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:01.440 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:01.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:01.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:01.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:01.582 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:03:01.582 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:03:01.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.582 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:206570 6.c 6:398d1f06:::benchmark_data_smithi012_182576_object101049:head [delete] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-09T10:03:01.582 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.562557+0000", 2023-10-09T10:03:01.582 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.010518947000000001, 2023-10-09T10:03:01.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010552063, 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 206570 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562557+0000", 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562557+0000", 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562555+0000", 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:03:01.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562565+0000", 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0036e-05 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562576+0000", 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1221e-05 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562587+0000", 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0393e-05 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562633+0000", 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6445e-05 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562795+0000", 2023-10-09T10:03:01.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016218299999999999 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.562964+0000", 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016856999999999999 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.566668+0000", 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037046259999999999 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.570388+0000", 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037195090000000002 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:206573 6.4 e180/173 6:26b3f227:::benchmark_data_smithi012_182576_object101052:head v 180'10167, mlcod=180'10165)", 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.572105+0000", 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00097046399999999998, 2023-10-09T10:03:01.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001101365, 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.572105+0000", 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.572105+0000", 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.572105+0000", 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.572113+0000", 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6850000000000007e-06 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.572127+0000", 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3837e-05 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.572135+0000", 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9019999999999999e-06 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.572181+0000", 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5853000000000003e-05 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.572202+0000", 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1183e-05 2023-10-09T10:03:01.592 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGRecoveryDeleteReply(6.7 e180,169 6:e386baf7:::benchmark_data_smithi012_182576_object64134:head,156'8749)", 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.573214+0000", 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9998622, 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5723999999999997e-05, 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573214+0000", 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573214+0000", 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573213+0000", 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573221+0000", 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7079999999999997e-06 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573239+0000", 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8627999999999999e-05 2023-10-09T10:03:01.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573263+0000", 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3623999999999999e-05 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:206566 6.8 e180/134)", 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.573295+0000", 2023-10-09T10:03:01.596 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9997811, 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4276e-05, 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573295+0000", 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573295+0000", 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573294+0000", 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573300+0000", 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6810000000000003e-06 2023-10-09T10:03:01.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573306+0000", 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2949999999999999e-06 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573310+0000", 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9689999999999996e-06 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.599 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:206578 6.18 e180/133)", 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.573325+0000", 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9997506, 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2799999999999994e-05, 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573325+0000", 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573325+0000", 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573325+0000", 2023-10-09T10:03:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573328+0000", 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5889999999999999e-06 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573332+0000", 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.602e-06 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573335+0000", 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0790000000000001e-06 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.34797.0:206570 6.c e180/169)", 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.573398+0000", 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9996781, 2023-10-09T10:03:01.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7660000000000001e-05, 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573398+0000", 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573398+0000", 2023-10-09T10:03:01.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573397+0000", 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573401+0000", 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7589999999999999e-06 2023-10-09T10:03:01.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573410+0000", 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0429999999999996e-06 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573416+0000", 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9660000000000001e-06 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.606 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:206572 6.5 e180/133)", 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.573426+0000", 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.99965, 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6771000000000002e-05, 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573426+0000", 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573426+0000", 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573426+0000", 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:01.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573429+0000", 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9349999999999999e-06 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573433+0000", 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8539999999999999e-06 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573435+0000", 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6979999999999998e-06 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573473+0000", 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.786e-05 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:01.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573493+0000", 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0336999999999999e-05 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.34797.0:206579 6.5 e180/133)", 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.573446+0000", 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.99963, 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010611399999999999, 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:01.611 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573446+0000", 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573446+0000", 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573446+0000", 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573449+0000", 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.382e-06 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573454+0000", 2023-10-09T10:03:01.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4360000000000001e-06 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573459+0000", 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3309999999999997e-06 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.34797.0:206570 6.c e180/169)", 2023-10-09T10:03:01.614 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:01.573480+0000", 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9995961, 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119397, 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573480+0000", 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573480+0000", 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573479+0000", 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:01.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573482+0000", 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5280000000000002e-06 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573488+0000", 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6089999999999997e-06 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:01.573493+0000", 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9509999999999997e-06 2023-10-09T10:03:01.617 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.618 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:01.618 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.618 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:01.618 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:03:01.618 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 9 2023-10-09T10:03:01.618 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:03:01.694 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:01.851 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:01 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:03:01.851 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:01 smithi012 ceph-mon[147926]: 6.8 deep-scrub starts 2023-10-09T10:03:01.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:01 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:03:01.852 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:01 smithi012 ceph-mon[152263]: 6.8 deep-scrub starts 2023-10-09T10:03:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:01 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:03:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:01 smithi110 ceph-mon[136032]: 6.8 deep-scrub starts 2023-10-09T10:03:02.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:03:02.305 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:02.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:02.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:02.471 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:03.006 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:03.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:03.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:02 smithi012 ceph-mon[152263]: pgmap v439: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 431 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 4790/20736 objects degraded (23.100%); 1109/20736 objects misplaced (5.348%); 5.4 MiB/s, 138 objects/s recovering 2023-10-09T10:03:03.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:02 smithi012 ceph-mon[152263]: 6.5 scrub starts 2023-10-09T10:03:03.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:02 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4790/20736 objects degraded (23.100%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:03.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:02 smithi012 ceph-mon[147926]: pgmap v439: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 431 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 4790/20736 objects degraded (23.100%); 1109/20736 objects misplaced (5.348%); 5.4 MiB/s, 138 objects/s recovering 2023-10-09T10:03:03.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:02 smithi012 ceph-mon[147926]: 6.5 scrub starts 2023-10-09T10:03:03.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:02 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4790/20736 objects degraded (23.100%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:03.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:03:03.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:03.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:02 smithi110 ceph-mon[136032]: pgmap v439: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 431 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 4790/20736 objects degraded (23.100%); 1109/20736 objects misplaced (5.348%); 5.4 MiB/s, 138 objects/s recovering 2023-10-09T10:03:03.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:02 smithi110 ceph-mon[136032]: 6.5 scrub starts 2023-10-09T10:03:03.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:02 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4790/20736 objects degraded (23.100%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:03.227 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:03.928 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:03.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:03.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:04.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:04.063 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:04.109 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:04.111 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:03 smithi110 ceph-mon[136032]: 6.18 scrub starts 2023-10-09T10:03:04.111 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:03 smithi110 ceph-mon[136032]: pgmap v440: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 303 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 4896/14592 objects degraded (33.553%); 690/14592 objects misplaced (4.729%); 695 KiB/s, 105 objects/s recovering 2023-10-09T10:03:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:04.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:04.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:03 smithi012 ceph-mon[152263]: 6.18 scrub starts 2023-10-09T10:03:04.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:03 smithi012 ceph-mon[152263]: pgmap v440: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 303 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 4896/14592 objects degraded (33.553%); 690/14592 objects misplaced (4.729%); 695 KiB/s, 105 objects/s recovering 2023-10-09T10:03:04.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:03 smithi012 ceph-mon[147926]: 6.18 scrub starts 2023-10-09T10:03:04.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:03 smithi012 ceph-mon[147926]: pgmap v440: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 303 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 4896/14592 objects degraded (33.553%); 690/14592 objects misplaced (4.729%); 695 KiB/s, 105 objects/s recovering 2023-10-09T10:03:04.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:04.320 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:05.364 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:05.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:05.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:05.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:05.577 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:05.735 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:05.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:05.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:05.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:05.875 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:05.898 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:05.914 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:05.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:05.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:05.986 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:06.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:06.043 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:06.088 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:06.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:06.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:06.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:06 smithi012 ceph-mon[147926]: pgmap v441: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 208 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 4896/10044 objects degraded (48.746%); 585/10044 objects misplaced (5.824%); 597 KiB/s, 90 objects/s recovering 2023-10-09T10:03:06.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3036271253' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:06.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:06 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:06.164 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:06 smithi110 ceph-mon[136032]: pgmap v441: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 208 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 4896/10044 objects degraded (48.746%); 585/10044 objects misplaced (5.824%); 597 KiB/s, 90 objects/s recovering 2023-10-09T10:03:06.164 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3036271253' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:06.165 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:06 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:06.174 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:03:06.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:06.272 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:06.436 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:06 smithi012 ceph-mon[152263]: pgmap v441: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 208 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 4896/10044 objects degraded (48.746%); 585/10044 objects misplaced (5.824%); 597 KiB/s, 90 objects/s recovering 2023-10-09T10:03:06.436 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3036271253' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:06.436 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:06 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:06.463 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T10:03:07.133 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T10:03:07.315 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:07.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:07.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:07.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:07.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: 6.5 deep-scrub starts 2023-10-09T10:03:07.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: 6.8 deep-scrub starts 2023-10-09T10:03:07.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:07.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2950017685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:03:07.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:03:07.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: 6.5 deep-scrub starts 2023-10-09T10:03:07.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: 6.8 deep-scrub starts 2023-10-09T10:03:07.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2950017685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:03:07.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:03:07.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: 6.5 deep-scrub starts 2023-10-09T10:03:07.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: 6.8 deep-scrub starts 2023-10-09T10:03:07.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:07.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:03:07.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T10:03:07.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: failed to encode map e181 with expected crc 2023-10-09T10:03:07.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2950017685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:03:07.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:03:07.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:07.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:07.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:07.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:07.606 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:07.634 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:07.684 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:07.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:07.788 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:07.813 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Removed 105518 objects 2023-10-09T10:03:07.813 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Clean up completed and total clean up time :155.405 2023-10-09T10:03:07.816 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:07.848 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:07.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:07.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:07.952 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:07.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:08.009 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:08.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:08.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:08.056 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:03:08.057 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:03:08.057 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:03:08.057 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:03:08.057 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:03:08.080 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:03:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:03:07] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-09T10:03:08.112 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:08.138 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:08.170 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:08.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:08.274 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:08.299 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:08.332 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:08.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:08.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:08.349 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:03:08.369 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:03:08.395 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:03:08.417 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:03:08.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:08 smithi012 ceph-mon[147926]: pgmap v443: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 169 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4896/8169 objects degraded (59.934%); 585/8169 objects misplaced (7.161%); 556 KiB/s, 84 objects/s recovering 2023-10-09T10:03:08.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4896/8169 objects degraded (59.934%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:08.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:08 smithi012 ceph-mon[152263]: pgmap v443: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 169 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4896/8169 objects degraded (59.934%); 585/8169 objects misplaced (7.161%); 556 KiB/s, 84 objects/s recovering 2023-10-09T10:03:08.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4896/8169 objects degraded (59.934%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:08.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:08.436 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:03:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:08 smithi110 ceph-mon[136032]: pgmap v443: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 169 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 4896/8169 objects degraded (59.934%); 585/8169 objects misplaced (7.161%); 556 KiB/s, 84 objects/s recovering 2023-10-09T10:03:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4896/8169 objects degraded (59.934%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:08.461 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:03:08.469 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:08.484 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:03:08.508 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:03:08.513 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:03:08.513 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:03:08.513 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:03:08.513 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:03:08.513 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:03:08.514 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:03:08.514 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:03:08.514 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:03:08.514 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:08.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:03:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:08.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:08.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:08.649 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:08.830 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:08.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:08.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:08.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:08.965 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:09.004 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:09.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:09.108 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:09.137 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:09.174 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:09.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:09.207 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:09.279 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:09.306 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:09.340 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:09.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:09.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:03:09.388 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3955540035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:09.388 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3955540035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:09.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3955540035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:09.444 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:09.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:09.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:09.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:09.604 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:09.630 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:09.662 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:09.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:09.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:09.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:09.798 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:09.835 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:09.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:09.938 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:09.970 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:10.061 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:10.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:10.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:10.193 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:10.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:10 smithi012 ceph-mon[147926]: pgmap v444: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 84 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 4998/4062 objects degraded (123.043%); 160/4062 objects misplaced (3.939%); 556 KiB/s, 65 objects/s recovering 2023-10-09T10:03:10.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:10 smithi012 ceph-mon[152263]: pgmap v444: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 84 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 4998/4062 objects degraded (123.043%); 160/4062 objects misplaced (3.939%); 556 KiB/s, 65 objects/s recovering 2023-10-09T10:03:10.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:10 smithi110 ceph-mon[136032]: pgmap v444: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 84 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 4998/4062 objects degraded (123.043%); 160/4062 objects misplaced (3.939%); 556 KiB/s, 65 objects/s recovering 2023-10-09T10:03:10.449 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T10:03:10.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T10:03:10.631 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:10.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:10.754 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:10.786 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:10.821 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:10.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:10.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:03:10.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:10.962 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:11.262 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:03:11.262 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:03:11.308 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:11.352 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:11.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:11.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:11.367 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:11.459 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:11.491 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:11.528 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:11.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:11.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:11.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:11.663 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:12.135 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 2, 4, 0] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:03:12.135 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:03:12.135 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 3, 6, 7, 2, 4, 0] 2023-10-09T10:03:12.135 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T10:03:12.400 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:12.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:12.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:12.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:12 smithi012 ceph-mon[147926]: pgmap v445: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 26 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 4998/1299 objects degraded (384.758%); 71/1299 objects misplaced (5.466%); 0 B/s, 57 objects/s recovering 2023-10-09T10:03:12.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:12 smithi012 ceph-mon[152263]: pgmap v445: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 26 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 4998/1299 objects degraded (384.758%); 71/1299 objects misplaced (5.466%); 0 B/s, 57 objects/s recovering 2023-10-09T10:03:12.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:12 smithi110 ceph-mon[136032]: pgmap v445: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 26 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 4998/1299 objects degraded (384.758%); 71/1299 objects misplaced (5.466%); 0 B/s, 57 objects/s recovering 2023-10-09T10:03:12.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:12.539 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:12.576 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:12.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:12.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:12.709 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:13.354 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4998/1299 objects degraded (384.758%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:13.355 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4998/1299 objects degraded (384.758%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:13.356 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:03:13.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4998/1299 objects degraded (384.758%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:13.546 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:13.549 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:13.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:13.651 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:13.678 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:13.711 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:13.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:13.808 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:13.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:13.857 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:13.963 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:14.099 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:14.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:14.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:14.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:14.228 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:14.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[147926]: pgmap v446: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 13 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 4679/651 objects degraded (718.740%); 24/651 objects misplaced (3.687%); 26 KiB/s, 168 objects/s recovering 2023-10-09T10:03:14.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[147926]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T10:03:14.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[147926]: failed to encode map e182 with expected crc 2023-10-09T10:03:14.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[147926]: failed to encode map e182 with expected crc 2023-10-09T10:03:14.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:14.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:14.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[152263]: pgmap v446: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 13 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 4679/651 objects degraded (718.740%); 24/651 objects misplaced (3.687%); 26 KiB/s, 168 objects/s recovering 2023-10-09T10:03:14.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[152263]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T10:03:14.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[152263]: failed to encode map e182 with expected crc 2023-10-09T10:03:14.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[152263]: failed to encode map e182 with expected crc 2023-10-09T10:03:14.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:14.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:15.161 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:15.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:14 smithi110 ceph-mon[136032]: pgmap v446: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 13 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 4679/651 objects degraded (718.740%); 24/651 objects misplaced (3.687%); 26 KiB/s, 168 objects/s recovering 2023-10-09T10:03:15.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:14 smithi110 ceph-mon[136032]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T10:03:15.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:14 smithi110 ceph-mon[136032]: failed to encode map e182 with expected crc 2023-10-09T10:03:15.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:14 smithi110 ceph-mon[136032]: failed to encode map e182 with expected crc 2023-10-09T10:03:15.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:15.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:15.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:15.265 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:15.290 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:15.322 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:15.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:15.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:15.459 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:15.551 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:15.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:15.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:15.682 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:15.718 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:15.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:15.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:15.752 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_0' removed 2023-10-09T10:03:15.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[152263]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[152263]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T10:03:15.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[152263]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[152263]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[152263]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/688015492' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:03:15.810 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:03:15.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[147926]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[147926]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T10:03:15.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[147926]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[147926]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[147926]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/688015492' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:03:15.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:03:15.822 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:15.855 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:15 smithi110 ceph-mon[136032]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:15 smithi110 ceph-mon[136032]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T10:03:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:15 smithi110 ceph-mon[136032]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:15 smithi110 ceph-mon[136032]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:15 smithi110 ceph-mon[136032]: failed to encode map e183 with expected crc 2023-10-09T10:03:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/688015492' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:03:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:03:15.944 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:15.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:15.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:16.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:16.078 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:16.596 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:03:16.596 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:03:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:16 smithi110 ceph-mon[136032]: pgmap v449: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 13 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 4679/651 objects degraded (718.740%); 24/651 objects misplaced (3.687%); 32 KiB/s, 210 objects/s recovering 2023-10-09T10:03:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:16 smithi110 ceph-mon[136032]: failed to encode map e184 with expected crc 2023-10-09T10:03:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:16 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:03:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:16 smithi110 ceph-mon[136032]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T10:03:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:16 smithi110 ceph-mon[136032]: failed to encode map e184 with expected crc 2023-10-09T10:03:17.077 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:03:17.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[147926]: pgmap v449: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 13 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 4679/651 objects degraded (718.740%); 24/651 objects misplaced (3.687%); 32 KiB/s, 210 objects/s recovering 2023-10-09T10:03:17.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[147926]: failed to encode map e184 with expected crc 2023-10-09T10:03:17.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:03:17.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[147926]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T10:03:17.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[147926]: failed to encode map e184 with expected crc 2023-10-09T10:03:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[152263]: pgmap v449: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 13 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 4679/651 objects degraded (718.740%); 24/651 objects misplaced (3.687%); 32 KiB/s, 210 objects/s recovering 2023-10-09T10:03:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[152263]: failed to encode map e184 with expected crc 2023-10-09T10:03:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:03:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[152263]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T10:03:17.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:16 smithi012 ceph-mon[152263]: failed to encode map e184 with expected crc 2023-10-09T10:03:17.391 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:17.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:17.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:17.523 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:17.559 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:17.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:17.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:17.670 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:17.678 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:03:17.678 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:03:17.678 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T10:03:17.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T10:03:17.694 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:17.828 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:17.829 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:17 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4679/651 objects degraded (718.740%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:17.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:17.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:17 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4679/651 objects degraded (718.740%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:17.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:17 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4679/651 objects degraded (718.740%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:03:17.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:17.965 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:18.105 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:18.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:03:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:03:17] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T10:03:18.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:18.245 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:18.262 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:03:18.288 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:18.291 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:03:18.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:18.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:18.311 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:03:18.338 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:03:18.365 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:03:18.389 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:03:18.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:18.409 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:03:18.421 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:18.429 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:03:18.430 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:03:18.430 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:03:18.430 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:03:18.430 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:03:18.430 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:03:18.430 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:03:18.431 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:03:18.431 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:03:18.445 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:18.457 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:18.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:18.562 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:18.591 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:18.677 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:18.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:18.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:18.758 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:03:18.766 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.2. 2023-10-09T10:03:18.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:18.809 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:18.844 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:18.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:18.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:18.868 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:03:18.868 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:03:18.868 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:03:18.868 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:03:18.868 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:03:18.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:03:18.591+0000 7f85a99d3700 -1 mon.a@0(leader).osd e184 definitely_dead 0 2023-10-09T10:03:18.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[147926]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:03:18.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4679/651 objects degraded (718.740%), 2 pgs degraded) 2023-10-09T10:03:18.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/255858494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:18.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/214842098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:18.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1891146547' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:03:18.900 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[147926]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:03:18.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[152263]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:03:18.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4679/651 objects degraded (718.740%), 2 pgs degraded) 2023-10-09T10:03:18.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/255858494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:18.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/214842098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:18.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1891146547' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:03:18.902 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:18 smithi012 ceph-mon[152263]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:03:18.948 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:18.976 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:19.013 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:18 smithi110 ceph-mon[136032]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:03:19.014 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:18 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4679/651 objects degraded (718.740%), 2 pgs degraded) 2023-10-09T10:03:19.014 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/255858494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:19.014 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/214842098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:19.014 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1891146547' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:03:19.014 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:18 smithi110 ceph-mon[136032]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:03:19.268 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:19.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:19.299 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:03:19.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:19.408 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:19.441 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:19.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:19.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:19.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:19.574 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:19.906 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:19.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:19.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:19.935 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:20.011 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:19 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:19 smithi110 ceph-mon[136032]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/255858494' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1891146547' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:19 smithi110 ceph-mon[136032]: osdmap e185: 8 total, 8 up, 6 in 2023-10-09T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:19 smithi110 ceph-mon[136032]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.036 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:19 smithi110 ceph-mon[136032]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.038 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:20.070 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:20.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:20.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:20.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:20.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[147926]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/255858494' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:20.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1891146547' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[147926]: osdmap e185: 8 total, 8 up, 6 in 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[147926]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[147926]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[152263]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/255858494' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1891146547' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[152263]: osdmap e185: 8 total, 8 up, 6 in 2023-10-09T10:03:20.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[152263]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.129 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:19 smithi012 ceph-mon[152263]: failed to encode map e185 with expected crc 2023-10-09T10:03:20.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:20.205 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:20.239 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:20.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:20.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:20.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:20.371 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:20.406 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:20.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:20.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:20.511 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:20.538 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:20.663 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:20.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:20.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:20.796 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:20.853 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:20.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:20.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:03:20.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:20.986 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:21.025 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:21.026 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_1' created 2023-10-09T10:03:21.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:21.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:21.054 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:20 smithi012 ceph-mon[152263]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:03:21.054 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:20 smithi012 ceph-mon[152263]: osdmap e186: 8 total, 8 up, 6 in 2023-10-09T10:03:21.054 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:20 smithi012 ceph-mon[152263]: failed to encode map e186 with expected crc 2023-10-09T10:03:21.054 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:20 smithi012 ceph-mon[152263]: failed to encode map e186 with expected crc 2023-10-09T10:03:21.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:20 smithi012 ceph-mon[147926]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:03:21.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:20 smithi012 ceph-mon[147926]: osdmap e186: 8 total, 8 up, 6 in 2023-10-09T10:03:21.055 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:20 smithi012 ceph-mon[147926]: failed to encode map e186 with expected crc 2023-10-09T10:03:21.055 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:20 smithi012 ceph-mon[147926]: failed to encode map e186 with expected crc 2023-10-09T10:03:21.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:21.161 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:21.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:03:21.225 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:03:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:03:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96930 6.e 6:710aa3be:::benchmark_data_smithi012_182576_object96929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.62887304500001, 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098520888000000001, 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96930 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:03:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593683+0000", 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.313e-05 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593746+0000", 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2289999999999992e-06 2023-10-09T10:03:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593756+0000", 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0159999999999999e-06 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593761+0000", 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8661999999999998e-05 2023-10-09T10:03:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593859+0000", 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146317 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.594006+0000", 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000179419 2023-10-09T10:03:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.594185+0000", 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081610680000000005 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602346+0000", 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061366659999999998 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608483+0000", 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079327541000000001 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.687810+0000", 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1600999999999995e-05 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.687872+0000", 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043339989999999998 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692206+0000", 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098520888000000001 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96932 6.e 6:7facdbe4:::benchmark_data_smithi012_182576_object96931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.62041679699999, 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094685332999999997, 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96932 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602140+0000", 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8144999999999999e-05 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602168+0000", 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2849999999999997e-06 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602175+0000", 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7560000000000001e-06 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602179+0000", 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000195152 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602374+0000", 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206584 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602581+0000", 2023-10-09T10:03:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000174067 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602755+0000", 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032482050000000001 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.606003+0000", 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024921650000000002 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608495+0000", 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083792852000000001 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692288+0000", 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6909000000000006e-05 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692355+0000", 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044717339999999998 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.696827+0000", 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094685332999999997 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96937 6.e 6:7e02d1d2:::benchmark_data_smithi012_182576_object96936:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.608811+0000", 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.61374696199999, 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092425560000000004, 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96937 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608811+0000", 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608811+0000", 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608809+0000", 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.863e-05 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608858+0000", 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6682000000000001e-05 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608875+0000", 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1960000000000008e-06 2023-10-09T10:03:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608883+0000", 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3739000000000003e-05 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608927+0000", 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017568999999999999 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.609102+0000", 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023748899999999999 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.609340+0000", 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095543239999999995 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.618894+0000", 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013873755 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.632768+0000", 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.064114811999999993 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.696883+0000", 2023-10-09T10:03:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4169999999999998e-05 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.696937+0000", 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042997770000000003 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.701237+0000", 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092425560000000004 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96944 6.e 6:71fd26c2:::benchmark_data_smithi012_182576_object96943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.59780237199999, 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094319584999999997, 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96944 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624753+0000", 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9678999999999994e-05 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624833+0000", 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0849000000000001e-05 2023-10-09T10:03:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624844+0000", 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5359999999999998e-06 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624850+0000", 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012482843 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637333+0000", 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016598700000000001 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637499+0000", 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027263199999999997 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637772+0000", 2023-10-09T10:03:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033034599999999998 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641075+0000", 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088881500000000005 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641964+0000", 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072603763000000002 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714568+0000", 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4980000000000005e-05 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714633+0000", 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044422960000000001 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.719075+0000", 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094319584999999997 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:105483 6.2 6:45b0cc0b:::benchmark_data_smithi012_182576_object105482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 171.49991765600001, 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1741883319999999, 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:03:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105483 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5788999999999999e-05 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722676+0000", 2023-10-09T10:03:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.089e-06 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722684+0000", 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5950000000000004e-06 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722690+0000", 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010708354 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733399+0000", 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013741599999999999 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733536+0000", 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017242200000000001 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:03:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733708+0000", 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003714143 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.737423+0000", 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015821828 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.753244+0000", 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036249568000000003 2023-10-09T10:03:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789494+0000", 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8799999999999997e-06 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789503+0000", 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1072112519999999 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.896714+0000", 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114647 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.896829+0000", 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1741883319999999 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:105508 6.e 6:77549acc:::benchmark_data_smithi012_182576_object105507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 171.447481286, 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1483757859999999, 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105508 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775075+0000", 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9004000000000003e-05 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775174+0000", 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.751e-05 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775192+0000", 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6340000000000001e-06 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775199+0000", 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4343000000000003e-05 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775274+0000", 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000152091 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775426+0000", 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019803600000000001 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775624+0000", 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057209089999999997 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.781345+0000", 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079732469999999993 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789318+0000", 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8120000000000002e-06 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789328+0000", 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.114660743 2023-10-09T10:03:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.903988+0000", 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000495179 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.904484+0000", 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018860534000000002 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.923344+0000", 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010841900000000001 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.923453+0000", 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1483757859999999 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:178976 6.e 6.4855639e (undecoded) ondisk+retry+write+known_if_redirected e158)", 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 64.256702098000005, 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.74206674699999997, 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178976 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965854+0000", 2023-10-09T10:03:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5659999999999997e-06 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965860+0000", 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9839999999999996e-06 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965870+0000", 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.72e-06 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965877+0000", 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020714689999999998 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.967948+0000", 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0054999999999999e-05 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.967958+0000", 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.73995176900000004 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.707910+0000", 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.277e-05 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.707923+0000", 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.74206674699999997 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179004 6.e 6.afd2ea6e (undecoded) ondisk+write+known_if_redirected e158)", 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 64.218092474000002, 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.703617838, 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179004 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:03:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004464+0000", 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9590000000000007e-06 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004474+0000", 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1376999999999999e-05 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004495+0000", 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4409999999999993e-06 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004504+0000", 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0488000000000002e-05 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004554+0000", 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0353999999999999e-05 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004564+0000", 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70350862300000006 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708073+0000", 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.04e-05 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708083+0000", 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.703617838 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179005 6.e 6.7a62357e (undecoded) ondisk+write+known_if_redirected e158)", 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:03:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 64.217982560999999, 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70354121400000003, 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179005 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8079999999999998e-06 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004579+0000", 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5859999999999999e-06 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004586+0000", 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5590000000000001e-06 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004590+0000", 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6616000000000002e-05 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004637+0000", 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2291e-05 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004649+0000", 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70345850600000004 2023-10-09T10:03:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708107+0000", 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3999999999999998e-06 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708117+0000", 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70354121400000003 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195777 6.12 6:4f83111c:::benchmark_data_smithi012_182576_object90256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 37.331145991, 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86944099200000002, 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195777 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:03:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891410+0000", 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4070000000000004e-06 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891418+0000", 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4187e-05 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891432+0000", 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6590000000000004e-06 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891438+0000", 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.266e-05 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891500+0000", 2023-10-09T10:03:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7346999999999998e-05 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891518+0000", 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.84995838999999995 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741476+0000", 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1224999999999995e-05 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741567+0000", 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034862599999999999 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741916+0000", 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076180900000000001 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749534+0000", 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011212183000000001 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760746+0000", 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6701000000000001e-05 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760773+0000", 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.125e-05 2023-10-09T10:03:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760814+0000", 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8943000000000003e-05 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760853+0000", 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86944099200000002 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195847 6.12 6:4f2f2e62:::benchmark_data_smithi012_182576_object90326:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 37.099013563, 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63759273800000005, 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195847 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123543+0000", 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.46e-06 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123550+0000", 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3708e-05 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123563+0000", 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8350000000000004e-06 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123571+0000", 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1295000000000002e-05 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123632+0000", 2023-10-09T10:03:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8449999999999998e-06 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123642+0000", 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.61961346399999995 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743256+0000", 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8935000000000003e-05 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743325+0000", 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017814799999999999 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743503+0000", 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060367659999999998 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749540+0000", 2023-10-09T10:03:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011359289 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760899+0000", 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015118900000000001 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761050+0000", 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.49e-05 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761095+0000", 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2329000000000001e-05 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761137+0000", 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63759273800000005 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195920 6.12 6:4be14f21:::benchmark_data_smithi012_182576_object90399:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.958904222999998, 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.49761078399999997, 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195920 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263652+0000", 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6619999999999999e-06 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263659+0000", 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.222e-05 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263671+0000", 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3700000000000003e-06 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263677+0000", 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5055999999999995e-05 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263742+0000", 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.256e-05 2023-10-09T10:03:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263754+0000", 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.48082169600000002 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744576+0000", 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114827 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744691+0000", 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026918100000000001 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744960+0000", 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096003400000000006 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754560+0000", 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063868149999999997 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760947+0000", 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024620099999999998 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761193+0000", 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4209000000000003e-05 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761238+0000", 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7093e-05 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761265+0000", 2023-10-09T10:03:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.49761078399999997 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196022 6.12 6:4bcd6600:::benchmark_data_smithi012_182576_object90501:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.826961439000002, 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.36575843200000002, 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.312 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196022 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395595+0000", 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2799999999999998e-06 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395602+0000", 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5897999999999999e-05 2023-10-09T10:03:21.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395618+0000", 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4340000000000003e-06 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395626+0000", 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1285999999999995e-05 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395697+0000", 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2662e-05 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395710+0000", 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35087969000000002 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746589+0000", 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010907400000000001 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746698+0000", 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024344699999999999 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746942+0000", 2023-10-09T10:03:21.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076297279999999997 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754572+0000", 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063959359999999996 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760967+0000", 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032883000000000001 2023-10-09T10:03:21.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761296+0000", 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6942999999999999e-05 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761323+0000", 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1780999999999999e-05 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761355+0000", 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.36575843200000002 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196042 6.12 6:4e976c7d:::benchmark_data_smithi012_182576_object90521:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.789147300000003, 2023-10-09T10:03:21.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.330443298, 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196042 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433409+0000", 2023-10-09T10:03:21.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3280000000000006e-06 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433417+0000", 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2411e-05 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433430+0000", 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3770000000000004e-06 2023-10-09T10:03:21.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433435+0000", 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5327999999999997e-05 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433481+0000", 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3749999999999997e-06 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433488+0000", 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.31462358499999998 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748112+0000", 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115394 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748227+0000", 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025816400000000002 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748485+0000", 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010564301 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759049+0000", 2023-10-09T10:03:21.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037354570000000002 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.762785+0000", 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0009886719999999999 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763774+0000", 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7079000000000001e-05 2023-10-09T10:03:21.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763821+0000", 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3420999999999997e-05 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763854+0000", 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.330443298 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.328 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196076 6.12 6:49ef5a44:::benchmark_data_smithi012_182576_object90555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.754088518000003, 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30399614400000002, 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.329 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196076 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:21.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468468+0000", 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.251e-06 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468474+0000", 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3427e-05 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468488+0000", 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6200000000000004e-06 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468493+0000", 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3034000000000006e-05 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468566+0000", 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2557e-05 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468579+0000", 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28100584000000001 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749585+0000", 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118539 2023-10-09T10:03:21.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749703+0000", 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025233699999999999 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749955+0000", 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009103547 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759059+0000", 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048303510000000001 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763889+0000", 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084814780000000006 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772371+0000", 2023-10-09T10:03:21.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2203999999999997e-05 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772423+0000", 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2608000000000003e-05 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772466+0000", 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30399614400000002 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.336 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196085 6.12 6:4d84fb9e:::benchmark_data_smithi012_182576_object90564:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.743408082000002, 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.298003501, 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.337 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196085 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:03:21.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479148+0000", 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7549999999999994e-06 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479156+0000", 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6019e-05 2023-10-09T10:03:21.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479172+0000", 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3470000000000004e-06 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479178+0000", 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6717999999999994e-05 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479245+0000", 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2296999999999999e-05 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479257+0000", 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27237020299999998 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.751627+0000", 2023-10-09T10:03:21.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114201 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.751741+0000", 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027028600000000001 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.752012+0000", 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070559289999999998 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759068+0000", 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013462363 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772530+0000", 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045611979999999998 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777091+0000", 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3906999999999999e-05 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777125+0000", 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8297e-05 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777153+0000", 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.298003501 2023-10-09T10:03:21.345 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196149 6.12 6:48b2c020:::benchmark_data_smithi012_182576_object90628:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.650015918000001, 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20477431600000001, 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.346 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196149 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572541+0000", 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572548+0000", 2023-10-09T10:03:21.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2437e-05 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572560+0000", 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4049999999999997e-06 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572567+0000", 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6699999999999995e-05 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572633+0000", 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1873e-05 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572645+0000", 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.180477941 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753123+0000", 2023-10-09T10:03:21.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5816999999999995e-05 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753209+0000", 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000185847 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753395+0000", 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070544839999999998 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760449+0000", 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012128781 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772578+0000", 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046280150000000001 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777206+0000", 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5373e-05 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777261+0000", 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5093999999999998e-05 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777316+0000", 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20477431600000001 2023-10-09T10:03:21.353 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196189 6.12 6:4bcf1e87:::benchmark_data_smithi012_182576_object90668:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.588965606999999, 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14382505400000001, 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.354 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196189 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633591+0000", 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2080000000000002e-06 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633596+0000", 2023-10-09T10:03:21.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9220000000000002e-06 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633602+0000", 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2490000000000001e-06 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633605+0000", 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8455000000000003e-05 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633653+0000", 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7867999999999999e-05 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633671+0000", 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12094508800000001 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754616+0000", 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2300999999999996e-05 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754709+0000", 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019090900000000001 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754899+0000", 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055585510000000001 2023-10-09T10:03:21.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760458+0000", 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012168912 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772627+0000", 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047358959999999999 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777363+0000", 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0698999999999999e-05 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777393+0000", 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4036000000000002e-05 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777418+0000", 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14382505400000001 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196191 6.12 6:48379c93:::benchmark_data_smithi012_182576_object90670:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.588942979999999, 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14395870599999999, 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.362 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196191 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.013e-06 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633617+0000", 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5360000000000001e-06 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633620+0000", 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3039999999999999e-06 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633623+0000", 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047045000000000002 2023-10-09T10:03:21.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634093+0000", 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0719999999999999e-06 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634098+0000", 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121978771 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756077+0000", 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104263 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756181+0000", 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000219302 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756400+0000", 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004065333 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760466+0000", 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012786838999999999 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.773253+0000", 2023-10-09T10:03:21.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042139990000000004 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777467+0000", 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7116e-05 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777524+0000", 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0078999999999999e-05 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777574+0000", 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14395870599999999 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196192 6.12 6:4fcc3e62:::benchmark_data_smithi012_182576_object90671:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.588915211, 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14401113700000001, 2023-10-09T10:03:21.370 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196192 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:03:21.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7549999999999999e-06 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633644+0000", 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3650000000000003e-06 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633650+0000", 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7549999999999999e-06 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633652+0000", 2023-10-09T10:03:21.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000899926 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634552+0000", 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3590000000000001e-06 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634557+0000", 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12309893399999999 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757656+0000", 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1304999999999995e-05 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757737+0000", 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019102100000000001 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:03:21.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757928+0000", 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027759170000000001 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760704+0000", 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013396351000000001 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.774100+0000", 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035033410000000001 2023-10-09T10:03:21.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777604+0000", 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7534000000000001e-05 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777631+0000", 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2878e-05 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:21.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:21.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777654+0000", 2023-10-09T10:03:21.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14401113700000001 2023-10-09T10:03:21.378 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.378 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.378 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.378 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:21.378 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:21.378 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:03:21.379 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:20 smithi110 ceph-mon[136032]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:03:21.379 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:20 smithi110 ceph-mon[136032]: osdmap e186: 8 total, 8 up, 6 in 2023-10-09T10:03:21.379 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:20 smithi110 ceph-mon[136032]: failed to encode map e186 with expected crc 2023-10-09T10:03:21.379 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:20 smithi110 ceph-mon[136032]: failed to encode map e186 with expected crc 2023-10-09T10:03:21.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:21.380 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:21.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:21.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T10:03:21.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:21.509 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:21.545 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:21.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:21.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:21.648 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:21.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:03:21.664 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:21.676 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:21.711 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:21.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:21.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:21.843 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:21.880 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:21.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:21.984 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:22.010 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:22.029 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:22.051 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:22.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:22.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:22.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3853194348' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:03:22.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: osdmap e187: 8 total, 8 up, 6 in 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[147926]: pgmap v456: 89 pgs: 16 unknown, 5 peering, 68 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.121 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3853194348' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: osdmap e187: 8 total, 8 up, 6 in 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:21 smithi012 ceph-mon[152263]: pgmap v456: 89 pgs: 16 unknown, 5 peering, 68 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:03:22.155 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:22.185 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:22.216 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:22.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: failed to encode map e186 with expected crc 2023-10-09T10:03:22.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3853194348' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:03:22.218 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:03:22.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T10:03:22.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: osdmap e187: 8 total, 8 up, 6 in 2023-10-09T10:03:22.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: failed to encode map e187 with expected crc 2023-10-09T10:03:22.219 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:21 smithi110 ceph-mon[136032]: pgmap v456: 89 pgs: 16 unknown, 5 peering, 68 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:03:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:22.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:22.275 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:22.321 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:22.351 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:23.065 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:23.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:23.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:23.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:22 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:03:23.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:22 smithi110 ceph-mon[136032]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:22 smithi110 ceph-mon[136032]: osdmap e188: 8 total, 8 up, 6 in 2023-10-09T10:03:23.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:22 smithi110 ceph-mon[136032]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:22 smithi110 ceph-mon[136032]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:23.284 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:23.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:23.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:23.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:23.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:03:23.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[147926]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[147926]: osdmap e188: 8 total, 8 up, 6 in 2023-10-09T10:03:23.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[147926]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.419 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[147926]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:03:23.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[152263]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[152263]: osdmap e188: 8 total, 8 up, 6 in 2023-10-09T10:03:23.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[152263]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:22 smithi012 ceph-mon[152263]: failed to encode map e188 with expected crc 2023-10-09T10:03:23.421 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:23.446 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:03:23.447 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:03:23.447 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:03:23.447 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:03:23.637 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:23.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:23.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:23.641 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:23.742 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:23.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:23.793 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:03:23.805 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:23.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:23.908 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:23.944 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:24.397 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:24 smithi012 ceph-mon[147926]: osdmap e189: 8 total, 8 up, 6 in 2023-10-09T10:03:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:24 smithi012 ceph-mon[147926]: pgmap v459: 89 pgs: 7 creating+peering, 5 unknown, 5 peering, 72 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 146 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T10:03:24.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:24 smithi012 ceph-mon[152263]: osdmap e189: 8 total, 8 up, 6 in 2023-10-09T10:03:24.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:24 smithi012 ceph-mon[152263]: pgmap v459: 89 pgs: 7 creating+peering, 5 unknown, 5 peering, 72 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 146 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T10:03:24.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:24 smithi110 ceph-mon[136032]: osdmap e189: 8 total, 8 up, 6 in 2023-10-09T10:03:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:24 smithi110 ceph-mon[136032]: pgmap v459: 89 pgs: 7 creating+peering, 5 unknown, 5 peering, 72 active+clean; 582 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 146 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T10:03:24.869 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:24.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:24.962 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0] out_osds: [5, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:03:24.963 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:03:24.963 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T10:03:24.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T10:03:24.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:25.042 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:25.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:25 smithi110 ceph-mon[136032]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:25 smithi110 ceph-mon[136032]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:25 smithi110 ceph-mon[136032]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:25 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:25.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:25 smithi110 ceph-mon[136032]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.356 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[147926]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.357 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[147926]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.357 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[147926]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.357 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:25.357 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[147926]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.357 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[152263]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.357 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[152263]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.357 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[152263]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.357 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:25.358 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:25 smithi012 ceph-mon[152263]: failed to encode map e189 with expected crc 2023-10-09T10:03:25.384 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:03:25.385 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:03:25.385 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:03:25.410 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:03:25.425 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:25.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:25.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:25.428 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:03:25.446 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:03:25.466 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:03:25.485 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:03:25.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:03:25.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:03:25.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:03:25.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:03:25.487 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:03:25.487 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:03:25.487 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:03:25.487 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:03:25.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:25.559 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:25.582 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:03:25.582 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:03:25.952 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:25.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:25.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:26.010 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:03:26.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:26.084 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:26.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:26 smithi012 ceph-mon[152263]: pgmap v460: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 110 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:03:26.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3182725251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:26.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:26 smithi012 ceph-mon[147926]: pgmap v460: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 110 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:03:26.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3182725251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:26.270 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:26.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:26.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:26.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:26.398 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:26 smithi110 ceph-mon[136032]: pgmap v460: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 110 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:03:26.399 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3182725251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:26.401 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:26.433 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:26.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:26.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:26.539 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:26.564 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:26.596 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:26.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:26.608 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:26.683 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:26.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:26.735 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:26.815 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:26.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:26.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:26.949 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:26.983 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:26.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:27.055 INFO:teuthology.orchestra.run.smithi012.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T10:03:27.055 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:03:27.083 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:27 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:03:27.083 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/283297514' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:03:27.083 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1325231444' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:03:27.083 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:27 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:03:27.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/283297514' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:03:27.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1325231444' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:03:27.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:27.111 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:27 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:03:27.112 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/283297514' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:03:27.112 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1325231444' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:03:27.115 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:27.149 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:27.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:27.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:27.282 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:27.798 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:03:27.798 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:03:27.828 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:27.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:27.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:27.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:27.961 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:27.998 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:28.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:28.059 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:28.060 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[147926]: failed to encode map e190 with expected crc 2023-10-09T10:03:28.060 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/283297514' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:03:28.060 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1325231444' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:03:28.060 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[147926]: osdmap e190: 8 total, 8 up, 6 in 2023-10-09T10:03:28.060 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[147926]: pgmap v462: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 92 B/s, 0 objects/s recovering 2023-10-09T10:03:28.060 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:03:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:03:27] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T10:03:28.061 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:28.061 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[152263]: failed to encode map e190 with expected crc 2023-10-09T10:03:28.061 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/283297514' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:03:28.061 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1325231444' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:03:28.061 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[152263]: osdmap e190: 8 total, 8 up, 6 in 2023-10-09T10:03:28.061 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-mon[152263]: pgmap v462: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 92 B/s, 0 objects/s recovering 2023-10-09T10:03:28.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:28.112 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:28 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:28.112 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:28 smithi110 ceph-mon[136032]: failed to encode map e190 with expected crc 2023-10-09T10:03:28.112 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/283297514' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:03:28.113 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1325231444' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:03:28.113 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:28 smithi110 ceph-mon[136032]: osdmap e190: 8 total, 8 up, 6 in 2023-10-09T10:03:28.113 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:28 smithi110 ceph-mon[136032]: pgmap v462: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 92 B/s, 0 objects/s recovering 2023-10-09T10:03:28.133 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:03:28.134 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:28.168 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:28.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:28.272 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:28.298 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:28.333 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:28.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:28.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:03:28.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:28.488 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:28.526 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:28.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:28.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:28.658 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:28.691 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:28.693 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:03:28.468+0000 7f85a99d3700 -1 mon.a@0(leader).osd e190 definitely_dead 0 2023-10-09T10:03:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:28.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:28.731 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:28.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:28.822 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:28.853 DEBUG:teuthology.orchestra.run.smithi012:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T10:03:28.855 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:03:28.855 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:03:28.856 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:28.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:28.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:28.909 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:hints = 1 2023-10-09T10:03:28.909 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:03:28.910 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object prefix: benchmark_data_smithi012_221426 2023-10-09T10:03:28.910 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:03:28.910 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:03:28.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:29.018 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:29.052 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:29.080 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:29.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:29.094 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.5. 2023-10-09T10:03:29.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:29 smithi110 ceph-mon[136032]: failed to encode map e190 with expected crc 2023-10-09T10:03:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:29 smithi110 ceph-mon[136032]: failed to encode map e190 with expected crc 2023-10-09T10:03:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3025905835' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:03:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:03:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:29.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:29.223 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:29.307 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:29.308 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[147926]: failed to encode map e190 with expected crc 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[147926]: failed to encode map e190 with expected crc 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3025905835' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[152263]: failed to encode map e190 with expected crc 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[152263]: failed to encode map e190 with expected crc 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3025905835' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:03:29.309 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:03:29.310 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:29.310 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:29.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:29.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:29.440 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:29.603 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:29.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:29.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:29.628 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T10:03:29.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:29.736 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:29.888 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:29.889 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:03:29.889 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:03:29.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:29.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:29.910 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 1 16 506 490 30.6225 30.625 0.0012822 0.0058484 2023-10-09T10:03:29.992 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:30.013 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:03:30.020 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:30.073 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:30.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:30.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:30.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T10:03:30.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:30 smithi012 ceph-mon[152263]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T10:03:30.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:30 smithi012 ceph-mon[152263]: pgmap v463: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:03:30.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:30 smithi012 ceph-mon[152263]: failed to encode map e191 with expected crc 2023-10-09T10:03:30.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:30.186 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T10:03:30.186 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:30 smithi012 ceph-mon[147926]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T10:03:30.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:30 smithi012 ceph-mon[147926]: pgmap v463: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:03:30.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:30 smithi012 ceph-mon[147926]: failed to encode map e191 with expected crc 2023-10-09T10:03:30.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:03:30.280 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:30.302 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T10:03:30.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:30 smithi110 ceph-mon[136032]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T10:03:30.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:30 smithi110 ceph-mon[136032]: pgmap v463: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:03:30.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:30 smithi110 ceph-mon[136032]: failed to encode map e191 with expected crc 2023-10-09T10:03:30.370 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:30.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:30.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:30.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:30.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:30.552 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:03:30.568 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:30.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:30.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:30.590 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:03:30.638 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:03:30.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:30.682 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:03:30.715 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:30.721 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:03:30.757 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:30.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:03:30.765 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:30.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:30.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:30.805 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:03:30.837 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:03:30.847 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:03:30.847 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:03:30.848 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:03:30.848 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:03:30.848 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:03:30.848 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:03:30.848 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:03:30.848 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:03:30.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:30.910 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 2 16 2673 2657 83.0245 135.438 0.00266977 0.0120271 2023-10-09T10:03:30.927 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:31.005 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:31.148 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e191 with expected crc 2023-10-09T10:03:31.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T10:03:31.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3438823718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e191 with expected crc 2023-10-09T10:03:31.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.172 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:31 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3438823718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:31.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e191 with expected crc 2023-10-09T10:03:31.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T10:03:31.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.414 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: failed to encode map e192 with expected crc 2023-10-09T10:03:31.414 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:31 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3438823718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:31.488 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:31.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:31.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:31.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:31.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:31.740 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:31.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:31.846 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:31.887 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:31.910 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 3 16 5572 5556 115.74 181.188 0.003935 0.0086273 2023-10-09T10:03:31.935 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:31.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:31.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:32.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:32.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:32 smithi012 ceph-mon[147926]: pgmap v466: 89 pgs: 18 peering, 71 active+clean; 12 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.8 MiB/s wr, 28 op/s 2023-10-09T10:03:32.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:32 smithi012 ceph-mon[152263]: pgmap v466: 89 pgs: 18 peering, 71 active+clean; 12 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.8 MiB/s wr, 28 op/s 2023-10-09T10:03:32.111 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:32.173 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:32.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:32.261 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:03:32.261 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:03:32.262 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:03:32.262 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:03:32.262 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:03:32.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:32.287 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:32 smithi110 ceph-mon[136032]: pgmap v466: 89 pgs: 18 peering, 71 active+clean; 12 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.8 MiB/s wr, 28 op/s 2023-10-09T10:03:32.326 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:32.386 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:32.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:32.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:32.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:32.559 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:32.614 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:32.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:32.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:03:32.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:32.774 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:32.837 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:32.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:32.913 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 4 16 8348 8332 130.092 173.5 0.00294522 0.00767672 2023-10-09T10:03:32.943 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:32.990 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:33.040 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:33.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:33.093 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3036530660' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:33.093 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:33 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:33.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3036530660' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:33.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:33 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:33.124 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:03:33.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:33.195 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:33.239 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:33.241 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3036530660' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:33.241 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:33 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:33.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:33.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:33.344 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:33.406 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:33.416 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:33.469 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:33.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:33.577 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:33.619 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:33.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:03:33.674 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:33.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:33.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:33.840 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:33.914 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 5 16 11067 11051 138.043 169.938 0.00508696 0.00723037 2023-10-09T10:03:33.937 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:33.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:33.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:34.046 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:34.084 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:34.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[147926]: pgmap v467: 89 pgs: 18 peering, 71 active+clean; 249 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 382 KiB/s, 6 objects/s recovering 2023-10-09T10:03:34.138 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:34.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:34.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:34.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:34.302 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:34.366 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:34.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:34.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:03:34.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:03:34.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[147926]: failed to encode map e193 with expected crc 2023-10-09T10:03:34.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:34.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[147926]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T10:03:34.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[152263]: pgmap v467: 89 pgs: 18 peering, 71 active+clean; 249 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 382 KiB/s, 6 objects/s recovering 2023-10-09T10:03:34.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:03:34.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:03:34.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[152263]: failed to encode map e193 with expected crc 2023-10-09T10:03:34.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:34.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:34 smithi012 ceph-mon[152263]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T10:03:34.431 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:34 smithi110 ceph-mon[136032]: pgmap v467: 89 pgs: 18 peering, 71 active+clean; 249 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 382 KiB/s, 6 objects/s recovering 2023-10-09T10:03:34.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:34 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:03:34.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:34 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:03:34.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:34 smithi110 ceph-mon[136032]: failed to encode map e193 with expected crc 2023-10-09T10:03:34.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:34 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:34.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:34 smithi110 ceph-mon[136032]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T10:03:34.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:34.525 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:34.529 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:34.582 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:34.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:34.629 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:03:34.629 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:03:34.630 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T10:03:34.630 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:03:34.630 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:03:34.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T10:03:34.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:34.696 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:03:34.696 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:03:34.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13899 6.c e103/97 6:39dbd6a1:::benchmark_data_smithi012_182576_object13898:head v 103'862, mlcod=103'860)", 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 269.14680585100001, 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109510709, 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:03:34.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545772+0000", 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4119999999999999e-05 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545806+0000", 2023-10-09T10:03:34.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4049000000000001e-05 2023-10-09T10:03:34.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545820+0000", 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1470000000000004e-06 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545827+0000", 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5004000000000002e-05 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545902+0000", 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3674999999999999e-05 2023-10-09T10:03:34.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545926+0000", 2023-10-09T10:03:34.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109217777 2023-10-09T10:03:34.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655143+0000", 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014009299999999999 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655284+0000", 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109510709 2023-10-09T10:03:34.703 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.704 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.704 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.704 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13895 6.3 e103/97 6:cb4c122f:::benchmark_data_smithi012_182576_object13894:head v 103'855, mlcod=103'853)", 2023-10-09T10:03:34.705 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:03:34.705 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 269.14627707800003, 2023-10-09T10:03:34.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108952985, 2023-10-09T10:03:34.705 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.705 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.706 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:03:34.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:03:34.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:34.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546301+0000", 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2656000000000001e-05 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546324+0000", 2023-10-09T10:03:34.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6180000000000001e-05 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546340+0000", 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9619999999999998e-06 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546346+0000", 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5816000000000003e-05 2023-10-09T10:03:34.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546391+0000", 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0018e-05 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546421+0000", 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108744009 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655165+0000", 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9089e-05 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655255+0000", 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108952985 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13902 6.5 e103/97 6:a6447830:::benchmark_data_smithi012_182576_object13901:head v 103'874, mlcod=103'872)", 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 269.14621871499997, 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110437017, 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0539000000000001e-05 2023-10-09T10:03:34.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546380+0000", 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8390000000000001e-06 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546387+0000", 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2400000000000001e-06 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546391+0000", 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1564999999999998e-05 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546443+0000", 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1464999999999999e-05 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546464+0000", 2023-10-09T10:03:34.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110295141 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656759+0000", 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7586999999999998e-05 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656797+0000", 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110437017 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.716 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13914 6.a e103/97 6:527234fb:::benchmark_data_smithi012_182576_object13913:head v 103'853, mlcod=103'851)", 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 269.141453818, 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.105616578, 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551124+0000", 2023-10-09T10:03:34.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4997000000000001e-05 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551149+0000", 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104127211 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655276+0000", 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4699999999999999e-06 2023-10-09T10:03:34.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655283+0000", 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9079000000000003e-05 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655322+0000", 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1195000000000002e-05 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655353+0000", 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013534339999999999 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656706+0000", 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5194000000000002e-05 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656741+0000", 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.105616578 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13926 6.1 e103/97 6:8de8341f:::benchmark_data_smithi012_182576_object13925:head v 103'823, mlcod=103'821)", 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:03:34.722 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 269.12875182099998, 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092614976000000002, 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563825+0000", 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014807399999999999 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563974+0000", 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.091111844999999997 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655085+0000", 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0784000000000002e-05 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655116+0000", 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017427 2023-10-09T10:03:34.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655290+0000", 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4281999999999999e-05 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655315+0000", 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010930600000000001 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656408+0000", 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4076999999999999e-05 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656442+0000", 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092614976000000002 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.727 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13929 6.c e103/97 6:39e98511:::benchmark_data_smithi012_182576_object13928:head v 103'863, mlcod=103'860)", 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 269.12859408200001, 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092504991999999994, 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:34.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563984+0000", 2023-10-09T10:03:34.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2965000000000004e-05 2023-10-09T10:03:34.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564047+0000", 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.091099323999999995 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655146+0000", 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0924000000000002e-05 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655167+0000", 2023-10-09T10:03:34.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016929299999999999 2023-10-09T10:03:34.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655336+0000", 2023-10-09T10:03:34.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0599999999999999e-05 2023-10-09T10:03:34.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655357+0000", 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011074889999999999 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656464+0000", 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5162999999999999e-05 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656490+0000", 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092504991999999994 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96028 6.f e118/117 6:f25f2574:::benchmark_data_smithi012_182576_object96027:head v 118'5972, mlcod=0'0)", 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:03:34.734 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 202.53323007200001, 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041325972000000002, 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:03:34.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159348+0000", 2023-10-09T10:03:34.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0521e-05 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159388+0000", 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9960000000000005e-06 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159398+0000", 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.981e-06 2023-10-09T10:03:34.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159404+0000", 2023-10-09T10:03:34.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0812e-05 2023-10-09T10:03:34.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159465+0000", 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2714999999999999e-05 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159488+0000", 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041139283999999998 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.200627+0000", 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7336000000000002e-05 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.200675+0000", 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041325972000000002 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.740 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96906 6.8 e119/117 6:1349a12f:::benchmark_data_smithi012_182576_object96905:head v 119'6178, mlcod=0'0)", 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.12893227500001, 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043713813999999997, 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.741 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4641e-05 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563671+0000", 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3489999999999997e-06 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563677+0000", 2023-10-09T10:03:34.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0620000000000002e-06 2023-10-09T10:03:34.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563681+0000", 2023-10-09T10:03:34.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024566689999999999 2023-10-09T10:03:34.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.566138+0000", 2023-10-09T10:03:34.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8526000000000001e-05 2023-10-09T10:03:34.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.566166+0000", 2023-10-09T10:03:34.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041156198999999997 2023-10-09T10:03:34.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.607322+0000", 2023-10-09T10:03:34.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7769999999999999e-05 2023-10-09T10:03:34.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.607360+0000", 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043713813999999997 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96908 6.8 e119/117 6:16fa2f2f:::benchmark_data_smithi012_182576_object96907:head v 119'6179, mlcod=0'0)", 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:03:34.747 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.12888710600001, 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052663933000000003, 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563691+0000", 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3872000000000001e-05 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563715+0000", 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0090000000000004e-06 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563721+0000", 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9999999999999998e-06 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563725+0000", 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010034111 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.573759+0000", 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.565e-05 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.573785+0000", 2023-10-09T10:03:34.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042522310000000001 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616307+0000", 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8340000000000001e-05 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616356+0000", 2023-10-09T10:03:34.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052663933000000003 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96912 6.8 e119/117 6:11471bcc:::benchmark_data_smithi012_182576_object96911:head v 119'6180, mlcod=0'0)", 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.121710949, 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04550949, 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.753 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:03:34.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570867+0000", 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5475000000000002e-05 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570903+0000", 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1657999999999999e-05 2023-10-09T10:03:34.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570914+0000", 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8560000000000001e-06 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570921+0000", 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027926853000000001 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.598848+0000", 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8767000000000001e-05 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.598877+0000", 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017483862999999999 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616361+0000", 2023-10-09T10:03:34.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6659000000000001e-05 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616377+0000", 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04550949 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96916 6.8 e119/117 6:11cbd0a6:::benchmark_data_smithi012_182576_object96915:head v 119'6181, mlcod=0'0)", 2023-10-09T10:03:34.758 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.09718701899999, 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042488638000000002, 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595390+0000", 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6316e-05 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595416+0000", 2023-10-09T10:03:34.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.643e-05 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595433+0000", 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0700000000000007e-06 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595441+0000", 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013293442000000001 2023-10-09T10:03:34.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608734+0000", 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4462000000000003e-05 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608769+0000", 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029077441999999998 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637846+0000", 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3998999999999998e-05 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637880+0000", 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042488638000000002 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96919 6.8 e119/117 6:16cabaa4:::benchmark_data_smithi012_182576_object96918:head v 119'6182, mlcod=0'0)", 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 201.09713004299999, 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042453923999999997, 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595448+0000", 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9534e-05 2023-10-09T10:03:34.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595468+0000", 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1179999999999997e-06 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595474+0000", 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2780000000000004e-06 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595478+0000", 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020955610999999999 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616434+0000", 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5219000000000003e-05 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616469+0000", 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021415979000000002 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637885+0000", 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7509000000000001e-05 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637903+0000", 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042453923999999997 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:98780 6.2 e119/117 6:46aa6df3:::benchmark_data_smithi012_182576_object98779:head v 119'6253, mlcod=0'0)", 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:03:34.769 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 198.01492705699999, 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053554971999999999, 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677650+0000", 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0443e-05 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677681+0000", 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8221999999999999e-05 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677699+0000", 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1400000000000002e-06 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677706+0000", 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070735980000000004 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.684780+0000", 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4288000000000001e-05 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.684814+0000", 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046346799000000001 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731161+0000", 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5666e-05 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731207+0000", 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053554971999999999 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:98784 6.2 e119/117 6:44e3e91a:::benchmark_data_smithi012_182576_object98783:head v 119'6254, mlcod=0'0)", 2023-10-09T10:03:34.775 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 198.01357120099999, 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052221194999999998, 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:03:34.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6214e-05 2023-10-09T10:03:34.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679033+0000", 2023-10-09T10:03:34.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6022999999999999e-05 2023-10-09T10:03:34.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679049+0000", 2023-10-09T10:03:34.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7379999999999997e-06 2023-10-09T10:03:34.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679056+0000", 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013390713 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.692446+0000", 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8688e-05 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.692475+0000", 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038735865000000001 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731211+0000", 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.783e-05 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731229+0000", 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052221194999999998 2023-10-09T10:03:34.781 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103547 6.8 e122/121 6:1259b0ac:::benchmark_data_smithi012_182576_object103546:head v 122'6626, mlcod=0'0)", 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 188.209456488, 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040879891000000002, 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.782 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483121+0000", 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105214 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483226+0000", 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9423999999999999e-05 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483246+0000", 2023-10-09T10:03:34.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4619999999999999e-06 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483253+0000", 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038037510000000002 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.487057+0000", 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.73e-05 2023-10-09T10:03:34.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.487084+0000", 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036875568999999997 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.523960+0000", 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2120000000000003e-05 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.524002+0000", 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040879891000000002 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103568 6.f e122/117 6:f1ce72d6:::benchmark_data_smithi012_182576_object103567:head v 122'6429, mlcod=0'0)", 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:03:34.787 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 188.172789322, 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047394631, 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7826e-05 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519827+0000", 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.409e-06 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519833+0000", 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1130000000000003e-06 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519837+0000", 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.008335238 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.528173+0000", 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7257e-05 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.528200+0000", 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038942002000000003 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.567142+0000", 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2160999999999998e-05 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.567184+0000", 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047394631 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.792 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103834 6.2 e122/117 6:481c5bb1:::benchmark_data_smithi012_182576_object103833:head v 122'6597, mlcod=0'0)", 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.73907725800001, 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040357787999999999, 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953500+0000", 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8389999999999997e-05 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953538+0000", 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6200000000000001e-05 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953554+0000", 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.303e-06 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953563+0000", 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011028718999999999 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.964591+0000", 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2540000000000001e-05 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.964614+0000", 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029196083000000001 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.993810+0000", 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9212999999999998e-05 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.993859+0000", 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040357787999999999 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.797 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:104205 6.f e122/117 6:f3405025:::benchmark_data_smithi012_182576_object104204:head v 122'6474, mlcod=0'0)", 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 187.113347621, 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048731662000000002, 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1229999999999997e-05 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579262+0000", 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1770000000000001e-05 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579274+0000", 2023-10-09T10:03:34.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8019999999999999e-06 2023-10-09T10:03:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579278+0000", 2023-10-09T10:03:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079426949999999996 2023-10-09T10:03:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.587221+0000", 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8532000000000001e-05 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.587250+0000", 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040668007999999999 2023-10-09T10:03:34.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.627918+0000", 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5040999999999998e-05 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.627963+0000", 2023-10-09T10:03:34.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048731662000000002 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:105353 6.2 e123/117 6:4730719a:::benchmark_data_smithi012_182576_object105352:head v 123'6695, mlcod=0'0)", 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 185.17199778400001, 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041308386000000002, 2023-10-09T10:03:34.804 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:03:34.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1875e-05 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520602+0000", 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4280000000000001e-06 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520609+0000", 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0799999999999999e-06 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520613+0000", 2023-10-09T10:03:34.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078419579999999996 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.528455+0000", 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0639999999999998e-05 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.528486+0000", 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033364444 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.561850+0000", 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9289999999999998e-05 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.561889+0000", 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041308386000000002 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(6.2 144/138 [PushOp(6:4011f362:::benchmark_data_smithi012_182576_object40319:head, version: 105'2578, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:4011f362:::benchmark_data_smithi012_182576_object40319:head@105'2578, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:03:34.810 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 127.72610366799999, 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048028721000000003, 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:03:34.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:03:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966474+0000", 2023-10-09T10:03:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8900000000000002e-05 2023-10-09T10:03:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966533+0000", 2023-10-09T10:03:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7698000000000001e-05 2023-10-09T10:03:34.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966561+0000", 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2663000000000001e-05 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966573+0000", 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.039901351000000002 2023-10-09T10:03:34.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.006475+0000", 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4449999999999996e-06 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.006483+0000", 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080206500000000007 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:34.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:03:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.014504+0000", 2023-10-09T10:03:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048028721000000003 2023-10-09T10:03:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:34.816 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:34.816 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:03:34.817 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:34.913 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 6 16 13452 13436 139.876 149.062 0.00424597 0.00714431 2023-10-09T10:03:35.013 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:35.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:35.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:35.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:03:35.117 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:35.159 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:35.218 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:35.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:35.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:35.365 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:35 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:35.373 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:35.427 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:35.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:35 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:35.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:35 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:35.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:35.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:35.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:03:35.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:35.590 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:35.786 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:35.913 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 7 16 15565 15549 138.759 132.062 0.0104758 0.00720128 2023-10-09T10:03:36.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:36 smithi012 ceph-mon[152263]: pgmap v469: 89 pgs: 89 active+clean; 540 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 1.2 MiB/s, 20 objects/s recovering 2023-10-09T10:03:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:36 smithi012 ceph-mon[147926]: pgmap v469: 89 pgs: 89 active+clean; 540 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 1.2 MiB/s, 20 objects/s recovering 2023-10-09T10:03:36.409 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:36 smithi110 ceph-mon[136032]: pgmap v469: 89 pgs: 89 active+clean; 540 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 1.2 MiB/s, 20 objects/s recovering 2023-10-09T10:03:36.914 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 8 16 18155 18139 141.645 161.875 0.00401244 0.00705492 2023-10-09T10:03:37.313 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:03:37.313 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:03:37.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.313 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35157.0:18954 7.c e193/187 7:395aa40f:::benchmark_data_smithi012_221426_object18953:head v 193'1185, mlcod=193'1181)", 2023-10-09T10:03:37.313 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:03:37.307627+0000", 2023-10-09T10:03:37.313 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.001664698, 2023-10-09T10:03:37.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001691667, 2023-10-09T10:03:37.313 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307627+0000", 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307627+0000", 2023-10-09T10:03:37.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307626+0000", 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1699000000000003e-05 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307658+0000", 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3011e-05 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307671+0000", 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5380000000000002e-06 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307676+0000", 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120631 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307797+0000", 2023-10-09T10:03:37.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.715e-05 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307814+0000", 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018758199999999999 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:18960 7.8 7:1369d8a7:::benchmark_data_smithi012_221426_object18959:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:03:37.307933+0000", 2023-10-09T10:03:37.317 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0013585470000000001, 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001459377, 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 18960 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:37.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307933+0000", 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307933+0000", 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307931+0000", 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9378000000000001e-05 2023-10-09T10:03:37.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307971+0000", 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4891e-05 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307985+0000", 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1800000000000001e-06 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:37.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.307992+0000", 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0130999999999999e-05 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308032+0000", 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012091000000000001 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308153+0000", 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000175553 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308328+0000", 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039543 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35157.0:18956 7.c e193/187)", 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:03:37.308715+0000", 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00057598000000000002, 2023-10-09T10:03:37.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00075009800000000004, 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308715+0000", 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308715+0000", 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:03:37.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308715+0000", 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5417999999999997e-05 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308770+0000", 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0755e-05 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308781+0000", 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.037e-06 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:37.308787+0000", 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1445999999999993e-05 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:03:37.325 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 3 2023-10-09T10:03:37.326 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:03:37.757 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:03:37.771 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:37.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:37.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:37.881 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:37.914 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 9 16 20297 20281 140.781 133.875 0.0097984 0.00709435 2023-10-09T10:03:37.918 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:37.980 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:37.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:37.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:38.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:38.146 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:03:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:03:37] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.33.4" 2023-10-09T10:03:38.150 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:38.493 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:38.916 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:38.916 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 10 16 22162 22146 138.334 116.562 0.00790611 0.00722262 2023-10-09T10:03:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:38.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:39.009 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:38 smithi110 ceph-mon[136032]: pgmap v470: 89 pgs: 89 active+clean; 827 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 928 KiB/s, 15 objects/s recovering 2023-10-09T10:03:39.009 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:38 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:03:39.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:39.078 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:38 smithi012 ceph-mon[147926]: pgmap v470: 89 pgs: 89 active+clean; 827 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 928 KiB/s, 15 objects/s recovering 2023-10-09T10:03:39.078 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:38 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:03:39.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:38 smithi012 ceph-mon[152263]: pgmap v470: 89 pgs: 89 active+clean; 827 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 928 KiB/s, 15 objects/s recovering 2023-10-09T10:03:39.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:38 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:03:39.086 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:39.332 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:39.333 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:39.338 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:39.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:39.439 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:39.494 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:39.606 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:39.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:39.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:39.716 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:39.762 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:39.916 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 11 16 23991 23975 136.144 114.312 0.0115837 0.00733847 2023-10-09T10:03:39.980 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:39.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:39.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:40.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:40.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:40.201 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:40.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:40.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:40.234 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:03:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:03:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:03:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:03:40.234 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:03:40.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:40.373 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:40.654 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:03:40.770 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:40.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:40.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:40.842 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:03:40.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:40.904 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:40.916 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 12 16 25672 25656 133.554 105.062 0.00210457 0.0074406 2023-10-09T10:03:40.937 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:03:40.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:40 smithi110 ceph-mon[136032]: pgmap v471: 89 pgs: 89 active+clean; 1023 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 112 MiB/s wr, 1.80k op/s; 824 KiB/s, 14 objects/s recovering 2023-10-09T10:03:40.979 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:03:41.020 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:41.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:41.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:40 smithi012 ceph-mon[152263]: pgmap v471: 89 pgs: 89 active+clean; 1023 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 112 MiB/s wr, 1.80k op/s; 824 KiB/s, 14 objects/s recovering 2023-10-09T10:03:41.028 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:03:41.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:41.029 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:40 smithi012 ceph-mon[147926]: pgmap v471: 89 pgs: 89 active+clean; 1023 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 112 MiB/s wr, 1.80k op/s; 824 KiB/s, 14 objects/s recovering 2023-10-09T10:03:41.061 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:03:41.099 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:03:41.130 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:41.135 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:03:41.174 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:03:41.183 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:03:41.183 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:03:41.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:03:41.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:03:41.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:03:41.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:03:41.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:03:41.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:03:41.195 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:41.508 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:03:41.605 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:41.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:41.714 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:41.750 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:41.916 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 13 16 27455 27439 131.852 111.438 0.00697838 0.00757954 2023-10-09T10:03:41.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1199301769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:41.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1199301769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1199301769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:42.875 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:42.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:42.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:42.917 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 14 16 29276 29260 130.564 113.812 0.0079237 0.00765311 2023-10-09T10:03:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:42 smithi012 ceph-mon[147926]: pgmap v472: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s; 742 KiB/s, 12 objects/s recovering 2023-10-09T10:03:42.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:42 smithi012 ceph-mon[152263]: pgmap v472: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s; 742 KiB/s, 12 objects/s recovering 2023-10-09T10:03:42.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:43.079 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:43.150 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:43.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:43.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:43.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:42 smithi110 ceph-mon[136032]: pgmap v472: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s; 742 KiB/s, 12 objects/s recovering 2023-10-09T10:03:43.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:03:43.284 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:03:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:03:43.364 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:43.779 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:03:43.919 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 15 16 30768 30752 128.057 93.25 0.0195242 0.00780098 2023-10-09T10:03:44.054 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:44.057 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1285180956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:44.057 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:44.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:44.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:44.059 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1285180956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:44.059 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:44.086 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1285180956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:44.086 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:03:44.160 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:44.211 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:44.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:03:44.261 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:44.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:44.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:44.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:44.439 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:44.517 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:44.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:44.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:44.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:44.661 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:44.707 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:44.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:44.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:44.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[147926]: pgmap v473: 89 pgs: 89 active+clean; 1.6 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s; 512 KiB/s, 8 objects/s recovering 2023-10-09T10:03:44.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[152263]: pgmap v473: 89 pgs: 89 active+clean; 1.6 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s; 512 KiB/s, 8 objects/s recovering 2023-10-09T10:03:44.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:44.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[152263]: failed to encode map e194 with expected crc 2023-10-09T10:03:44.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:03:44.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:44.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:44.919 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 16 16 32420 32404 126.506 103.25 0.00443336 0.00789874 2023-10-09T10:03:44.994 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:45.023 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:45.025 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:44 smithi110 ceph-mon[136032]: pgmap v473: 89 pgs: 89 active+clean; 1.6 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s; 512 KiB/s, 8 objects/s recovering 2023-10-09T10:03:45.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:44 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:45.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:44 smithi110 ceph-mon[136032]: failed to encode map e194 with expected crc 2023-10-09T10:03:45.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:44 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:03:45.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:44 smithi110 ceph-mon[136032]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T10:03:45.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:44 smithi110 ceph-mon[136032]: failed to encode map e194 with expected crc 2023-10-09T10:03:45.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:45.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:45.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:45.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[147926]: failed to encode map e194 with expected crc 2023-10-09T10:03:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:03:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[147926]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T10:03:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[147926]: failed to encode map e194 with expected crc 2023-10-09T10:03:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:45.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[152263]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T10:03:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[152263]: failed to encode map e194 with expected crc 2023-10-09T10:03:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:03:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:45.179 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:45.325 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:45.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:45.383 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:03:45.383 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:03:45.428 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:45.493 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:45.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:03:45.919 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 17 16 34056 34040 125.08 102.25 0.00745523 0.00798986 2023-10-09T10:03:45.935 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:45.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:46.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:46.171 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:46.199 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:03:46.919 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 18 16 35601 35585 123.496 96.5625 0.00923107 0.00809235 2023-10-09T10:03:47.069 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:47.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:47.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:47.077 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:46 smithi012 ceph-mon[147926]: pgmap v475: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T10:03:47.077 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:46 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:47.077 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:46 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:47.078 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:46 smithi012 ceph-mon[152263]: pgmap v475: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T10:03:47.078 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:46 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:47.078 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:46 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:47.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:47.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:46 smithi110 ceph-mon[136032]: pgmap v475: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T10:03:47.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:46 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:47.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:46 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:47.239 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:47.662 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:47.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:47.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:47.772 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:47.825 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:47.871 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:47.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:47.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:47.919 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 19 16 37240 37224 122.387 102.438 0.0108429 0.00816399 2023-10-09T10:03:47.920 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:03:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:03:47] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.33.4" 2023-10-09T10:03:47.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:48.050 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:48.085 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:03:48.128 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:48.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:48.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:48.135 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:03:48.173 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:03:48.212 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:48.224 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:03:48.234 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:48.259 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:03:48.269 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:48.280 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:48.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:03:48.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:48.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:48.351 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:03:48.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:03:48.405 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:03:48.405 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:03:48.405 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:03:48.406 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:03:48.406 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:03:48.406 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:03:48.406 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:03:48.406 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:03:48.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:48.475 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:48.680 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:48.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:48.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:48.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:48.869 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:48.919 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:03:48.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T10:03:48.920 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:03:48.920128+0000 min lat: 0.00114602 max lat: 1.02658 avg lat: 0.00831232 2023-10-09T10:03:48.920 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:03:48.920 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 20 16 38512 38496 120.243 79.5 0.0120104 0.00831232 2023-10-09T10:03:48.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:48 smithi012 ceph-mon[147926]: pgmap v476: 89 pgs: 89 active+clean; 2.0 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-10-09T10:03:48.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4029438881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:48.979 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:48 smithi012 ceph-mon[152263]: pgmap v476: 89 pgs: 89 active+clean; 2.0 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-10-09T10:03:48.979 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4029438881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:48 smithi110 ceph-mon[136032]: pgmap v476: 89 pgs: 89 active+clean; 2.0 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-10-09T10:03:49.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4029438881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:49.210 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:49.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:49.316 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:49.354 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:49.403 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:49.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:49.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:49.508 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:49.585 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:49.920 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 21 16 39872 39856 118.565 85 0.0158477 0.00842929 2023-10-09T10:03:50.037 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:50.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:50.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:50.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:50.177 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:50.217 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:50.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:50.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:50.323 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:50.354 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:50.392 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:50.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:50.426 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:03:50.426 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:03:50.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.426 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:40532 7.5 7:aa280335:::benchmark_data_smithi012_221426_object40531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T10:03:50.426 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.418993+0000", 2023-10-09T10:03:50.426 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0056139129999999999, 2023-10-09T10:03:50.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005650062, 2023-10-09T10:03:50.426 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40532 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.418993+0000", 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.418993+0000", 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.418992+0000", 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419023+0000", 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1769e-05 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419039+0000", 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5681000000000001e-05 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419045+0000", 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1240000000000002e-06 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419684+0000", 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00063897200000000002 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419815+0000", 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130419 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419935+0000", 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012089100000000001 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:40533 7.5 7:ab895d81:::benchmark_data_smithi012_221426_object40532:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.419054+0000", 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.005552235, 2023-10-09T10:03:50.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056755729999999997, 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40533 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419054+0000", 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419054+0000", 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419054+0000", 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419085+0000", 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1553000000000001e-05 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419092+0000", 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.139e-06 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419096+0000", 2023-10-09T10:03:50.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.126e-06 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.421545+0000", 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024494849999999999 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.421682+0000", 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013712899999999999 2023-10-09T10:03:50.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.421825+0000", 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014227 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:40535 7.9 7:97879944:::benchmark_data_smithi012_221426_object40534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.419101+0000", 2023-10-09T10:03:50.436 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0055059230000000002, 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0057106860000000004, 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40535 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419101+0000", 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419101+0000", 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419100+0000", 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:50.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419122+0000", 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2002e-05 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419130+0000", 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7770000000000002e-06 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419137+0000", 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5339999999999996e-06 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419216+0000", 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9288000000000004e-05 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419364+0000", 2023-10-09T10:03:50.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000147937 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419533+0000", 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016936100000000001 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.421529+0000", 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019962270000000002 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.424436+0000", 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029066719999999999 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:40537 7.4 7:2d73c18e:::benchmark_data_smithi012_221426_object40536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.419298+0000", 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0053086619999999996, 2023-10-09T10:03:50.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056146629999999998, 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40537 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419298+0000", 2023-10-09T10:03:50.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419298+0000", 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419297+0000", 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:03:50.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419358+0000", 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1061000000000004e-05 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419368+0000", 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9490000000000008e-06 2023-10-09T10:03:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419384+0000", 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5911000000000001e-05 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419449+0000", 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5314999999999994e-05 2023-10-09T10:03:50.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:50.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419595+0000", 2023-10-09T10:03:50.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014534300000000001 2023-10-09T10:03:50.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:03:50.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419750+0000", 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015559499999999999 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.424569+0000", 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048182449999999996 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:40538 7.5 7:abc1d50b:::benchmark_data_smithi012_221426_object40537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.419403+0000", 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0052039460000000001, 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056039219999999999, 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:50.449 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40538 2023-10-09T10:03:50.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.450 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419403+0000", 2023-10-09T10:03:50.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419403+0000", 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419402+0000", 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419434+0000", 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2008000000000003e-05 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419442+0000", 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2940000000000006e-06 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419449+0000", 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5339999999999996e-06 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423309+0000", 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003860455 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:50.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423442+0000", 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013211599999999999 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423577+0000", 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013525700000000001 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:40539 7.7 7:effd4cc7:::benchmark_data_smithi012_221426_object40538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.419455+0000", 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0051514860000000003, 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056384099999999999, 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:03:50.455 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40539 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419455+0000", 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419455+0000", 2023-10-09T10:03:50.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419455+0000", 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419486+0000", 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0966000000000001e-05 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419494+0000", 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8140000000000007e-06 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419501+0000", 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3770000000000001e-06 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:50.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419671+0000", 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017041499999999999 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419789+0000", 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011778900000000001 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419925+0000", 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013623199999999999 2023-10-09T10:03:50.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.424231+0000", 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043052100000000003 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:40540 7.5 7.af0c5c15 (undecoded) ondisk+write+known_if_redirected e194)", 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.419517+0000", 2023-10-09T10:03:50.460 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0050894759999999999, 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056639730000000001, 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40540 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419517+0000", 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419517+0000", 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419516+0000", 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:03:50.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419539+0000", 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3331000000000001e-05 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419547+0000", 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4429999999999997e-06 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.419554+0000", 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9949999999999999e-06 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:40534 7.d e194/187 7:bf0382ea:::benchmark_data_smithi012_221426_object40533:head v 194'2508, mlcod=194'2506)", 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.420895+0000", 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0037112439999999998, 2023-10-09T10:03:50.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043419419999999997, 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.420895+0000", 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.420895+0000", 2023-10-09T10:03:50.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.420895+0000", 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.420933+0000", 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8720999999999997e-05 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.420945+0000", 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1698e-05 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.420952+0000", 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2840000000000001e-06 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:50.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.421420+0000", 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000467381 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.421444+0000", 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4165e-05 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.35157.0:40532 7.5 e194/187)", 2023-10-09T10:03:50.468 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.423947+0000", 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00065940599999999997, 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001345974, 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423947+0000", 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423947+0000", 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423947+0000", 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423950+0000", 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9779999999999999e-06 2023-10-09T10:03:50.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423962+0000", 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1476e-05 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.423970+0000", 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9340000000000006e-06 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:40531 7.6 7:6e4cc522:::benchmark_data_smithi012_221426_object40530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:03:50.415477+0000", 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.009129379, 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0098887060000000006, 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 40531 2023-10-09T10:03:50.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.415477+0000", 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.415477+0000", 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.415476+0000", 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.415502+0000", 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5950000000000001e-05 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:03:50.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.415514+0000", 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2152e-05 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.415522+0000", 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0190000000000006e-06 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.416676+0000", 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011542239999999999 2023-10-09T10:03:50.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.416820+0000", 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014304899999999999 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.416981+0000", 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016126700000000001 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.418840+0000", 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018588770000000001 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:03:50.421575+0000", 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002735131 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:03:50.477 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:03:50.478 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 10 2023-10-09T10:03:50.478 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:03:50.482 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:50.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:50.528 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:50.566 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:50.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:50.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:50.670 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:03:50.704 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:50.743 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:03:50.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:50.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:03:50.810 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:50 smithi110 ceph-mon[136032]: pgmap v477: 89 pgs: 89 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T10:03:50.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:50 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:50.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:50 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:50.813 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:50 smithi012 ceph-mon[147926]: pgmap v477: 89 pgs: 89 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T10:03:50.817 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:50 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:50.817 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:50 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:50.847 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:50.883 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:50.925 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 22 16 41186 41170 116.882 82.125 0.00696451 0.00854969 2023-10-09T10:03:50.925 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:50.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:51.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:51.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:03:51.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:50 smithi012 ceph-mon[152263]: pgmap v477: 89 pgs: 89 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T10:03:51.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:50 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:51.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:50 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:51.185 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:51.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:03:51.268 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:51.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:51.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:51.374 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:51.404 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:51.446 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:51.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:51.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:51.637 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:51.801 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:51.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:51.843 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:03:51.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4048496456' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:51.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:51 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:03:51.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:51 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:03:51.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:51 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:51.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4048496456' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:51.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:51 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:03:51.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:51 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:03:51.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:51 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:51.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:51.924 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 23 16 42289 42273 114.799 68.9375 0.0184071 0.00870316 2023-10-09T10:03:51.996 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:52.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4048496456' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:03:52.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:51 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:03:52.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:51 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:03:52.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:51 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:03:52.304 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:52.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:52.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:52.410 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:52.446 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:52.498 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:52.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:52.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:52.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:52.674 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:52.706 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:52.777 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:52.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:52.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:52.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:52.925 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 24 16 43653 43637 113.568 85.25 0.0124166 0.00879969 2023-10-09T10:03:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[147926]: pgmap v478: 89 pgs: 89 active+clean; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T10:03:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[147926]: failed to encode map e195 with expected crc 2023-10-09T10:03:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4048496456' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[147926]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T10:03:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[147926]: failed to encode map e195 with expected crc 2023-10-09T10:03:52.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[152263]: pgmap v478: 89 pgs: 89 active+clean; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T10:03:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[152263]: failed to encode map e195 with expected crc 2023-10-09T10:03:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4048496456' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[152263]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T10:03:52.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:52 smithi012 ceph-mon[152263]: failed to encode map e195 with expected crc 2023-10-09T10:03:52.943 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:52.993 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:53.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:52 smithi110 ceph-mon[136032]: pgmap v478: 89 pgs: 89 active+clean; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T10:03:53.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:52 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:03:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:52 smithi110 ceph-mon[136032]: failed to encode map e195 with expected crc 2023-10-09T10:03:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4048496456' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:03:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:52 smithi110 ceph-mon[136032]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T10:03:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:52 smithi110 ceph-mon[136032]: failed to encode map e195 with expected crc 2023-10-09T10:03:53.581 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:53.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:53.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:53.763 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:53.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3868167861' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:03:53.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3868167861' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:03:53.865 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T10:03:53.925 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 25 16 45109 45093 112.665 91 0.00777095 0.00887089 2023-10-09T10:03:54.022 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:54.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:54.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:54.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3868167861' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:03:54.198 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:54.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:03:54.676 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:54.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:54.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:54.782 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:54.816 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:54.862 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:54.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:54.925 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 26 16 46458 46442 111.575 84.3125 0.00330874 0.00895859 2023-10-09T10:03:54.967 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:55.008 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:55.053 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:55.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:55.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:55.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[147926]: pgmap v480: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T10:03:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3868167861' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:03:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[147926]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T10:03:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[147926]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[147926]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[147926]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[147926]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[147926]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[152263]: pgmap v480: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T10:03:55.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3868167861' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:03:55.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[152263]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T10:03:55.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[152263]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[152263]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[152263]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[152263]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:54 smithi012 ceph-mon[152263]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:54 smithi110 ceph-mon[136032]: pgmap v480: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T10:03:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:54 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3868167861' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:03:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:54 smithi110 ceph-mon[136032]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T10:03:55.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:54 smithi110 ceph-mon[136032]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:54 smithi110 ceph-mon[136032]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:54 smithi110 ceph-mon[136032]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:54 smithi110 ceph-mon[136032]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:54 smithi110 ceph-mon[136032]: failed to encode map e196 with expected crc 2023-10-09T10:03:55.213 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:55.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T10:03:55.925 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 27 16 47876 47860 110.725 88.625 0.00247334 0.0090136 2023-10-09T10:03:56.387 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:56.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:56.502 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:56.543 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:56.604 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:56.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:56.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:56.709 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:56.741 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:56.781 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:56.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:56.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:03:56.856 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:56 smithi110 ceph-mon[136032]: pgmap v482: 89 pgs: 89 active+clean; 2.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:03:56.886 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:03:56.925 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 28 16 49306 49290 109.963 89.375 0.0076196 0.00908896 2023-10-09T10:03:56.926 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:56.975 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:03:56.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:56.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:03:57.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:03:57.138 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:56 smithi012 ceph-mon[152263]: pgmap v482: 89 pgs: 89 active+clean; 2.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:03:57.142 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:56 smithi012 ceph-mon[147926]: pgmap v482: 89 pgs: 89 active+clean; 2.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:03:57.146 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:57.763 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:03:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:57.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:03:57.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:03:57.924 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:57.928 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 29 16 50706 50690 109.176 87.5 0.0107362 0.00915361 2023-10-09T10:03:58.093 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:03:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:03:57] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.33.4" 2023-10-09T10:03:58.307 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:03:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:58.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:03:58.320 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:03:58.321 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:03:58.321 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:03:58.321 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:03:58.321 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:03:58.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:58.447 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:03:58.486 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:58.490 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:03:58.503 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:58.529 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:03:58.571 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:03:58.613 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:03:58.647 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:03:58.682 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:03:58.724 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:03:58.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:03:58.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:03:58.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:03:58.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:03:58.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:03:58.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:03:58.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:03:58.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:03:58.755 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:58.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:58.862 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:03:58.891 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:03:58.929 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 30 16 51877 51861 107.977 73.1875 0.0300567 0.00925358 2023-10-09T10:03:58.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:03:58.952 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:59.023 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:58 smithi012 ceph-mon[147926]: pgmap v483: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T10:03:59.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/111400119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:59.024 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:58 smithi012 ceph-mon[152263]: pgmap v483: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T10:03:59.024 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/111400119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:59.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:58 smithi110 ceph-mon[136032]: pgmap v483: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T10:03:59.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/111400119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:03:59.311 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:03:59.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:59.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:03:59.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:03:59.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:59.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:03:59.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:59.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:03:59.605 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:03:59.688 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:59.799 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:03:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:59.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:03:59.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:03:59.929 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 31 16 53079 53063 106.917 75.125 0.0090114 0.00934776 2023-10-09T10:03:59.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:03:59.978 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:03:59.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:03:59.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:00.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:00.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:03:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:00.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:00.158 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-10-09T10:04:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:03:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:00.626 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:00.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:00.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:00.750 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:00.803 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:00.914 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:00.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:00.930 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 32 16 54239 54223 105.842 72.5 0.0132852 0.00944199 2023-10-09T10:04:01.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:01.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[147926]: pgmap v484: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T10:04:01.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[147926]: failed to encode map e197 with expected crc 2023-10-09T10:04:01.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[147926]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[147926]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[152263]: pgmap v484: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[152263]: failed to encode map e197 with expected crc 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[152263]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T10:04:01.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:00 smithi012 ceph-mon[152263]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T10:04:01.153 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:00 smithi110 ceph-mon[136032]: pgmap v484: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T10:04:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:00 smithi110 ceph-mon[136032]: failed to encode map e197 with expected crc 2023-10-09T10:04:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:04:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2023-10-09T10:04:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:00 smithi110 ceph-mon[136032]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T10:04:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:00 smithi110 ceph-mon[136032]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T10:04:01.222 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:01.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:01.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:01.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:04:01.362 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:01.404 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:01.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:01.509 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:01.548 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:01.588 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:01.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:01.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:01.778 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:01.845 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:01.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:01.934 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T10:04:01.936 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 33 16 55306 55290 104.644 66.6875 0.0160216 0.00954948 2023-10-09T10:04:01.948 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:04:01.992 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:02.034 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:02.073 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:02.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:02.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:02.083 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.083 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.083 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.083 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.083 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.083 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: pgmap v487: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T10:04:02.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/592127129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:02.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:02.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/542194217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:04:02.084 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:04:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: pgmap v487: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T10:04:02.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/592127129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:02.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:02.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/542194217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:04:02.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:04:02.177 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: failed to encode map e198 with expected crc 2023-10-09T10:04:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: pgmap v487: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T10:04:02.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/592127129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:02.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:02.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/542194217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:04:02.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:04:02.215 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:02.262 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:02.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:02.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:02.368 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:02.404 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:02.422 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:04:02.423 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:04:02.451 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:02.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:02.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:02.632 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:02.919 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:02.920 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[147926]: failed to encode map e199 with expected crc 2023-10-09T10:04:02.933 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 34 16 56464 56448 103.695 72.375 0.0109477 0.00963813 2023-10-09T10:04:03.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:04:03.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:04:03.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[147926]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T10:04:03.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[147926]: failed to encode map e199 with expected crc 2023-10-09T10:04:03.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:03.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[152263]: failed to encode map e199 with expected crc 2023-10-09T10:04:03.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:04:03.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:04:03.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[152263]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T10:04:03.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:02 smithi012 ceph-mon[152263]: failed to encode map e199 with expected crc 2023-10-09T10:04:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:02 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:02 smithi110 ceph-mon[136032]: failed to encode map e199 with expected crc 2023-10-09T10:04:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:04:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:04:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:02 smithi110 ceph-mon[136032]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T10:04:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:02 smithi110 ceph-mon[136032]: failed to encode map e199 with expected crc 2023-10-09T10:04:03.933 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 35 16 57825 57809 103.163 85.0625 0.0101592 0.00968707 2023-10-09T10:04:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:03 smithi012 ceph-mon[147926]: pgmap v489: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T10:04:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:03 smithi012 ceph-mon[152263]: pgmap v489: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T10:04:04.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:03 smithi110 ceph-mon[136032]: pgmap v489: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T10:04:04.933 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 36 16 59095 59079 102.502 79.375 0.0170685 0.00975074 2023-10-09T10:04:05.701 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:05.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:05.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:05.803 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:05.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T10:04:05.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:05.904 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:05.933 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 37 16 60350 60334 101.852 78.4375 0.0285359 0.00981248 2023-10-09T10:04:06.154 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:06 smithi012 ceph-mon[147926]: pgmap v490: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T10:04:06.154 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:06 smithi012 ceph-mon[152263]: pgmap v490: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T10:04:06.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:06 smithi110 ceph-mon[136032]: pgmap v490: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T10:04:06.570 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:06.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:06.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:06.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:06.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:04:06.800 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:06.882 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:06.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:06.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:06.933 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 38 16 61550 61534 101.146 75 0.00573938 0.00988183 2023-10-09T10:04:06.991 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:07.024 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:07.064 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:07.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:07.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:07.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:07.226 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:07.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:04:07.416 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:07.465 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:07.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:07.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:07.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:07.665 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:07.802 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:07.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:07.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:07.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:07.939 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 39 15 62528 62513 100.12 61.1875 0.00952165 0.00998203 2023-10-09T10:04:07.997 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:08.030 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:08.144 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:08.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:08.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:08.151 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:04:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:04:07] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.33.4" 2023-10-09T10:04:08.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:04:08.200 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:04:08.252 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:08.253 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:04:08.309 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:04:08.317 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:08.341 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:04:08.379 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:04:08.415 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:04:08.459 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:04:08.465 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:04:08.465 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:04:08.465 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:04:08.465 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:04:08.465 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:04:08.465 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:04:08.465 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:04:08.465 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:04:08.801 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:08.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:08.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:08.847 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:08.910 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:08.934 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:04:08.934455+0000 min lat: 0.00114602 max lat: 1.02658 avg lat: 0.010074 2023-10-09T10:04:08.934 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:04:08.934 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 40 16 63546 63530 99.2067 63.5625 0.0097449 0.010074 2023-10-09T10:04:08.949 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:08.995 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:08.997 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:08 smithi110 ceph-mon[136032]: pgmap v491: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T10:04:08.997 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1296867701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:08.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:09.099 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:09.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:09.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:08 smithi012 ceph-mon[147926]: pgmap v491: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T10:04:09.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1296867701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:09.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:08 smithi012 ceph-mon[152263]: pgmap v491: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T10:04:09.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1296867701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:09.184 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:09.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:09.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:09.350 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:09.638 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:09.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:09.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:09.753 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:09.824 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:09.934 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 41 16 64613 64597 98.4137 66.6875 0.00631634 0.0101559 2023-10-09T10:04:10.071 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:10.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:10.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:10.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:10.249 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:10.325 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:10.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:10.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:10.503 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:10.529 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:04:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:04:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:65165 7.a e199/191 7:5ab2004d:::benchmark_data_smithi012_221426_object65164:head v 199'4084, mlcod=199'4081)", 2023-10-09T10:04:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:10.515549+0000", 2023-10-09T10:04:10.529 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.011463285, 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011496671999999999, 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.515549+0000", 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.515549+0000", 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.515549+0000", 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.515573+0000", 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3722000000000001e-05 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.515581+0000", 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2090000000000005e-06 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:10.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.515587+0000", 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2090000000000002e-06 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.518011+0000", 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024245629999999998 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.518033+0000", 2023-10-09T10:04:10.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1954e-05 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:65170 7.5 e199/187 7:abbd6e5f:::benchmark_data_smithi012_221426_object65169:head v 199'4035, mlcod=199'4032)", 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:10.524299+0000", 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0027138689999999998, 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028172919999999999, 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:10.534 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.524299+0000", 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.524299+0000", 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:10.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.524298+0000", 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.524336+0000", 2023-10-09T10:04:10.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8305e-05 2023-10-09T10:04:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.524353+0000", 2023-10-09T10:04:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6858999999999999e-05 2023-10-09T10:04:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.524361+0000", 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.602e-06 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.524407+0000", 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5915999999999999e-05 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.524430+0000", 2023-10-09T10:04:10.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2599000000000002e-05 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:65175 7.5 e199/187 7:a571ec9c:::benchmark_data_smithi012_221426_object65174:head v 199'4036, mlcod=199'4032)", 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:10.525319+0000", 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0016933129999999999, 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001866489, 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:10.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525319+0000", 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525319+0000", 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525318+0000", 2023-10-09T10:04:10.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:04:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525349+0000", 2023-10-09T10:04:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0988000000000002e-05 2023-10-09T10:04:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525365+0000", 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5784999999999999e-05 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525373+0000", 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2699999999999999e-06 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:10.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.526705+0000", 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00133204 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.526722+0000", 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7856999999999999e-05 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:65169 7.a e199/191 7:513fe359:::benchmark_data_smithi012_221426_object65168:head v 199'4085, mlcod=199'4082)", 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:10.525536+0000", 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.001476871, 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001709253, 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:10.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525536+0000", 2023-10-09T10:04:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525536+0000", 2023-10-09T10:04:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:10.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525535+0000", 2023-10-09T10:04:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:04:10.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525554+0000", 2023-10-09T10:04:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9259999999999999e-05 2023-10-09T10:04:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525565+0000", 2023-10-09T10:04:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0769e-05 2023-10-09T10:04:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525570+0000", 2023-10-09T10:04:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9969999999999998e-06 2023-10-09T10:04:10.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525662+0000", 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2232999999999996e-05 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:10.525684+0000", 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1985999999999999e-05 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.549 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:04:10.550 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 4 2023-10-09T10:04:10.550 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:04:10.670 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:10.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:10.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:10.801 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T10:04:10.851 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:10.917 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:10.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:10.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:10.923 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:10 smithi012 ceph-mon[147926]: pgmap v492: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T10:04:10.923 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2658546119' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:04:10.923 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:10 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:04:10.924 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:10 smithi012 ceph-mon[152263]: pgmap v492: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T10:04:10.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2658546119' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:04:10.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:10 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:04:10.934 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 42 16 65599 65583 97.5381 61.625 0.0179464 0.010247 2023-10-09T10:04:11.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:11.106 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:11.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:10 smithi110 ceph-mon[136032]: pgmap v492: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T10:04:11.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2658546119' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:04:11.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:10 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:04:11.556 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:04:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:04:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[152263]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T10:04:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[152263]: failed to encode map e200 with expected crc 2023-10-09T10:04:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[152263]: failed to encode map e200 with expected crc 2023-10-09T10:04:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[152263]: failed to encode map e200 with expected crc 2023-10-09T10:04:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[152263]: failed to encode map e200 with expected crc 2023-10-09T10:04:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:04:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[147926]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T10:04:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[147926]: failed to encode map e200 with expected crc 2023-10-09T10:04:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[147926]: failed to encode map e200 with expected crc 2023-10-09T10:04:11.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[147926]: failed to encode map e200 with expected crc 2023-10-09T10:04:11.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:11 smithi012 ceph-mon[147926]: failed to encode map e200 with expected crc 2023-10-09T10:04:11.936 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 43 16 66761 66745 96.9538 72.625 0.00714048 0.0103092 2023-10-09T10:04:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:11 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:04:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:11 smithi110 ceph-mon[136032]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T10:04:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:11 smithi110 ceph-mon[136032]: failed to encode map e200 with expected crc 2023-10-09T10:04:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:11 smithi110 ceph-mon[136032]: failed to encode map e200 with expected crc 2023-10-09T10:04:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:11 smithi110 ceph-mon[136032]: failed to encode map e200 with expected crc 2023-10-09T10:04:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:11 smithi110 ceph-mon[136032]: failed to encode map e200 with expected crc 2023-10-09T10:04:12.936 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 44 16 67945 67929 96.4322 74 0.00322827 0.0103527 2023-10-09T10:04:13.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:12 smithi012 ceph-mon[147926]: pgmap v494: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T10:04:13.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:12 smithi012 ceph-mon[152263]: pgmap v494: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T10:04:13.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T10:04:13.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:12 smithi110 ceph-mon[136032]: pgmap v494: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T10:04:13.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:04:13.802 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1489169622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:13.831 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:04:13.901 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:13.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:13.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:13.936 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 45 16 68870 68854 95.5743 57.8125 0.0125198 0.0104553 2023-10-09T10:04:14.000 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:14.013 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:14.085 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1489169622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:14.431 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1489169622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:14.431 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:14.432 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:14.432 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:14.937 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 46 16 69819 69803 94.7861 59.3125 0.0102333 0.0105428 2023-10-09T10:04:15.102 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:15.110 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: pgmap v495: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:04:15.110 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:04:15.110 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:04:15.110 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1489169622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:15.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T10:04:15.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:15.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:15.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:15.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:15.113 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: pgmap v495: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:04:15.113 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:04:15.113 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:04:15.113 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.114 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1489169622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:15.114 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T10:04:15.114 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.114 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.114 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.114 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:15.114 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:15.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: pgmap v495: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1489169622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: failed to encode map e201 with expected crc 2023-10-09T10:04:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:15.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:15.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:15.292 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:15.916 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:15.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:15.940 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 47 16 70951 70935 94.2679 70.75 0.026125 0.0106026 2023-10-09T10:04:16.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:16.077 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:16.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:16 smithi012 ceph-mon[152263]: pgmap v497: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T10:04:16.919 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:16.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:16.940 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 48 16 72054 72038 93.7404 68.9375 0.00546892 0.0106623 2023-10-09T10:04:17.028 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:17.061 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:17.067 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:16 smithi012 ceph-mon[147926]: pgmap v497: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T10:04:17.104 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:17.107 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:16 smithi110 ceph-mon[136032]: pgmap v497: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T10:04:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:17.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:17.210 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:17.294 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:17.467 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:17.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:17.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:17.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:17.659 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:17.736 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:17.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:17.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:17.845 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:17.882 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:17.899 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:04:17.917 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:04:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:04:17] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T10:04:17.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:17.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:17.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:17.943 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 49 16 73017 73001 93.051 60.1875 0.0217194 0.0107408 2023-10-09T10:04:17.945 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:04:17.983 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:04:18.029 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:04:18.034 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:18.071 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:18.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:04:18.121 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:04:18.122 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:18.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:18.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:18.158 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:04:18.201 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:04:18.223 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:04:18.223 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:04:18.224 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:04:18.224 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:04:18.224 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:04:18.224 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:04:18.224 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:04:18.224 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:04:18.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:18.310 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:18.667 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:18.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:18.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:18.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:04:18.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:18.800 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:18.839 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:18.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:18.886 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:18 smithi012 ceph-mon[147926]: pgmap v498: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T10:04:18.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/442330696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:18.887 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:18 smithi012 ceph-mon[152263]: pgmap v498: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T10:04:18.887 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/442330696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:18.943 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 50 16 73967 73951 92.3778 59.375 0.01148 0.0108213 2023-10-09T10:04:18.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:18.970 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:04:18.970 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:04:19.020 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:19.083 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:19.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:19.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:19.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:19.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:18 smithi110 ceph-mon[136032]: pgmap v498: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T10:04:19.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/442330696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:19.229 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:19.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:19.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:19.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:19.378 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:19.415 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:19.461 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:19.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:19.566 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:19.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:04:19.595 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:19.636 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:19.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:19.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:19.730 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:19.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:19.859 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:19.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2177444121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:04:19.885 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:19 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:04:19.916 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T10:04:19.943 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 51 16 74830 74814 91.6244 53.9375 0.0150089 0.0109078 2023-10-09T10:04:20.064 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:20.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:20.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:20.091 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2177444121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:04:20.092 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:19 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:04:20.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2177444121' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:04:20.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:19 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:04:20.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:20.253 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:20.385 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:20.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:20.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:20.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T10:04:20.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:20.607 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:20.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:04:20.935 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:20.943 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 52 16 75781 75765 91.0057 59.4375 0.0328237 0.0109829 2023-10-09T10:04:20.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:20.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:21.003 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:21.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:21.139 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[147926]: pgmap v499: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:04:21.139 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:04:21.139 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[147926]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T10:04:21.140 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[147926]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.140 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[147926]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.140 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[147926]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.140 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[147926]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.140 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[147926]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.140 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[152263]: pgmap v499: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:04:21.140 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:04:21.140 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[152263]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T10:04:21.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[152263]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[152263]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[152263]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[152263]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.141 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:20 smithi012 ceph-mon[152263]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.149 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:20 smithi110 ceph-mon[136032]: pgmap v499: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:04:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:20 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:04:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:20 smithi110 ceph-mon[136032]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T10:04:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:20 smithi110 ceph-mon[136032]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:20 smithi110 ceph-mon[136032]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:20 smithi110 ceph-mon[136032]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:20 smithi110 ceph-mon[136032]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:20 smithi110 ceph-mon[136032]: failed to encode map e202 with expected crc 2023-10-09T10:04:21.443 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:21.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:21.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:21.592 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:21.663 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:21.890 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:21.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:21.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:21.947 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 53 16 76665 76649 90.3255 55.25 0.00459055 0.0110656 2023-10-09T10:04:22.003 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:22.033 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:22.075 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:22.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:22.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:21 smithi012 ceph-mon[152263]: pgmap v501: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-09T10:04:22.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:22.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:21 smithi012 ceph-mon[147926]: pgmap v501: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-09T10:04:22.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:21 smithi110 ceph-mon[136032]: pgmap v501: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-09T10:04:22.264 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:22.336 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:22.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:22.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:22.467 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:22.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:22.536 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:22.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:22.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:22.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:22.707 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:22.781 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:22.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:22.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:22.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:22.934 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:22.946 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:04:22.951 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1664505237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:04:22.951 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:04:22.953 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 54 16 77653 77637 89.785 61.75 0.0160626 0.0111328 2023-10-09T10:04:22.974 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:22.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:23.077 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:23.116 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:23.166 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:23.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:23.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1664505237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:04:23.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:04:23.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1664505237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:04:23.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:04:23.273 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:23.316 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:23.362 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:23.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:23.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:23.468 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:23.536 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:23.955 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 55 16 78635 78619 89.2675 61.375 0.022224 0.011197 2023-10-09T10:04:24.232 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:24.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:24.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:24.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:24.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[152263]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:04:24.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[152263]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T10:04:24.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[152263]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[152263]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[152263]: pgmap v503: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:04:24.256 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:24.256 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[147926]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.256 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:04:24.256 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[147926]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T10:04:24.256 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[147926]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.256 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[147926]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.256 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:23 smithi012 ceph-mon[147926]: pgmap v503: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:04:24.262 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:04:24.262 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:04:24.352 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:24.380 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:23 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:24.380 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:23 smithi110 ceph-mon[136032]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.380 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:04:24.381 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:23 smithi110 ceph-mon[136032]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T10:04:24.381 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:23 smithi110 ceph-mon[136032]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.381 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:23 smithi110 ceph-mon[136032]: failed to encode map e203 with expected crc 2023-10-09T10:04:24.381 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:23 smithi110 ceph-mon[136032]: pgmap v503: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:04:24.383 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:24.427 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:24.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:24.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:24.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:24.581 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:24.633 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:24.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:24.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:24.737 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:24.778 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:24.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:24.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:24.935 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:24.954 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 56 16 79633 79617 88.7875 62.375 0.00899364 0.0112574 2023-10-09T10:04:24.972 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:25.016 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:25.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:25.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:04:25.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:25.198 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:25.407 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:25.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:25.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:25.513 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:25.533 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:04:25.544 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:25.563 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:04:25.590 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:25.595 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:04:25.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:25.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:25.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:04:25.664 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:04:25.699 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:25.701 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:04:25.737 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:25.743 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:04:25.782 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:04:25.788 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:25.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:25.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:25.811 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:04:25.811 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:04:25.811 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:04:25.811 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:04:25.811 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:04:25.811 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:04:25.811 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:04:25.811 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:04:25.896 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:25.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:25.955 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 57 16 80483 80467 88.1622 53.125 0.0129457 0.0113375 2023-10-09T10:04:25.977 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:25.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:25.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:25.987 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:26.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:26.149 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:26.202 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T10:04:26.219 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:26.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:26.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:26.264 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:04:26.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:26.408 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:26 smithi012 ceph-mon[147926]: pgmap v504: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-09T10:04:26.408 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2251001302' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:04:26.409 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:04:26.409 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1458741815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:26.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:26 smithi012 ceph-mon[152263]: pgmap v504: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-09T10:04:26.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2251001302' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:04:26.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:04:26.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1458741815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:26.428 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:26.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:26 smithi110 ceph-mon[136032]: pgmap v504: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-09T10:04:26.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2251001302' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:04:26.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:04:26.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1458741815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:26.620 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:26.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:26.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:26.809 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:26.956 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 58 16 81382 81366 87.611 56.1875 0.00775264 0.0114072 2023-10-09T10:04:26.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T10:04:27.073 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:27.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:27.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:27.182 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:27.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-mon[147926]: failed to encode map e204 with expected crc 2023-10-09T10:04:27.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-mon[147926]: failed to encode map e204 with expected crc 2023-10-09T10:04:27.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:04:27.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-mon[147926]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T10:04:27.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-mon[152263]: failed to encode map e204 with expected crc 2023-10-09T10:04:27.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-mon[152263]: failed to encode map e204 with expected crc 2023-10-09T10:04:27.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:04:27.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-mon[152263]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T10:04:27.289 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:27.428 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:27.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:27.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:27 smithi110 ceph-mon[136032]: failed to encode map e204 with expected crc 2023-10-09T10:04:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:27 smithi110 ceph-mon[136032]: failed to encode map e204 with expected crc 2023-10-09T10:04:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:04:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:27 smithi110 ceph-mon[136032]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T10:04:27.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:27.619 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:27.798 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:27.803 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:27.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:27.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:27.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:27.956 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 59 16 82271 82255 87.0667 55.5625 0.0130448 0.011481 2023-10-09T10:04:27.961 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:28.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:28.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:28.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:28.068 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:28.125 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:04:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:04:27] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T10:04:28.211 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:28.594 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:28.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:28.672 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:04:28.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:28.827 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:28.936 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:28 smithi012 ceph-mon[147926]: pgmap v506: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:04:28.936 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:28 smithi012 ceph-mon[152263]: pgmap v506: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:04:28.956 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:04:28.956742+0000 min lat: 0.00114602 max lat: 1.02658 avg lat: 0.0115367 2023-10-09T10:04:28.957 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:04:28.957 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 60 16 83253 83237 86.6387 61.375 0.0288335 0.0115367 2023-10-09T10:04:29.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:28 smithi110 ceph-mon[136032]: pgmap v506: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:04:29.307 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:29.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:29.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:29.368 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:04:29.368 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:04:29.369 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:04:29.369 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:04:29.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:29.504 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:29.587 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:29.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:29.694 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:29.740 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:29.778 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:29.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:29.882 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:29.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:29.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:29.922 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:29.956 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 61 16 84130 84114 86.1172 54.8125 0.0193624 0.011607 2023-10-09T10:04:29.975 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:29.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:30.083 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:30.115 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:30.115 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:30.119 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:30.164 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:30.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:30.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:30.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:30.269 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:30.302 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:30.351 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:30.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:30.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:30.491 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:30.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:04:30.537 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:30.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:30.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:30.720 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:30.834 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:30.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:30.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:30.843 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:30.902 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:30 smithi110 ceph-mon[136032]: pgmap v507: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T10:04:30.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:30.956 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 62 16 85032 85016 85.6378 56.375 0.0209726 0.0116697 2023-10-09T10:04:30.964 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:30 smithi012 ceph-mon[147926]: pgmap v507: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T10:04:30.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:30 smithi012 ceph-mon[152263]: pgmap v507: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T10:04:31.025 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:31.096 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:31.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:31.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:31.203 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:31.234 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:31.276 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:31.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:31.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:31.379 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:31.410 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:31.456 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:31.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:31.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:31.602 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:31.638 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:31.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:31.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:31.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:31.835 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:31.942 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T10:04:31.965 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 63 16 85843 85827 85.0747 50.6875 0.00476955 0.0117487 2023-10-09T10:04:32.084 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:32.091 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:31 smithi012 ceph-mon[147926]: pgmap v508: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T10:04:32.091 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:31 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2851011379' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:04:32.091 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:31 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:04:32.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:32.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:32.092 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:31 smithi012 ceph-mon[152263]: pgmap v508: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T10:04:32.092 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:31 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2851011379' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:04:32.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:31 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:04:32.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:31 smithi110 ceph-mon[136032]: pgmap v508: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T10:04:32.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:31 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2851011379' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:04:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:31 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:04:32.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:32.198 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:32.268 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:32.671 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:32.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:32.717 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T10:04:32.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:32.954 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:32.963 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 64 16 86703 86687 84.5856 53.75 0.0176408 0.0118159 2023-10-09T10:04:32.978 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:04:33.066 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:33.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:33.116 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[152263]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.116 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:04:33.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[152263]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T10:04:33.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[152263]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[152263]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[152263]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2653701227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:33.117 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:33.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[147926]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:04:33.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[147926]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T10:04:33.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[147926]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[147926]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[147926]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2653701227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:33.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:32 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:33.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:32 smithi110 ceph-mon[136032]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:32 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:04:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:32 smithi110 ceph-mon[136032]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T10:04:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:32 smithi110 ceph-mon[136032]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:32 smithi110 ceph-mon[136032]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:32 smithi110 ceph-mon[136032]: failed to encode map e205 with expected crc 2023-10-09T10:04:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2653701227' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:32 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:33.227 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:04:33.269 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:04:33.289 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:33.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:04:33.346 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:04:33.350 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:33.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:33.382 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:04:33.414 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:04:33.457 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:04:33.461 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:33.502 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:04:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:04:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:04:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:04:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:04:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:04:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:04:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:04:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:04:33.521 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:33.966 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 65 16 87518 87502 84.0646 50.9375 0.00845295 0.0118905 2023-10-09T10:04:34.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:34.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:34.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T10:04:34.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: pgmap v511: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2782178609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: pgmap v511: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:04:34.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2782178609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T10:04:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: failed to encode map e206 with expected crc 2023-10-09T10:04:34.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: pgmap v511: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:04:34.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2782178609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:34.215 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:34.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:34.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:34.441 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:34.514 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:34.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:34.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:34.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:34.665 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:34.710 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:34.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:34.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:34.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:34.852 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:34.894 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:34.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:34.966 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 66 16 88483 88467 83.7049 60.3125 0.0216685 0.0119413 2023-10-09T10:04:34.989 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:35.000 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:35.032 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:35.037 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:04:35.037 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:04:35.075 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:35.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:35.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:35.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:35.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:35.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:35.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:35.446 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:35.571 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:35.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:35.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:04:35.677 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:35.803 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:35.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:04:35.966 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 67 16 89366 89350 83.2786 55.1875 0.0203435 0.0120023 2023-10-09T10:04:36.141 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:36.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:36.261 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:36.291 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:36.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:36.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:36.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:04:36.425 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:36 smithi012 ceph-mon[147926]: pgmap v512: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T10:04:36.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:36 smithi012 ceph-mon[152263]: pgmap v512: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T10:04:36.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:36.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:36 smithi110 ceph-mon[136032]: pgmap v512: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T10:04:36.553 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:36.572 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:36.630 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:36.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:36.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:36.734 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:36.823 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:36.968 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 68 16 90080 90064 82.7096 44.625 0.0216011 0.0120842 2023-10-09T10:04:36.988 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:36.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:37.096 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:37.220 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:37.537 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:37.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:37.595 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:37.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:37.680 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:37.735 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:37.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:37.777 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:37.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:37.872 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:37.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:37.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:37.968 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 69 16 90907 90891 82.2602 51.6875 0.0104575 0.0121512 2023-10-09T10:04:38.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:38.109 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:38.169 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:04:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:04:37] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.33.4" 2023-10-09T10:04:38.225 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T10:04:38.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:38 smithi012 ceph-mon[147926]: pgmap v513: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T10:04:38.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2717639853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:04:38.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:38 smithi012 ceph-mon[152263]: pgmap v513: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T10:04:38.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2717639853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:04:38.563 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:04:38.563 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:04:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:38 smithi110 ceph-mon[136032]: pgmap v513: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T10:04:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2717639853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:04:38.967 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 70 16 91781 91765 81.8657 54.625 0.011895 0.0122094 2023-10-09T10:04:38.978 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:38.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:38.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:39.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:39.132 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T10:04:39.132 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:04:39.184 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:39.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2717639853' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:04:39.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T10:04:39.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[152263]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2717639853' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T10:04:39.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:39 smithi012 ceph-mon[147926]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.636 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:04:39.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2717639853' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:04:39.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T10:04:39.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:39 smithi110 ceph-mon[136032]: failed to encode map e207 with expected crc 2023-10-09T10:04:39.759 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:39.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:39.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:39.870 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:39.905 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:39.952 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:39.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:39.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:39.967 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 71 16 92594 92578 81.4285 50.8125 0.0261879 0.012275 2023-10-09T10:04:40.056 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:40.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:40.137 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:40.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:40.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:40.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:40.306 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:40.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:40 smithi012 ceph-mon[147926]: pgmap v515: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T10:04:40.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:40 smithi012 ceph-mon[152263]: pgmap v515: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T10:04:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:40 smithi110 ceph-mon[136032]: pgmap v515: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T10:04:40.970 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 72 16 93520 93504 81.1015 57.875 0.017181 0.0123255 2023-10-09T10:04:41.303 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:04:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/505785129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:41 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/505785129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:41 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:41.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/505785129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:41 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:04:41.699 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:41.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:41.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:41.804 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:41.810 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:41.869 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:41.915 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:41.968 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 73 16 94396 94380 80.7407 54.75 0.019521 0.0123801 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: pgmap v516: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[152263]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: pgmap v516: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s 2023-10-09T10:04:42.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:42.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:04:42.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T10:04:42.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:42 smithi012 ceph-mon[147926]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: pgmap v516: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s 2023-10-09T10:04:42.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:04:42.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:04:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T10:04:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:42 smithi110 ceph-mon[136032]: failed to encode map e208 with expected crc 2023-10-09T10:04:42.968 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 74 16 95251 95235 80.3715 53.4375 0.0127533 0.0124374 2023-10-09T10:04:43.140 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:43.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:43.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:43.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:43.345 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:43.823 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:43.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:43.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:43.938 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:43.968 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 75 16 96117 96101 80.0217 54.125 0.0213405 0.0124913 2023-10-09T10:04:43.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:44.023 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:44.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:44.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:44.044 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:04:44.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:04:44.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:44.138 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:04:44.156 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:44.175 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:04:44.196 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:44.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:44.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:44.212 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:04:44.249 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:04:44.291 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:04:44.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:44.330 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:04:44.330 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:04:44.331 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:04:44.331 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:04:44.331 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:04:44.331 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:04:44.331 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:04:44.331 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:04:44.331 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:04:44.387 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:44.727 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:44.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:44.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:44.837 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:44.885 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:44.932 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:44.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:44.971 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 76 16 96966 96950 79.6645 53.0625 0.0124831 0.0125475 2023-10-09T10:04:45.037 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:45.074 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:44 smithi110 ceph-mon[136032]: pgmap v518: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:04:45.075 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:45.075 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2275678707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:45.077 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:45.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:44 smithi012 ceph-mon[147926]: pgmap v518: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:04:45.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:45.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2275678707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:45.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:44 smithi012 ceph-mon[152263]: pgmap v518: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:04:45.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:45.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2275678707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:45.125 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:45.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:45.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:45.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:45.314 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:45.590 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:04:45.590 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:04:45.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.590 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:97481 7.0 e208/191 7:086bb76c:::benchmark_data_smithi012_221426_object97480:head v 208'6252, mlcod=208'6249)", 2023-10-09T10:04:45.590 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:45.579404+0000", 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0094446159999999994, 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094712270000000005, 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579404+0000", 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579404+0000", 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579403+0000", 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579442+0000", 2023-10-09T10:04:45.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8569000000000003e-05 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579454+0000", 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.152e-05 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579461+0000", 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5179999999999998e-06 2023-10-09T10:04:45.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579511+0000", 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9675000000000001e-05 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579530+0000", 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9760000000000001e-05 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:45.594 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:97470 7.6 e208/187 7:6d3a8616:::benchmark_data_smithi012_221426_object97469:head v 208'6141, mlcod=208'6139)", 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:45.582319+0000", 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0065299850000000003, 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066066040000000003, 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:45.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582319+0000", 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582319+0000", 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582318+0000", 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:04:45.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582345+0000", 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.728e-05 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582385+0000", 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9648999999999999e-05 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:45.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582393+0000", 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6939999999999995e-06 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582616+0000", 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00022353100000000001 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582637+0000", 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0937999999999998e-05 2023-10-09T10:04:45.598 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:97475 7.6 e208/187 7:63a59058:::benchmark_data_smithi012_221426_object97474:head v 208'6142, mlcod=208'6139)", 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:45.586534+0000", 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0023145829999999998, 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024269119999999998, 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:45.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586534+0000", 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586534+0000", 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586534+0000", 2023-10-09T10:04:45.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586558+0000", 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4091999999999999e-05 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586566+0000", 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1899999999999995e-06 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586572+0000", 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8719999999999999e-06 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586771+0000", 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000199104 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:45.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586792+0000", 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0985e-05 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:97482 7.6 e208/187)", 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:45.586593+0000", 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.002255792, 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024153, 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:45.603 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586593+0000", 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586593+0000", 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586593+0000", 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586615+0000", 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2555999999999999e-05 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:45.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586623+0000", 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0129999999999993e-06 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.586628+0000", 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1290000000000004e-06 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.606 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:97477 7.2 7:4bfb945a:::benchmark_data_smithi012_221426_object97476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:45.574951+0000", 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.013897687000000001, 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014093009, 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 97477 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:45.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.574951+0000", 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.574951+0000", 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.574949+0000", 2023-10-09T10:04:45.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.574977+0000", 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7509e-05 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.574993+0000", 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5597e-05 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575000+0000", 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8620000000000003e-06 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575797+0000", 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00079667599999999996 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575950+0000", 2023-10-09T10:04:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000153237 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.576103+0000", 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000152366 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.582380+0000", 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006277141 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.585387+0000", 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030068870000000002 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:97480 7.a 7:5ab54c20:::benchmark_data_smithi012_221426_object97479:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:04:45.575842+0000", 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.013007143000000001, 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013272806999999999, 2023-10-09T10:04:45.612 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 97480 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575842+0000", 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575842+0000", 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575841+0000", 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:04:45.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575871+0000", 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9777999999999998e-05 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575880+0000", 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9330000000000007e-06 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.575886+0000", 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4030000000000004e-06 2023-10-09T10:04:45.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.578864+0000", 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029782210000000001 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.578988+0000", 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012364200000000001 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:04:45.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.579106+0000", 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117828 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.585452+0000", 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0063460959999999999 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:04:45.585460+0000", 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0620000000000001e-06 2023-10-09T10:04:45.617 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.618 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:04:45.618 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.618 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:04:45.618 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:04:45.618 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 6 2023-10-09T10:04:45.618 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:04:45.621 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:45.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:45.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:45.728 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:45.771 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:45.817 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:45.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:45.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:45.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:04:45.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:45.971 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 77 16 97794 97778 79.3021 51.75 0.0167462 0.012604 2023-10-09T10:04:46.007 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:46.202 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:46.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:46.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:46.268 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:04:46.307 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:46.335 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:46.375 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:46.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:46.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:46.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:46.615 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:46.643 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:04:46.679 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:04:46.682 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:46.684 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:46.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:46.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:46.798 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:46.837 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:46.890 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:46.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:46.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:46.896 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:46 smithi012 ceph-mon[152263]: pgmap v519: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:04:46.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:46 smithi012 ceph-mon[147926]: pgmap v519: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:04:46.971 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 78 16 98489 98473 78.8426 43.4375 0.00696877 0.0126787 2023-10-09T10:04:46.994 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:47.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:46 smithi110 ceph-mon[136032]: pgmap v519: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:04:47.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:47.086 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:47.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:47.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:47.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:47.206 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":519,"stamp":"2023-10-09T10:04:45.116675+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6250436512,"num_objects":95389,"num_object_clones":0,"num_object_copies":286167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95389,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":95625,"num_write_kb":6105805,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":179,"num_bytes_recovered":8791084,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95559,"ondisk_log_size":95559,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20294696,"kb_used_data":18317424,"kb_used_omap":44,"kb_used_meta":1977171,"kb_avail":635921368,"statfs":{"total":671965249536,"available":651183480832,"internally_reserved":0,"allocated":18757042176,"data_stored":18752615265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":2024623521},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":453378048,"num_objects":6918,"num_object_clones":0,"num_object_copies":20754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6918,"num_write_kb":442752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003309"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":68,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220426+0000","last_change":"2023-10-09T10:03:19.763993+0000","last_active":"2023-10-09T10:04:38.220426+0000","last_peered":"2023-10-09T10:04:38.220426+0000","last_clean":"2023-10-09T10:04:38.220426+0000","last_became_active":"2023-10-09T10:03:19.763623+0000","last_became_peered":"2023-10-09T10:03:19.763623+0000","last_unstale":"2023-10-09T10:04:38.220426+0000","last_undegraded":"2023-10-09T10:04:38.220426+0000","last_fullsized":"2023-10-09T10:04:38.220426+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":195,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212662+0000","last_change":"2023-10-09T10:02:17.718080+0000","last_active":"2023-10-09T10:04:38.212662+0000","last_peered":"2023-10-09T10:04:38.212662+0000","last_clean":"2023-10-09T10:04:38.212662+0000","last_became_active":"2023-10-09T10:02:17.717746+0000","last_became_peered":"2023-10-09T10:02:17.717746+0000","last_unstale":"2023-10-09T10:04:38.212662+0000","last_undegraded":"2023-10-09T10:04:38.212662+0000","last_fullsized":"2023-10-09T10:04:38.212662+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:59.375104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":182,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217867+0000","last_change":"2023-10-09T10:02:17.740018+0000","last_active":"2023-10-09T10:04:38.217867+0000","last_peered":"2023-10-09T10:04:38.217867+0000","last_clean":"2023-10-09T10:04:38.217867+0000","last_became_active":"2023-10-09T10:02:17.739691+0000","last_became_peered":"2023-10-09T10:02:17.739691+0000","last_unstale":"2023-10-09T10:04:38.217867+0000","last_undegraded":"2023-10-09T10:04:38.217867+0000","last_fullsized":"2023-10-09T10:04:38.217867+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.836600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":57,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212697+0000","last_change":"2023-10-09T10:02:29.904188+0000","last_active":"2023-10-09T10:04:38.212697+0000","last_peered":"2023-10-09T10:04:38.212697+0000","last_clean":"2023-10-09T10:04:38.212697+0000","last_became_active":"2023-10-09T10:02:29.899658+0000","last_became_peered":"2023-10-09T10:02:29.899658+0000","last_unstale":"2023-10-09T10:04:38.212697+0000","last_undegraded":"2023-10-09T10:04:38.212697+0000","last_fullsized":"2023-10-09T10:04:38.212697+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:35.455601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":211,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212636+0000","last_change":"2023-10-09T10:02:29.885613+0000","last_active":"2023-10-09T10:04:38.212636+0000","last_peered":"2023-10-09T10:04:38.212636+0000","last_clean":"2023-10-09T10:04:38.212636+0000","last_became_active":"2023-10-09T10:02:29.885185+0000","last_became_peered":"2023-10-09T10:02:29.885185+0000","last_unstale":"2023-10-09T10:04:38.212636+0000","last_undegraded":"2023-10-09T10:04:38.212636+0000","last_fullsized":"2023-10-09T10:04:38.212636+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:01:44.101600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":26,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.219296+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:04:38.219296+0000","last_peered":"2023-10-09T10:04:38.219296+0000","last_clean":"2023-10-09T10:04:38.219296+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:04:38.219296+0000","last_undegraded":"2023-10-09T10:04:38.219296+0000","last_fullsized":"2023-10-09T10:04:38.219296+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":74,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217256+0000","last_change":"2023-10-09T10:03:30.080811+0000","last_active":"2023-10-09T10:04:38.217256+0000","last_peered":"2023-10-09T10:04:38.217256+0000","last_clean":"2023-10-09T10:04:38.217256+0000","last_became_active":"2023-10-09T10:03:30.080435+0000","last_became_peered":"2023-10-09T10:03:30.080435+0000","last_unstale":"2023-10-09T10:04:38.217256+0000","last_undegraded":"2023-10-09T10:04:38.217256+0000","last_fullsized":"2023-10-09T10:04:38.217256+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:31.031546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":177,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212715+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T10:04:38.212715+0000","last_peered":"2023-10-09T10:04:38.212715+0000","last_clean":"2023-10-09T10:04:38.212715+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T10:04:38.212715+0000","last_undegraded":"2023-10-09T10:04:38.212715+0000","last_fullsized":"2023-10-09T10:04:38.212715+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":26,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.219136+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:04:38.219136+0000","last_peered":"2023-10-09T10:04:38.219136+0000","last_clean":"2023-10-09T10:04:38.219136+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:04:38.219136+0000","last_undegraded":"2023-10-09T10:04:38.219136+0000","last_fullsized":"2023-10-09T10:04:38.219136+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":138,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.213174+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T10:04:38.213174+0000","last_peered":"2023-10-09T10:04:38.213174+0000","last_clean":"2023-10-09T10:04:38.213174+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T10:04:38.213174+0000","last_undegraded":"2023-10-09T10:04:38.213174+0000","last_fullsized":"2023-10-09T10:04:38.213174+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":221,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212703+0000","last_change":"2023-10-09T10:02:18.222658+0000","last_active":"2023-10-09T10:04:38.212703+0000","last_peered":"2023-10-09T10:04:38.212703+0000","last_clean":"2023-10-09T10:04:38.212703+0000","last_became_active":"2023-10-09T10:02:17.734984+0000","last_became_peered":"2023-10-09T10:02:17.734984+0000","last_unstale":"2023-10-09T10:04:38.212703+0000","last_undegraded":"2023-10-09T10:04:38.212703+0000","last_fullsized":"2023-10-09T10:04:38.212703+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":191,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218267+0000","last_change":"2023-10-09T10:03:19.764825+0000","last_active":"2023-10-09T10:04:38.218267+0000","last_peered":"2023-10-09T10:04:38.218267+0000","last_clean":"2023-10-09T10:04:38.218267+0000","last_became_active":"2023-10-09T10:03:19.764466+0000","last_became_peered":"2023-10-09T10:03:19.764466+0000","last_unstale":"2023-10-09T10:04:38.218267+0000","last_undegraded":"2023-10-09T10:04:38.218267+0000","last_fullsized":"2023-10-09T10:04:38.218267+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:37.631717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":69,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220795+0000","last_change":"2023-10-09T10:02:18.088005+0000","last_active":"2023-10-09T10:04:38.220795+0000","last_peered":"2023-10-09T10:04:38.220795+0000","last_clean":"2023-10-09T10:04:38.220795+0000","last_became_active":"2023-10-09T10:02:17.722407+0000","last_became_peered":"2023-10-09T10:02:17.722407+0000","last_unstale":"2023-10-09T10:04:38.220795+0000","last_undegraded":"2023-10-09T10:04:38.220795+0000","last_fullsized":"2023-10-09T10:04:38.220795+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220583+0000","last_change":"2023-10-09T10:02:17.731192+0000","last_active":"2023-10-09T10:04:38.220583+0000","last_peered":"2023-10-09T10:04:38.220583+0000","last_clean":"2023-10-09T10:04:38.220583+0000","last_became_active":"2023-10-09T10:02:17.730757+0000","last_became_peered":"2023-10-09T10:02:17.730757+0000","last_unstale":"2023-10-09T10:04:38.220583+0000","last_undegraded":"2023-10-09T10:04:38.220583+0000","last_fullsized":"2023-10-09T10:04:38.220583+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":250,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212320+0000","last_change":"2023-10-09T10:03:19.765719+0000","last_active":"2023-10-09T10:04:38.212320+0000","last_peered":"2023-10-09T10:04:38.212320+0000","last_clean":"2023-10-09T10:04:38.212320+0000","last_became_active":"2023-10-09T10:03:19.761432+0000","last_became_peered":"2023-10-09T10:03:19.761432+0000","last_unstale":"2023-10-09T10:04:38.212320+0000","last_undegraded":"2023-10-09T10:04:38.212320+0000","last_fullsized":"2023-10-09T10:04:38.212320+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":186,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212597+0000","last_change":"2023-10-09T10:04:00.899350+0000","last_active":"2023-10-09T10:04:38.212597+0000","last_peered":"2023-10-09T10:04:38.212597+0000","last_clean":"2023-10-09T10:04:38.212597+0000","last_became_active":"2023-10-09T10:04:00.899001+0000","last_became_peered":"2023-10-09T10:04:00.899001+0000","last_unstale":"2023-10-09T10:04:38.212597+0000","last_undegraded":"2023-10-09T10:04:38.212597+0000","last_fullsized":"2023-10-09T10:04:38.212597+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:49.346314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":56,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.209835+0000","last_change":"2023-10-09T10:02:29.904731+0000","last_active":"2023-10-09T10:04:38.209835+0000","last_peered":"2023-10-09T10:04:38.209835+0000","last_clean":"2023-10-09T10:04:38.209835+0000","last_became_active":"2023-10-09T10:02:29.904290+0000","last_became_peered":"2023-10-09T10:02:29.904290+0000","last_unstale":"2023-10-09T10:04:38.209835+0000","last_undegraded":"2023-10-09T10:04:38.209835+0000","last_fullsized":"2023-10-09T10:04:38.209835+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:00.376061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":26,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218180+0000","last_change":"2023-10-09T10:03:30.082426+0000","last_active":"2023-10-09T10:04:38.218180+0000","last_peered":"2023-10-09T10:04:38.218180+0000","last_clean":"2023-10-09T10:04:38.218180+0000","last_became_active":"2023-10-09T10:03:30.082112+0000","last_became_peered":"2023-10-09T10:03:30.082112+0000","last_unstale":"2023-10-09T10:04:38.218180+0000","last_undegraded":"2023-10-09T10:04:38.218180+0000","last_fullsized":"2023-10-09T10:04:38.218180+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220607+0000","last_change":"2023-10-09T10:02:17.730673+0000","last_active":"2023-10-09T10:04:38.220607+0000","last_peered":"2023-10-09T10:04:38.220607+0000","last_clean":"2023-10-09T10:04:38.220607+0000","last_became_active":"2023-10-09T10:02:17.729848+0000","last_became_peered":"2023-10-09T10:02:17.729848+0000","last_unstale":"2023-10-09T10:04:38.220607+0000","last_undegraded":"2023-10-09T10:04:38.220607+0000","last_fullsized":"2023-10-09T10:04:38.220607+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":56,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212481+0000","last_change":"2023-10-09T10:02:29.900023+0000","last_active":"2023-10-09T10:04:38.212481+0000","last_peered":"2023-10-09T10:04:38.212481+0000","last_clean":"2023-10-09T10:04:38.212481+0000","last_became_active":"2023-10-09T10:02:29.899702+0000","last_became_peered":"2023-10-09T10:02:29.899702+0000","last_unstale":"2023-10-09T10:04:38.212481+0000","last_undegraded":"2023-10-09T10:04:38.212481+0000","last_fullsized":"2023-10-09T10:04:38.212481+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:39.396593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":94,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.209724+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:04:38.209724+0000","last_peered":"2023-10-09T10:04:38.209724+0000","last_clean":"2023-10-09T10:04:38.209724+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:04:38.209724+0000","last_undegraded":"2023-10-09T10:04:38.209724+0000","last_fullsized":"2023-10-09T10:04:38.209724+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":177,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.213979+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T10:04:38.213979+0000","last_peered":"2023-10-09T10:04:38.213979+0000","last_clean":"2023-10-09T10:04:38.213979+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T10:04:38.213979+0000","last_undegraded":"2023-10-09T10:04:38.213979+0000","last_fullsized":"2023-10-09T10:04:38.213979+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"208'6048","reported_seq":6157,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:44.231395+0000","last_change":"2023-10-09T10:03:30.261125+0000","last_active":"2023-10-09T10:04:44.231395+0000","last_peered":"2023-10-09T10:04:44.231395+0000","last_clean":"2023-10-09T10:04:44.231395+0000","last_became_active":"2023-10-09T10:03:30.095093+0000","last_became_peered":"2023-10-09T10:03:30.095093+0000","last_unstale":"2023-10-09T10:04:44.231395+0000","last_undegraded":"2023-10-09T10:04:44.231395+0000","last_fullsized":"2023-10-09T10:04:44.231395+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6048,"log_dups_size":0,"ondisk_log_size":6048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:57.462339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396361728,"num_objects":6048,"num_object_clones":0,"num_object_copies":18144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6048,"num_write_kb":387072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":2228224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":226,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212914+0000","last_change":"2023-10-09T10:02:29.908231+0000","last_active":"2023-10-09T10:04:38.212914+0000","last_peered":"2023-10-09T10:04:38.212914+0000","last_clean":"2023-10-09T10:04:38.212914+0000","last_became_active":"2023-10-09T10:02:29.880421+0000","last_became_peered":"2023-10-09T10:02:29.880421+0000","last_unstale":"2023-10-09T10:04:38.212914+0000","last_undegraded":"2023-10-09T10:04:38.212914+0000","last_fullsized":"2023-10-09T10:04:38.212914+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":26,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217798+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:04:38.217798+0000","last_peered":"2023-10-09T10:04:38.217798+0000","last_clean":"2023-10-09T10:04:38.217798+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:04:38.217798+0000","last_undegraded":"2023-10-09T10:04:38.217798+0000","last_fullsized":"2023-10-09T10:04:38.217798+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"208'5963","reported_seq":6074,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:42.581568+0000","last_change":"2023-10-09T10:03:30.249521+0000","last_active":"2023-10-09T10:04:42.581568+0000","last_peered":"2023-10-09T10:04:42.581568+0000","last_clean":"2023-10-09T10:04:42.581568+0000","last_became_active":"2023-10-09T10:03:30.082956+0000","last_became_peered":"2023-10-09T10:03:30.082956+0000","last_unstale":"2023-10-09T10:04:42.581568+0000","last_undegraded":"2023-10-09T10:04:42.581568+0000","last_fullsized":"2023-10-09T10:04:42.581568+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5963,"log_dups_size":0,"ondisk_log_size":5963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:18:43.459243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390791168,"num_objects":5963,"num_object_clones":0,"num_object_copies":17889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5963,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":214,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212759+0000","last_change":"2023-10-09T10:02:18.135878+0000","last_active":"2023-10-09T10:04:38.212759+0000","last_peered":"2023-10-09T10:04:38.212759+0000","last_clean":"2023-10-09T10:04:38.212759+0000","last_became_active":"2023-10-09T10:02:17.734932+0000","last_became_peered":"2023-10-09T10:02:17.734932+0000","last_unstale":"2023-10-09T10:04:38.212759+0000","last_undegraded":"2023-10-09T10:04:38.212759+0000","last_fullsized":"2023-10-09T10:04:38.212759+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220543+0000","last_change":"2023-10-09T10:02:17.743834+0000","last_active":"2023-10-09T10:04:38.220543+0000","last_peered":"2023-10-09T10:04:38.220543+0000","last_clean":"2023-10-09T10:04:38.220543+0000","last_became_active":"2023-10-09T10:02:17.743066+0000","last_became_peered":"2023-10-09T10:02:17.743066+0000","last_unstale":"2023-10-09T10:04:38.220543+0000","last_undegraded":"2023-10-09T10:04:38.220543+0000","last_fullsized":"2023-10-09T10:04:38.220543+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"208'5795","reported_seq":5825,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.855973+0000","last_change":"2023-10-09T10:03:22.033916+0000","last_active":"2023-10-09T10:04:41.855973+0000","last_peered":"2023-10-09T10:04:41.855973+0000","last_clean":"2023-10-09T10:04:41.855973+0000","last_became_active":"2023-10-09T10:03:22.033637+0000","last_became_peered":"2023-10-09T10:03:22.033637+0000","last_unstale":"2023-10-09T10:04:41.855973+0000","last_undegraded":"2023-10-09T10:04:41.855973+0000","last_fullsized":"2023-10-09T10:04:41.855973+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5795,"log_dups_size":0,"ondisk_log_size":5795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:33:02.120588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379781120,"num_objects":5795,"num_object_clones":0,"num_object_copies":17385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5795,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":30,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218225+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:04:38.218225+0000","last_peered":"2023-10-09T10:04:38.218225+0000","last_clean":"2023-10-09T10:04:38.218225+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:04:38.218225+0000","last_undegraded":"2023-10-09T10:04:38.218225+0000","last_fullsized":"2023-10-09T10:04:38.218225+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":113,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218368+0000","last_change":"2023-10-09T10:01:36.025782+0000","last_active":"2023-10-09T10:04:38.218368+0000","last_peered":"2023-10-09T10:04:38.218368+0000","last_clean":"2023-10-09T10:04:38.218368+0000","last_became_active":"2023-10-09T10:01:36.025196+0000","last_became_peered":"2023-10-09T10:01:36.025196+0000","last_unstale":"2023-10-09T10:04:38.218368+0000","last_undegraded":"2023-10-09T10:04:38.218368+0000","last_fullsized":"2023-10-09T10:04:38.218368+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:04.486165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"208'6037","reported_seq":6066,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:44.928730+0000","last_change":"2023-10-09T10:03:22.032901+0000","last_active":"2023-10-09T10:04:44.928730+0000","last_peered":"2023-10-09T10:04:44.928730+0000","last_clean":"2023-10-09T10:04:44.928730+0000","last_became_active":"2023-10-09T10:03:22.032596+0000","last_became_peered":"2023-10-09T10:03:22.032596+0000","last_unstale":"2023-10-09T10:04:44.928730+0000","last_undegraded":"2023-10-09T10:04:44.928730+0000","last_fullsized":"2023-10-09T10:04:44.928730+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6037,"log_dups_size":0,"ondisk_log_size":6037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395640832,"num_objects":6037,"num_object_clones":0,"num_object_copies":18111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6037,"num_write_kb":386368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":26,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218042+0000","last_change":"2023-10-09T10:03:30.092246+0000","last_active":"2023-10-09T10:04:38.218042+0000","last_peered":"2023-10-09T10:04:38.218042+0000","last_clean":"2023-10-09T10:04:38.218042+0000","last_became_active":"2023-10-09T10:03:30.091926+0000","last_became_peered":"2023-10-09T10:03:30.091926+0000","last_unstale":"2023-10-09T10:04:38.218042+0000","last_undegraded":"2023-10-09T10:04:38.218042+0000","last_fullsized":"2023-10-09T10:04:38.218042+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":57,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.210285+0000","last_change":"2023-10-09T10:02:29.889376+0000","last_active":"2023-10-09T10:04:38.210285+0000","last_peered":"2023-10-09T10:04:38.210285+0000","last_clean":"2023-10-09T10:04:38.210285+0000","last_became_active":"2023-10-09T10:02:29.888420+0000","last_became_peered":"2023-10-09T10:02:29.888420+0000","last_unstale":"2023-10-09T10:04:38.210285+0000","last_undegraded":"2023-10-09T10:04:38.210285+0000","last_fullsized":"2023-10-09T10:04:38.210285+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:16.976540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"208'5906","reported_seq":5936,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.777466+0000","last_change":"2023-10-09T10:03:22.032070+0000","last_active":"2023-10-09T10:04:41.777466+0000","last_peered":"2023-10-09T10:04:41.777466+0000","last_clean":"2023-10-09T10:04:41.777466+0000","last_became_active":"2023-10-09T10:03:22.031790+0000","last_became_peered":"2023-10-09T10:03:22.031790+0000","last_unstale":"2023-10-09T10:04:41.777466+0000","last_undegraded":"2023-10-09T10:04:41.777466+0000","last_fullsized":"2023-10-09T10:04:41.777466+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5906,"log_dups_size":0,"ondisk_log_size":5906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:57.360330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387055616,"num_objects":5906,"num_object_clones":0,"num_object_copies":17718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5906,"num_write_kb":377984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":228,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.213007+0000","last_change":"2023-10-09T10:02:29.946556+0000","last_active":"2023-10-09T10:04:38.213007+0000","last_peered":"2023-10-09T10:04:38.213007+0000","last_clean":"2023-10-09T10:04:38.213007+0000","last_became_active":"2023-10-09T10:02:29.884952+0000","last_became_peered":"2023-10-09T10:02:29.884952+0000","last_unstale":"2023-10-09T10:04:38.213007+0000","last_undegraded":"2023-10-09T10:04:38.213007+0000","last_fullsized":"2023-10-09T10:04:38.213007+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:32.438007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220690+0000","last_change":"2023-10-09T10:02:17.743630+0000","last_active":"2023-10-09T10:04:38.220690+0000","last_peered":"2023-10-09T10:04:38.220690+0000","last_clean":"2023-10-09T10:04:38.220690+0000","last_became_active":"2023-10-09T10:02:17.742997+0000","last_became_peered":"2023-10-09T10:02:17.742997+0000","last_unstale":"2023-10-09T10:04:38.220690+0000","last_undegraded":"2023-10-09T10:04:38.220690+0000","last_fullsized":"2023-10-09T10:04:38.220690+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"208'5975","reported_seq":6031,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:44.929875+0000","last_change":"2023-10-09T10:03:30.230552+0000","last_active":"2023-10-09T10:04:44.929875+0000","last_peered":"2023-10-09T10:04:44.929875+0000","last_clean":"2023-10-09T10:04:44.929875+0000","last_became_active":"2023-10-09T10:03:30.085250+0000","last_became_peered":"2023-10-09T10:03:30.085250+0000","last_unstale":"2023-10-09T10:04:44.929875+0000","last_undegraded":"2023-10-09T10:04:44.929875+0000","last_fullsized":"2023-10-09T10:04:44.929875+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391577600,"num_objects":5975,"num_object_clones":0,"num_object_copies":17925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5975,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220529+0000","last_change":"2023-10-09T10:02:17.733313+0000","last_active":"2023-10-09T10:04:38.220529+0000","last_peered":"2023-10-09T10:04:38.220529+0000","last_clean":"2023-10-09T10:04:38.220529+0000","last_became_active":"2023-10-09T10:02:17.732658+0000","last_became_peered":"2023-10-09T10:02:17.732658+0000","last_unstale":"2023-10-09T10:04:38.220529+0000","last_undegraded":"2023-10-09T10:04:38.220529+0000","last_fullsized":"2023-10-09T10:04:38.220529+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":68,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220879+0000","last_change":"2023-10-09T10:03:19.762981+0000","last_active":"2023-10-09T10:04:38.220879+0000","last_peered":"2023-10-09T10:04:38.220879+0000","last_clean":"2023-10-09T10:04:38.220879+0000","last_became_active":"2023-10-09T10:03:19.762647+0000","last_became_peered":"2023-10-09T10:03:19.762647+0000","last_unstale":"2023-10-09T10:04:38.220879+0000","last_undegraded":"2023-10-09T10:04:38.220879+0000","last_fullsized":"2023-10-09T10:04:38.220879+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"208'5984","reported_seq":6014,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:44.185642+0000","last_change":"2023-10-09T10:03:22.356577+0000","last_active":"2023-10-09T10:04:44.185642+0000","last_peered":"2023-10-09T10:04:44.185642+0000","last_clean":"2023-10-09T10:04:44.185642+0000","last_became_active":"2023-10-09T10:03:22.356253+0000","last_became_peered":"2023-10-09T10:03:22.356253+0000","last_unstale":"2023-10-09T10:04:44.185642+0000","last_undegraded":"2023-10-09T10:04:44.185642+0000","last_fullsized":"2023-10-09T10:04:44.185642+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5984,"log_dups_size":0,"ondisk_log_size":5984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:14.903436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392167424,"num_objects":5984,"num_object_clones":0,"num_object_copies":17952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5984,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":68,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220431+0000","last_change":"2023-10-09T10:03:19.760276+0000","last_active":"2023-10-09T10:04:38.220431+0000","last_peered":"2023-10-09T10:04:38.220431+0000","last_clean":"2023-10-09T10:04:38.220431+0000","last_became_active":"2023-10-09T10:03:19.759911+0000","last_became_peered":"2023-10-09T10:03:19.759911+0000","last_unstale":"2023-10-09T10:04:38.220431+0000","last_undegraded":"2023-10-09T10:04:38.220431+0000","last_fullsized":"2023-10-09T10:04:38.220431+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220802+0000","last_change":"2023-10-09T10:02:17.732379+0000","last_active":"2023-10-09T10:04:38.220802+0000","last_peered":"2023-10-09T10:04:38.220802+0000","last_clean":"2023-10-09T10:04:38.220802+0000","last_became_active":"2023-10-09T10:02:17.729410+0000","last_became_peered":"2023-10-09T10:02:17.729410+0000","last_unstale":"2023-10-09T10:04:38.220802+0000","last_undegraded":"2023-10-09T10:04:38.220802+0000","last_fullsized":"2023-10-09T10:04:38.220802+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"208'5944","reported_seq":5974,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:42.562058+0000","last_change":"2023-10-09T10:03:22.356576+0000","last_active":"2023-10-09T10:04:42.562058+0000","last_peered":"2023-10-09T10:04:42.562058+0000","last_clean":"2023-10-09T10:04:42.562058+0000","last_became_active":"2023-10-09T10:03:22.356253+0000","last_became_peered":"2023-10-09T10:03:22.356253+0000","last_unstale":"2023-10-09T10:04:42.562058+0000","last_undegraded":"2023-10-09T10:04:42.562058+0000","last_fullsized":"2023-10-09T10:04:42.562058+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5944,"log_dups_size":0,"ondisk_log_size":5944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:58.353155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389545984,"num_objects":5944,"num_object_clones":0,"num_object_copies":17832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5944,"num_write_kb":380416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":179,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212477+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T10:04:38.212477+0000","last_peered":"2023-10-09T10:04:38.212477+0000","last_clean":"2023-10-09T10:04:38.212477+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T10:04:38.212477+0000","last_undegraded":"2023-10-09T10:04:38.212477+0000","last_fullsized":"2023-10-09T10:04:38.212477+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":182,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.221121+0000","last_change":"2023-10-09T10:02:17.726854+0000","last_active":"2023-10-09T10:04:38.221121+0000","last_peered":"2023-10-09T10:04:38.221121+0000","last_clean":"2023-10-09T10:04:38.221121+0000","last_became_active":"2023-10-09T10:02:17.726570+0000","last_became_peered":"2023-10-09T10:02:17.726570+0000","last_unstale":"2023-10-09T10:04:38.221121+0000","last_undegraded":"2023-10-09T10:04:38.221121+0000","last_fullsized":"2023-10-09T10:04:38.221121+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:41.295112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"208'6038","reported_seq":6067,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:44.892325+0000","last_change":"2023-10-09T10:03:22.358239+0000","last_active":"2023-10-09T10:04:44.892325+0000","last_peered":"2023-10-09T10:04:44.892325+0000","last_clean":"2023-10-09T10:04:44.892325+0000","last_became_active":"2023-10-09T10:03:22.357886+0000","last_became_peered":"2023-10-09T10:03:22.357886+0000","last_unstale":"2023-10-09T10:04:44.892325+0000","last_undegraded":"2023-10-09T10:04:44.892325+0000","last_fullsized":"2023-10-09T10:04:44.892325+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6038,"log_dups_size":0,"ondisk_log_size":6038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395706368,"num_objects":6038,"num_object_clones":0,"num_object_copies":18114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6038,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":207,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217957+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:04:38.217957+0000","last_peered":"2023-10-09T10:04:38.217957+0000","last_clean":"2023-10-09T10:04:38.217957+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:04:38.217957+0000","last_undegraded":"2023-10-09T10:04:38.217957+0000","last_fullsized":"2023-10-09T10:04:38.217957+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":26,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218140+0000","last_change":"2023-10-09T10:03:30.093765+0000","last_active":"2023-10-09T10:04:38.218140+0000","last_peered":"2023-10-09T10:04:38.218140+0000","last_clean":"2023-10-09T10:04:38.218140+0000","last_became_active":"2023-10-09T10:03:30.093294+0000","last_became_peered":"2023-10-09T10:03:30.093294+0000","last_unstale":"2023-10-09T10:04:38.218140+0000","last_undegraded":"2023-10-09T10:04:38.218140+0000","last_fullsized":"2023-10-09T10:04:38.218140+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":149,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.219379+0000","last_change":"2023-10-09T10:03:30.081023+0000","last_active":"2023-10-09T10:04:38.219379+0000","last_peered":"2023-10-09T10:04:38.219379+0000","last_clean":"2023-10-09T10:04:38.219379+0000","last_became_active":"2023-10-09T10:03:30.080737+0000","last_became_peered":"2023-10-09T10:03:30.080737+0000","last_unstale":"2023-10-09T10:04:38.219379+0000","last_undegraded":"2023-10-09T10:04:38.219379+0000","last_fullsized":"2023-10-09T10:04:38.219379+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:04.667262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"208'5857","reported_seq":5887,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.845885+0000","last_change":"2023-10-09T10:03:22.356731+0000","last_active":"2023-10-09T10:04:41.845885+0000","last_peered":"2023-10-09T10:04:41.845885+0000","last_clean":"2023-10-09T10:04:41.845885+0000","last_became_active":"2023-10-09T10:03:22.356414+0000","last_became_peered":"2023-10-09T10:03:22.356414+0000","last_unstale":"2023-10-09T10:04:41.845885+0000","last_undegraded":"2023-10-09T10:04:41.845885+0000","last_fullsized":"2023-10-09T10:04:41.845885+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5857,"log_dups_size":0,"ondisk_log_size":5857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:54.939596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383844352,"num_objects":5857,"num_object_clones":0,"num_object_copies":17571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5857,"num_write_kb":374848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":26,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218276+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:04:38.218276+0000","last_peered":"2023-10-09T10:04:38.218276+0000","last_clean":"2023-10-09T10:04:38.218276+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:04:38.218276+0000","last_undegraded":"2023-10-09T10:04:38.218276+0000","last_fullsized":"2023-10-09T10:04:38.218276+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":68,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220657+0000","last_change":"2023-10-09T10:02:29.891638+0000","last_active":"2023-10-09T10:04:38.220657+0000","last_peered":"2023-10-09T10:04:38.220657+0000","last_clean":"2023-10-09T10:04:38.220657+0000","last_became_active":"2023-10-09T10:02:29.891265+0000","last_became_peered":"2023-10-09T10:02:29.891265+0000","last_unstale":"2023-10-09T10:04:38.220657+0000","last_undegraded":"2023-10-09T10:04:38.220657+0000","last_fullsized":"2023-10-09T10:04:38.220657+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":206,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217887+0000","last_change":"2023-10-09T10:03:19.763071+0000","last_active":"2023-10-09T10:04:38.217887+0000","last_peered":"2023-10-09T10:04:38.217887+0000","last_clean":"2023-10-09T10:04:38.217887+0000","last_became_active":"2023-10-09T10:03:19.762353+0000","last_became_peered":"2023-10-09T10:03:19.762353+0000","last_unstale":"2023-10-09T10:04:38.217887+0000","last_undegraded":"2023-10-09T10:04:38.217887+0000","last_fullsized":"2023-10-09T10:04:38.217887+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:50.245039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"208'5804","reported_seq":5834,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.829940+0000","last_change":"2023-10-09T10:03:22.033981+0000","last_active":"2023-10-09T10:04:41.829940+0000","last_peered":"2023-10-09T10:04:41.829940+0000","last_clean":"2023-10-09T10:04:41.829940+0000","last_became_active":"2023-10-09T10:03:22.033707+0000","last_became_peered":"2023-10-09T10:03:22.033707+0000","last_unstale":"2023-10-09T10:04:41.829940+0000","last_undegraded":"2023-10-09T10:04:41.829940+0000","last_fullsized":"2023-10-09T10:04:41.829940+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5804,"log_dups_size":0,"ondisk_log_size":5804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:05:34.341309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380370944,"num_objects":5804,"num_object_clones":0,"num_object_copies":17412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5804,"num_write_kb":371456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":125,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.213303+0000","last_change":"2023-10-09T10:03:19.759400+0000","last_active":"2023-10-09T10:04:38.213303+0000","last_peered":"2023-10-09T10:04:38.213303+0000","last_clean":"2023-10-09T10:04:38.213303+0000","last_became_active":"2023-10-09T10:03:19.759059+0000","last_became_peered":"2023-10-09T10:03:19.759059+0000","last_unstale":"2023-10-09T10:04:38.213303+0000","last_undegraded":"2023-10-09T10:04:38.213303+0000","last_fullsized":"2023-10-09T10:04:38.213303+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:03:20.450835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":191,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.269261+0000","last_change":"2023-10-09T10:03:19.764744+0000","last_active":"2023-10-09T10:04:41.269261+0000","last_peered":"2023-10-09T10:04:41.269261+0000","last_clean":"2023-10-09T10:04:41.269261+0000","last_became_active":"2023-10-09T10:03:19.764382+0000","last_became_peered":"2023-10-09T10:03:19.764382+0000","last_unstale":"2023-10-09T10:04:41.269261+0000","last_undegraded":"2023-10-09T10:04:41.269261+0000","last_fullsized":"2023-10-09T10:04:41.269261+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:34.922940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":220,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218464+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:04:38.218464+0000","last_peered":"2023-10-09T10:04:38.218464+0000","last_clean":"2023-10-09T10:04:38.218464+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:04:38.218464+0000","last_undegraded":"2023-10-09T10:04:38.218464+0000","last_fullsized":"2023-10-09T10:04:38.218464+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"208'5932","reported_seq":5962,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.777201+0000","last_change":"2023-10-09T10:03:22.032323+0000","last_active":"2023-10-09T10:04:41.777201+0000","last_peered":"2023-10-09T10:04:41.777201+0000","last_clean":"2023-10-09T10:04:41.777201+0000","last_became_active":"2023-10-09T10:03:22.031949+0000","last_became_peered":"2023-10-09T10:03:22.031949+0000","last_unstale":"2023-10-09T10:04:41.777201+0000","last_undegraded":"2023-10-09T10:04:41.777201+0000","last_fullsized":"2023-10-09T10:04:41.777201+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:14.245332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388759552,"num_objects":5932,"num_object_clones":0,"num_object_copies":17796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5932,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":26,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218473+0000","last_change":"2023-10-09T10:03:30.082150+0000","last_active":"2023-10-09T10:04:38.218473+0000","last_peered":"2023-10-09T10:04:38.218473+0000","last_clean":"2023-10-09T10:04:38.218473+0000","last_became_active":"2023-10-09T10:03:30.081805+0000","last_became_peered":"2023-10-09T10:03:30.081805+0000","last_unstale":"2023-10-09T10:04:38.218473+0000","last_undegraded":"2023-10-09T10:04:38.218473+0000","last_fullsized":"2023-10-09T10:04:38.218473+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":455,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212182+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:04:38.212182+0000","last_peered":"2023-10-09T10:04:38.212182+0000","last_clean":"2023-10-09T10:04:38.212182+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:04:38.212182+0000","last_undegraded":"2023-10-09T10:04:38.212182+0000","last_fullsized":"2023-10-09T10:04:38.212182+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217299+0000","last_change":"2023-10-09T10:03:30.089277+0000","last_active":"2023-10-09T10:04:38.217299+0000","last_peered":"2023-10-09T10:04:38.217299+0000","last_clean":"2023-10-09T10:04:38.217299+0000","last_became_active":"2023-10-09T10:03:30.088960+0000","last_became_peered":"2023-10-09T10:03:30.088960+0000","last_unstale":"2023-10-09T10:04:38.217299+0000","last_undegraded":"2023-10-09T10:04:38.217299+0000","last_fullsized":"2023-10-09T10:04:38.217299+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:27.610770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220962+0000","last_change":"2023-10-09T10:02:17.734226+0000","last_active":"2023-10-09T10:04:38.220962+0000","last_peered":"2023-10-09T10:04:38.220962+0000","last_clean":"2023-10-09T10:04:38.220962+0000","last_became_active":"2023-10-09T10:02:17.733854+0000","last_became_peered":"2023-10-09T10:02:17.733854+0000","last_unstale":"2023-10-09T10:04:38.220962+0000","last_undegraded":"2023-10-09T10:04:38.220962+0000","last_fullsized":"2023-10-09T10:04:38.220962+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"208'5879","reported_seq":5909,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.857612+0000","last_change":"2023-10-09T10:03:22.032500+0000","last_active":"2023-10-09T10:04:41.857612+0000","last_peered":"2023-10-09T10:04:41.857612+0000","last_clean":"2023-10-09T10:04:41.857612+0000","last_became_active":"2023-10-09T10:03:22.032139+0000","last_became_peered":"2023-10-09T10:03:22.032139+0000","last_unstale":"2023-10-09T10:04:41.857612+0000","last_undegraded":"2023-10-09T10:04:41.857612+0000","last_fullsized":"2023-10-09T10:04:41.857612+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5879,"log_dups_size":0,"ondisk_log_size":5879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:01.747725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385286144,"num_objects":5879,"num_object_clones":0,"num_object_copies":17637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5879,"num_write_kb":376256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":180,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217729+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:04:38.217729+0000","last_peered":"2023-10-09T10:04:38.217729+0000","last_clean":"2023-10-09T10:04:38.217729+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:04:38.217729+0000","last_undegraded":"2023-10-09T10:04:38.217729+0000","last_fullsized":"2023-10-09T10:04:38.217729+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":57,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.211687+0000","last_change":"2023-10-09T10:02:29.953999+0000","last_active":"2023-10-09T10:04:38.211687+0000","last_peered":"2023-10-09T10:04:38.211687+0000","last_clean":"2023-10-09T10:04:38.211687+0000","last_became_active":"2023-10-09T10:02:29.953244+0000","last_became_peered":"2023-10-09T10:02:29.953244+0000","last_unstale":"2023-10-09T10:04:38.211687+0000","last_undegraded":"2023-10-09T10:04:38.211687+0000","last_fullsized":"2023-10-09T10:04:38.211687+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:05.612343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":84,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274836+0000","last_change":"2023-10-09T10:04:00.900774+0000","last_active":"2023-10-09T10:04:41.274836+0000","last_peered":"2023-10-09T10:04:41.274836+0000","last_clean":"2023-10-09T10:04:41.274836+0000","last_became_active":"2023-10-09T10:04:00.900387+0000","last_became_peered":"2023-10-09T10:04:00.900387+0000","last_unstale":"2023-10-09T10:04:41.274836+0000","last_undegraded":"2023-10-09T10:04:41.274836+0000","last_fullsized":"2023-10-09T10:04:41.274836+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:27:43.957142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"208'6220","reported_seq":6271,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:44.900832+0000","last_change":"2023-10-09T10:03:30.133157+0000","last_active":"2023-10-09T10:04:44.900832+0000","last_peered":"2023-10-09T10:04:44.900832+0000","last_clean":"2023-10-09T10:04:44.900832+0000","last_became_active":"2023-10-09T10:03:30.078454+0000","last_became_peered":"2023-10-09T10:03:30.078454+0000","last_unstale":"2023-10-09T10:04:44.900832+0000","last_undegraded":"2023-10-09T10:04:44.900832+0000","last_fullsized":"2023-10-09T10:04:44.900832+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6220,"log_dups_size":0,"ondisk_log_size":6220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407633920,"num_objects":6220,"num_object_clones":0,"num_object_copies":18660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6220,"num_write_kb":398080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":1376256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":213,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212613+0000","last_change":"2023-10-09T10:02:29.882693+0000","last_active":"2023-10-09T10:04:38.212613+0000","last_peered":"2023-10-09T10:04:38.212613+0000","last_clean":"2023-10-09T10:04:38.212613+0000","last_became_active":"2023-10-09T10:02:29.882300+0000","last_became_peered":"2023-10-09T10:02:29.882300+0000","last_unstale":"2023-10-09T10:04:38.212613+0000","last_undegraded":"2023-10-09T10:04:38.212613+0000","last_fullsized":"2023-10-09T10:04:38.212613+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":209,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218041+0000","last_change":"2023-10-09T10:03:30.080560+0000","last_active":"2023-10-09T10:04:38.218041+0000","last_peered":"2023-10-09T10:04:38.218041+0000","last_clean":"2023-10-09T10:04:38.218041+0000","last_became_active":"2023-10-09T10:03:30.080189+0000","last_became_peered":"2023-10-09T10:03:30.080189+0000","last_unstale":"2023-10-09T10:04:38.218041+0000","last_undegraded":"2023-10-09T10:04:38.218041+0000","last_fullsized":"2023-10-09T10:04:38.218041+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:51.220323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.210160+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:04:38.210160+0000","last_peered":"2023-10-09T10:04:38.210160+0000","last_clean":"2023-10-09T10:04:38.210160+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:04:38.210160+0000","last_undegraded":"2023-10-09T10:04:38.210160+0000","last_fullsized":"2023-10-09T10:04:38.210160+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"208'5925","reported_seq":5955,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:42.533683+0000","last_change":"2023-10-09T10:03:22.356605+0000","last_active":"2023-10-09T10:04:42.533683+0000","last_peered":"2023-10-09T10:04:42.533683+0000","last_clean":"2023-10-09T10:04:42.533683+0000","last_became_active":"2023-10-09T10:03:22.356305+0000","last_became_peered":"2023-10-09T10:03:22.356305+0000","last_unstale":"2023-10-09T10:04:42.533683+0000","last_undegraded":"2023-10-09T10:04:42.533683+0000","last_fullsized":"2023-10-09T10:04:42.533683+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":5925,"log_dups_size":0,"ondisk_log_size":5925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:30.705942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388300800,"num_objects":5925,"num_object_clones":0,"num_object_copies":17775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5925,"num_write_kb":379200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":202,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.213903+0000","last_change":"2023-10-09T10:02:29.972836+0000","last_active":"2023-10-09T10:04:38.213903+0000","last_peered":"2023-10-09T10:04:38.213903+0000","last_clean":"2023-10-09T10:04:38.213903+0000","last_became_active":"2023-10-09T10:02:29.888134+0000","last_became_peered":"2023-10-09T10:02:29.888134+0000","last_unstale":"2023-10-09T10:04:38.213903+0000","last_undegraded":"2023-10-09T10:04:38.213903+0000","last_fullsized":"2023-10-09T10:04:38.213903+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:22.244154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":166,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218468+0000","last_change":"2023-10-09T10:02:17.723233+0000","last_active":"2023-10-09T10:04:38.218468+0000","last_peered":"2023-10-09T10:04:38.218468+0000","last_clean":"2023-10-09T10:04:38.218468+0000","last_became_active":"2023-10-09T10:02:17.722944+0000","last_became_peered":"2023-10-09T10:02:17.722944+0000","last_unstale":"2023-10-09T10:04:38.218468+0000","last_undegraded":"2023-10-09T10:04:38.218468+0000","last_fullsized":"2023-10-09T10:04:38.218468+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:56.963212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":191,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.212554+0000","last_change":"2023-10-09T10:02:29.886514+0000","last_active":"2023-10-09T10:04:38.212554+0000","last_peered":"2023-10-09T10:04:38.212554+0000","last_clean":"2023-10-09T10:04:38.212554+0000","last_became_active":"2023-10-09T10:02:29.885706+0000","last_became_peered":"2023-10-09T10:02:29.885706+0000","last_unstale":"2023-10-09T10:04:38.212554+0000","last_undegraded":"2023-10-09T10:04:38.212554+0000","last_fullsized":"2023-10-09T10:04:38.212554+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:28:06.195775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"208'6058","reported_seq":6088,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:42.581378+0000","last_change":"2023-10-09T10:03:22.032464+0000","last_active":"2023-10-09T10:04:42.581378+0000","last_peered":"2023-10-09T10:04:42.581378+0000","last_clean":"2023-10-09T10:04:42.581378+0000","last_became_active":"2023-10-09T10:03:22.031750+0000","last_became_peered":"2023-10-09T10:03:22.031750+0000","last_unstale":"2023-10-09T10:04:42.581378+0000","last_undegraded":"2023-10-09T10:04:42.581378+0000","last_fullsized":"2023-10-09T10:04:42.581378+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6058,"log_dups_size":0,"ondisk_log_size":6058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:23.160842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397017088,"num_objects":6058,"num_object_clones":0,"num_object_copies":18174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6058,"num_write_kb":387712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":33,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.213708+0000","last_change":"2023-10-09T10:03:19.760664+0000","last_active":"2023-10-09T10:04:38.213708+0000","last_peered":"2023-10-09T10:04:38.213708+0000","last_clean":"2023-10-09T10:04:38.213708+0000","last_became_active":"2023-10-09T10:03:19.760358+0000","last_became_peered":"2023-10-09T10:03:19.760358+0000","last_unstale":"2023-10-09T10:04:38.213708+0000","last_undegraded":"2023-10-09T10:04:38.213708+0000","last_fullsized":"2023-10-09T10:04:38.213708+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:35.382690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":62,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274994+0000","last_change":"2023-10-09T10:02:29.922084+0000","last_active":"2023-10-09T10:04:41.274994+0000","last_peered":"2023-10-09T10:04:41.274994+0000","last_clean":"2023-10-09T10:04:41.274994+0000","last_became_active":"2023-10-09T10:02:29.886231+0000","last_became_peered":"2023-10-09T10:02:29.886231+0000","last_unstale":"2023-10-09T10:04:41.274994+0000","last_undegraded":"2023-10-09T10:04:41.274994+0000","last_fullsized":"2023-10-09T10:04:41.274994+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:02:16.587891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220440+0000","last_change":"2023-10-09T10:03:30.090128+0000","last_active":"2023-10-09T10:04:38.220440+0000","last_peered":"2023-10-09T10:04:38.220440+0000","last_clean":"2023-10-09T10:04:38.220440+0000","last_became_active":"2023-10-09T10:03:30.089771+0000","last_became_peered":"2023-10-09T10:03:30.089771+0000","last_unstale":"2023-10-09T10:04:38.220440+0000","last_undegraded":"2023-10-09T10:04:38.220440+0000","last_fullsized":"2023-10-09T10:04:38.220440+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":56,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.210018+0000","last_change":"2023-10-09T10:02:29.889587+0000","last_active":"2023-10-09T10:04:38.210018+0000","last_peered":"2023-10-09T10:04:38.210018+0000","last_clean":"2023-10-09T10:04:38.210018+0000","last_became_active":"2023-10-09T10:02:29.888869+0000","last_became_peered":"2023-10-09T10:02:29.888869+0000","last_unstale":"2023-10-09T10:04:38.210018+0000","last_undegraded":"2023-10-09T10:04:38.210018+0000","last_fullsized":"2023-10-09T10:04:38.210018+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:22.220539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":124,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218061+0000","last_change":"2023-10-09T10:03:19.762707+0000","last_active":"2023-10-09T10:04:38.218061+0000","last_peered":"2023-10-09T10:04:38.218061+0000","last_clean":"2023-10-09T10:04:38.218061+0000","last_became_active":"2023-10-09T10:03:19.762140+0000","last_became_peered":"2023-10-09T10:03:19.762140+0000","last_unstale":"2023-10-09T10:04:38.218061+0000","last_undegraded":"2023-10-09T10:04:38.218061+0000","last_fullsized":"2023-10-09T10:04:38.218061+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:03.706004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":30,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217999+0000","last_change":"2023-10-09T10:03:30.094629+0000","last_active":"2023-10-09T10:04:38.217999+0000","last_peered":"2023-10-09T10:04:38.217999+0000","last_clean":"2023-10-09T10:04:38.217999+0000","last_became_active":"2023-10-09T10:03:30.082732+0000","last_became_peered":"2023-10-09T10:03:30.082732+0000","last_unstale":"2023-10-09T10:04:38.217999+0000","last_undegraded":"2023-10-09T10:04:38.217999+0000","last_fullsized":"2023-10-09T10:04:38.217999+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220794+0000","last_change":"2023-10-09T10:02:17.743821+0000","last_active":"2023-10-09T10:04:38.220794+0000","last_peered":"2023-10-09T10:04:38.220794+0000","last_clean":"2023-10-09T10:04:38.220794+0000","last_became_active":"2023-10-09T10:02:17.743215+0000","last_became_peered":"2023-10-09T10:02:17.743215+0000","last_unstale":"2023-10-09T10:04:38.220794+0000","last_undegraded":"2023-10-09T10:04:38.220794+0000","last_fullsized":"2023-10-09T10:04:38.220794+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":30,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.218098+0000","last_change":"2023-10-09T10:03:30.132633+0000","last_active":"2023-10-09T10:04:38.218098+0000","last_peered":"2023-10-09T10:04:38.218098+0000","last_clean":"2023-10-09T10:04:38.218098+0000","last_became_active":"2023-10-09T10:03:30.092696+0000","last_became_peered":"2023-10-09T10:03:30.092696+0000","last_unstale":"2023-10-09T10:04:38.218098+0000","last_undegraded":"2023-10-09T10:04:38.218098+0000","last_fullsized":"2023-10-09T10:04:38.218098+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":68,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220522+0000","last_change":"2023-10-09T10:02:29.890059+0000","last_active":"2023-10-09T10:04:38.220522+0000","last_peered":"2023-10-09T10:04:38.220522+0000","last_clean":"2023-10-09T10:04:38.220522+0000","last_became_active":"2023-10-09T10:02:29.889678+0000","last_became_peered":"2023-10-09T10:02:29.889678+0000","last_unstale":"2023-10-09T10:04:38.220522+0000","last_undegraded":"2023-10-09T10:04:38.220522+0000","last_fullsized":"2023-10-09T10:04:38.220522+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":211,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.213693+0000","last_change":"2023-10-09T10:03:19.762851+0000","last_active":"2023-10-09T10:04:38.213693+0000","last_peered":"2023-10-09T10:04:38.213693+0000","last_clean":"2023-10-09T10:04:38.213693+0000","last_became_active":"2023-10-09T10:03:19.762439+0000","last_became_peered":"2023-10-09T10:03:19.762439+0000","last_unstale":"2023-10-09T10:04:38.213693+0000","last_undegraded":"2023-10-09T10:04:38.213693+0000","last_fullsized":"2023-10-09T10:04:38.213693+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:50.254919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":250,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.213097+0000","last_change":"2023-10-09T10:03:30.083372+0000","last_active":"2023-10-09T10:04:38.213097+0000","last_peered":"2023-10-09T10:04:38.213097+0000","last_clean":"2023-10-09T10:04:38.213097+0000","last_became_active":"2023-10-09T10:03:30.082995+0000","last_became_peered":"2023-10-09T10:03:30.082995+0000","last_unstale":"2023-10-09T10:04:38.213097+0000","last_undegraded":"2023-10-09T10:04:38.213097+0000","last_fullsized":"2023-10-09T10:04:38.213097+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:27.827812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":108,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.217073+0000","last_change":"2023-10-09T10:03:30.099986+0000","last_active":"2023-10-09T10:04:38.217073+0000","last_peered":"2023-10-09T10:04:38.217073+0000","last_clean":"2023-10-09T10:04:38.217073+0000","last_became_active":"2023-10-09T10:03:30.091487+0000","last_became_peered":"2023-10-09T10:03:30.091487+0000","last_unstale":"2023-10-09T10:04:38.217073+0000","last_undegraded":"2023-10-09T10:04:38.217073+0000","last_fullsized":"2023-10-09T10:04:38.217073+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:48.095705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T10:04:38.220321+0000","last_change":"2023-10-09T10:02:17.720763+0000","last_active":"2023-10-09T10:04:38.220321+0000","last_peered":"2023-10-09T10:04:38.220321+0000","last_clean":"2023-10-09T10:04:38.220321+0000","last_became_active":"2023-10-09T10:02:17.720442+0000","last_became_peered":"2023-10-09T10:02:17.720442+0000","last_unstale":"2023-10-09T10:04:38.220321+0000","last_undegraded":"2023-10-09T10:04:38.220321+0000","last_fullsized":"2023-10-09T10:04:38.220321+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6249840640,"num_objects":95365,"num_object_clones":0,"num_object_copies":286095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":95365,"num_write_kb":6103360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":7602176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18859425792,"data_stored":18859425792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95365,"ondisk_log_size":95365,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8172,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430421,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2638372,"kb_used_data":2332956,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91106780,"statfs":{"total":95995035648,"available":93293342720,"internally_reserved":0,"allocated":2388946944,"data_stored":2388439183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":5.9329999999999998,"5min":2.5310000000000001,"15min":1.24},"min":{"1min":0.252,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":50.591999999999999,"5min":50.591999999999999,"15min":50.591999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":6.2679999999999998,"5min":2.4580000000000002,"15min":1.1839999999999999},"min":{"1min":0.33900000000000002,"5min":0.245,"15min":0.245},"max":{"1min":50.747,"5min":50.747,"15min":50.747},"last":0.81899999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":1.7949999999999999,"15min":0.96199999999999997},"min":{"1min":0.248,"5min":0.246,"15min":0.246},"max":{"1min":3.847,"5min":25.338000000000001,"15min":25.338000000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.4079999999999999,"15min":0.85899999999999999},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.855,"5min":12.996,"15min":12.996},"last":0.48299999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.673,"15min":0.96099999999999997},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.5129999999999999,"5min":16.302,"15min":16.302},"last":0.314},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.8680000000000001,"15min":0.995},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":4.8239999999999998,"5min":13.295,"15min":13.295},"last":1.1499999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:03:59 2023","interfaces":[{"interface":"back","average":{"1min":3.2679999999999998,"5min":2.2330000000000001,"15min":1.1339999999999999},"min":{"1min":0.38300000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":28.614000000000001,"5min":28.614000000000001,"15min":28.614000000000001},"last":59.302999999999997},{"interface":"front","average":{"1min":3.0699999999999998,"5min":2.3809999999999998,"15min":1.1850000000000001},"min":{"1min":0.32000000000000001,"5min":0.312,"15min":0.312},"max":{"1min":28.943000000000001,"5min":28.943000000000001,"15min":28.943000000000001},"last":0.378}]},{"osd":4,"last update":"Mon Oct 9 10:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.85999999999999999,"15min":0.621},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.25900000000000001},"max":{"1min":4.9829999999999997,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.78700000000000003,"15min":0.59599999999999997},"min":{"1min":0.29099999999999998,"5min":0.28899999999999998,"15min":0.26500000000000001},"max":{"1min":4.8579999999999997,"5min":4.8579999999999997,"15min":4.8579999999999997},"last":0.33800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.147,"15min":0.72099999999999997},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.278,"5min":12.958,"15min":12.958},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.88500000000000001,"5min":1.0609999999999999,"15min":0.71399999999999997},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.319,"5min":6.6120000000000001,"15min":6.6120000000000001},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.93899999999999995,"15min":0.68899999999999995},"min":{"1min":0.35099999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.1200000000000001,"5min":5.4630000000000001,"15min":5.4630000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.89900000000000002,"15min":0.67700000000000005},"min":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.33100000000000002},"max":{"1min":5.0810000000000004,"5min":5.0810000000000004,"15min":5.0810000000000004},"last":0.46700000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626649,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2616796,"kb_used_data":2303188,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":91128356,"statfs":{"total":95995035648,"available":93315436544,"internally_reserved":0,"allocated":2358464512,"data_stored":2357965070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.421,"15min":0.86199999999999999},"min":{"1min":0.23300000000000001,"5min":0.222,"15min":0.222},"max":{"1min":3.8279999999999998,"5min":8.8650000000000002,"15min":8.8650000000000002},"last":0.246},{"interface":"front","average":{"1min":2.1240000000000001,"5min":1.5589999999999999,"15min":0.89000000000000001},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.1929999999999996,"5min":12.439,"15min":12.439},"last":0.41199999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.881,"15min":0.99399999999999999},"min":{"1min":0.29299999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.0629999999999997,"5min":18.853000000000002,"15min":18.853000000000002},"last":1.8},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.595,"15min":0.89000000000000001},"min":{"1min":0.30499999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.8500000000000001,"5min":12.380000000000001,"15min":12.380000000000001},"last":0.33400000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":2.1659999999999999,"15min":1.091},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":10.451000000000001,"5min":18.800000000000001,"15min":18.800000000000001},"last":1.012},{"interface":"front","average":{"1min":1.6859999999999999,"5min":2.0640000000000001,"15min":1.0980000000000001},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.2430000000000003,"5min":12.413,"15min":12.413},"last":4.7050000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:04:06 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.8720000000000001,"15min":1.034},"min":{"1min":0.33300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.9219999999999997,"5min":12.337,"15min":12.337},"last":0.437},{"interface":"front","average":{"1min":1.093,"5min":1.6870000000000001,"15min":0.96199999999999997},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.774,"5min":8.0530000000000008,"15min":8.0530000000000008},"last":0.47099999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:04:24 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":1.3280000000000001,"15min":0.75600000000000001},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.246,"5min":23.446000000000002,"15min":23.446000000000002},"last":0.375},{"interface":"front","average":{"1min":0.505,"5min":0.97699999999999998,"15min":0.65600000000000003},"min":{"1min":0.29499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":1.821,"5min":8.2379999999999995,"15min":8.2379999999999995},"last":0.32600000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:04:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.94699999999999995,"15min":0.64800000000000002},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.9930000000000001,"5min":14.09,"15min":14.09},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.91200000000000003,"15min":0.65200000000000002},"min":{"1min":0.311,"5min":0.254,"15min":0.254},"max":{"1min":2.3250000000000002,"5min":6.3540000000000001,"15min":6.3540000000000001},"last":0.42799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.92700000000000005,"15min":0.67300000000000004},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0539999999999998,"5min":5.9089999999999998,"15min":5.9089999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.96999999999999997,"15min":0.73099999999999998},"min":{"1min":0.35199999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.1059999999999999,"5min":6.0860000000000003,"15min":6.0860000000000003},"last":0.44900000000000001}]}]},{"osd":1,"up_from":84,"seq":360777252962,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3287384,"kb_used_data":3066128,"kb_used_omap":9,"kb_used_meta":221238,"kb_avail":90457768,"statfs":{"total":95995035648,"available":92628754432,"internally_reserved":0,"allocated":3139715072,"data_stored":3138921919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":226548713},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":2.1659999999999999,"15min":0.95599999999999996},"min":{"1min":0.219,"5min":0.185,"15min":0.185},"max":{"1min":3.613,"5min":16.324999999999999,"15min":16.324999999999999},"last":8.8000000000000007},{"interface":"front","average":{"1min":1.298,"5min":2.125,"15min":0.96899999999999997},"min":{"1min":0.23000000000000001,"5min":0.192,"15min":0.192},"max":{"1min":3.7530000000000001,"5min":15.573,"15min":15.573},"last":0.28299999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:04:34 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":2.4300000000000002,"15min":1.0880000000000001},"min":{"1min":0.436,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.1079999999999997,"5min":18.109000000000002,"15min":18.109000000000002},"last":0.31900000000000001},{"interface":"front","average":{"1min":3.0630000000000002,"5min":2.7469999999999999,"15min":1.1910000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.248},"max":{"1min":9.5069999999999997,"5min":12.069000000000001,"15min":12.069000000000001},"last":15.484}]},{"osd":3,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":3.133,"15min":1.296},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.8149999999999999,"5min":25.129999999999999,"15min":25.129999999999999},"last":6.0910000000000002},{"interface":"front","average":{"1min":2.2320000000000002,"5min":3.101,"15min":1.296},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.496,"5min":22.847999999999999,"15min":22.847999999999999},"last":16.068999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.0019999999999998,"15min":1.073},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":3.3159999999999998,"5min":11.311999999999999,"15min":11.311999999999999},"last":15.532999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.7410000000000001,"15min":1.0109999999999999},"min":{"1min":0.311,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.6360000000000001,"5min":16.053000000000001,"15min":16.053000000000001},"last":0.375}]},{"osd":5,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":2.1000000000000001,"15min":1.115},"min":{"1min":0.40400000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.4210000000000003,"5min":15.531000000000001,"15min":15.531000000000001},"last":0.501},{"interface":"front","average":{"1min":1.54,"5min":2.4220000000000002,"15min":1.2430000000000001},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.30299999999999999},"max":{"1min":10.090999999999999,"5min":19.706,"15min":19.706},"last":15.605}]},{"osd":6,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.964,"15min":1.1160000000000001},"min":{"1min":0.34699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":3.5369999999999999,"5min":16.193999999999999,"15min":16.193999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.92600000000000005,"5min":2.129,"15min":1.1559999999999999},"min":{"1min":0.40100000000000002,"5min":0.38,"15min":0.32000000000000001},"max":{"1min":3.7850000000000001,"5min":15.481999999999999,"15min":15.481999999999999},"last":0.53600000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.2400000000000002,"15min":1.171},"min":{"1min":0.45600000000000002,"5min":0.379,"15min":0.379},"max":{"1min":5.5170000000000003,"5min":12.766999999999999,"15min":12.766999999999999},"last":15.571999999999999},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.0760000000000001,"15min":1.1519999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.34799999999999998},"max":{"1min":5.3890000000000002,"5min":16.100999999999999,"15min":16.100999999999999},"last":0.439}]}]},{"osd":0,"up_from":79,"seq":339302416488,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3140444,"kb_used_data":2986580,"kb_used_omap":28,"kb_used_meta":153827,"kb_avail":90604708,"statfs":{"total":95995035648,"available":92779220992,"internally_reserved":0,"allocated":3058257920,"data_stored":3057459750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":157519696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:04:02 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.9119999999999999,"15min":0.88800000000000001},"min":{"1min":0.26700000000000002,"5min":0.20300000000000001,"15min":0.191},"max":{"1min":3.0619999999999998,"5min":11.73,"15min":11.73},"last":0.23000000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":2.1389999999999998,"15min":0.97699999999999998},"min":{"1min":0.23899999999999999,"5min":0.224,"15min":0.224},"max":{"1min":4.9109999999999996,"5min":14.728,"15min":14.728},"last":1.0449999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":2.5939999999999999,"15min":1.1359999999999999},"min":{"1min":0.23999999999999999,"5min":0.224,"15min":0.216},"max":{"1min":5.4059999999999997,"5min":38.575000000000003,"15min":38.575000000000003},"last":0.31},{"interface":"front","average":{"1min":2.21,"5min":3.194,"15min":1.3580000000000001},"min":{"1min":0.27400000000000002,"5min":0.27200000000000002,"15min":0.23899999999999999},"max":{"1min":8.3640000000000008,"5min":38.332000000000001,"15min":38.332000000000001},"last":1.075}]},{"osd":3,"last update":"Mon Oct 9 10:04:02 2023","interfaces":[{"interface":"back","average":{"1min":4.5430000000000001,"5min":4.0890000000000004,"15min":1.617},"min":{"1min":0.30099999999999999,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":16.745000000000001,"5min":59.524999999999999,"15min":59.524999999999999},"last":5.21},{"interface":"front","average":{"1min":2.7200000000000002,"5min":4.0199999999999996,"15min":1.633},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":8.4090000000000007,"5min":56.609999999999999,"15min":56.609999999999999},"last":6.5019999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.472,"15min":0.93100000000000005},"min":{"1min":0.35599999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.266,"5min":11.587999999999999,"15min":11.587999999999999},"last":0.435},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.0680000000000001,"15min":1.145},"min":{"1min":0.34599999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.5039999999999996,"5min":14.653,"15min":14.653},"last":1.141}]},{"osd":5,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":2.0459999999999998,"15min":1.123},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":6.1920000000000002,"5min":10.837,"15min":10.837},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.667,"15min":0.996},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.312},"max":{"1min":2.9580000000000002,"5min":11.667,"15min":11.667},"last":0.45600000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":2.0129999999999999,"15min":1.1240000000000001},"min":{"1min":0.48399999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.5170000000000003,"5min":11.699,"15min":11.699},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.198,"5min":2.52,"15min":1.3},"min":{"1min":0.46400000000000002,"5min":0.379,"15min":0.379},"max":{"1min":6.4530000000000003,"5min":15.108000000000001,"15min":15.108000000000001},"last":1.1919999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":2.278,"15min":1.2110000000000001},"min":{"1min":0.34300000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.3700000000000001,"5min":26.978000000000002,"15min":26.978000000000002},"last":1.165},{"interface":"front","average":{"1min":1.5740000000000001,"5min":2.2709999999999999,"15min":1.2110000000000001},"min":{"1min":0.41799999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":5.665,"5min":26.768000000000001,"15min":26.768000000000001},"last":0.70399999999999996}]}]},{"osd":2,"up_from":0,"seq":382252089435,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":94,"seq":403726925908,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4526924,"kb_used_data":4158724,"kb_used_omap":7,"kb_used_meta":368184,"kb_avail":89218228,"statfs":{"total":95995035648,"available":91359465472,"internally_reserved":0,"allocated":4258533376,"data_stored":4257693229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":377021032},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":2.5579999999999998,"15min":1.516},"min":{"1min":0.26600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.1900000000000004,"5min":11.925000000000001,"15min":11.958},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.03,"5min":2.8130000000000002,"15min":1.238},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.2670000000000003,"5min":21.285,"15min":21.285},"last":1.052}]},{"osd":1,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":2.6589999999999998,"15min":1.165},"min":{"1min":0.25900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":5.8310000000000004,"5min":21.341999999999999,"15min":21.341999999999999},"last":3.585},{"interface":"front","average":{"1min":2.1789999999999998,"5min":3.1179999999999999,"15min":1.3160000000000001},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.2919999999999998,"5min":15.59,"15min":15.59},"last":0.34699999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":2.8730000000000002,"15min":1.655},"min":{"1min":0.32500000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.0720000000000001,"5min":15.632999999999999,"15min":15.632999999999999},"last":9.3450000000000006},{"interface":"front","average":{"1min":1.452,"5min":2.794,"15min":1.236},"min":{"1min":0.29699999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.6660000000000004,"5min":11.879,"15min":11.879},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.6830000000000001,"15min":0.99099999999999999},"min":{"1min":0.374,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.4080000000000004,"5min":11.976000000000001,"15min":11.976000000000001},"last":0.37},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.952,"15min":1.109},"min":{"1min":0.35999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":4.3460000000000001,"5min":21.061,"15min":21.061},"last":0.42599999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.0390000000000001,"15min":1.139},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.3010000000000002,"5min":21.015000000000001,"15min":21.015000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.296,"5min":2.161,"15min":1.4219999999999999},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.1340000000000003,"5min":10.127000000000001,"15min":10.127000000000001},"last":5.5650000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":2.5579999999999998,"15min":1.3089999999999999},"min":{"1min":0.38800000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.3239999999999998,"5min":21.317,"15min":21.317},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.266,"5min":2.4289999999999998,"15min":1.514},"min":{"1min":0.41699999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":7.1100000000000003,"5min":12.106999999999999,"15min":12.106999999999999},"last":5.6120000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.2269999999999999,"15min":1.4379999999999999},"min":{"1min":0.42899999999999999,"5min":0.307,"15min":0.307},"max":{"1min":7.1559999999999997,"5min":9.4719999999999995,"15min":9.4719999999999995},"last":5.6500000000000004},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.9830000000000001,"15min":1.0980000000000001},"min":{"1min":0.442,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":4.548,"5min":11.951000000000001,"15min":11.951000000000001},"last":0.53400000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019105,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2224348,"kb_used_data":1918932,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91520804,"statfs":{"total":95995035648,"available":93717303296,"internally_reserved":0,"allocated":1964986368,"data_stored":1964486185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.448,"15min":0.84899999999999998},"min":{"1min":0.23799999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":31.059000000000001,"5min":31.059000000000001,"15min":31.059000000000001},"last":2.3610000000000002},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.7070000000000001,"15min":0.96099999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":31.279,"5min":31.279,"15min":31.279},"last":3.137}]},{"osd":1,"last update":"Mon Oct 9 10:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.4370000000000001,"15min":0.82499999999999996},"min":{"1min":0.28999999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":13.898999999999999,"5min":13.898999999999999,"15min":13.898999999999999},"last":2.4470000000000001},{"interface":"front","average":{"1min":1.508,"5min":1.7569999999999999,"15min":0.96899999999999997},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":3.3460000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":1.7130000000000001,"15min":0.95199999999999996},"min":{"1min":0.29999999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":9.5709999999999997,"5min":9.5709999999999997,"15min":9.5709999999999997},"last":1.7010000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.607,"15min":0.92500000000000004},"min":{"1min":0.37,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.8919999999999999,"5min":11.999000000000001,"15min":11.999000000000001},"last":3.1589999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":2.7799999999999998,"15min":1.274},"min":{"1min":0.376,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.008,"5min":67.156999999999996,"15min":67.156999999999996},"last":3.5329999999999999},{"interface":"front","average":{"1min":1.1759999999999999,"5min":2.331,"15min":1.139},"min":{"1min":0.312,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.7850000000000001,"5min":70.238,"15min":70.238},"last":4.5259999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:04:10 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.91600000000000004,"15min":0.60999999999999999},"min":{"1min":0.27200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.4420000000000002,"5min":12.970000000000001,"15min":12.970000000000001},"last":3.0870000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.0389999999999999,"15min":0.67300000000000004},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.9199999999999999,"5min":14.773,"15min":14.773},"last":1.6719999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.72399999999999998,"15min":0.57699999999999996},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":2.6030000000000002,"5min":9.4079999999999995,"15min":9.4079999999999995},"last":2.391},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.028,"15min":0.69599999999999995},"min":{"1min":0.34000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.6600000000000001,"5min":9.0109999999999992,"15min":9.0109999999999992},"last":3.117}]},{"osd":7,"last update":"Mon Oct 9 10:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.028,"15min":0.70799999999999996},"min":{"1min":0.38800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.5509999999999999,"5min":14.81,"15min":14.81},"last":1.649},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.66000000000000003,"15min":0.60499999999999998},"min":{"1min":0.372,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.577,"5min":4.0350000000000001,"15min":4.0350000000000001},"last":2.323}]}]},{"osd":5,"up_from":36,"seq":154618822877,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1860428,"kb_used_data":1550916,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91884724,"statfs":{"total":95995035648,"available":94089957376,"internally_reserved":0,"allocated":1588137984,"data_stored":1587649929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.8520000000000001,"15min":0.996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":6.0540000000000003,"5min":15.773999999999999,"15min":15.773999999999999},"last":2.1560000000000001},{"interface":"front","average":{"1min":1.4179999999999999,"5min":2.1960000000000002,"15min":1.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":7.0259999999999998,"5min":15.699,"15min":15.699},"last":3.1970000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:03:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.04,"15min":1.054},"min":{"1min":0.254,"5min":0.248,"15min":0.248},"max":{"1min":5.5330000000000004,"5min":15.689,"15min":15.689},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.016,"5min":2.0960000000000001,"15min":1.0760000000000001},"min":{"1min":0.30199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8490000000000002,"5min":15.896000000000001,"15min":15.896000000000001},"last":5.3440000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:04:35 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.7090000000000001,"15min":1.0349999999999999},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8200000000000003,"5min":12.582000000000001,"15min":12.582000000000001},"last":1.899},{"interface":"front","average":{"1min":1.764,"5min":2.0830000000000002,"15min":1.1619999999999999},"min":{"1min":0.36899999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.8490000000000002,"5min":12.438000000000001,"15min":12.438000000000001},"last":3.226}]},{"osd":3,"last update":"Mon Oct 9 10:04:02 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":2.351,"15min":1.2370000000000001},"min":{"1min":0.374,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.1689999999999996,"5min":39.345999999999997,"15min":39.345999999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.7909999999999999,"5min":3.044,"15min":1.4450000000000001},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.081,"5min":38.906999999999996,"15min":38.906999999999996},"last":3.7010000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:04:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.151,"15min":0.69699999999999995},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.4830000000000001,"5min":12.304,"15min":12.304},"last":3.1539999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":1.2549999999999999,"15min":0.746},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.224,"5min":27.768000000000001,"15min":27.768000000000001},"last":3.0019999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.139,"15min":0.73899999999999999},"min":{"1min":0.30099999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.694,"5min":27.716000000000001,"15min":27.716000000000001},"last":2.8740000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.85499999999999998,"15min":0.65500000000000003},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.28499999999999998},"max":{"1min":0.98299999999999998,"5min":10.173999999999999,"15min":10.173999999999999},"last":3.3439999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.94599999999999995,"15min":0.71199999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.81399999999999995,"5min":10.334,"15min":10.334},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.302,"15min":0.81100000000000005},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.317},"max":{"1min":3.3889999999999998,"5min":12.356,"15min":12.356},"last":3.3889999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3071148032,"data_stored":3071148032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3163291648,"data_stored":3163291648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4289331200,"data_stored":4289331200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1968635904,"data_stored":1968635904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1596719104,"data_stored":1596719104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2376663040,"data_stored":2376663040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2393636864,"data_stored":2393636864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:04:47.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:04:47.279 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:47.368 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:47.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:47.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:47.478 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:47.519 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:47.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:47.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:47.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:47.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:04:47.668 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:47.703 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:47.745 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:47.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:47.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:47.785 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:47 smithi110 ceph-mon[136032]: from='client.35259 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:04:47.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:47.918 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:47.929 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:47.974 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 79 16 99252 99236 78.4485 47.6875 0.0346473 0.0127413 2023-10-09T10:04:48.010 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:48.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:48.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:47 smithi012 ceph-mon[147926]: from='client.35259 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:04:48.018 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:04:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:04:47] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T10:04:48.018 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:47 smithi012 ceph-mon[152263]: from='client.35259 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:04:48.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:48.120 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:48.202 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:48.562 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:48.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:48.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:48.651 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:48.670 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:48.699 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:48.739 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:48.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:48.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:48.878 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:48.923 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:48.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:48.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:48 smithi012 ceph-mon[147926]: pgmap v520: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T10:04:48.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:48 smithi012 ceph-mon[152263]: pgmap v520: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T10:04:48.980 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:04:48.977345+0000 min lat: 0.00114602 max lat: 1.02658 avg lat: 0.0128035 2023-10-09T10:04:48.980 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:04:48.980 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 80 16 100033 100017 78.0732 48.8125 0.027338 0.0128035 2023-10-09T10:04:49.031 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:49.061 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:48 smithi110 ceph-mon[136032]: pgmap v520: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T10:04:49.066 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:49.113 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:49.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:49.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:49.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:49.332 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:49.510 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:49.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:49.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:49.618 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:49.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:49.685 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:49.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:49.790 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:04:49.858 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:04:49.879 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:49.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3205061283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:49.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:49 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:49.980 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 81 16 100816 100800 77.7103 48.9375 0.0425196 0.01286 2023-10-09T10:04:50.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3205061283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:50.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:49 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:50.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3205061283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:50.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:49 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:04:50.980 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 82 16 101618 101602 77.3741 50.125 0.00816744 0.0129198 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: pgmap v521: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[152263]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: pgmap v521: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:50 smithi012 ceph-mon[147926]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: pgmap v521: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T10:04:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:04:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:04:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:04:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T10:04:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:50 smithi110 ceph-mon[136032]: failed to encode map e209 with expected crc 2023-10-09T10:04:51.981 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 83 16 102535 102519 77.1325 57.3125 0.0197114 0.0129591 2023-10-09T10:04:52.149 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:51 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:04:52.152 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:51 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:04:52.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:51 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:04:52.982 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 84 16 103446 103430 76.8907 56.9375 0.0231221 0.0130004 2023-10-09T10:04:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:52 smithi012 ceph-mon[147926]: pgmap v523: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T10:04:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:52 smithi012 ceph-mon[152263]: pgmap v523: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T10:04:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:53.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:52 smithi110 ceph-mon[136032]: pgmap v523: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T10:04:53.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:53.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:53.982 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 85 16 104272 104256 76.5936 51.625 0.0142534 0.0130508 2023-10-09T10:04:53.997 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:04:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:54.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:04:54.122 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:54.169 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:54.214 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:54.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:54.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:54.246 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:54.317 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:54.370 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:54.418 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:54.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:54.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:54.523 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:54.555 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:54.603 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:54.891 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:54 smithi012 ceph-mon[152263]: pgmap v524: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T10:04:54.891 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:54 smithi012 ceph-mon[147926]: pgmap v524: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T10:04:54.891 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:54 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:54.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:54.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:54.982 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 86 16 105006 104990 76.2366 45.875 0.0163829 0.013112 2023-10-09T10:04:54.993 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:55.006 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:04:55.006 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:04:55.006 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:04:55.007 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:04:55.007 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:04:55.032 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:55.085 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:55.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:55.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:55.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:54 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:55.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:54 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:55.128 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:54 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:55.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:54 smithi110 ceph-mon[136032]: pgmap v524: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-09T10:04:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:54 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:55.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:54 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:55.291 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:55.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:04:55.692 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:55.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:55.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:55.917 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:55.986 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 87 16 105750 105734 75.8923 46.5 0.0270968 0.0131717 2023-10-09T10:04:56.097 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:56.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:56.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:56.113 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:04:56.137 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:04:56.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:04:56.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:56.314 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:56.410 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:56.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:56.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:56.515 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:04:56.557 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:56.596 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:04:56.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:56.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:04:56.700 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:56.734 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:56.826 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:56.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:56.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:56.839 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":525,"stamp":"2023-10-09T10:04:55.120311+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6783178656,"num_objects":103518,"num_object_clones":0,"num_object_copies":310554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103518,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":103754,"num_write_kb":6626061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":179,"num_bytes_recovered":8791084,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103688,"ondisk_log_size":103688,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":21857196,"kb_used_data":19904500,"kb_used_omap":44,"kb_used_meta":1952595,"kb_avail":634358868,"statfs":{"total":671965249536,"available":649583480832,"internally_reserved":0,"allocated":20382208000,"data_stored":20377767432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1999457697},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,2,6,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":532742144,"num_objects":8129,"num_object_clones":0,"num_object_copies":24387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8129,"num_write_kb":520256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003635"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":69,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.285751+0000","last_change":"2023-10-09T10:03:19.763993+0000","last_active":"2023-10-09T10:04:41.285751+0000","last_peered":"2023-10-09T10:04:41.285751+0000","last_clean":"2023-10-09T10:04:41.285751+0000","last_became_active":"2023-10-09T10:03:19.763623+0000","last_became_peered":"2023-10-09T10:03:19.763623+0000","last_unstale":"2023-10-09T10:04:41.285751+0000","last_undegraded":"2023-10-09T10:04:41.285751+0000","last_fullsized":"2023-10-09T10:04:41.285751+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":196,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274114+0000","last_change":"2023-10-09T10:02:17.718080+0000","last_active":"2023-10-09T10:04:41.274114+0000","last_peered":"2023-10-09T10:04:41.274114+0000","last_clean":"2023-10-09T10:04:41.274114+0000","last_became_active":"2023-10-09T10:02:17.717746+0000","last_became_peered":"2023-10-09T10:02:17.717746+0000","last_unstale":"2023-10-09T10:04:41.274114+0000","last_undegraded":"2023-10-09T10:04:41.274114+0000","last_fullsized":"2023-10-09T10:04:41.274114+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:59.375104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":183,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.268970+0000","last_change":"2023-10-09T10:02:17.740018+0000","last_active":"2023-10-09T10:04:41.268970+0000","last_peered":"2023-10-09T10:04:41.268970+0000","last_clean":"2023-10-09T10:04:41.268970+0000","last_became_active":"2023-10-09T10:02:17.739691+0000","last_became_peered":"2023-10-09T10:02:17.739691+0000","last_unstale":"2023-10-09T10:04:41.268970+0000","last_undegraded":"2023-10-09T10:04:41.268970+0000","last_fullsized":"2023-10-09T10:04:41.268970+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.836600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":58,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274907+0000","last_change":"2023-10-09T10:02:29.904188+0000","last_active":"2023-10-09T10:04:41.274907+0000","last_peered":"2023-10-09T10:04:41.274907+0000","last_clean":"2023-10-09T10:04:41.274907+0000","last_became_active":"2023-10-09T10:02:29.899658+0000","last_became_peered":"2023-10-09T10:02:29.899658+0000","last_unstale":"2023-10-09T10:04:41.274907+0000","last_undegraded":"2023-10-09T10:04:41.274907+0000","last_fullsized":"2023-10-09T10:04:41.274907+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:35.455601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":212,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.272270+0000","last_change":"2023-10-09T10:02:29.885613+0000","last_active":"2023-10-09T10:04:41.272270+0000","last_peered":"2023-10-09T10:04:41.272270+0000","last_clean":"2023-10-09T10:04:41.272270+0000","last_became_active":"2023-10-09T10:02:29.885185+0000","last_became_peered":"2023-10-09T10:02:29.885185+0000","last_unstale":"2023-10-09T10:04:41.272270+0000","last_undegraded":"2023-10-09T10:04:41.272270+0000","last_fullsized":"2023-10-09T10:04:41.272270+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:01:44.101600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":27,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.278922+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:04:41.278922+0000","last_peered":"2023-10-09T10:04:41.278922+0000","last_clean":"2023-10-09T10:04:41.278922+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:04:41.278922+0000","last_undegraded":"2023-10-09T10:04:41.278922+0000","last_fullsized":"2023-10-09T10:04:41.278922+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":75,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.275419+0000","last_change":"2023-10-09T10:03:30.080811+0000","last_active":"2023-10-09T10:04:41.275419+0000","last_peered":"2023-10-09T10:04:41.275419+0000","last_clean":"2023-10-09T10:04:41.275419+0000","last_became_active":"2023-10-09T10:03:30.080435+0000","last_became_peered":"2023-10-09T10:03:30.080435+0000","last_unstale":"2023-10-09T10:04:41.275419+0000","last_undegraded":"2023-10-09T10:04:41.275419+0000","last_fullsized":"2023-10-09T10:04:41.275419+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:31.031546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":179,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:47.593446+0000","last_change":"2023-10-09T09:57:49.624665+0000","last_active":"2023-10-09T10:04:47.593446+0000","last_peered":"2023-10-09T10:04:47.593446+0000","last_clean":"2023-10-09T10:04:47.593446+0000","last_became_active":"2023-10-09T09:57:49.623883+0000","last_became_peered":"2023-10-09T09:57:49.623883+0000","last_unstale":"2023-10-09T10:04:47.593446+0000","last_undegraded":"2023-10-09T10:04:47.593446+0000","last_fullsized":"2023-10-09T10:04:47.593446+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:14.598420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":27,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.277498+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:04:41.277498+0000","last_peered":"2023-10-09T10:04:41.277498+0000","last_clean":"2023-10-09T10:04:41.277498+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:04:41.277498+0000","last_undegraded":"2023-10-09T10:04:41.277498+0000","last_fullsized":"2023-10-09T10:04:41.277498+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":140,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:47.593475+0000","last_change":"2023-10-09T09:57:49.625511+0000","last_active":"2023-10-09T10:04:47.593475+0000","last_peered":"2023-10-09T10:04:47.593475+0000","last_clean":"2023-10-09T10:04:47.593475+0000","last_became_active":"2023-10-09T09:57:49.624960+0000","last_became_peered":"2023-10-09T09:57:49.624960+0000","last_unstale":"2023-10-09T10:04:47.593475+0000","last_undegraded":"2023-10-09T10:04:47.593475+0000","last_fullsized":"2023-10-09T10:04:47.593475+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:59.374794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":222,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.282335+0000","last_change":"2023-10-09T10:02:18.222658+0000","last_active":"2023-10-09T10:04:41.282335+0000","last_peered":"2023-10-09T10:04:41.282335+0000","last_clean":"2023-10-09T10:04:41.282335+0000","last_became_active":"2023-10-09T10:02:17.734984+0000","last_became_peered":"2023-10-09T10:02:17.734984+0000","last_unstale":"2023-10-09T10:04:41.282335+0000","last_undegraded":"2023-10-09T10:04:41.282335+0000","last_fullsized":"2023-10-09T10:04:41.282335+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":192,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.268956+0000","last_change":"2023-10-09T10:03:19.764825+0000","last_active":"2023-10-09T10:04:41.268956+0000","last_peered":"2023-10-09T10:04:41.268956+0000","last_clean":"2023-10-09T10:04:41.268956+0000","last_became_active":"2023-10-09T10:03:19.764466+0000","last_became_peered":"2023-10-09T10:03:19.764466+0000","last_unstale":"2023-10-09T10:04:41.268956+0000","last_undegraded":"2023-10-09T10:04:41.268956+0000","last_fullsized":"2023-10-09T10:04:41.268956+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:37.631717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":70,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286560+0000","last_change":"2023-10-09T10:02:18.088005+0000","last_active":"2023-10-09T10:04:41.286560+0000","last_peered":"2023-10-09T10:04:41.286560+0000","last_clean":"2023-10-09T10:04:41.286560+0000","last_became_active":"2023-10-09T10:02:17.722407+0000","last_became_peered":"2023-10-09T10:02:17.722407+0000","last_unstale":"2023-10-09T10:04:41.286560+0000","last_undegraded":"2023-10-09T10:04:41.286560+0000","last_fullsized":"2023-10-09T10:04:41.286560+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286143+0000","last_change":"2023-10-09T10:02:17.731192+0000","last_active":"2023-10-09T10:04:41.286143+0000","last_peered":"2023-10-09T10:04:41.286143+0000","last_clean":"2023-10-09T10:04:41.286143+0000","last_became_active":"2023-10-09T10:02:17.730757+0000","last_became_peered":"2023-10-09T10:02:17.730757+0000","last_unstale":"2023-10-09T10:04:41.286143+0000","last_undegraded":"2023-10-09T10:04:41.286143+0000","last_fullsized":"2023-10-09T10:04:41.286143+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":251,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.282126+0000","last_change":"2023-10-09T10:03:19.765719+0000","last_active":"2023-10-09T10:04:41.282126+0000","last_peered":"2023-10-09T10:04:41.282126+0000","last_clean":"2023-10-09T10:04:41.282126+0000","last_became_active":"2023-10-09T10:03:19.761432+0000","last_became_peered":"2023-10-09T10:03:19.761432+0000","last_unstale":"2023-10-09T10:04:41.282126+0000","last_undegraded":"2023-10-09T10:04:41.282126+0000","last_fullsized":"2023-10-09T10:04:41.282126+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":188,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:49.816587+0000","last_change":"2023-10-09T10:04:00.899350+0000","last_active":"2023-10-09T10:04:49.816587+0000","last_peered":"2023-10-09T10:04:49.816587+0000","last_clean":"2023-10-09T10:04:49.816587+0000","last_became_active":"2023-10-09T10:04:00.899001+0000","last_became_peered":"2023-10-09T10:04:00.899001+0000","last_unstale":"2023-10-09T10:04:49.816587+0000","last_undegraded":"2023-10-09T10:04:49.816587+0000","last_fullsized":"2023-10-09T10:04:49.816587+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:49.346314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:49.821506+0000","last_change":"2023-10-09T10:02:29.904731+0000","last_active":"2023-10-09T10:04:49.821506+0000","last_peered":"2023-10-09T10:04:49.821506+0000","last_clean":"2023-10-09T10:04:49.821506+0000","last_became_active":"2023-10-09T10:02:29.904290+0000","last_became_peered":"2023-10-09T10:02:29.904290+0000","last_unstale":"2023-10-09T10:04:49.821506+0000","last_undegraded":"2023-10-09T10:04:49.821506+0000","last_fullsized":"2023-10-09T10:04:49.821506+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:00.376061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":27,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.280439+0000","last_change":"2023-10-09T10:03:30.082426+0000","last_active":"2023-10-09T10:04:41.280439+0000","last_peered":"2023-10-09T10:04:41.280439+0000","last_clean":"2023-10-09T10:04:41.280439+0000","last_became_active":"2023-10-09T10:03:30.082112+0000","last_became_peered":"2023-10-09T10:03:30.082112+0000","last_unstale":"2023-10-09T10:04:41.280439+0000","last_undegraded":"2023-10-09T10:04:41.280439+0000","last_fullsized":"2023-10-09T10:04:41.280439+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286024+0000","last_change":"2023-10-09T10:02:17.730673+0000","last_active":"2023-10-09T10:04:41.286024+0000","last_peered":"2023-10-09T10:04:41.286024+0000","last_clean":"2023-10-09T10:04:41.286024+0000","last_became_active":"2023-10-09T10:02:17.729848+0000","last_became_peered":"2023-10-09T10:02:17.729848+0000","last_unstale":"2023-10-09T10:04:41.286024+0000","last_undegraded":"2023-10-09T10:04:41.286024+0000","last_fullsized":"2023-10-09T10:04:41.286024+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":57,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274702+0000","last_change":"2023-10-09T10:02:29.900023+0000","last_active":"2023-10-09T10:04:41.274702+0000","last_peered":"2023-10-09T10:04:41.274702+0000","last_clean":"2023-10-09T10:04:41.274702+0000","last_became_active":"2023-10-09T10:02:29.899702+0000","last_became_peered":"2023-10-09T10:02:29.899702+0000","last_unstale":"2023-10-09T10:04:41.274702+0000","last_undegraded":"2023-10-09T10:04:41.274702+0000","last_fullsized":"2023-10-09T10:04:41.274702+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:39.396593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":96,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:49.821365+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:04:49.821365+0000","last_peered":"2023-10-09T10:04:49.821365+0000","last_clean":"2023-10-09T10:04:49.821365+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:04:49.821365+0000","last_undegraded":"2023-10-09T10:04:49.821365+0000","last_fullsized":"2023-10-09T10:04:49.821365+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":178,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274960+0000","last_change":"2023-10-09T09:57:49.623806+0000","last_active":"2023-10-09T10:04:41.274960+0000","last_peered":"2023-10-09T10:04:41.274960+0000","last_clean":"2023-10-09T10:04:41.274960+0000","last_became_active":"2023-10-09T09:57:49.621759+0000","last_became_peered":"2023-10-09T09:57:49.621759+0000","last_unstale":"2023-10-09T10:04:41.274960+0000","last_undegraded":"2023-10-09T10:04:41.274960+0000","last_fullsized":"2023-10-09T10:04:41.274960+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:42.732361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"209'6573","reported_seq":6683,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:54.231434+0000","last_change":"2023-10-09T10:03:30.261125+0000","last_active":"2023-10-09T10:04:54.231434+0000","last_peered":"2023-10-09T10:04:54.231434+0000","last_clean":"2023-10-09T10:04:54.231434+0000","last_became_active":"2023-10-09T10:03:30.095093+0000","last_became_peered":"2023-10-09T10:03:30.095093+0000","last_unstale":"2023-10-09T10:04:54.231434+0000","last_undegraded":"2023-10-09T10:04:54.231434+0000","last_fullsized":"2023-10-09T10:04:54.231434+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:57.462339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430768128,"num_objects":6573,"num_object_clones":0,"num_object_copies":19719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6573,"num_write_kb":420672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":2228224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":227,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.282576+0000","last_change":"2023-10-09T10:02:29.908231+0000","last_active":"2023-10-09T10:04:41.282576+0000","last_peered":"2023-10-09T10:04:41.282576+0000","last_clean":"2023-10-09T10:04:41.282576+0000","last_became_active":"2023-10-09T10:02:29.880421+0000","last_became_peered":"2023-10-09T10:02:29.880421+0000","last_unstale":"2023-10-09T10:04:41.282576+0000","last_undegraded":"2023-10-09T10:04:41.282576+0000","last_fullsized":"2023-10-09T10:04:41.282576+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":27,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.276129+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:04:41.276129+0000","last_peered":"2023-10-09T10:04:41.276129+0000","last_clean":"2023-10-09T10:04:41.276129+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:04:41.276129+0000","last_undegraded":"2023-10-09T10:04:41.276129+0000","last_fullsized":"2023-10-09T10:04:41.276129+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"209'6476","reported_seq":6588,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:52.589804+0000","last_change":"2023-10-09T10:03:30.249521+0000","last_active":"2023-10-09T10:04:52.589804+0000","last_peered":"2023-10-09T10:04:52.589804+0000","last_clean":"2023-10-09T10:04:52.589804+0000","last_became_active":"2023-10-09T10:03:30.082956+0000","last_became_peered":"2023-10-09T10:03:30.082956+0000","last_unstale":"2023-10-09T10:04:52.589804+0000","last_undegraded":"2023-10-09T10:04:52.589804+0000","last_fullsized":"2023-10-09T10:04:52.589804+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6476,"log_dups_size":0,"ondisk_log_size":6476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:18:43.459243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424411136,"num_objects":6476,"num_object_clones":0,"num_object_copies":19428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6476,"num_write_kb":414464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":215,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.282001+0000","last_change":"2023-10-09T10:02:18.135878+0000","last_active":"2023-10-09T10:04:41.282001+0000","last_peered":"2023-10-09T10:04:41.282001+0000","last_clean":"2023-10-09T10:04:41.282001+0000","last_became_active":"2023-10-09T10:02:17.734932+0000","last_became_peered":"2023-10-09T10:02:17.734932+0000","last_unstale":"2023-10-09T10:04:41.282001+0000","last_undegraded":"2023-10-09T10:04:41.282001+0000","last_fullsized":"2023-10-09T10:04:41.282001+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286002+0000","last_change":"2023-10-09T10:02:17.743834+0000","last_active":"2023-10-09T10:04:41.286002+0000","last_peered":"2023-10-09T10:04:41.286002+0000","last_clean":"2023-10-09T10:04:41.286002+0000","last_became_active":"2023-10-09T10:02:17.743066+0000","last_became_peered":"2023-10-09T10:02:17.743066+0000","last_unstale":"2023-10-09T10:04:41.286002+0000","last_undegraded":"2023-10-09T10:04:41.286002+0000","last_fullsized":"2023-10-09T10:04:41.286002+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"209'6301","reported_seq":6332,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:51.872341+0000","last_change":"2023-10-09T10:03:22.033916+0000","last_active":"2023-10-09T10:04:51.872341+0000","last_peered":"2023-10-09T10:04:51.872341+0000","last_clean":"2023-10-09T10:04:51.872341+0000","last_became_active":"2023-10-09T10:03:22.033637+0000","last_became_peered":"2023-10-09T10:03:22.033637+0000","last_unstale":"2023-10-09T10:04:51.872341+0000","last_undegraded":"2023-10-09T10:04:51.872341+0000","last_fullsized":"2023-10-09T10:04:51.872341+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:33:02.120588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412942336,"num_objects":6301,"num_object_clones":0,"num_object_copies":18903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6301,"num_write_kb":403264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":31,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.280530+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:04:41.280530+0000","last_peered":"2023-10-09T10:04:41.280530+0000","last_clean":"2023-10-09T10:04:41.280530+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:04:41.280530+0000","last_undegraded":"2023-10-09T10:04:41.280530+0000","last_fullsized":"2023-10-09T10:04:41.280530+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":114,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.269076+0000","last_change":"2023-10-09T10:01:36.025782+0000","last_active":"2023-10-09T10:04:41.269076+0000","last_peered":"2023-10-09T10:04:41.269076+0000","last_clean":"2023-10-09T10:04:41.269076+0000","last_became_active":"2023-10-09T10:01:36.025196+0000","last_became_peered":"2023-10-09T10:01:36.025196+0000","last_unstale":"2023-10-09T10:04:41.269076+0000","last_undegraded":"2023-10-09T10:04:41.269076+0000","last_fullsized":"2023-10-09T10:04:41.269076+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:04.486165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"209'6522","reported_seq":6552,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:54.920764+0000","last_change":"2023-10-09T10:03:22.032901+0000","last_active":"2023-10-09T10:04:54.920764+0000","last_peered":"2023-10-09T10:04:54.920764+0000","last_clean":"2023-10-09T10:04:54.920764+0000","last_became_active":"2023-10-09T10:03:22.032596+0000","last_became_peered":"2023-10-09T10:03:22.032596+0000","last_unstale":"2023-10-09T10:04:54.920764+0000","last_undegraded":"2023-10-09T10:04:54.920764+0000","last_fullsized":"2023-10-09T10:04:54.920764+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427425792,"num_objects":6522,"num_object_clones":0,"num_object_copies":19566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6522,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":27,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.278888+0000","last_change":"2023-10-09T10:03:30.092246+0000","last_active":"2023-10-09T10:04:41.278888+0000","last_peered":"2023-10-09T10:04:41.278888+0000","last_clean":"2023-10-09T10:04:41.278888+0000","last_became_active":"2023-10-09T10:03:30.091926+0000","last_became_peered":"2023-10-09T10:03:30.091926+0000","last_unstale":"2023-10-09T10:04:41.278888+0000","last_undegraded":"2023-10-09T10:04:41.278888+0000","last_fullsized":"2023-10-09T10:04:41.278888+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":59,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:46.867397+0000","last_change":"2023-10-09T10:02:29.889376+0000","last_active":"2023-10-09T10:04:46.867397+0000","last_peered":"2023-10-09T10:04:46.867397+0000","last_clean":"2023-10-09T10:04:46.867397+0000","last_became_active":"2023-10-09T10:02:29.888420+0000","last_became_peered":"2023-10-09T10:02:29.888420+0000","last_unstale":"2023-10-09T10:04:46.867397+0000","last_undegraded":"2023-10-09T10:04:46.867397+0000","last_fullsized":"2023-10-09T10:04:46.867397+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:16.976540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"209'6421","reported_seq":6452,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:51.848156+0000","last_change":"2023-10-09T10:03:22.032070+0000","last_active":"2023-10-09T10:04:51.848156+0000","last_peered":"2023-10-09T10:04:51.848156+0000","last_clean":"2023-10-09T10:04:51.848156+0000","last_became_active":"2023-10-09T10:03:22.031790+0000","last_became_peered":"2023-10-09T10:03:22.031790+0000","last_unstale":"2023-10-09T10:04:51.848156+0000","last_undegraded":"2023-10-09T10:04:51.848156+0000","last_fullsized":"2023-10-09T10:04:51.848156+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6421,"log_dups_size":0,"ondisk_log_size":6421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:57.360330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420806656,"num_objects":6421,"num_object_clones":0,"num_object_copies":19263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6421,"num_write_kb":410944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":229,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274054+0000","last_change":"2023-10-09T10:02:29.946556+0000","last_active":"2023-10-09T10:04:41.274054+0000","last_peered":"2023-10-09T10:04:41.274054+0000","last_clean":"2023-10-09T10:04:41.274054+0000","last_became_active":"2023-10-09T10:02:29.884952+0000","last_became_peered":"2023-10-09T10:02:29.884952+0000","last_unstale":"2023-10-09T10:04:41.274054+0000","last_undegraded":"2023-10-09T10:04:41.274054+0000","last_fullsized":"2023-10-09T10:04:41.274054+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:32.438007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286466+0000","last_change":"2023-10-09T10:02:17.743630+0000","last_active":"2023-10-09T10:04:41.286466+0000","last_peered":"2023-10-09T10:04:41.286466+0000","last_clean":"2023-10-09T10:04:41.286466+0000","last_became_active":"2023-10-09T10:02:17.742997+0000","last_became_peered":"2023-10-09T10:02:17.742997+0000","last_unstale":"2023-10-09T10:04:41.286466+0000","last_undegraded":"2023-10-09T10:04:41.286466+0000","last_fullsized":"2023-10-09T10:04:41.286466+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"209'6473","reported_seq":6530,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:54.921399+0000","last_change":"2023-10-09T10:03:30.230552+0000","last_active":"2023-10-09T10:04:54.921399+0000","last_peered":"2023-10-09T10:04:54.921399+0000","last_clean":"2023-10-09T10:04:54.921399+0000","last_became_active":"2023-10-09T10:03:30.085250+0000","last_became_peered":"2023-10-09T10:03:30.085250+0000","last_unstale":"2023-10-09T10:04:54.921399+0000","last_undegraded":"2023-10-09T10:04:54.921399+0000","last_fullsized":"2023-10-09T10:04:54.921399+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424214528,"num_objects":6473,"num_object_clones":0,"num_object_copies":19419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6473,"num_write_kb":414272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.285888+0000","last_change":"2023-10-09T10:02:17.733313+0000","last_active":"2023-10-09T10:04:41.285888+0000","last_peered":"2023-10-09T10:04:41.285888+0000","last_clean":"2023-10-09T10:04:41.285888+0000","last_became_active":"2023-10-09T10:02:17.732658+0000","last_became_peered":"2023-10-09T10:02:17.732658+0000","last_unstale":"2023-10-09T10:04:41.285888+0000","last_undegraded":"2023-10-09T10:04:41.285888+0000","last_fullsized":"2023-10-09T10:04:41.285888+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":69,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286631+0000","last_change":"2023-10-09T10:03:19.762981+0000","last_active":"2023-10-09T10:04:41.286631+0000","last_peered":"2023-10-09T10:04:41.286631+0000","last_clean":"2023-10-09T10:04:41.286631+0000","last_became_active":"2023-10-09T10:03:19.762647+0000","last_became_peered":"2023-10-09T10:03:19.762647+0000","last_unstale":"2023-10-09T10:04:41.286631+0000","last_undegraded":"2023-10-09T10:04:41.286631+0000","last_fullsized":"2023-10-09T10:04:41.286631+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"209'6500","reported_seq":6531,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:54.214854+0000","last_change":"2023-10-09T10:03:22.356577+0000","last_active":"2023-10-09T10:04:54.214854+0000","last_peered":"2023-10-09T10:04:54.214854+0000","last_clean":"2023-10-09T10:04:54.214854+0000","last_became_active":"2023-10-09T10:03:22.356253+0000","last_became_peered":"2023-10-09T10:03:22.356253+0000","last_unstale":"2023-10-09T10:04:54.214854+0000","last_undegraded":"2023-10-09T10:04:54.214854+0000","last_fullsized":"2023-10-09T10:04:54.214854+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6500,"log_dups_size":0,"ondisk_log_size":6500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:14.903436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425984000,"num_objects":6500,"num_object_clones":0,"num_object_copies":19500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6500,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.285865+0000","last_change":"2023-10-09T10:03:19.760276+0000","last_active":"2023-10-09T10:04:41.285865+0000","last_peered":"2023-10-09T10:04:41.285865+0000","last_clean":"2023-10-09T10:04:41.285865+0000","last_became_active":"2023-10-09T10:03:19.759911+0000","last_became_peered":"2023-10-09T10:03:19.759911+0000","last_unstale":"2023-10-09T10:04:41.285865+0000","last_undegraded":"2023-10-09T10:04:41.285865+0000","last_fullsized":"2023-10-09T10:04:41.285865+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286735+0000","last_change":"2023-10-09T10:02:17.732379+0000","last_active":"2023-10-09T10:04:41.286735+0000","last_peered":"2023-10-09T10:04:41.286735+0000","last_clean":"2023-10-09T10:04:41.286735+0000","last_became_active":"2023-10-09T10:02:17.729410+0000","last_became_peered":"2023-10-09T10:02:17.729410+0000","last_unstale":"2023-10-09T10:04:41.286735+0000","last_undegraded":"2023-10-09T10:04:41.286735+0000","last_fullsized":"2023-10-09T10:04:41.286735+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"209'6447","reported_seq":6478,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:52.559378+0000","last_change":"2023-10-09T10:03:22.356576+0000","last_active":"2023-10-09T10:04:52.559378+0000","last_peered":"2023-10-09T10:04:52.559378+0000","last_clean":"2023-10-09T10:04:52.559378+0000","last_became_active":"2023-10-09T10:03:22.356253+0000","last_became_peered":"2023-10-09T10:03:22.356253+0000","last_unstale":"2023-10-09T10:04:52.559378+0000","last_undegraded":"2023-10-09T10:04:52.559378+0000","last_fullsized":"2023-10-09T10:04:52.559378+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:58.353155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422510592,"num_objects":6447,"num_object_clones":0,"num_object_copies":19341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6447,"num_write_kb":412608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":180,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.272121+0000","last_change":"2023-10-09T09:57:49.624469+0000","last_active":"2023-10-09T10:04:41.272121+0000","last_peered":"2023-10-09T10:04:41.272121+0000","last_clean":"2023-10-09T10:04:41.272121+0000","last_became_active":"2023-10-09T09:57:49.623104+0000","last_became_peered":"2023-10-09T09:57:49.623104+0000","last_unstale":"2023-10-09T10:04:41.272121+0000","last_undegraded":"2023-10-09T10:04:41.272121+0000","last_fullsized":"2023-10-09T10:04:41.272121+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:55.756185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":183,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.268571+0000","last_change":"2023-10-09T10:02:17.726854+0000","last_active":"2023-10-09T10:04:41.268571+0000","last_peered":"2023-10-09T10:04:41.268571+0000","last_clean":"2023-10-09T10:04:41.268571+0000","last_became_active":"2023-10-09T10:02:17.726570+0000","last_became_peered":"2023-10-09T10:02:17.726570+0000","last_unstale":"2023-10-09T10:04:41.268571+0000","last_undegraded":"2023-10-09T10:04:41.268571+0000","last_fullsized":"2023-10-09T10:04:41.268571+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:41.295112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"209'6550","reported_seq":6580,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:54.895914+0000","last_change":"2023-10-09T10:03:22.358239+0000","last_active":"2023-10-09T10:04:54.895914+0000","last_peered":"2023-10-09T10:04:54.895914+0000","last_clean":"2023-10-09T10:04:54.895914+0000","last_became_active":"2023-10-09T10:03:22.357886+0000","last_became_peered":"2023-10-09T10:03:22.357886+0000","last_unstale":"2023-10-09T10:04:54.895914+0000","last_undegraded":"2023-10-09T10:04:54.895914+0000","last_fullsized":"2023-10-09T10:04:54.895914+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429260800,"num_objects":6550,"num_object_clones":0,"num_object_copies":19650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6550,"num_write_kb":419200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":208,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.270550+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:04:41.270550+0000","last_peered":"2023-10-09T10:04:41.270550+0000","last_clean":"2023-10-09T10:04:41.270550+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:04:41.270550+0000","last_undegraded":"2023-10-09T10:04:41.270550+0000","last_fullsized":"2023-10-09T10:04:41.270550+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":27,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.280368+0000","last_change":"2023-10-09T10:03:30.093765+0000","last_active":"2023-10-09T10:04:41.280368+0000","last_peered":"2023-10-09T10:04:41.280368+0000","last_clean":"2023-10-09T10:04:41.280368+0000","last_became_active":"2023-10-09T10:03:30.093294+0000","last_became_peered":"2023-10-09T10:03:30.093294+0000","last_unstale":"2023-10-09T10:04:41.280368+0000","last_undegraded":"2023-10-09T10:04:41.280368+0000","last_fullsized":"2023-10-09T10:04:41.280368+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":150,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.269217+0000","last_change":"2023-10-09T10:03:30.081023+0000","last_active":"2023-10-09T10:04:41.269217+0000","last_peered":"2023-10-09T10:04:41.269217+0000","last_clean":"2023-10-09T10:04:41.269217+0000","last_became_active":"2023-10-09T10:03:30.080737+0000","last_became_peered":"2023-10-09T10:03:30.080737+0000","last_unstale":"2023-10-09T10:04:41.269217+0000","last_undegraded":"2023-10-09T10:04:41.269217+0000","last_fullsized":"2023-10-09T10:04:41.269217+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:04.667262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"209'6342","reported_seq":6373,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:51.819521+0000","last_change":"2023-10-09T10:03:22.356731+0000","last_active":"2023-10-09T10:04:51.819521+0000","last_peered":"2023-10-09T10:04:51.819521+0000","last_clean":"2023-10-09T10:04:51.819521+0000","last_became_active":"2023-10-09T10:03:22.356414+0000","last_became_peered":"2023-10-09T10:03:22.356414+0000","last_unstale":"2023-10-09T10:04:51.819521+0000","last_undegraded":"2023-10-09T10:04:51.819521+0000","last_fullsized":"2023-10-09T10:04:51.819521+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6342,"log_dups_size":0,"ondisk_log_size":6342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:54.939596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415629312,"num_objects":6342,"num_object_clones":0,"num_object_copies":19026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6342,"num_write_kb":405888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":27,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.276474+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:04:41.276474+0000","last_peered":"2023-10-09T10:04:41.276474+0000","last_clean":"2023-10-09T10:04:41.276474+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:04:41.276474+0000","last_undegraded":"2023-10-09T10:04:41.276474+0000","last_fullsized":"2023-10-09T10:04:41.276474+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286214+0000","last_change":"2023-10-09T10:02:29.891638+0000","last_active":"2023-10-09T10:04:41.286214+0000","last_peered":"2023-10-09T10:04:41.286214+0000","last_clean":"2023-10-09T10:04:41.286214+0000","last_became_active":"2023-10-09T10:02:29.891265+0000","last_became_peered":"2023-10-09T10:02:29.891265+0000","last_unstale":"2023-10-09T10:04:41.286214+0000","last_undegraded":"2023-10-09T10:04:41.286214+0000","last_fullsized":"2023-10-09T10:04:41.286214+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":207,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.270484+0000","last_change":"2023-10-09T10:03:19.763071+0000","last_active":"2023-10-09T10:04:41.270484+0000","last_peered":"2023-10-09T10:04:41.270484+0000","last_clean":"2023-10-09T10:04:41.270484+0000","last_became_active":"2023-10-09T10:03:19.762353+0000","last_became_peered":"2023-10-09T10:03:19.762353+0000","last_unstale":"2023-10-09T10:04:41.270484+0000","last_undegraded":"2023-10-09T10:04:41.270484+0000","last_fullsized":"2023-10-09T10:04:41.270484+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:50.245039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"209'6331","reported_seq":6362,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:51.837218+0000","last_change":"2023-10-09T10:03:22.033981+0000","last_active":"2023-10-09T10:04:51.837218+0000","last_peered":"2023-10-09T10:04:51.837218+0000","last_clean":"2023-10-09T10:04:51.837218+0000","last_became_active":"2023-10-09T10:03:22.033707+0000","last_became_peered":"2023-10-09T10:03:22.033707+0000","last_unstale":"2023-10-09T10:04:51.837218+0000","last_undegraded":"2023-10-09T10:04:51.837218+0000","last_fullsized":"2023-10-09T10:04:51.837218+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6331,"log_dups_size":0,"ondisk_log_size":6331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:05:34.341309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414908416,"num_objects":6331,"num_object_clones":0,"num_object_copies":18993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6331,"num_write_kb":405184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":126,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.272306+0000","last_change":"2023-10-09T10:03:19.759400+0000","last_active":"2023-10-09T10:04:41.272306+0000","last_peered":"2023-10-09T10:04:41.272306+0000","last_clean":"2023-10-09T10:04:41.272306+0000","last_became_active":"2023-10-09T10:03:19.759059+0000","last_became_peered":"2023-10-09T10:03:19.759059+0000","last_unstale":"2023-10-09T10:04:41.272306+0000","last_undegraded":"2023-10-09T10:04:41.272306+0000","last_fullsized":"2023-10-09T10:04:41.272306+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:03:20.450835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":192,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:44.242199+0000","last_change":"2023-10-09T10:03:19.764744+0000","last_active":"2023-10-09T10:04:44.242199+0000","last_peered":"2023-10-09T10:04:44.242199+0000","last_clean":"2023-10-09T10:04:44.242199+0000","last_became_active":"2023-10-09T10:03:19.764382+0000","last_became_peered":"2023-10-09T10:03:19.764382+0000","last_unstale":"2023-10-09T10:04:44.242199+0000","last_undegraded":"2023-10-09T10:04:44.242199+0000","last_fullsized":"2023-10-09T10:04:44.242199+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:34.922940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":221,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.268497+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:04:41.268497+0000","last_peered":"2023-10-09T10:04:41.268497+0000","last_clean":"2023-10-09T10:04:41.268497+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:04:41.268497+0000","last_undegraded":"2023-10-09T10:04:41.268497+0000","last_fullsized":"2023-10-09T10:04:41.268497+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"209'6470","reported_seq":6501,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:51.847475+0000","last_change":"2023-10-09T10:03:22.032323+0000","last_active":"2023-10-09T10:04:51.847475+0000","last_peered":"2023-10-09T10:04:51.847475+0000","last_clean":"2023-10-09T10:04:51.847475+0000","last_became_active":"2023-10-09T10:03:22.031949+0000","last_became_peered":"2023-10-09T10:03:22.031949+0000","last_unstale":"2023-10-09T10:04:51.847475+0000","last_undegraded":"2023-10-09T10:04:51.847475+0000","last_fullsized":"2023-10-09T10:04:51.847475+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:14.245332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424017920,"num_objects":6470,"num_object_clones":0,"num_object_copies":19410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6470,"num_write_kb":414080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":27,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.276620+0000","last_change":"2023-10-09T10:03:30.082150+0000","last_active":"2023-10-09T10:04:41.276620+0000","last_peered":"2023-10-09T10:04:41.276620+0000","last_clean":"2023-10-09T10:04:41.276620+0000","last_became_active":"2023-10-09T10:03:30.081805+0000","last_became_peered":"2023-10-09T10:03:30.081805+0000","last_unstale":"2023-10-09T10:04:41.276620+0000","last_undegraded":"2023-10-09T10:04:41.276620+0000","last_fullsized":"2023-10-09T10:04:41.276620+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":456,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.281658+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:04:41.281658+0000","last_peered":"2023-10-09T10:04:41.281658+0000","last_clean":"2023-10-09T10:04:41.281658+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:04:41.281658+0000","last_undegraded":"2023-10-09T10:04:41.281658+0000","last_fullsized":"2023-10-09T10:04:41.281658+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":158,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.279028+0000","last_change":"2023-10-09T10:03:30.089277+0000","last_active":"2023-10-09T10:04:41.279028+0000","last_peered":"2023-10-09T10:04:41.279028+0000","last_clean":"2023-10-09T10:04:41.279028+0000","last_became_active":"2023-10-09T10:03:30.088960+0000","last_became_peered":"2023-10-09T10:03:30.088960+0000","last_unstale":"2023-10-09T10:04:41.279028+0000","last_undegraded":"2023-10-09T10:04:41.279028+0000","last_fullsized":"2023-10-09T10:04:41.279028+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:27.610770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286722+0000","last_change":"2023-10-09T10:02:17.734226+0000","last_active":"2023-10-09T10:04:41.286722+0000","last_peered":"2023-10-09T10:04:41.286722+0000","last_clean":"2023-10-09T10:04:41.286722+0000","last_became_active":"2023-10-09T10:02:17.733854+0000","last_became_peered":"2023-10-09T10:02:17.733854+0000","last_unstale":"2023-10-09T10:04:41.286722+0000","last_undegraded":"2023-10-09T10:04:41.286722+0000","last_fullsized":"2023-10-09T10:04:41.286722+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"209'6387","reported_seq":6418,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:51.837220+0000","last_change":"2023-10-09T10:03:22.032500+0000","last_active":"2023-10-09T10:04:51.837220+0000","last_peered":"2023-10-09T10:04:51.837220+0000","last_clean":"2023-10-09T10:04:51.837220+0000","last_became_active":"2023-10-09T10:03:22.032139+0000","last_became_peered":"2023-10-09T10:03:22.032139+0000","last_unstale":"2023-10-09T10:04:51.837220+0000","last_undegraded":"2023-10-09T10:04:51.837220+0000","last_fullsized":"2023-10-09T10:04:51.837220+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6387,"log_dups_size":0,"ondisk_log_size":6387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:01.747725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418578432,"num_objects":6387,"num_object_clones":0,"num_object_copies":19161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6387,"num_write_kb":408768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":181,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.269379+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:04:41.269379+0000","last_peered":"2023-10-09T10:04:41.269379+0000","last_clean":"2023-10-09T10:04:41.269379+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:04:41.269379+0000","last_undegraded":"2023-10-09T10:04:41.269379+0000","last_fullsized":"2023-10-09T10:04:41.269379+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":58,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274742+0000","last_change":"2023-10-09T10:02:29.953999+0000","last_active":"2023-10-09T10:04:41.274742+0000","last_peered":"2023-10-09T10:04:41.274742+0000","last_clean":"2023-10-09T10:04:41.274742+0000","last_became_active":"2023-10-09T10:02:29.953244+0000","last_became_peered":"2023-10-09T10:02:29.953244+0000","last_unstale":"2023-10-09T10:04:41.274742+0000","last_undegraded":"2023-10-09T10:04:41.274742+0000","last_fullsized":"2023-10-09T10:04:41.274742+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:05.612343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":85,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.865535+0000","last_change":"2023-10-09T10:04:00.900774+0000","last_active":"2023-10-09T10:04:41.865535+0000","last_peered":"2023-10-09T10:04:41.865535+0000","last_clean":"2023-10-09T10:04:41.865535+0000","last_became_active":"2023-10-09T10:04:00.900387+0000","last_became_peered":"2023-10-09T10:04:00.900387+0000","last_unstale":"2023-10-09T10:04:41.865535+0000","last_undegraded":"2023-10-09T10:04:41.865535+0000","last_fullsized":"2023-10-09T10:04:41.865535+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:27:43.957142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"209'6724","reported_seq":6776,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:54.902426+0000","last_change":"2023-10-09T10:03:30.133157+0000","last_active":"2023-10-09T10:04:54.902426+0000","last_peered":"2023-10-09T10:04:54.902426+0000","last_clean":"2023-10-09T10:04:54.902426+0000","last_became_active":"2023-10-09T10:03:30.078454+0000","last_became_peered":"2023-10-09T10:03:30.078454+0000","last_unstale":"2023-10-09T10:04:54.902426+0000","last_undegraded":"2023-10-09T10:04:54.902426+0000","last_fullsized":"2023-10-09T10:04:54.902426+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":440664064,"num_objects":6724,"num_object_clones":0,"num_object_copies":20172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6724,"num_write_kb":430336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":1376256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":214,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.282249+0000","last_change":"2023-10-09T10:02:29.882693+0000","last_active":"2023-10-09T10:04:41.282249+0000","last_peered":"2023-10-09T10:04:41.282249+0000","last_clean":"2023-10-09T10:04:41.282249+0000","last_became_active":"2023-10-09T10:02:29.882300+0000","last_became_peered":"2023-10-09T10:02:29.882300+0000","last_unstale":"2023-10-09T10:04:41.282249+0000","last_undegraded":"2023-10-09T10:04:41.282249+0000","last_fullsized":"2023-10-09T10:04:41.282249+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":210,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.270623+0000","last_change":"2023-10-09T10:03:30.080560+0000","last_active":"2023-10-09T10:04:41.270623+0000","last_peered":"2023-10-09T10:04:41.270623+0000","last_clean":"2023-10-09T10:04:41.270623+0000","last_became_active":"2023-10-09T10:03:30.080189+0000","last_became_peered":"2023-10-09T10:03:30.080189+0000","last_unstale":"2023-10-09T10:04:41.270623+0000","last_undegraded":"2023-10-09T10:04:41.270623+0000","last_fullsized":"2023-10-09T10:04:41.270623+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:51.220323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":76,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.275070+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:04:41.275070+0000","last_peered":"2023-10-09T10:04:41.275070+0000","last_clean":"2023-10-09T10:04:41.275070+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:04:41.275070+0000","last_undegraded":"2023-10-09T10:04:41.275070+0000","last_fullsized":"2023-10-09T10:04:41.275070+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"209'6416","reported_seq":6447,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:52.561837+0000","last_change":"2023-10-09T10:03:22.356605+0000","last_active":"2023-10-09T10:04:52.561837+0000","last_peered":"2023-10-09T10:04:52.561837+0000","last_clean":"2023-10-09T10:04:52.561837+0000","last_became_active":"2023-10-09T10:03:22.356305+0000","last_became_peered":"2023-10-09T10:03:22.356305+0000","last_unstale":"2023-10-09T10:04:52.561837+0000","last_undegraded":"2023-10-09T10:04:52.561837+0000","last_fullsized":"2023-10-09T10:04:52.561837+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6416,"log_dups_size":0,"ondisk_log_size":6416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:30.705942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420478976,"num_objects":6416,"num_object_clones":0,"num_object_copies":19248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6416,"num_write_kb":410624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":203,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274892+0000","last_change":"2023-10-09T10:02:29.972836+0000","last_active":"2023-10-09T10:04:41.274892+0000","last_peered":"2023-10-09T10:04:41.274892+0000","last_clean":"2023-10-09T10:04:41.274892+0000","last_became_active":"2023-10-09T10:02:29.888134+0000","last_became_peered":"2023-10-09T10:02:29.888134+0000","last_unstale":"2023-10-09T10:04:41.274892+0000","last_undegraded":"2023-10-09T10:04:41.274892+0000","last_fullsized":"2023-10-09T10:04:41.274892+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:22.244154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":167,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.269941+0000","last_change":"2023-10-09T10:02:17.723233+0000","last_active":"2023-10-09T10:04:41.269941+0000","last_peered":"2023-10-09T10:04:41.269941+0000","last_clean":"2023-10-09T10:04:41.269941+0000","last_became_active":"2023-10-09T10:02:17.722944+0000","last_became_peered":"2023-10-09T10:02:17.722944+0000","last_unstale":"2023-10-09T10:04:41.269941+0000","last_undegraded":"2023-10-09T10:04:41.269941+0000","last_fullsized":"2023-10-09T10:04:41.269941+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:56.963212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":192,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.272194+0000","last_change":"2023-10-09T10:02:29.886514+0000","last_active":"2023-10-09T10:04:41.272194+0000","last_peered":"2023-10-09T10:04:41.272194+0000","last_clean":"2023-10-09T10:04:41.272194+0000","last_became_active":"2023-10-09T10:02:29.885706+0000","last_became_peered":"2023-10-09T10:02:29.885706+0000","last_unstale":"2023-10-09T10:04:41.272194+0000","last_undegraded":"2023-10-09T10:04:41.272194+0000","last_fullsized":"2023-10-09T10:04:41.272194+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:28:06.195775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"209'6561","reported_seq":6592,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T10:04:52.508534+0000","last_change":"2023-10-09T10:03:22.032464+0000","last_active":"2023-10-09T10:04:52.508534+0000","last_peered":"2023-10-09T10:04:52.508534+0000","last_clean":"2023-10-09T10:04:52.508534+0000","last_became_active":"2023-10-09T10:03:22.031750+0000","last_became_peered":"2023-10-09T10:03:22.031750+0000","last_unstale":"2023-10-09T10:04:52.508534+0000","last_undegraded":"2023-10-09T10:04:52.508534+0000","last_fullsized":"2023-10-09T10:04:52.508534+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:23.160842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429981696,"num_objects":6561,"num_object_clones":0,"num_object_copies":19683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6561,"num_write_kb":419904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":34,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274714+0000","last_change":"2023-10-09T10:03:19.760664+0000","last_active":"2023-10-09T10:04:41.274714+0000","last_peered":"2023-10-09T10:04:41.274714+0000","last_clean":"2023-10-09T10:04:41.274714+0000","last_became_active":"2023-10-09T10:03:19.760358+0000","last_became_peered":"2023-10-09T10:03:19.760358+0000","last_unstale":"2023-10-09T10:04:41.274714+0000","last_undegraded":"2023-10-09T10:04:41.274714+0000","last_fullsized":"2023-10-09T10:04:41.274714+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:35.382690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":63,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.865563+0000","last_change":"2023-10-09T10:02:29.922084+0000","last_active":"2023-10-09T10:04:41.865563+0000","last_peered":"2023-10-09T10:04:41.865563+0000","last_clean":"2023-10-09T10:04:41.865563+0000","last_became_active":"2023-10-09T10:02:29.886231+0000","last_became_peered":"2023-10-09T10:02:29.886231+0000","last_unstale":"2023-10-09T10:04:41.865563+0000","last_undegraded":"2023-10-09T10:04:41.865563+0000","last_fullsized":"2023-10-09T10:04:41.865563+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:02:16.587891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.285936+0000","last_change":"2023-10-09T10:03:30.090128+0000","last_active":"2023-10-09T10:04:41.285936+0000","last_peered":"2023-10-09T10:04:41.285936+0000","last_clean":"2023-10-09T10:04:41.285936+0000","last_became_active":"2023-10-09T10:03:30.089771+0000","last_became_peered":"2023-10-09T10:03:30.089771+0000","last_unstale":"2023-10-09T10:04:41.285936+0000","last_undegraded":"2023-10-09T10:04:41.285936+0000","last_fullsized":"2023-10-09T10:04:41.285936+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":57,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274923+0000","last_change":"2023-10-09T10:02:29.889587+0000","last_active":"2023-10-09T10:04:41.274923+0000","last_peered":"2023-10-09T10:04:41.274923+0000","last_clean":"2023-10-09T10:04:41.274923+0000","last_became_active":"2023-10-09T10:02:29.888869+0000","last_became_peered":"2023-10-09T10:02:29.888869+0000","last_unstale":"2023-10-09T10:04:41.274923+0000","last_undegraded":"2023-10-09T10:04:41.274923+0000","last_fullsized":"2023-10-09T10:04:41.274923+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:22.220539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":125,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.270694+0000","last_change":"2023-10-09T10:03:19.762707+0000","last_active":"2023-10-09T10:04:41.270694+0000","last_peered":"2023-10-09T10:04:41.270694+0000","last_clean":"2023-10-09T10:04:41.270694+0000","last_became_active":"2023-10-09T10:03:19.762140+0000","last_became_peered":"2023-10-09T10:03:19.762140+0000","last_unstale":"2023-10-09T10:04:41.270694+0000","last_undegraded":"2023-10-09T10:04:41.270694+0000","last_fullsized":"2023-10-09T10:04:41.270694+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:03.706004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":31,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.278837+0000","last_change":"2023-10-09T10:03:30.094629+0000","last_active":"2023-10-09T10:04:41.278837+0000","last_peered":"2023-10-09T10:04:41.278837+0000","last_clean":"2023-10-09T10:04:41.278837+0000","last_became_active":"2023-10-09T10:03:30.082732+0000","last_became_peered":"2023-10-09T10:03:30.082732+0000","last_unstale":"2023-10-09T10:04:41.278837+0000","last_undegraded":"2023-10-09T10:04:41.278837+0000","last_fullsized":"2023-10-09T10:04:41.278837+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.286307+0000","last_change":"2023-10-09T10:02:17.743821+0000","last_active":"2023-10-09T10:04:41.286307+0000","last_peered":"2023-10-09T10:04:41.286307+0000","last_clean":"2023-10-09T10:04:41.286307+0000","last_became_active":"2023-10-09T10:02:17.743215+0000","last_became_peered":"2023-10-09T10:02:17.743215+0000","last_unstale":"2023-10-09T10:04:41.286307+0000","last_undegraded":"2023-10-09T10:04:41.286307+0000","last_fullsized":"2023-10-09T10:04:41.286307+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":31,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.280289+0000","last_change":"2023-10-09T10:03:30.132633+0000","last_active":"2023-10-09T10:04:41.280289+0000","last_peered":"2023-10-09T10:04:41.280289+0000","last_clean":"2023-10-09T10:04:41.280289+0000","last_became_active":"2023-10-09T10:03:30.092696+0000","last_became_peered":"2023-10-09T10:03:30.092696+0000","last_unstale":"2023-10-09T10:04:41.280289+0000","last_undegraded":"2023-10-09T10:04:41.280289+0000","last_fullsized":"2023-10-09T10:04:41.280289+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":69,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.285834+0000","last_change":"2023-10-09T10:02:29.890059+0000","last_active":"2023-10-09T10:04:41.285834+0000","last_peered":"2023-10-09T10:04:41.285834+0000","last_clean":"2023-10-09T10:04:41.285834+0000","last_became_active":"2023-10-09T10:02:29.889678+0000","last_became_peered":"2023-10-09T10:02:29.889678+0000","last_unstale":"2023-10-09T10:04:41.285834+0000","last_undegraded":"2023-10-09T10:04:41.285834+0000","last_fullsized":"2023-10-09T10:04:41.285834+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":212,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.274265+0000","last_change":"2023-10-09T10:03:19.762851+0000","last_active":"2023-10-09T10:04:41.274265+0000","last_peered":"2023-10-09T10:04:41.274265+0000","last_clean":"2023-10-09T10:04:41.274265+0000","last_became_active":"2023-10-09T10:03:19.762439+0000","last_became_peered":"2023-10-09T10:03:19.762439+0000","last_unstale":"2023-10-09T10:04:41.274265+0000","last_undegraded":"2023-10-09T10:04:41.274265+0000","last_fullsized":"2023-10-09T10:04:41.274265+0000","mapping_epoch":185,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:50.254919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":251,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.272138+0000","last_change":"2023-10-09T10:03:30.083372+0000","last_active":"2023-10-09T10:04:41.272138+0000","last_peered":"2023-10-09T10:04:41.272138+0000","last_clean":"2023-10-09T10:04:41.272138+0000","last_became_active":"2023-10-09T10:03:30.082995+0000","last_became_peered":"2023-10-09T10:03:30.082995+0000","last_unstale":"2023-10-09T10:04:41.272138+0000","last_undegraded":"2023-10-09T10:04:41.272138+0000","last_fullsized":"2023-10-09T10:04:41.272138+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:27.827812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":109,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.275250+0000","last_change":"2023-10-09T10:03:30.099986+0000","last_active":"2023-10-09T10:04:41.275250+0000","last_peered":"2023-10-09T10:04:41.275250+0000","last_clean":"2023-10-09T10:04:41.275250+0000","last_became_active":"2023-10-09T10:03:30.091487+0000","last_became_peered":"2023-10-09T10:03:30.091487+0000","last_unstale":"2023-10-09T10:04:41.275250+0000","last_undegraded":"2023-10-09T10:04:41.275250+0000","last_fullsized":"2023-10-09T10:04:41.275250+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:48.095705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T10:04:41.285633+0000","last_change":"2023-10-09T10:02:17.720763+0000","last_active":"2023-10-09T10:04:41.285633+0000","last_peered":"2023-10-09T10:04:41.285633+0000","last_clean":"2023-10-09T10:04:41.285633+0000","last_became_active":"2023-10-09T10:02:17.720442+0000","last_became_peered":"2023-10-09T10:02:17.720442+0000","last_unstale":"2023-10-09T10:04:41.285633+0000","last_undegraded":"2023-10-09T10:04:41.285633+0000","last_fullsized":"2023-10-09T10:04:41.285633+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6782582784,"num_objects":103494,"num_object_clones":0,"num_object_copies":310482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":103494,"num_write_kb":6623616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":7602176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20454834176,"data_stored":20454834176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103494,"ondisk_log_size":103494,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8172,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430423,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2822252,"kb_used_data":2525028,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":90922900,"statfs":{"total":95995035648,"available":93105049600,"internally_reserved":0,"allocated":2585628672,"data_stored":2585120586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":5.9329999999999998,"5min":2.5310000000000001,"15min":1.24},"min":{"1min":0.252,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":50.591999999999999,"5min":50.591999999999999,"15min":50.591999999999999},"last":4.4279999999999999},{"interface":"front","average":{"1min":6.2679999999999998,"5min":2.4580000000000002,"15min":1.1839999999999999},"min":{"1min":0.33900000000000002,"5min":0.245,"15min":0.245},"max":{"1min":50.747,"5min":50.747,"15min":50.747},"last":3.8839999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":1.7949999999999999,"15min":0.96199999999999997},"min":{"1min":0.248,"5min":0.246,"15min":0.246},"max":{"1min":3.847,"5min":25.338000000000001,"15min":25.338000000000001},"last":4.452},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.4079999999999999,"15min":0.85899999999999999},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.855,"5min":12.996,"15min":12.996},"last":0.5}]},{"osd":2,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.673,"15min":0.96099999999999997},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.5129999999999999,"5min":16.302,"15min":16.302},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.8680000000000001,"15min":0.995},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":4.8239999999999998,"5min":13.295,"15min":13.295},"last":0.35899999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:03:59 2023","interfaces":[{"interface":"back","average":{"1min":3.2679999999999998,"5min":2.2330000000000001,"15min":1.1339999999999999},"min":{"1min":0.38300000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":28.614000000000001,"5min":28.614000000000001,"15min":28.614000000000001},"last":1.1439999999999999},{"interface":"front","average":{"1min":3.0699999999999998,"5min":2.3809999999999998,"15min":1.1850000000000001},"min":{"1min":0.32000000000000001,"5min":0.312,"15min":0.312},"max":{"1min":28.943000000000001,"5min":28.943000000000001,"15min":28.943000000000001},"last":4.5579999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.85999999999999999,"15min":0.621},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.25900000000000001},"max":{"1min":4.9829999999999997,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.78700000000000003,"15min":0.59599999999999997},"min":{"1min":0.29099999999999998,"5min":0.28899999999999998,"15min":0.26500000000000001},"max":{"1min":4.8579999999999997,"5min":4.8579999999999997,"15min":4.8579999999999997},"last":0.29999999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.147,"15min":0.72099999999999997},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.278,"5min":12.958,"15min":12.958},"last":4.391},{"interface":"front","average":{"1min":0.88500000000000001,"5min":1.0609999999999999,"15min":0.71399999999999997},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.319,"5min":6.6120000000000001,"15min":6.6120000000000001},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.93899999999999995,"15min":0.68899999999999995},"min":{"1min":0.35099999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.1200000000000001,"5min":5.4630000000000001,"15min":5.4630000000000001},"last":3.9199999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.89900000000000002,"15min":0.67700000000000005},"min":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.33100000000000002},"max":{"1min":5.0810000000000004,"5min":5.0810000000000004,"15min":5.0810000000000004},"last":0.47899999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626651,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2802788,"kb_used_data":2497372,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90942364,"statfs":{"total":95995035648,"available":93124980736,"internally_reserved":0,"allocated":2557308928,"data_stored":2556809161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.421,"15min":0.86199999999999999},"min":{"1min":0.23300000000000001,"5min":0.222,"15min":0.222},"max":{"1min":3.8279999999999998,"5min":8.8650000000000002,"15min":8.8650000000000002},"last":3.2210000000000001},{"interface":"front","average":{"1min":2.1240000000000001,"5min":1.5589999999999999,"15min":0.89000000000000001},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.1929999999999996,"5min":12.439,"15min":12.439},"last":2.1709999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:04:53 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":2.04,"15min":1.0860000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":10.664,"5min":18.853000000000002,"15min":18.853000000000002},"last":2.0790000000000002},{"interface":"front","average":{"1min":2.5630000000000002,"5min":1.917,"15min":1.0249999999999999},"min":{"1min":0.308,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":11.311,"5min":12.380000000000001,"15min":12.380000000000001},"last":0.93300000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":2.1659999999999999,"15min":1.091},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":10.451000000000001,"5min":18.800000000000001,"15min":18.800000000000001},"last":1.3759999999999999},{"interface":"front","average":{"1min":1.6859999999999999,"5min":2.0640000000000001,"15min":1.0980000000000001},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.2430000000000003,"5min":12.413,"15min":12.413},"last":2.0369999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:04:06 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.8720000000000001,"15min":1.034},"min":{"1min":0.33300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.9219999999999997,"5min":12.337,"15min":12.337},"last":3.7469999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.6870000000000001,"15min":0.96199999999999997},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.774,"5min":8.0530000000000008,"15min":8.0530000000000008},"last":1.526}]},{"osd":4,"last update":"Mon Oct 9 10:04:24 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":1.3280000000000001,"15min":0.75600000000000001},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.246,"5min":23.446000000000002,"15min":23.446000000000002},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.97699999999999998,"15min":0.65600000000000003},"min":{"1min":0.29499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":1.821,"5min":8.2379999999999995,"15min":8.2379999999999995},"last":1.254}]},{"osd":5,"last update":"Mon Oct 9 10:04:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.94699999999999995,"15min":0.64800000000000002},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.9930000000000001,"5min":14.09,"15min":14.09},"last":1.167},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.91200000000000003,"15min":0.65200000000000002},"min":{"1min":0.311,"5min":0.254,"15min":0.254},"max":{"1min":2.3250000000000002,"5min":6.3540000000000001,"15min":6.3540000000000001},"last":1.127}]},{"osd":7,"last update":"Mon Oct 9 10:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.92700000000000005,"15min":0.67300000000000004},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0539999999999998,"5min":5.9089999999999998,"15min":5.9089999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.96999999999999997,"15min":0.73099999999999998},"min":{"1min":0.35199999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.1059999999999999,"5min":6.0860000000000003,"15min":6.0860000000000003},"last":0.61499999999999999}]}]},{"osd":1,"up_from":84,"seq":360777252964,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3570660,"kb_used_data":3353500,"kb_used_omap":9,"kb_used_meta":217142,"kb_avail":90174492,"statfs":{"total":95995035648,"available":92338679808,"internally_reserved":0,"allocated":3433984000,"data_stored":3433186728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":222354409},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":2.1659999999999999,"15min":0.95599999999999996},"min":{"1min":0.219,"5min":0.185,"15min":0.185},"max":{"1min":3.613,"5min":16.324999999999999,"15min":16.324999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.298,"5min":2.125,"15min":0.96899999999999997},"min":{"1min":0.23000000000000001,"5min":0.192,"15min":0.192},"max":{"1min":3.7530000000000001,"5min":15.573,"15min":15.573},"last":4.9710000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:04:34 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":2.4300000000000002,"15min":1.0880000000000001},"min":{"1min":0.436,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.1079999999999997,"5min":18.109000000000002,"15min":18.109000000000002},"last":4.9109999999999996},{"interface":"front","average":{"1min":3.0630000000000002,"5min":2.7469999999999999,"15min":1.1910000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.248},"max":{"1min":9.5069999999999997,"5min":12.069000000000001,"15min":12.069000000000001},"last":9.8279999999999994}]},{"osd":3,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":3.133,"15min":1.296},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.8149999999999999,"5min":25.129999999999999,"15min":25.129999999999999},"last":4.9450000000000003},{"interface":"front","average":{"1min":2.2320000000000002,"5min":3.101,"15min":1.296},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.496,"5min":22.847999999999999,"15min":22.847999999999999},"last":9.6920000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":2.0019999999999998,"15min":1.073},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":3.3159999999999998,"5min":11.311999999999999,"15min":11.311999999999999},"last":9.5700000000000003},{"interface":"front","average":{"1min":1.087,"5min":1.7410000000000001,"15min":1.0109999999999999},"min":{"1min":0.311,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.6360000000000001,"5min":16.053000000000001,"15min":16.053000000000001},"last":0.754}]},{"osd":5,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":2.1000000000000001,"15min":1.115},"min":{"1min":0.40400000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.4210000000000003,"5min":15.531000000000001,"15min":15.531000000000001},"last":4.8380000000000001},{"interface":"front","average":{"1min":1.54,"5min":2.4220000000000002,"15min":1.2430000000000001},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.30299999999999999},"max":{"1min":10.090999999999999,"5min":19.706,"15min":19.706},"last":9.6199999999999992}]},{"osd":6,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.964,"15min":1.1160000000000001},"min":{"1min":0.34699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":3.5369999999999999,"5min":16.193999999999999,"15min":16.193999999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.92600000000000005,"5min":2.129,"15min":1.1559999999999999},"min":{"1min":0.40100000000000002,"5min":0.38,"15min":0.32000000000000001},"max":{"1min":3.7850000000000001,"5min":15.481999999999999,"15min":15.481999999999999},"last":4.8700000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.2400000000000002,"15min":1.171},"min":{"1min":0.45600000000000002,"5min":0.379,"15min":0.379},"max":{"1min":5.5170000000000003,"5min":12.766999999999999,"15min":12.766999999999999},"last":9.6530000000000005},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.0760000000000001,"15min":1.1519999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.34799999999999998},"max":{"1min":5.3890000000000002,"5min":16.100999999999999,"15min":16.100999999999999},"last":0.78600000000000003}]}]},{"osd":0,"up_from":79,"seq":339302416490,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3414120,"kb_used_data":3247968,"kb_used_omap":28,"kb_used_meta":166115,"kb_avail":90331032,"statfs":{"total":95995035648,"available":92498976768,"internally_reserved":0,"allocated":3325919232,"data_stored":3325116943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":170102608},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:04:02 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.9119999999999999,"15min":0.88800000000000001},"min":{"1min":0.26700000000000002,"5min":0.20300000000000001,"15min":0.191},"max":{"1min":3.0619999999999998,"5min":11.73,"15min":11.73},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":2.1389999999999998,"15min":0.97699999999999998},"min":{"1min":0.23899999999999999,"5min":0.224,"15min":0.224},"max":{"1min":4.9109999999999996,"5min":14.728,"15min":14.728},"last":5.5960000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":2.5939999999999999,"15min":1.1359999999999999},"min":{"1min":0.23999999999999999,"5min":0.224,"15min":0.216},"max":{"1min":5.4059999999999997,"5min":38.575000000000003,"15min":38.575000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.21,"5min":3.194,"15min":1.3580000000000001},"min":{"1min":0.27400000000000002,"5min":0.27200000000000002,"15min":0.23899999999999999},"max":{"1min":8.3640000000000008,"5min":38.332000000000001,"15min":38.332000000000001},"last":5.6580000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:04:02 2023","interfaces":[{"interface":"back","average":{"1min":4.5430000000000001,"5min":4.0890000000000004,"15min":1.617},"min":{"1min":0.30099999999999999,"5min":0.27400000000000002,"15min":0.25700000000000001},"max":{"1min":16.745000000000001,"5min":59.524999999999999,"15min":59.524999999999999},"last":5.7839999999999998},{"interface":"front","average":{"1min":2.7200000000000002,"5min":4.0199999999999996,"15min":1.633},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":8.4090000000000007,"5min":56.609999999999999,"15min":56.609999999999999},"last":8.4280000000000008}]},{"osd":4,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.472,"15min":0.93100000000000005},"min":{"1min":0.35599999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.266,"5min":11.587999999999999,"15min":11.587999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.0680000000000001,"15min":1.145},"min":{"1min":0.34599999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.5039999999999996,"5min":14.653,"15min":14.653},"last":5.694}]},{"osd":5,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":2.0459999999999998,"15min":1.123},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":6.1920000000000002,"5min":10.837,"15min":10.837},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.667,"15min":0.996},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.312},"max":{"1min":2.9580000000000002,"5min":11.667,"15min":11.667},"last":0.56000000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":2.0129999999999999,"15min":1.1240000000000001},"min":{"1min":0.48399999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.5170000000000003,"5min":11.699,"15min":11.699},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.198,"5min":2.52,"15min":1.3},"min":{"1min":0.46400000000000002,"5min":0.379,"15min":0.379},"max":{"1min":6.4530000000000003,"5min":15.108000000000001,"15min":15.108000000000001},"last":5.7300000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:04:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":2.278,"15min":1.2110000000000001},"min":{"1min":0.34300000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.3700000000000001,"5min":26.978000000000002,"15min":26.978000000000002},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.5740000000000001,"5min":2.2709999999999999,"15min":1.2110000000000001},"min":{"1min":0.41799999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":5.665,"5min":26.768000000000001,"15min":26.768000000000001},"last":0.65900000000000003}]}]},{"osd":2,"up_from":0,"seq":382252089437,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":94,"seq":403726925910,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4885848,"kb_used_data":4517648,"kb_used_omap":7,"kb_used_meta":368184,"kb_avail":88859304,"statfs":{"total":95995035648,"available":90991927296,"internally_reserved":0,"allocated":4626071552,"data_stored":4625227286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":377021032},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":2.5579999999999998,"15min":1.516},"min":{"1min":0.26600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.1900000000000004,"5min":11.925000000000001,"15min":11.958},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.03,"5min":2.8130000000000002,"15min":1.238},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.2670000000000003,"5min":21.285,"15min":21.285},"last":0.76500000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":2.6589999999999998,"15min":1.165},"min":{"1min":0.25900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":5.8310000000000004,"5min":21.341999999999999,"15min":21.341999999999999},"last":0.80400000000000005},{"interface":"front","average":{"1min":2.1789999999999998,"5min":3.1179999999999999,"15min":1.3160000000000001},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.2919999999999998,"5min":15.59,"15min":15.59},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":2.8730000000000002,"15min":1.655},"min":{"1min":0.32500000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.0720000000000001,"5min":15.632999999999999,"15min":15.632999999999999},"last":0.625},{"interface":"front","average":{"1min":1.452,"5min":2.794,"15min":1.236},"min":{"1min":0.29699999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.6660000000000004,"5min":11.879,"15min":11.879},"last":5.0019999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.6830000000000001,"15min":0.99099999999999999},"min":{"1min":0.374,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.4080000000000004,"5min":11.976000000000001,"15min":11.976000000000001},"last":4.9660000000000002},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.952,"15min":1.109},"min":{"1min":0.35999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":4.3460000000000001,"5min":21.061,"15min":21.061},"last":1.9219999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.0390000000000001,"15min":1.139},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.3010000000000002,"5min":21.015000000000001,"15min":21.015000000000001},"last":2.028},{"interface":"front","average":{"1min":1.296,"5min":2.161,"15min":1.4219999999999999},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.1340000000000003,"5min":10.127000000000001,"15min":10.127000000000001},"last":3.948}]},{"osd":6,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":2.5579999999999998,"15min":1.3089999999999999},"min":{"1min":0.38800000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.3239999999999998,"5min":21.317,"15min":21.317},"last":2.7480000000000002},{"interface":"front","average":{"1min":1.266,"5min":2.4289999999999998,"15min":1.514},"min":{"1min":0.41699999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":7.1100000000000003,"5min":12.106999999999999,"15min":12.106999999999999},"last":4.0339999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.2269999999999999,"15min":1.4379999999999999},"min":{"1min":0.42899999999999999,"5min":0.307,"15min":0.307},"max":{"1min":7.1559999999999997,"5min":9.4719999999999995,"15min":9.4719999999999995},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.9830000000000001,"15min":1.0980000000000001},"min":{"1min":0.442,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":4.548,"5min":11.951000000000001,"15min":11.951000000000001},"last":0.70999999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019107,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2377828,"kb_used_data":2080604,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91367324,"statfs":{"total":95995035648,"available":93560139776,"internally_reserved":0,"allocated":2130538496,"data_stored":2130037988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.448,"15min":0.84899999999999998},"min":{"1min":0.23799999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":31.059000000000001,"5min":31.059000000000001,"15min":31.059000000000001},"last":0.998},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.7070000000000001,"15min":0.96099999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":31.279,"5min":31.279,"15min":31.279},"last":2.1779999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:04:52 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.4910000000000001,"15min":0.875},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.4989999999999997,"5min":13.898999999999999,"15min":13.898999999999999},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.885,"15min":1.0600000000000001},"min":{"1min":0.373,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.0419999999999998,"5min":15.353,"15min":15.353},"last":0.377}]},{"osd":2,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":1.7130000000000001,"15min":0.95199999999999996},"min":{"1min":0.29999999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":9.5709999999999997,"5min":9.5709999999999997,"15min":9.5709999999999997},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.607,"15min":0.92500000000000004},"min":{"1min":0.37,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.8919999999999999,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.432}]},{"osd":3,"last update":"Mon Oct 9 10:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":2.7799999999999998,"15min":1.274},"min":{"1min":0.376,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.008,"5min":67.156999999999996,"15min":67.156999999999996},"last":2.125},{"interface":"front","average":{"1min":1.1759999999999999,"5min":2.331,"15min":1.139},"min":{"1min":0.312,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.7850000000000001,"5min":70.238,"15min":70.238},"last":2.355}]},{"osd":5,"last update":"Mon Oct 9 10:04:10 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.91600000000000004,"15min":0.60999999999999999},"min":{"1min":0.27200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.4420000000000002,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.0389999999999999,"15min":0.67300000000000004},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.9199999999999999,"5min":14.773,"15min":14.773},"last":0.38500000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:04:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.72399999999999998,"15min":0.57699999999999996},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":2.6030000000000002,"5min":9.4079999999999995,"15min":9.4079999999999995},"last":1.153},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.028,"15min":0.69599999999999995},"min":{"1min":0.34000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.6600000000000001,"5min":9.0109999999999992,"15min":9.0109999999999992},"last":0.57799999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:04:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.0880000000000001,"15min":0.746},"min":{"1min":0.42599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.944,"5min":14.81,"15min":14.81},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.70399999999999996,"15min":0.624},"min":{"1min":0.38600000000000001,"5min":0.32800000000000001,"15min":0.32200000000000001},"max":{"1min":2.323,"5min":4.0350000000000001,"15min":4.0350000000000001},"last":1.1830000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822879,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1983700,"kb_used_data":1682380,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91761452,"statfs":{"total":95995035648,"available":93963726848,"internally_reserved":0,"allocated":1722757120,"data_stored":1722268740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:04:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.8520000000000001,"15min":0.996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":6.0540000000000003,"5min":15.773999999999999,"15min":15.773999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.4179999999999999,"5min":2.1960000000000002,"15min":1.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":7.0259999999999998,"5min":15.699,"15min":15.699},"last":2.968}]},{"osd":1,"last update":"Mon Oct 9 10:03:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.04,"15min":1.054},"min":{"1min":0.254,"5min":0.248,"15min":0.248},"max":{"1min":5.5330000000000004,"5min":15.689,"15min":15.689},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.016,"5min":2.0960000000000001,"15min":1.0760000000000001},"min":{"1min":0.30199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8490000000000002,"5min":15.896000000000001,"15min":15.896000000000001},"last":1.875}]},{"osd":2,"last update":"Mon Oct 9 10:04:35 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.7090000000000001,"15min":1.0349999999999999},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8200000000000003,"5min":12.582000000000001,"15min":12.582000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.764,"5min":2.0830000000000002,"15min":1.1619999999999999},"min":{"1min":0.36899999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.8490000000000002,"5min":12.438000000000001,"15min":12.438000000000001},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:04:02 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":2.351,"15min":1.2370000000000001},"min":{"1min":0.374,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.1689999999999996,"5min":39.345999999999997,"15min":39.345999999999997},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.7909999999999999,"5min":3.044,"15min":1.4450000000000001},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.081,"5min":38.906999999999996,"15min":38.906999999999996},"last":2.3460000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:04:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.151,"15min":0.69699999999999995},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.4830000000000001,"5min":12.304,"15min":12.304},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":1.2549999999999999,"15min":0.746},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.224,"5min":27.768000000000001,"15min":27.768000000000001},"last":0.45900000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.139,"15min":0.73899999999999999},"min":{"1min":0.30099999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.694,"5min":27.716000000000001,"15min":27.716000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.85499999999999998,"15min":0.65500000000000003},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.28499999999999998},"max":{"1min":0.98299999999999998,"5min":10.173999999999999,"15min":10.173999999999999},"last":0.439}]},{"osd":7,"last update":"Mon Oct 9 10:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.94599999999999995,"15min":0.71199999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.81399999999999995,"5min":10.334,"15min":10.334},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.302,"15min":0.81100000000000005},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.317},"max":{"1min":3.3889999999999998,"5min":12.356,"15min":12.356},"last":0.57099999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3339714560,"data_stored":3339714560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3433299968,"data_stored":3433299968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4655546368,"data_stored":4655546368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2132344832,"data_stored":2132344832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1731198976,"data_stored":1731198976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2573074432,"data_stored":2573074432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2589655040,"data_stored":2589655040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:04:56.848 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T10:04:56.848 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:04:56.848 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T10:04:56.848 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:04:56.848 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:04:56.849 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T10:04:56.931 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:04:56.970 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:56.976 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:04:56.985 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 88 16 106383 106367 75.4798 39.5625 0.0324292 0.0132428 2023-10-09T10:04:57.018 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:04:57.019 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:04:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:57.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:04:57.056 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:57.072 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:04:57.100 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:56 smithi012 ceph-mon[147926]: pgmap v525: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T10:04:57.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:56 smithi012 ceph-mon[152263]: pgmap v525: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T10:04:57.119 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:04:57.129 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:04:57.161 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:56 smithi110 ceph-mon[136032]: pgmap v525: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T10:04:57.162 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:04:57.166 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:57.214 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:04:57.222 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:04:57.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:57.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:04:57.254 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:04:57.289 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:04:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:04:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:04:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:04:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:04:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:04:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:04:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:04:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:04:57.326 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:04:57.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:57.432 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:57.716 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:04:57.747 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:57.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:57.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:57.858 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:57.901 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:57.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:04:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:04:57] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T10:04:57.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:57 smithi012 ceph-mon[147926]: from='client.45224 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:04:57.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1512253866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:57.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:57 smithi012 ceph-mon[152263]: from='client.45224 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:04:57.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1512253866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:57.959 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:57.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:57.986 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 89 16 107100 107084 75.1354 44.8125 0.0215488 0.0133048 2023-10-09T10:04:58.063 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:04:58.133 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:58.171 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:57 smithi110 ceph-mon[136032]: from='client.45224 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:04:58.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1512253866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:04:58.266 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:04:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:58.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:04:58.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:04:58.486 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:58.986 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total time run: 90.0167 2023-10-09T10:04:58.986 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total writes made: 107775 2023-10-09T10:04:58.986 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Write size: 65536 2023-10-09T10:04:58.986 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object size: 65536 2023-10-09T10:04:58.986 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Bandwidth (MB/sec): 74.8299 2023-10-09T10:04:58.986 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Bandwidth: 31.6825 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max bandwidth (MB/sec): 181.188 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min bandwidth (MB/sec): 30.625 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average IOPS: 1197 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev IOPS: 506.919 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max IOPS: 2899 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min IOPS: 490 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average Latency(s): 0.0133599 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Latency(s): 0.0148146 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max latency(s): 1.02658 2023-10-09T10:04:58.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min latency(s): 0.00114602 2023-10-09T10:04:58.992 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:04:59.024 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:04:59.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:59.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:04:59.100 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:04:59.100 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-10-09T10:04:59.101 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:04:59.123 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:58 smithi110 ceph-mon[136032]: pgmap v526: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T10:04:59.123 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:58 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.124 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:58 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.124 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:58 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.124 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:58 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:59.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[147926]: pgmap v526: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T10:04:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:59.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[152263]: pgmap v526: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T10:04:59.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:04:59.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:58 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:04:59.220 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:59.540 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:04:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:59.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:04:59.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:04:59.690 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:59.741 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:04:59.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:04:59.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:04:59.848 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:04:59.906 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:04:59.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:04:59 smithi110 ceph-mon[136032]: pgmap v527: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T10:05:00.028 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:00.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:00.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:00.038 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:04:59 smithi012 ceph-mon[147926]: pgmap v527: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T10:05:00.039 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:04:59 smithi012 ceph-mon[152263]: pgmap v527: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T10:05:00.136 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:00.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:00.215 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:00.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:00.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:00.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:05:00.324 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:00.354 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:00.391 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:00.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:00.422 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:05:00.423 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:05:00.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:00.628 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:00.853 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:00.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:00.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:00.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2682297935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:00.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:00 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:00.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:05:00.984 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:05:00.988 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:05:01.061 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:01.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2682297935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:01.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:00 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2682297935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:00 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:01.498 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:01.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:01.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:01.605 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:01.637 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:01.662 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:01.663 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:01.685 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:01.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:01.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:01.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:01.857 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:01.980 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:01.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:02.090 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:02.130 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:02.176 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:05:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[147926]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T10:05:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[147926]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[147926]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[147926]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[147926]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[147926]: pgmap v529: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 779 op/s 2023-10-09T10:05:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:05:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[152263]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T10:05:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[152263]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[152263]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[152263]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[152263]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:01 smithi012 ceph-mon[152263]: pgmap v529: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 779 op/s 2023-10-09T10:05:02.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:02.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:01 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:05:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:01 smithi110 ceph-mon[136032]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T10:05:02.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:01 smithi110 ceph-mon[136032]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:01 smithi110 ceph-mon[136032]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:01 smithi110 ceph-mon[136032]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:01 smithi110 ceph-mon[136032]: failed to encode map e210 with expected crc 2023-10-09T10:05:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:01 smithi110 ceph-mon[136032]: pgmap v529: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 779 op/s 2023-10-09T10:05:02.281 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:02.317 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:02.473 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:02.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:02.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:02.580 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:02.646 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:02.739 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:02.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:02.850 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:02.895 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:02.971 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:02.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:03.042 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:05:03.042 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:05:03.079 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:03.128 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:03.386 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:03.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:03.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:03.405 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:05:03.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:05:03.443 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:05:03.473 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:05:03.491 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:03.517 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:05:03.561 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:05:03.566 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:03.593 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:05:03.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:05:03.614 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:03.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:03.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:03.633 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:05:03.672 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:05:03.683 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:05:03.683 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:05:03.683 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:05:03.683 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:05:03.683 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:05:03.683 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:05:03.683 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:05:03.683 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:05:03.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:03.825 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:03.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1232668306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:03.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:03 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:03.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:03 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:03.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1232668306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:03.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:03 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:03.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:03 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:04.084 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:04.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:04.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:04.164 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:04.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:05:04.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1232668306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:04.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:03 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:04.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:03 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:04.275 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:04.338 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:04.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:04.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:04.503 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:05:04.503 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:05:04.504 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T10:05:04.504 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:05:04.504 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:05:04.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T10:05:04.517 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:04.533 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:04.568 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:04.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:04.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:04.802 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:05.032 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:04 smithi110 ceph-mon[136032]: pgmap v530: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 758 op/s 2023-10-09T10:05:05.032 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:04 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:05:05.032 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:04 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:05:05.032 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:04 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:05.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:04 smithi012 ceph-mon[147926]: pgmap v530: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 758 op/s 2023-10-09T10:05:05.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:04 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:05:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:04 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:05:05.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:04 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:05.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:04 smithi012 ceph-mon[152263]: pgmap v530: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 758 op/s 2023-10-09T10:05:05.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:04 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:05:05.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:04 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:05:05.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:04 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:05.541 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:05.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:05.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:05.551 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:05:05.551 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:05:05.551 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:05:05.551 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:05:05.551 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:05:05.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:05.709 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:05.886 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:05.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:05.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:05:05.990 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:06.045 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:06.115 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:06.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:06.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:06.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:05:06.221 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:06.274 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:06.328 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:06.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:06.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:06.540 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:06.611 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:06.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:06.681 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:06.715 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:06.748 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:06.795 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:06.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:06.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:06.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:06.965 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:07.036 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:07.040 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:06 smithi012 ceph-mon[147926]: pgmap v531: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 742 op/s 2023-10-09T10:05:07.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:06 smithi012 ceph-mon[152263]: pgmap v531: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 742 op/s 2023-10-09T10:05:07.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:07.080 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:06 smithi110 ceph-mon[136032]: pgmap v531: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 742 op/s 2023-10-09T10:05:07.142 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:07.142 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:07.184 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:07.192 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:07.449 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:07.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:07.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:07.616 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:07.711 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:07.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:07.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:07.915 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:07.967 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:05:07.967 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:05:07.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:05:07.967 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:05:07.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.967 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96930 6.e 6:710aa3be:::benchmark_data_smithi012_182576_object96929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:05:07.967 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:05:07.967 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 294.37052980499999, 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098520888000000001, 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96930 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:07.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593683+0000", 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.313e-05 2023-10-09T10:05:07.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593746+0000", 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2289999999999992e-06 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593756+0000", 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0159999999999999e-06 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593761+0000", 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8661999999999998e-05 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593859+0000", 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146317 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.594006+0000", 2023-10-09T10:05:07.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000179419 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.594185+0000", 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081610680000000005 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602346+0000", 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061366659999999998 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608483+0000", 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079327541000000001 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.687810+0000", 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1600999999999995e-05 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:07.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.687872+0000", 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043339989999999998 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692206+0000", 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098520888000000001 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.974 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96932 6.e 6:7facdbe4:::benchmark_data_smithi012_182576_object96931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 294.36207355699997, 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094685332999999997, 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96932 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:07.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602140+0000", 2023-10-09T10:05:07.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8144999999999999e-05 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602168+0000", 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2849999999999997e-06 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602175+0000", 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7560000000000001e-06 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602179+0000", 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000195152 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602374+0000", 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206584 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:07.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602581+0000", 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000174067 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602755+0000", 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032482050000000001 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.606003+0000", 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024921650000000002 2023-10-09T10:05:07.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608495+0000", 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083792852000000001 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692288+0000", 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6909000000000006e-05 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692355+0000", 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044717339999999998 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.696827+0000", 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094685332999999997 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96944 6.e 6:71fd26c2:::benchmark_data_smithi012_182576_object96943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 294.339459132, 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094319584999999997, 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96944 2023-10-09T10:05:07.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624753+0000", 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9678999999999994e-05 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624833+0000", 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0849000000000001e-05 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:07.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624844+0000", 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5359999999999998e-06 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624850+0000", 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012482843 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637333+0000", 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016598700000000001 2023-10-09T10:05:07.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637499+0000", 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027263199999999997 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637772+0000", 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033034599999999998 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641075+0000", 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088881500000000005 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641964+0000", 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072603763000000002 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:07.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714568+0000", 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4980000000000005e-05 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714633+0000", 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044422960000000001 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.719075+0000", 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094319584999999997 2023-10-09T10:05:07.988 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:105483 6.2 6:45b0cc0b:::benchmark_data_smithi012_182576_object105482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 278.24157441599999, 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1741883319999999, 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:07.989 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105483 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:05:07.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5788999999999999e-05 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722676+0000", 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.089e-06 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722684+0000", 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5950000000000004e-06 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722690+0000", 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010708354 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:07.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733399+0000", 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013741599999999999 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733536+0000", 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017242200000000001 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733708+0000", 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003714143 2023-10-09T10:05:07.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.737423+0000", 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015821828 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.753244+0000", 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036249568000000003 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789494+0000", 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8799999999999997e-06 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789503+0000", 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1072112519999999 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.896714+0000", 2023-10-09T10:05:07.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114647 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.896829+0000", 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1741883319999999 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:105508 6.e 6:77549acc:::benchmark_data_smithi012_182576_object105507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:05:07.996 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 278.18913804599998, 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1483757859999999, 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105508 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775075+0000", 2023-10-09T10:05:07.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9004000000000003e-05 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775174+0000", 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.751e-05 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775192+0000", 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6340000000000001e-06 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:07.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775199+0000", 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4343000000000003e-05 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775274+0000", 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000152091 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775426+0000", 2023-10-09T10:05:08.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019803600000000001 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775624+0000", 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057209089999999997 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.781345+0000", 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079732469999999993 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789318+0000", 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8120000000000002e-06 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789328+0000", 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.114660743 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.903988+0000", 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000495179 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.904484+0000", 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018860534000000002 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.923344+0000", 2023-10-09T10:05:08.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010841900000000001 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.923453+0000", 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1483757859999999 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:178976 6.e 6.4855639e (undecoded) ondisk+retry+write+known_if_redirected e158)", 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:05:08.004 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 170.99835885799999, 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.74206674699999997, 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178976 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965854+0000", 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5659999999999997e-06 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965860+0000", 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9839999999999996e-06 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965870+0000", 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.72e-06 2023-10-09T10:05:08.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965877+0000", 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020714689999999998 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.967948+0000", 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0054999999999999e-05 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.967958+0000", 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.73995176900000004 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.707910+0000", 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.277e-05 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.707923+0000", 2023-10-09T10:05:08.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.74206674699999997 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179004 6.e 6.afd2ea6e (undecoded) ondisk+write+known_if_redirected e158)", 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 170.95974923399999, 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.703617838, 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:05:08.010 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179004 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004464+0000", 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9590000000000007e-06 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004474+0000", 2023-10-09T10:05:08.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1376999999999999e-05 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004495+0000", 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4409999999999993e-06 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004504+0000", 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0488000000000002e-05 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004554+0000", 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0353999999999999e-05 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004564+0000", 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70350862300000006 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708073+0000", 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.04e-05 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708083+0000", 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.703617838 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.015 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179005 6.e 6.7a62357e (undecoded) ondisk+write+known_if_redirected e158)", 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 170.959639321, 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70354121400000003, 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179005 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:05:08.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8079999999999998e-06 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004579+0000", 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5859999999999999e-06 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004586+0000", 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5590000000000001e-06 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004590+0000", 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6616000000000002e-05 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004637+0000", 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2291e-05 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004649+0000", 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70345850600000004 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708107+0000", 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3999999999999998e-06 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708117+0000", 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70354121400000003 2023-10-09T10:05:08.020 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195777 6.12 6:4f83111c:::benchmark_data_smithi012_182576_object90256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 144.07280275100001, 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86944099200000002, 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.021 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195777 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891410+0000", 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4070000000000004e-06 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891418+0000", 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4187e-05 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891432+0000", 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6590000000000004e-06 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891438+0000", 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.266e-05 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891500+0000", 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7346999999999998e-05 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891518+0000", 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.84995838999999995 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741476+0000", 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1224999999999995e-05 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741567+0000", 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034862599999999999 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741916+0000", 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076180900000000001 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749534+0000", 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011212183000000001 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760746+0000", 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6701000000000001e-05 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760773+0000", 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.125e-05 2023-10-09T10:05:08.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760814+0000", 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8943000000000003e-05 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760853+0000", 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86944099200000002 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.028 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195847 6.12 6:4f2f2e62:::benchmark_data_smithi012_182576_object90326:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.84067032300001, 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63759273800000005, 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.029 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195847 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:08.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123543+0000", 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.46e-06 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123550+0000", 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3708e-05 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123563+0000", 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8350000000000004e-06 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123571+0000", 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1295000000000002e-05 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123632+0000", 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8449999999999998e-06 2023-10-09T10:05:08.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123642+0000", 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.61961346399999995 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743256+0000", 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8935000000000003e-05 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743325+0000", 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017814799999999999 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743503+0000", 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060367659999999998 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749540+0000", 2023-10-09T10:05:08.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011359289 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760899+0000", 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015118900000000001 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761050+0000", 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.49e-05 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761095+0000", 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2329000000000001e-05 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761137+0000", 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63759273800000005 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195920 6.12 6:4be14f21:::benchmark_data_smithi012_182576_object90399:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.700560983, 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.49761078399999997, 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195920 2023-10-09T10:05:08.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263652+0000", 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6619999999999999e-06 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263659+0000", 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.222e-05 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263671+0000", 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3700000000000003e-06 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263677+0000", 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5055999999999995e-05 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263742+0000", 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.256e-05 2023-10-09T10:05:08.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263754+0000", 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.48082169600000002 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744576+0000", 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114827 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744691+0000", 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026918100000000001 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744960+0000", 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096003400000000006 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754560+0000", 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063868149999999997 2023-10-09T10:05:08.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760947+0000", 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024620099999999998 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761193+0000", 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4209000000000003e-05 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761238+0000", 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7093e-05 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761265+0000", 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.49761078399999997 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.044 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196022 6.12 6:4bcd6600:::benchmark_data_smithi012_182576_object90501:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.56861819900001, 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.36575843200000002, 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196022 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395595+0000", 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2799999999999998e-06 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395602+0000", 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5897999999999999e-05 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395618+0000", 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4340000000000003e-06 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395626+0000", 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1285999999999995e-05 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395697+0000", 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2662e-05 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395710+0000", 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35087969000000002 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746589+0000", 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010907400000000001 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746698+0000", 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024344699999999999 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746942+0000", 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076297279999999997 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754572+0000", 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063959359999999996 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760967+0000", 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032883000000000001 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761296+0000", 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6942999999999999e-05 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761323+0000", 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1780999999999999e-05 2023-10-09T10:05:08.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761355+0000", 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.36575843200000002 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196042 6.12 6:4e976c7d:::benchmark_data_smithi012_182576_object90521:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.53080406000001, 2023-10-09T10:05:08.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.330443298, 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196042 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433409+0000", 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3280000000000006e-06 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433417+0000", 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2411e-05 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433430+0000", 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3770000000000004e-06 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433435+0000", 2023-10-09T10:05:08.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5327999999999997e-05 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433481+0000", 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3749999999999997e-06 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433488+0000", 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.31462358499999998 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748112+0000", 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115394 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748227+0000", 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025816400000000002 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748485+0000", 2023-10-09T10:05:08.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010564301 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759049+0000", 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037354570000000002 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.762785+0000", 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0009886719999999999 2023-10-09T10:05:08.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763774+0000", 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7079000000000001e-05 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763821+0000", 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3420999999999997e-05 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763854+0000", 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.330443298 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196076 6.12 6:49ef5a44:::benchmark_data_smithi012_182576_object90555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.49574527799999, 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30399614400000002, 2023-10-09T10:05:08.060 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196076 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468468+0000", 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.251e-06 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468474+0000", 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3427e-05 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468488+0000", 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6200000000000004e-06 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468493+0000", 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3034000000000006e-05 2023-10-09T10:05:08.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468566+0000", 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2557e-05 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468579+0000", 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28100584000000001 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749585+0000", 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118539 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749703+0000", 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025233699999999999 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749955+0000", 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009103547 2023-10-09T10:05:08.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759059+0000", 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048303510000000001 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763889+0000", 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084814780000000006 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772371+0000", 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2203999999999997e-05 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772423+0000", 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2608000000000003e-05 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772466+0000", 2023-10-09T10:05:08.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30399614400000002 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196085 6.12 6:4d84fb9e:::benchmark_data_smithi012_182576_object90564:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.48506484200001, 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.298003501, 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.068 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196085 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:05:08.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479148+0000", 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7549999999999994e-06 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479156+0000", 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6019e-05 2023-10-09T10:05:08.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479172+0000", 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3470000000000004e-06 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479178+0000", 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6717999999999994e-05 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479245+0000", 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2296999999999999e-05 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479257+0000", 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27237020299999998 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.751627+0000", 2023-10-09T10:05:08.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114201 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.751741+0000", 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027028600000000001 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.752012+0000", 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070559289999999998 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759068+0000", 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013462363 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772530+0000", 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045611979999999998 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777091+0000", 2023-10-09T10:05:08.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3906999999999999e-05 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777125+0000", 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8297e-05 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777153+0000", 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.298003501 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.075 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196149 6.12 6:48b2c020:::benchmark_data_smithi012_182576_object90628:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.39167267799999, 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20477431600000001, 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.076 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196149 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:08.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572541+0000", 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572548+0000", 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2437e-05 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572560+0000", 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4049999999999997e-06 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572567+0000", 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6699999999999995e-05 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572633+0000", 2023-10-09T10:05:08.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1873e-05 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572645+0000", 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.180477941 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753123+0000", 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5816999999999995e-05 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753209+0000", 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000185847 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753395+0000", 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070544839999999998 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760449+0000", 2023-10-09T10:05:08.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012128781 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772578+0000", 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046280150000000001 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777206+0000", 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5373e-05 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777261+0000", 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5093999999999998e-05 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777316+0000", 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20477431600000001 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196189 6.12 6:4bcf1e87:::benchmark_data_smithi012_182576_object90668:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.33062236699999, 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14382505400000001, 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196189 2023-10-09T10:05:08.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633591+0000", 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2080000000000002e-06 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633596+0000", 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9220000000000002e-06 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633602+0000", 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2490000000000001e-06 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633605+0000", 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8455000000000003e-05 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633653+0000", 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7867999999999999e-05 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633671+0000", 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12094508800000001 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754616+0000", 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2300999999999996e-05 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754709+0000", 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019090900000000001 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754899+0000", 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055585510000000001 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760458+0000", 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012168912 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772627+0000", 2023-10-09T10:05:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047358959999999999 2023-10-09T10:05:08.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777363+0000", 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0698999999999999e-05 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777393+0000", 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4036000000000002e-05 2023-10-09T10:05:08.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777418+0000", 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14382505400000001 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196191 6.12 6:48379c93:::benchmark_data_smithi012_182576_object90670:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:05:08.092 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.33059974, 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14395870599999999, 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196191 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:05:08.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.013e-06 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633617+0000", 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5360000000000001e-06 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633620+0000", 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3039999999999999e-06 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633623+0000", 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047045000000000002 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634093+0000", 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0719999999999999e-06 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634098+0000", 2023-10-09T10:05:08.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121978771 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756077+0000", 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104263 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756181+0000", 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000219302 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756400+0000", 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004065333 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760466+0000", 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012786838999999999 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.773253+0000", 2023-10-09T10:05:08.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042139990000000004 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777467+0000", 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7116e-05 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777524+0000", 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0078999999999999e-05 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777574+0000", 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14395870599999999 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196192 6.12 6:4fcc3e62:::benchmark_data_smithi012_182576_object90671:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 143.33057197100001, 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14401113700000001, 2023-10-09T10:05:08.100 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196192 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7549999999999999e-06 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633644+0000", 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3650000000000003e-06 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633650+0000", 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7549999999999999e-06 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633652+0000", 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000899926 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634552+0000", 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3590000000000001e-06 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634557+0000", 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12309893399999999 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757656+0000", 2023-10-09T10:05:08.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1304999999999995e-05 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757737+0000", 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019102100000000001 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757928+0000", 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027759170000000001 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760704+0000", 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013396351000000001 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.774100+0000", 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035033410000000001 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777604+0000", 2023-10-09T10:05:08.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7534000000000001e-05 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777631+0000", 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2878e-05 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777654+0000", 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14401113700000001 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.107 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:13667 7.1 7:8e32afd2:::benchmark_data_smithi012_221426_object13666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:03:35.011644+0000", 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 92.952571266000007, 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11403589, 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:05:08.108 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13667 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011644+0000", 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011644+0000", 2023-10-09T10:05:08.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011642+0000", 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000147603 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011789+0000", 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6086000000000002e-05 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011805+0000", 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9779999999999999e-06 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011812+0000", 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2852000000000002e-05 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011865+0000", 2023-10-09T10:05:08.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130438 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011996+0000", 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000165916 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.012162+0000", 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00091800400000000004 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.013080+0000", 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001552908 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.014633+0000", 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110859579 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.125492+0000", 2023-10-09T10:05:08.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7923000000000006e-05 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.125570+0000", 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109425 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.125679+0000", 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11403589 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.114 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.115 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.115 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:08.115 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:08.115 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:05:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:08.116 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:08.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:08.116 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:05:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:05:07] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T10:05:08.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:08.268 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:08.327 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:05:08.401 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:08.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:08.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:08.598 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:08.749 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:08.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:08.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:08.848 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:08 smithi012 ceph-mon[147926]: pgmap v532: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 734 op/s 2023-10-09T10:05:08.849 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2195579475' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:08.849 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:08.858 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:08.861 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:05:08.892 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:08.939 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:08.941 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:08.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:08.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:09.046 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:09.090 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:09.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:08 smithi012 ceph-mon[152263]: pgmap v532: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 734 op/s 2023-10-09T10:05:09.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2195579475' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:09.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:09.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:08 smithi110 ceph-mon[136032]: pgmap v532: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 734 op/s 2023-10-09T10:05:09.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2195579475' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:09.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:09.198 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:09.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:09.304 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:09.341 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:09.560 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:05:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "" 2023-10-09T10:05:09.560 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:05:09.782 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:09.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:09.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:09.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:09.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:09.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[152263]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T10:05:09.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[152263]: failed to encode map e211 with expected crc 2023-10-09T10:05:09.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[152263]: failed to encode map e211 with expected crc 2023-10-09T10:05:09.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[152263]: failed to encode map e211 with expected crc 2023-10-09T10:05:09.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[152263]: failed to encode map e211 with expected crc 2023-10-09T10:05:09.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[152263]: failed to encode map e211 with expected crc 2023-10-09T10:05:09.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:09.981 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:10.005 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:05:10.046 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:05:10.052 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:09 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:10.052 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:09 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:10.052 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:09 smithi110 ceph-mon[136032]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T10:05:10.052 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:09 smithi110 ceph-mon[136032]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.052 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:09 smithi110 ceph-mon[136032]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.053 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:09 smithi110 ceph-mon[136032]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.053 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:09 smithi110 ceph-mon[136032]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.053 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:09 smithi110 ceph-mon[136032]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.077 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:05:10.114 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:10.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:10.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:10.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:10.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:10.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[147926]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T10:05:10.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[147926]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[147926]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[147926]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[147926]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:09 smithi012 ceph-mon[147926]: failed to encode map e211 with expected crc 2023-10-09T10:05:10.122 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:05:10.157 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:05:10.201 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:05:10.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:10.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:05:10.272 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:10.289 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:05:10.308 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:05:10.308 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:05:10.308 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:05:10.308 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:05:10.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:05:10.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:05:10.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:05:10.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:05:10.312 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:05:10.312 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:05:10.328 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:10.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:10.437 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:10.480 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:10.661 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:10.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:10.707 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:05:10.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:05:10.843 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:10.909 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:10.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:10.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:10.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:10 smithi110 ceph-mon[136032]: pgmap v534: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.2 MiB/s wr, 745 op/s 2023-10-09T10:05:10.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3483811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:11.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:05:11.074 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:11.142 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:11.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:11.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:11.153 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:10 smithi012 ceph-mon[147926]: pgmap v534: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.2 MiB/s wr, 745 op/s 2023-10-09T10:05:11.154 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3483811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:11.154 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:10 smithi012 ceph-mon[152263]: pgmap v534: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.2 MiB/s wr, 745 op/s 2023-10-09T10:05:11.154 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3483811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:11.254 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:11.306 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:11.349 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:11.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:11.375 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:11.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:11.499 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:11.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:05:11.546 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:11.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:11.650 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:11.707 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:11.763 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:11.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:11.866 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:11.911 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:12.032 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:12.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:12.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:12.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:12.183 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:12.339 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:12.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:12.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:12.390 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:12.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:12.524 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:12.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:05:12.697 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:12.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:12.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:12.805 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:12.834 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:05:12.836 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:05:12.901 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:13.029 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:13.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:13.042 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:12 smithi012 ceph-mon[147926]: pgmap v535: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 5.1 MiB/s wr, 750 op/s 2023-10-09T10:05:13.042 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:12 smithi012 ceph-mon[152263]: pgmap v535: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 5.1 MiB/s wr, 750 op/s 2023-10-09T10:05:13.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:13.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:13.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:12 smithi110 ceph-mon[136032]: pgmap v535: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 5.1 MiB/s wr, 750 op/s 2023-10-09T10:05:13.226 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:13.248 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:05:13.412 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:13.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:13.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:13.576 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:13.641 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:13.644 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:13.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:13.755 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:13.784 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:13.861 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:13.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:13.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:13.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:13.981 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:14.058 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:14.159 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:14.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:14.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:14.261 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:14.292 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:14.340 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:14.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:14.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:14.445 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:14.489 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:14.938 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:05:14.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:14.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:14.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:14.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[147926]: pgmap v536: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T10:05:14.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:14.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:14.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1978761928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:14.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:14.974 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[152263]: pgmap v536: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T10:05:14.974 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:14.974 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:14.974 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1978761928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:14.974 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:15.058 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:15.091 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:15.154 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:05:15.155 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:05:15.155 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:05:15.155 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T10:05:15.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T10:05:15.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:14 smithi110 ceph-mon[136032]: pgmap v536: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 764 op/s 2023-10-09T10:05:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1978761928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:15.362 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:05:15.399 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:05:15.429 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:05:15.476 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:05:15.480 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:15 smithi012 systemd[1]: Stopping Ceph osd.2 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:05:15.511 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:05:15.557 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:05:15.611 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:05:15.649 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:05:15.650 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:05:15.650 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:05:15.650 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:05:15.650 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:05:15.650 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:05:15.651 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:05:15.651 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:05:15.651 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:05:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:15 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:05:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:15 smithi110 ceph-mon[136032]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T10:05:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:15 smithi110 ceph-mon[136032]: failed to encode map e212 with expected crc 2023-10-09T10:05:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:15 smithi110 ceph-mon[136032]: failed to encode map e212 with expected crc 2023-10-09T10:05:15.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:15 smithi110 ceph-mon[136032]: failed to encode map e212 with expected crc 2023-10-09T10:05:15.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:15 smithi110 ceph-mon[136032]: failed to encode map e212 with expected crc 2023-10-09T10:05:15.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2282538391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:16.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:16.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:05:16.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[147926]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T10:05:16.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[147926]: failed to encode map e212 with expected crc 2023-10-09T10:05:16.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[147926]: failed to encode map e212 with expected crc 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[147926]: failed to encode map e212 with expected crc 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[147926]: failed to encode map e212 with expected crc 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2282538391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[152263]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[152263]: failed to encode map e212 with expected crc 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[152263]: failed to encode map e212 with expected crc 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[152263]: failed to encode map e212 with expected crc 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[152263]: failed to encode map e212 with expected crc 2023-10-09T10:05:16.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2282538391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:16.050 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[166832]: 2023-10-09T10:05:16.044+0000 7f3c2b3b5700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:05:16.051 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[166832]: 2023-10-09T10:05:16.044+0000 7f3c2b3b5700 -1 osd.2 212 *** Got signal Terminated *** 2023-10-09T10:05:16.051 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:16 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[166832]: 2023-10-09T10:05:16.044+0000 7f3c2b3b5700 -1 osd.2 212 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:05:17.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:16 smithi110 ceph-mon[136032]: pgmap v538: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T10:05:17.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:16 smithi110 ceph-mon[136032]: osd.2 marked itself down and dead 2023-10-09T10:05:17.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:16 smithi012 ceph-mon[152263]: pgmap v538: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T10:05:17.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:16 smithi012 ceph-mon[152263]: osd.2 marked itself down and dead 2023-10-09T10:05:17.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:16 smithi012 ceph-mon[147926]: pgmap v538: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T10:05:17.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:16 smithi012 ceph-mon[147926]: osd.2 marked itself down and dead 2023-10-09T10:05:17.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:05:17.625 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:17.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:17.729 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:17.772 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:17.917 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:05:17] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T10:05:18.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:05:18.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[147926]: osdmap e213: 8 total, 7 up, 7 in 2023-10-09T10:05:18.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[147926]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[147926]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[147926]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[147926]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[147926]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[147926]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[147926]: pgmap v540: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T10:05:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[152263]: osdmap e213: 8 total, 7 up, 7 in 2023-10-09T10:05:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[152263]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[152263]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[152263]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[152263]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[152263]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[152263]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:17 smithi012 ceph-mon[152263]: pgmap v540: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T10:05:18.183 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:18.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:18.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:18.276 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:17 smithi110 ceph-mon[136032]: osdmap e213: 8 total, 7 up, 7 in 2023-10-09T10:05:18.276 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:17 smithi110 ceph-mon[136032]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.277 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:17 smithi110 ceph-mon[136032]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.277 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:17 smithi110 ceph-mon[136032]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.277 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:17 smithi110 ceph-mon[136032]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.277 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:17 smithi110 ceph-mon[136032]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.277 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:17 smithi110 ceph-mon[136032]: failed to encode map e213 with expected crc 2023-10-09T10:05:18.277 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:17 smithi110 ceph-mon[136032]: pgmap v540: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T10:05:18.288 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:18.340 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:18.442 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:18.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:18.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:18.528 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:58570 6.a 6:5886f2aa:::benchmark_data_smithi012_182576_object58569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:33.117243+0000", 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.40810644499999, 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42935664200000001, 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.529 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 58570 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117243+0000", 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117243+0000", 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117241+0000", 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5274e-05 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117297+0000", 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0071000000000001e-05 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117307+0000", 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5940000000000004e-06 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117311+0000", 2023-10-09T10:05:18.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9631000000000001e-05 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117341+0000", 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1067000000000002e-05 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117422+0000", 2023-10-09T10:05:18.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012439499999999999 2023-10-09T10:05:18.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:05:18.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.117546+0000", 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020504949999999998 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119597+0000", 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2861000000000001e-05 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119650+0000", 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42687205900000003 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.546522+0000", 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3847000000000002e-05 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.546566+0000", 2023-10-09T10:05:18.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3587000000000002e-05 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.546599+0000", 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42935664200000001 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:58575 6.7 6:e1098f94:::benchmark_data_smithi012_182576_object58574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:33.119469+0000", 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.40588007600002, 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42885122599999997, 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.538 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 58575 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119469+0000", 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119469+0000", 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119468+0000", 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139605 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119607+0000", 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0729999999999994e-06 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119616+0000", 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1460000000000004e-06 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119621+0000", 2023-10-09T10:05:18.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2410999999999997e-05 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119674+0000", 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011231299999999999 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119786+0000", 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014668600000000001 2023-10-09T10:05:18.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119933+0000", 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42420786599999999 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.544141+0000", 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4984999999999998e-05 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.544176+0000", 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040581430000000002 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.548234+0000", 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7042000000000001e-05 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.548281+0000", 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9334e-05 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.548320+0000", 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42885122599999997 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.545 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:58576 6.7 6:e6120b98:::benchmark_data_smithi012_182576_object58575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:33.119639+0000", 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.40570998099997, 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42996529700000002, 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 58576 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.546 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119639+0000", 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119639+0000", 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119638+0000", 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2552999999999998e-05 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119671+0000", 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8959999999999997e-06 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119678+0000", 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.177e-06 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119682+0000", 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42220631800000002 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.541888+0000", 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000172337 2023-10-09T10:05:18.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.542061+0000", 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000187219 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.542248+0000", 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059804869999999996 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.548228+0000", 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103349 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.548332+0000", 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012183109999999999 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.549550+0000", 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8725e-05 2023-10-09T10:05:18.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.549579+0000", 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5525e-05 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.549604+0000", 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42996529700000002 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:58578 6.7 6:e43b6cc8:::benchmark_data_smithi012_182576_object58577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:33.119690+0000", 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.40565924600003, 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.430006269, 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.553 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 58578 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119690+0000", 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119690+0000", 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119689+0000", 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9646e-05 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119729+0000", 2023-10-09T10:05:18.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5760000000000002e-06 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119735+0000", 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9960000000000004e-06 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.119739+0000", 2023-10-09T10:05:18.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42445596800000002 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.544195+0000", 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119867 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.544315+0000", 2023-10-09T10:05:18.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015087600000000001 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.544466+0000", 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038704479999999999 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.548337+0000", 2023-10-09T10:05:18.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4140000000000001e-05 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.548381+0000", 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012623739999999999 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.549643+0000", 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0426999999999999e-05 2023-10-09T10:05:18.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.549674+0000", 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2503000000000001e-05 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.549696+0000", 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.430006269 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.560 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:58571 6.4 e109/97 6:23a304fd:::benchmark_data_smithi012_182576_object58570:head v 109'3650, mlcod=109'3647)", 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:33.123504+0000", 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 345.40184520999998, 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.426818904, 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.123504+0000", 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.123504+0000", 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.123503+0000", 2023-10-09T10:05:18.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0213999999999997e-05 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.123553+0000", 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.41817929500000001 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.541733+0000", 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.031e-05 2023-10-09T10:05:18.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.541753+0000", 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000135437 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.541888+0000", 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1309e-05 2023-10-09T10:05:18.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.541930+0000", 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083604869999999998 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.550290+0000", 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2521999999999998e-05 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:33.550323+0000", 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.426818904 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:124332 6.17 6:eb9d4fcd:::benchmark_data_smithi012_182576_object18811:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:59.922816+0000", 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 258.60253258400002, 2023-10-09T10:05:18.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1875801510000001, 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124332 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922816+0000", 2023-10-09T10:05:18.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922816+0000", 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922814+0000", 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4519999999999997e-06 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922823+0000", 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6069000000000001e-05 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922839+0000", 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3810000000000004e-06 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922845+0000", 2023-10-09T10:05:18.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026576590000000001 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.925503+0000", 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3853999999999999e-05 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.925527+0000", 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98917412900000001 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.914701+0000", 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2989e-05 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.914714+0000", 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.18281733 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.097531+0000", 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1924999999999998e-05 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.097623+0000", 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7617999999999999e-05 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.097721+0000", 2023-10-09T10:05:18.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034036079999999998 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.101124+0000", 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9483e-05 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.101174+0000", 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079896740000000004 2023-10-09T10:05:18.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.109163+0000", 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8235999999999998e-05 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.109202+0000", 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011949619999999999 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.110397+0000", 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1875801510000001 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:124344 6.7 6:e214d4f7:::benchmark_data_smithi012_182576_object18823:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:59.948524+0000", 2023-10-09T10:05:18.575 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 258.57682475600001, 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.157156058, 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124344 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948524+0000", 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948524+0000", 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948523+0000", 2023-10-09T10:05:18.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3069999999999999e-06 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948529+0000", 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1191e-05 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948540+0000", 2023-10-09T10:05:18.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7899999999999999e-06 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948545+0000", 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4625e-05 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948590+0000", 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6800000000000004e-06 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948596+0000", 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96706273700000001 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.915659+0000", 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.619e-06 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.915668+0000", 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1799493320000001 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.095617+0000", 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111927 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.095729+0000", 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000116551 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.095845+0000", 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027622850000000002 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.098608+0000", 2023-10-09T10:05:18.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023244139999999999 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.100932+0000", 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037536100000000001 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.104686+0000", 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011579200000000001 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.104802+0000", 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00087872500000000001 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.105680+0000", 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.157156058 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:124349 6.17 6:ea801b85:::benchmark_data_smithi012_182576_object18828:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:59.957332+0000", 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 258.56801718000003, 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1541461819999999, 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124349 2023-10-09T10:05:18.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957332+0000", 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957332+0000", 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957330+0000", 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8789999999999998e-06 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957340+0000", 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2724999999999999e-05 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957353+0000", 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2390000000000002e-06 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957358+0000", 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8668999999999997e-05 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957417+0000", 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3715e-05 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957431+0000", 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95730658400000002 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.914737+0000", 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9449999999999999e-06 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.914746+0000", 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.183858906 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.098605+0000", 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011053 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.098716+0000", 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010681800000000001 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.098822+0000", 2023-10-09T10:05:18.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023075550000000002 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.101130+0000", 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2167000000000002e-05 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.101202+0000", 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0092419330000000008 2023-10-09T10:05:18.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.110444+0000", 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3112000000000003e-05 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.110487+0000", 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099079500000000004 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.111478+0000", 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1541461819999999 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:168952 6.d 6:bf19fa5d:::benchmark_data_smithi012_182576_object63431:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.166442+0000", 2023-10-09T10:05:18.594 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.358907488, 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.787552816, 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168952 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.166442+0000", 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.166442+0000", 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.166440+0000", 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6510000000000001e-06 2023-10-09T10:05:18.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.166448+0000", 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0608000000000001e-05 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.166459+0000", 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3799999999999999e-06 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.166465+0000", 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034817049999999999 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.169947+0000", 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.157e-06 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.169955+0000", 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.522283367 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692238+0000", 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0318e-05 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692248+0000", 2023-10-09T10:05:18.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.214338436 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.906587+0000", 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9846999999999995e-05 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.906677+0000", 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00061615300000000003 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.907293+0000", 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021427383000000001 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.928720+0000", 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010293775999999999 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.939014+0000", 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077362309999999997 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.946750+0000", 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00702296 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.953773+0000", 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2706000000000006e-05 2023-10-09T10:05:18.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.953836+0000", 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000158419 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.953994+0000", 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.787552816 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.603 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:168959 6.a 6:5b4f3c85:::benchmark_data_smithi012_182576_object63438:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.187477+0000", 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.337872175, 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.83604466, 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.604 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168959 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187477+0000", 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187477+0000", 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187475+0000", 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9630000000000001e-06 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187482+0000", 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3553000000000001e-05 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187496+0000", 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5649999999999998e-06 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187502+0000", 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5254000000000003e-05 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187558+0000", 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7540000000000001e-06 2023-10-09T10:05:18.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187567+0000", 2023-10-09T10:05:18.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50423419199999997 2023-10-09T10:05:18.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.691802+0000", 2023-10-09T10:05:18.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.269e-05 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.691814+0000", 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.266860579 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.958675+0000", 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3975999999999997e-05 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.958759+0000", 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00059436199999999997 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2,6", 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.959353+0000", 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028813258000000001 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.988166+0000", 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042397880000000004 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.992406+0000", 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012154336 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.004561+0000", 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018792485000000001 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023353+0000", 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5186999999999998e-05 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023418+0000", 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103275 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023522+0000", 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.83604466 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:168975 6.d 6:bb404b94:::benchmark_data_smithi012_182576_object63454:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.214508+0000", 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.31084119299999, 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.739661313, 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.615 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168975 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214508+0000", 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214508+0000", 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214506+0000", 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3237999999999999e-05 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214519+0000", 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3148e-05 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214533+0000", 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0739999999999998e-06 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214539+0000", 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5145999999999997e-05 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214594+0000", 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0696000000000001e-05 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214604+0000", 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.477666278 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692271+0000", 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9840000000000001e-06 2023-10-09T10:05:18.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692279+0000", 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2282862539999999 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.920565+0000", 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013848500000000001 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.920703+0000", 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017785899999999999 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.920881+0000", 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010182902000000001 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.931064+0000", 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079830509999999997 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.939047+0000", 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077500099999999999 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.946797+0000", 2023-10-09T10:05:18.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0072460090000000003 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954043+0000", 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5778999999999997e-05 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954079+0000", 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0038999999999996e-05 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954169+0000", 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.739661313 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:168976 6.a 6:55d3563c:::benchmark_data_smithi012_182576_object63455:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.214551+0000", 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.31079802400001, 2023-10-09T10:05:18.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8091393010000001, 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168976 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214551+0000", 2023-10-09T10:05:18.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214551+0000", 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214550+0000", 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0939999999999999e-06 2023-10-09T10:05:18.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214554+0000", 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3249999999999998e-06 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214559+0000", 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8389999999999997e-06 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214564+0000", 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8831000000000001e-05 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214613+0000", 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0939999999999997e-06 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.214622+0000", 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.479627523 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.694249+0000", 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2242000000000002e-05 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.694271+0000", 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.282174983 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.976446+0000", 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000140116 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.976587+0000", 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000160579 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2,6", 2023-10-09T10:05:18.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.976747+0000", 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015582472 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.992330+0000", 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013754800000000001 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.992467+0000", 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014036464 2023-10-09T10:05:18.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.006504+0000", 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017067602000000001 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023571+0000", 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8991e-05 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023610+0000", 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0113999999999999e-05 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023690+0000", 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8091393010000001 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:168995 6.d 6:b283bdcf:::benchmark_data_smithi012_182576_object63474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.250483+0000", 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.27486589700001, 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7039414370000001, 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168995 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.250483+0000", 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.250483+0000", 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.250482+0000", 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5660000000000001e-06 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.250489+0000", 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2048999999999999e-05 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.250501+0000", 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2490000000000004e-06 2023-10-09T10:05:18.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.250509+0000", 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0535000000000001e-05 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.250579+0000", 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0589999999999992e-06 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.250588+0000", 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.44171128300000001 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692300+0000", 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6380000000000004e-06 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692307+0000", 2023-10-09T10:05:18.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2317673899999999 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.924075+0000", 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9967000000000002e-05 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.924155+0000", 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7829000000000001e-05 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.924252+0000", 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006817254 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.931070+0000", 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015390401999999999 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.946460+0000", 2023-10-09T10:05:18.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037104899999999998 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.946831+0000", 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073908569999999998 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954222+0000", 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8989000000000001e-05 2023-10-09T10:05:18.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954281+0000", 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000143616 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954425+0000", 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7039414370000001 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.642 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:169002 6.d 6:bc6efe83:::benchmark_data_smithi012_182576_object63481:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.258641+0000", 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.26670808, 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.695927647, 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.643 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 169002 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.258641+0000", 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.258641+0000", 2023-10-09T10:05:18.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.258640+0000", 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1459999999999999e-06 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.258647+0000", 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3825000000000001e-05 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.258661+0000", 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3540000000000002e-06 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.258668+0000", 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9270000000000006e-05 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.258737+0000", 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0669999999999999e-05 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.258748+0000", 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.43358000099999999 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692328+0000", 2023-10-09T10:05:18.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.571e-06 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692335+0000", 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2344693579999999 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.926805+0000", 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9478000000000001e-05 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.926874+0000", 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6967999999999998e-05 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.926961+0000", 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041126119999999999 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.931074+0000", 2023-10-09T10:05:18.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015417231 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.946491+0000", 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037957300000000002 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.946871+0000", 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075925380000000002 2023-10-09T10:05:18.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954463+0000", 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0670000000000003e-05 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954494+0000", 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4678000000000006e-05 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954569+0000", 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.695927647 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:169004 6.a 6:5c46f5f5:::benchmark_data_smithi012_182576_object63483:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.263050+0000", 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.26229853199999, 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7608272839999999, 2023-10-09T10:05:18.652 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 169004 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.263050+0000", 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.263050+0000", 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.263049+0000", 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1300000000000001e-06 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.263057+0000", 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5028000000000001e-05 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.263072+0000", 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8580000000000002e-06 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.263079+0000", 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5401999999999995e-05 2023-10-09T10:05:18.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.263145+0000", 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.295e-05 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.263158+0000", 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.43130616700000002 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.694464+0000", 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0217999999999999e-05 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.694474+0000", 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.285822359 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.980296+0000", 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110859 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.980407+0000", 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120123 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2,6", 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.980527+0000", 2023-10-09T10:05:18.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011810905999999999 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.992338+0000", 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019050100000000001 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.992529+0000", 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014018688 2023-10-09T10:05:18.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.006547+0000", 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017187228999999998 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023735+0000", 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2453000000000003e-05 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023767+0000", 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011076199999999999 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023878+0000", 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7608272839999999 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:169015 6.d 6:b59c5f48:::benchmark_data_smithi012_182576_object63494:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.283308+0000", 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 196.24204149100001, 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.67149492, 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.662 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 169015 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.283308+0000", 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.283308+0000", 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.283306+0000", 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6699999999999994e-06 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.283314+0000", 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0312000000000001e-05 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.283324+0000", 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4080000000000003e-06 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.283330+0000", 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4768000000000001e-05 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.283385+0000", 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2388999999999999e-05 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.283398+0000", 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.40895865399999998 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692356+0000", 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1300000000000001e-06 2023-10-09T10:05:18.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.692364+0000", 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2363750929999999 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.928739+0000", 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6773000000000006e-05 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.928806+0000", 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010918500000000001 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.928915+0000", 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022470290000000002 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.931162+0000", 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015370376999999999 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.946533+0000", 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036220300000000001 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.946895+0000", 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077152879999999998 2023-10-09T10:05:18.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954610+0000", 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7939000000000001e-05 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954678+0000", 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012426200000000001 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.954802+0000", 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.67149492 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:178998 6.17 6:ef725fe1:::benchmark_data_smithi012_182576_object73477:head [delete] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:16.992521+0000", 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 181.53282767900001, 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86934053499999997, 2023-10-09T10:05:18.671 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178998 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.992521+0000", 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.992521+0000", 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999876 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.992509+0000", 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8935000000000001e-05 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.992528+0000", 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3441999999999999e-05 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.992541+0000", 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1819999999999997e-06 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.992546+0000", 2023-10-09T10:05:18.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7514000000000003e-05 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.992634+0000", 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3553000000000001e-05 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.992648+0000", 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.85315569199999997 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.845803+0000", 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3194000000000003e-05 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.845896+0000", 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035912299999999998 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.846256+0000", 2023-10-09T10:05:18.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033102650000000002 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.849566+0000", 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053829560000000004 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.854949+0000", 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0066389600000000002 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.861588+0000", 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3943999999999998e-05 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.861632+0000", 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023022799999999999 2023-10-09T10:05:18.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.861862+0000", 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86934053499999997 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179002 6.17 6:eb095bab:::benchmark_data_smithi012_182576_object73481:head [delete] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-09T10:05:18.679 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:16.996474+0000", 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 181.528874565, 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86920166099999996, 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179002 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.680 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.996474+0000", 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.996474+0000", 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.996473+0000", 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0170000000000002e-06 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.996482+0000", 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8910999999999999e-05 2023-10-09T10:05:18.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.996501+0000", 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8090000000000004e-06 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.996508+0000", 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3077000000000006e-05 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.996571+0000", 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7110000000000007e-06 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.996580+0000", 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.85117619099999997 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.847757+0000", 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8518999999999996e-05 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.847855+0000", 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025677600000000001 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.848112+0000", 2023-10-09T10:05:18.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002063058 2023-10-09T10:05:18.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.850175+0000", 2023-10-09T10:05:18.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048073650000000001 2023-10-09T10:05:18.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.854982+0000", 2023-10-09T10:05:18.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010448282 2023-10-09T10:05:18.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.865431+0000", 2023-10-09T10:05:18.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3748999999999997e-05 2023-10-09T10:05:18.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.865474+0000", 2023-10-09T10:05:18.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000201807 2023-10-09T10:05:18.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.865676+0000", 2023-10-09T10:05:18.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86920166099999996 2023-10-09T10:05:18.689 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.689 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.689 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179020 6.7 6:e4fe1083:::benchmark_data_smithi012_182576_object73499:head [delete] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:17.038897+0000", 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 181.486452493, 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.811043599, 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.690 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179020 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.038897+0000", 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.038897+0000", 2023-10-09T10:05:18.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.038896+0000", 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9130000000000002e-06 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.038900+0000", 2023-10-09T10:05:18.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0592e-05 2023-10-09T10:05:18.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.038911+0000", 2023-10-09T10:05:18.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8999999999999997e-06 2023-10-09T10:05:18.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.038916+0000", 2023-10-09T10:05:18.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4923999999999999e-05 2023-10-09T10:05:18.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.038961+0000", 2023-10-09T10:05:18.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4370000000000007e-06 2023-10-09T10:05:18.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.038969+0000", 2023-10-09T10:05:18.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.78457943699999999 2023-10-09T10:05:18.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.823549+0000", 2023-10-09T10:05:18.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114282 2023-10-09T10:05:18.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.823663+0000", 2023-10-09T10:05:18.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036500200000000001 2023-10-09T10:05:18.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:05:18.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.824028+0000", 2023-10-09T10:05:18.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02176604 2023-10-09T10:05:18.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.845794+0000", 2023-10-09T10:05:18.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037513109999999998 2023-10-09T10:05:18.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.849545+0000", 2023-10-09T10:05:18.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012300899999999999 2023-10-09T10:05:18.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.849668+0000", 2023-10-09T10:05:18.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5660999999999994e-05 2023-10-09T10:05:18.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.849734+0000", 2023-10-09T10:05:18.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020609599999999999 2023-10-09T10:05:18.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.849940+0000", 2023-10-09T10:05:18.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.811043599 2023-10-09T10:05:18.703 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179022 6.7 6:e2bfeb85:::benchmark_data_smithi012_182576_object73501:head [delete] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-09T10:05:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:17.050768+0000", 2023-10-09T10:05:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 181.474581086, 2023-10-09T10:05:18.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.79946869300000001, 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179022 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.050768+0000", 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:05:18.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.050768+0000", 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.050767+0000", 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0662999999999999e-05 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.050777+0000", 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4087000000000001e-05 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.050791+0000", 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2429999999999996e-06 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.050797+0000", 2023-10-09T10:05:18.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5107e-05 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.050842+0000", 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3919999999999992e-06 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.050851+0000", 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.77496472400000005 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.825816+0000", 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012402499999999999 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.825940+0000", 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025461199999999998 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.826194+0000", 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023297099000000002 2023-10-09T10:05:18.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.849492+0000", 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112033 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.849604+0000", 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039308799999999999 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.849997+0000", 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4969999999999997e-05 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.850052+0000", 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018502199999999999 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:05:18.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.850237+0000", 2023-10-09T10:05:18.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.79946869300000001 2023-10-09T10:05:18.712 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.712 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.712 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.712 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:05:18.712 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:05:18.712 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:05:18.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:18.755 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:18.864 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:18.869 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:05:18.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:18.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:18.971 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:19.040 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:19.123 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:19.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:19.177 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:18 smithi012 bash[236617]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T10:05:19.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:19.312 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:19.654 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:20.427 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:20.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:20.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:20.442 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:20 smithi012 ceph-mon[147926]: pgmap v541: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T10:05:20.449 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:20 smithi012 ceph-mon[152263]: pgmap v541: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T10:05:20.543 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:20.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:20 smithi110 ceph-mon[136032]: pgmap v541: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T10:05:20.580 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:20.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:20.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:20.765 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:20.770 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:20.776 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:20.811 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:20.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:20.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:21.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:21.011 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:05:21.011 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:05:21.011 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:05:21.011 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:05:21.035 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:21.071 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:21.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:21.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:21.175 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:21.215 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:21.396 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:21.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:21.419 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:05:21.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:21.555 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:21.618 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:21.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:21.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:21.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:21.805 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:22.090 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:22.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:22.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:22.197 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:22.205 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:22.265 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:22.362 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:22.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:22.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:22 smithi012 ceph-mon[147926]: pgmap v542: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T10:05:22.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:22 smithi012 ceph-mon[152263]: pgmap v542: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T10:05:22.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:22.484 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:22 smithi110 ceph-mon[136032]: pgmap v542: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T10:05:22.549 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:22.908 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:22.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:22.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:23.016 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:23.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:23.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:23.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:23.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:23.253 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:23.309 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:23.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:23.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:23.546 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:05:23.546 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:05:23.546 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:05:23.546 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:05:23.546 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:05:23.571 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:23.674 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:23.754 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:23.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:23.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:23.868 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:23.935 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:23.996 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:05:24.128 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:24.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:24.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:24.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:24.276 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:24.322 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:24.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:24.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:24.541 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:24.633 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:05:24.690 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:05:24.697 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:05:24.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:05:24.769 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:05:24.814 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:05:24.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T10:05:24.847 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:05:24.885 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:05:24.885 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:05:24.885 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:05:24.886 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T10:05:24.886 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:05:24.886 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:05:24.886 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:05:24.886 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:05:24.886 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:05:24.886 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:24.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:24.998 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:25.007 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:05:25.007 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T10:05:25.042 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:25.137 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:24 smithi110 ceph-mon[136032]: pgmap v543: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T10:05:25.137 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2227040951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:25.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:24 smithi012 ceph-mon[147926]: pgmap v543: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T10:05:25.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2227040951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:25.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:24 smithi012 ceph-mon[152263]: pgmap v543: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T10:05:25.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2227040951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:25.184 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:25.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:25.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:25.298 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:25.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:05:25.421 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:05:25.473 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:25.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:25.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:25.584 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:25.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:25.740 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:25.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:25.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:25.845 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:25.920 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:25.981 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:25.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:25.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:25.994 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:05:25.995 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:05:26.085 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:26.156 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:26.437 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:26.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:26.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:26.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:05:26.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:26.588 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:26.634 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:26.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:26.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:26.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:26.816 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:26.857 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:05:26.917 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:26.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:27.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:27.077 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:27.125 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:27.127 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:26 smithi110 ceph-mon[136032]: pgmap v544: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T10:05:27.127 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1813482494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:27.127 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:27.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:27.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:27.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:26 smithi012 ceph-mon[147926]: pgmap v544: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T10:05:27.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1813482494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:27.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:27.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:26 smithi012 ceph-mon[152263]: pgmap v544: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T10:05:27.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1813482494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:27.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:27.229 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:27.268 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:27.313 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:27.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:27.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:27.417 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:27.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:05:27.456 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:27.506 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:27.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:27.521 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:27.612 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:27.646 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:27.687 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:27.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:27.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:27.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:27.905 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:27.968 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:27.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:28.074 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:28.103 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:27 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:05:28.103 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:27 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:05:28.103 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:28.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:27 smithi110 ceph-mon[136032]: osdmap e214: 8 total, 7 up, 7 in 2023-10-09T10:05:28.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:27 smithi110 ceph-mon[136032]: failed to encode map e214 with expected crc 2023-10-09T10:05:28.104 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:27 smithi110 ceph-mon[136032]: failed to encode map e214 with expected crc 2023-10-09T10:05:28.106 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:28.112 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:05:28.112 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:05:28.112 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:28.112 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[147926]: osdmap e214: 8 total, 7 up, 7 in 2023-10-09T10:05:28.112 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[147926]: failed to encode map e214 with expected crc 2023-10-09T10:05:28.112 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[147926]: failed to encode map e214 with expected crc 2023-10-09T10:05:28.113 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:05:28.113 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:05:28.113 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:28.113 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[152263]: osdmap e214: 8 total, 7 up, 7 in 2023-10-09T10:05:28.113 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[152263]: failed to encode map e214 with expected crc 2023-10-09T10:05:28.113 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-mon[152263]: failed to encode map e214 with expected crc 2023-10-09T10:05:28.113 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:05:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:05:27] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T10:05:28.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:28.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:28.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:28.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:28.299 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:28.351 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:28.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:28.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:28.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:28.491 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:28.533 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:28.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:28.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:28.640 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:28.681 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:28.733 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:28.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:28.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:28.828 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:28.838 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:28.938 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:28 smithi110 ceph-mon[136032]: pgmap v546: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 780 op/s 2023-10-09T10:05:29.066 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:29.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:29.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:29.171 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:29.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:28 smithi012 ceph-mon[147926]: pgmap v546: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 780 op/s 2023-10-09T10:05:29.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:28 smithi012 ceph-mon[152263]: pgmap v546: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 780 op/s 2023-10-09T10:05:29.207 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:29.255 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:29.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:29.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:29.307 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:05:29.308 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:05:29.317 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:05:29.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:29.389 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:29.433 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:29.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:29.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:29.539 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:29.579 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:29.623 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:29.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:29.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:29.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:29.817 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:29.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:05:29.922 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:29.922 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:29.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:29.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:29.995 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:29.995 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:30.008 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5] out_osds: [2] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-10-09T10:05:30.008 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:05:30.008 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-10-09T10:05:30.008 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-10-09T10:05:30.008 DEBUG:teuthology.orchestra.run.smithi012:osd.2> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T10:05:30.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl start ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T10:05:30.092 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:30.095 INFO:journalctl@ceph.osd.2.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T10:05:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:30.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:30.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:30.243 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:30.283 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:30.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:30.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:30.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:30.427 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:30 smithi012 systemd[1]: Starting Ceph osd.2 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:05:30.455 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:30.528 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:30.925 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:30.933 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:05:30.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:30.971 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:05:31.014 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:05:31.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:31.061 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:05:31.102 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:05:31.104 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:31.135 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:30 smithi012 ceph-mon[147926]: pgmap v547: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T10:05:31.135 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/366921496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:31.136 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:30 smithi012 ceph-mon[152263]: pgmap v547: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T10:05:31.136 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/366921496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:31.136 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:05:31.165 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:31.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:31.177 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:05:31.177 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:05:31.178 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:05:31.178 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T10:05:31.178 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:05:31.178 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:05:31.178 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:05:31.178 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:05:31.178 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:05:31.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:30 smithi110 ceph-mon[136032]: pgmap v547: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T10:05:31.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/366921496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:31.278 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:31.310 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:31.342 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:31.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:31.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:31.478 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:31.523 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:31.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:31.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:31.628 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:31.712 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:05:31.805 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:05:31.911 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:31.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:32.021 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:32.119 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:32.146 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:32.196 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:32.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:32.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:32.314 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:32.404 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:32.589 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:32.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:32.673 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:32 smithi012 bash[238137]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T10:05:32.673 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:32 smithi012 bash[238137]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T10:05:32.673 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:32 smithi012 bash[238137]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T10:05:32.673 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:32 smithi012 bash[238137]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T10:05:32.673 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:32 smithi012 bash[238137]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T10:05:32.673 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:32 smithi012 bash[238137]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T10:05:32.673 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:32 smithi012 bash[238137]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T10:05:32.673 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:32 smithi012 bash[238137]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T10:05:32.700 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:32.739 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:32.787 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:32.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:32.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:32.971 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:33.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:32 smithi110 ceph-mon[136032]: pgmap v548: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T10:05:33.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:32 smithi012 ceph-mon[147926]: pgmap v548: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T10:05:33.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:32 smithi012 ceph-mon[152263]: pgmap v548: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T10:05:33.526 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:33.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:33.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:33.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:33.716 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:33.795 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:33.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:33.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:33.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:05:33.903 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:33.934 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:33.975 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:33.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:34.029 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:05:34.057 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2137847389' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:34.057 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:33 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:34.062 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2137847389' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:34.062 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:33 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:34.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:34.193 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2137847389' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:33 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:05:34.281 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:34.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:34.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:34.309 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:05:34.309 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:05:34.309 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:05:34.309 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:05:34.309 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:05:34.330 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:34 smithi012 podman[238787]: 2023-10-09T10:05:34.387 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:34.420 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:34.456 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:34.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:34.563 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:34.660 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:34.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:05:34.927 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:34.927 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:34 smithi012 bash[238787]: ffcd0df4995a3968fe0e2a50a7771bd56a74615e1ca9e0736a2043be3986f7df 2023-10-09T10:05:34.928 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[238878]: 2023-10-09T10:05:34.710+0000 7f8f335ae640 -1 Falling back to public interface 2023-10-09T10:05:34.928 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:34 smithi012 systemd[1]: Started Ceph osd.2 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T10:05:34.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:34.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:35.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:35.102 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:35.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[152263]: pgmap v549: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 744 op/s 2023-10-09T10:05:35.313 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:35.313 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[152263]: failed to encode map e215 with expected crc 2023-10-09T10:05:35.313 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:05:35.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[152263]: osdmap e215: 8 total, 7 up, 7 in 2023-10-09T10:05:35.314 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[152263]: failed to encode map e215 with expected crc 2023-10-09T10:05:35.315 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[147926]: pgmap v549: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 744 op/s 2023-10-09T10:05:35.315 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:35.315 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[147926]: failed to encode map e215 with expected crc 2023-10-09T10:05:35.315 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:05:35.315 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[147926]: osdmap e215: 8 total, 7 up, 7 in 2023-10-09T10:05:35.315 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:34 smithi012 ceph-mon[147926]: failed to encode map e215 with expected crc 2023-10-09T10:05:35.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:35.316 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:35.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:35.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:05:35.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:35.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:34 smithi110 ceph-mon[136032]: pgmap v549: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 744 op/s 2023-10-09T10:05:35.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:34 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:35.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:34 smithi110 ceph-mon[136032]: failed to encode map e215 with expected crc 2023-10-09T10:05:35.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:34 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:05:35.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:34 smithi110 ceph-mon[136032]: osdmap e215: 8 total, 7 up, 7 in 2023-10-09T10:05:35.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:34 smithi110 ceph-mon[136032]: failed to encode map e215 with expected crc 2023-10-09T10:05:35.477 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:35.494 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:35.589 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:35.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:35.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:35.700 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:35.821 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:35.933 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:35.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:35.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:36.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:36.114 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:35 smithi012 ceph-mon[147926]: pgmap v551: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T10:05:36.114 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:35 smithi012 ceph-mon[152263]: pgmap v551: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T10:05:36.139 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:36.206 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:36.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:36.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:36.316 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:36.343 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:35 smithi110 ceph-mon[136032]: pgmap v551: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T10:05:36.349 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:36.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:36.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:36.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:36.575 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:36.973 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:36.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:36.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:37.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:05:37.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:37.201 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:37.305 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:37.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:37.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:37.417 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:37.456 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:37.495 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:37.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:37.599 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:37.627 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:37.668 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:37.669 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:37.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:37.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:37.757 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:05:37.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:37.819 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:05:37.862 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:05:37.890 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:37.904 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:05:37.935 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:no valid command found; 10 closest matches: 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:0 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:1 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:2 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:abort 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:assert 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs debug_inject_read_zeros 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs files list 2023-10-09T10:05:37.982 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs stats 2023-10-09T10:05:37.983 INFO:teuthology.orchestra.run.smithi012.stderr:bluestore allocator dump block 2023-10-09T10:05:37.983 INFO:teuthology.orchestra.run.smithi012.stderr:bluestore allocator fragmentation block 2023-10-09T10:05:37.983 INFO:teuthology.orchestra.run.smithi012.stderr:admin_socket: invalid command 2023-10-09T10:05:37.989 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:05:38.025 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:05:38.025 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:05:38.025 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:05:38.025 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T10:05:38.025 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:05:38.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:05:38.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:05:38.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:05:38.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:05:38.130 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:05:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:05:37] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.33.4" 2023-10-09T10:05:38.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:38 smithi012 ceph-mon[147926]: pgmap v552: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T10:05:38.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3641487033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:38.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:38 smithi012 ceph-mon[152263]: pgmap v552: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T10:05:38.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3641487033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:38.432 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:38 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[238878]: 2023-10-09T10:05:38.339+0000 7f8f335ae640 -1 osd.2 212 log_to_monitors true 2023-10-09T10:05:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:38 smithi110 ceph-mon[136032]: pgmap v552: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T10:05:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3641487033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:39.203 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:39.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:39.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:05:39.302 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:05:39.309 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:39.354 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:39.411 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:39.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:39.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:39.518 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:39.549 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:39 smithi110 ceph-mon[136032]: from='osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:05:39.550 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:39 smithi110 ceph-mon[136032]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:05:39.552 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:39.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:39 smithi012 ceph-mon[147926]: from='osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:05:39.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:39 smithi012 ceph-mon[147926]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:05:39.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:39 smithi012 ceph-mon[152263]: from='osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:05:39.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:39 smithi012 ceph-mon[152263]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:05:39.608 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:05:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[238878]: 2023-10-09T10:05:39.279+0000 7f8f23217700 -1 osd.2 212 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:05:39.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:39.613 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:39.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:39.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:05:39.713 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-2, ['dump_ops_in_flight'] 2023-10-09T10:05:39.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:39.806 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:39.912 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:39.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:39.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:40.021 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:40.111 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:40.352 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:40.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:40.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:40.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:40.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:40.548 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:40.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:40.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:40.552 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: pgmap v553: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T10:05:40.552 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:05:40.552 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: osdmap e216: 8 total, 7 up, 7 in 2023-10-09T10:05:40.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: from='osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:05:40.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:05:40.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:40 smithi110 ceph-mon[136032]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.653 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:40.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: pgmap v553: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: osdmap e216: 8 total, 7 up, 7 in 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: from='osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[147926]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: pgmap v553: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: osdmap e216: 8 total, 7 up, 7 in 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: from='osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:05:40.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:40 smithi012 ceph-mon[152263]: failed to encode map e216 with expected crc 2023-10-09T10:05:40.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:40.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:40.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:40.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:40.931 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:41.098 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:41.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:41.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:41.228 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:05:41.229 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:05:41.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:05:41.308 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:41.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052] boot 2023-10-09T10:05:41.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: osdmap e217: 8 total, 8 up, 7 in 2023-10-09T10:05:41.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:05:41.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.392 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[147926]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052] boot 2023-10-09T10:05:41.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: osdmap e217: 8 total, 8 up, 7 in 2023-10-09T10:05:41.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:05:41.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:41 smithi012 ceph-mon[152263]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.438 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:41.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:41.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:41.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:41.588 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: osd.2 [v2:172.21.15.12:6818/4176449052,v1:172.21.15.12:6819/4176449052] boot 2023-10-09T10:05:41.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: osdmap e217: 8 total, 8 up, 7 in 2023-10-09T10:05:41.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:05:41.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.590 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.590 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.590 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:41 smithi110 ceph-mon[136032]: failed to encode map e217 with expected crc 2023-10-09T10:05:41.596 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:41.634 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:41.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:41.737 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:41.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:41.809 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:41.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:41.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:41.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:05:41.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:41.945 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:41.988 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:41.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:42.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:42.208 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:42.215 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:42.291 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:42.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:42.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:42.330 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:05:42.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:42.474 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:42 smithi012 ceph-mon[147926]: pgmap v556: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T10:05:42.475 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1159952052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:42.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:42 smithi012 ceph-mon[152263]: pgmap v556: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T10:05:42.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1159952052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:42.477 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:42 smithi110 ceph-mon[136032]: pgmap v556: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T10:05:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1159952052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:42.701 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:42.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:42.814 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:42.862 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:42.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:42.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:43.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:43.083 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:43.109 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:43.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:43.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1159952052' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:43.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[147926]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T10:05:43.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[147926]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[147926]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[147926]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1159952052' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[152263]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T10:05:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[152263]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[152263]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:43 smithi012 ceph-mon[152263]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.456 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:43.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:43.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:43.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:43.637 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:43.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:43 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1159952052' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:43 smithi110 ceph-mon[136032]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T10:05:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:43 smithi110 ceph-mon[136032]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:43 smithi110 ceph-mon[136032]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:43 smithi110 ceph-mon[136032]: failed to encode map e218 with expected crc 2023-10-09T10:05:43.804 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:43.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:43.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:43.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:43.945 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:43.991 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:43.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:44.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:44.183 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:44.387 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:44.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:44.494 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:44.576 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:05:44.624 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:44.624 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:05:44.681 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:05:44.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:05:44.720 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:05:44.747 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:05:44.752 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:05:44.788 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:05:44.832 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:05:44.874 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:05:44.913 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:05:44.913 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:05:44.914 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:05:44.914 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:05:44.914 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:05:44.914 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:05:44.914 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:05:44.914 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:05:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:44 smithi012 ceph-mon[147926]: pgmap v558: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T10:05:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:45.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2765118249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:45.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:44 smithi012 ceph-mon[152263]: pgmap v558: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T10:05:45.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:45.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:45.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2765118249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:45.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:44 smithi110 ceph-mon[136032]: pgmap v558: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T10:05:45.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:05:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:05:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2765118249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:45.848 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:45.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:45.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:45.952 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:46.025 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:46.873 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:46.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:46.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:46.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:47.015 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:47.067 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:47.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:47.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:47.174 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:47.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:46 smithi012 ceph-mon[147926]: pgmap v559: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T10:05:47.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:46 smithi012 ceph-mon[152263]: pgmap v559: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T10:05:47.213 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:46 smithi110 ceph-mon[136032]: pgmap v559: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T10:05:47.222 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:47.271 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:47.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:47.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:47.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:47.482 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:47.500 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:05:47.500 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:05:47.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.500 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:143951 7.2 e218/187 7:4ecedbd8:::benchmark_data_smithi012_221426_object36173:head v 218'9177, mlcod=218'9174)", 2023-10-09T10:05:47.500 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:05:47.479599+0000", 2023-10-09T10:05:47.501 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.004992242, 2023-10-09T10:05:47.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050268600000000002, 2023-10-09T10:05:47.501 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:05:47.501 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:05:47.501 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:05:47.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:05:47.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479599+0000", 2023-10-09T10:05:47.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:05:47.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479599+0000", 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479597+0000", 2023-10-09T10:05:47.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:05:47.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:05:47.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479610+0000", 2023-10-09T10:05:47.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2471e-05 2023-10-09T10:05:47.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:05:47.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479632+0000", 2023-10-09T10:05:47.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2436999999999999e-05 2023-10-09T10:05:47.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:05:47.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479642+0000", 2023-10-09T10:05:47.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9860000000000006e-06 2023-10-09T10:05:47.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:05:47.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479703+0000", 2023-10-09T10:05:47.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0773999999999998e-05 2023-10-09T10:05:47.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:05:47.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479731+0000", 2023-10-09T10:05:47.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7665999999999999e-05 2023-10-09T10:05:47.507 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.507 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:05:47.507 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.507 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:143948 7.0 e218/191 7:01018f3f:::benchmark_data_smithi012_221426_object36170:head v 218'9286, mlcod=218'9283)", 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:05:47.479725+0000", 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0048664440000000002, 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049825090000000004, 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:05:47.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479725+0000", 2023-10-09T10:05:47.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:05:47.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:05:47.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479725+0000", 2023-10-09T10:05:47.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:05:47.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:05:47.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479724+0000", 2023-10-09T10:05:47.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:05:47.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:05:47.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479736+0000", 2023-10-09T10:05:47.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1630999999999999e-05 2023-10-09T10:05:47.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:05:47.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479759+0000", 2023-10-09T10:05:47.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2742999999999999e-05 2023-10-09T10:05:47.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:05:47.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479766+0000", 2023-10-09T10:05:47.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2389999999999996e-06 2023-10-09T10:05:47.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:05:47.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479934+0000", 2023-10-09T10:05:47.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016783100000000001 2023-10-09T10:05:47.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:05:47.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479954+0000", 2023-10-09T10:05:47.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0381999999999999e-05 2023-10-09T10:05:47.514 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.514 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:05:47.514 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.515 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:143937 7.4 e218/187 7:2dcf4812:::benchmark_data_smithi012_221426_object36159:head v 218'8957, mlcod=218'8955)", 2023-10-09T10:05:47.515 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:05:47.479791+0000", 2023-10-09T10:05:47.515 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0048008149999999999, 2023-10-09T10:05:47.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004996248, 2023-10-09T10:05:47.515 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:05:47.515 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:05:47.515 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:05:47.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:05:47.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479791+0000", 2023-10-09T10:05:47.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:05:47.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:05:47.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479791+0000", 2023-10-09T10:05:47.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:05:47.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:05:47.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479790+0000", 2023-10-09T10:05:47.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:05:47.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:05:47.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479795+0000", 2023-10-09T10:05:47.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6199999999999998e-06 2023-10-09T10:05:47.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:05:47.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479802+0000", 2023-10-09T10:05:47.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7989999999999996e-06 2023-10-09T10:05:47.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:05:47.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479806+0000", 2023-10-09T10:05:47.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4159999999999997e-06 2023-10-09T10:05:47.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:05:47.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479861+0000", 2023-10-09T10:05:47.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5390999999999999e-05 2023-10-09T10:05:47.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:05:47.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.479886+0000", 2023-10-09T10:05:47.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4502e-05 2023-10-09T10:05:47.521 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.521 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:05:47.521 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.522 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:143953 7.d 7:b9092fc5:::benchmark_data_smithi012_221426_object36175:head [delete] snapc 0=[] ondisk+write+known_if_redirected e218)", 2023-10-09T10:05:47.522 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:05:47.482449+0000", 2023-10-09T10:05:47.522 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0021427920000000001, 2023-10-09T10:05:47.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024123349999999998, 2023-10-09T10:05:47.522 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:05:47.522 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:05:47.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:05:47.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:05:47.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:05:47.523 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 143953 2023-10-09T10:05:47.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.523 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:05:47.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:05:47.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482449+0000", 2023-10-09T10:05:47.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:05:47.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:05:47.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482449+0000", 2023-10-09T10:05:47.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:05:47.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:05:47.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482447+0000", 2023-10-09T10:05:47.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:05:47.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:05:47.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482456+0000", 2023-10-09T10:05:47.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0890000000000006e-06 2023-10-09T10:05:47.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:05:47.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482476+0000", 2023-10-09T10:05:47.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0044e-05 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482486+0000", 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6800000000000005e-06 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482564+0000", 2023-10-09T10:05:47.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8339999999999999e-05 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482719+0000", 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015544500000000001 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:05:47.482850+0000", 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013058899999999999 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.528 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:05:47.529 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.529 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:05:47.529 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:05:47.529 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 4 2023-10-09T10:05:47.529 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:05:47.539 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:47.926 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:47.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:48.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:48.054 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:48.063 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:05:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:05:47] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.33.4" 2023-10-09T10:05:48.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:05:48.138 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:48.244 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:48.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:48.348 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:48.379 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:48.422 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:48.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:48.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:48.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:48.619 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:49.023 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:48 smithi012 ceph-mon[147926]: pgmap v560: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 791 op/s 2023-10-09T10:05:49.023 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:48 smithi012 ceph-mon[152263]: pgmap v560: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 791 op/s 2023-10-09T10:05:49.050 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:49.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:49.159 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:49.190 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:48 smithi110 ceph-mon[136032]: pgmap v560: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 791 op/s 2023-10-09T10:05:49.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:49.243 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:49.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:49.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:49.349 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:49.386 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:05:49.390 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:49.450 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:49.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:49.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:49.557 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:49.584 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:49.625 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:49.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:49.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:49.733 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:49.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:49.817 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:49.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:49.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:49.884 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:49.924 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:49.951 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:49.992 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:49.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:50.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:50.131 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:50.176 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:50.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:50.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:50.245 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:05:50.245 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:05:50.245 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:05:50.245 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:05:50.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:50.407 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:50.667 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:50.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:50.776 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:50.819 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:50.828 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:50.859 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:50.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:50.872 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:05:50.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:50 smithi012 ceph-mon[147926]: pgmap v561: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T10:05:50.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3129509718' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:50.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:50 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:50.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:50 smithi012 ceph-mon[152263]: pgmap v561: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T10:05:50.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3129509718' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:50.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:50 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:50.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:05:51.060 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:51.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:50 smithi110 ceph-mon[136032]: pgmap v561: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T10:05:51.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3129509718' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:51.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:50 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:05:51.337 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:51.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:51.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:51.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:51.538 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:51.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:51.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:05:51.731 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:51.812 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:51.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:51.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:51.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:51 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:51.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:51 smithi110 ceph-mon[136032]: failed to encode map e219 with expected crc 2023-10-09T10:05:51.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:05:51.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:51.988 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:52.114 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:52.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:52.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[147926]: failed to encode map e219 with expected crc 2023-10-09T10:05:52.122 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:05:52.122 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[147926]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T10:05:52.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[147926]: failed to encode map e219 with expected crc 2023-10-09T10:05:52.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:05:52.124 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[152263]: failed to encode map e219 with expected crc 2023-10-09T10:05:52.124 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:05:52.124 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[152263]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T10:05:52.124 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:51 smithi012 ceph-mon[152263]: failed to encode map e219 with expected crc 2023-10-09T10:05:52.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:52.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:52.156 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:05:52.157 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:05:52.157 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:05:52.157 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:05:52.157 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:05:52.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:51 smithi110 ceph-mon[136032]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T10:05:52.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:51 smithi110 ceph-mon[136032]: failed to encode map e219 with expected crc 2023-10-09T10:05:52.227 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:52.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:52.324 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:52.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:52.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:52.429 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:52.476 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:52.526 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:52.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:52.631 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:52.662 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:52.707 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:52.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:52.814 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:52.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:05:52.846 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:52.901 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:52.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:53.005 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:53.036 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:53.081 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:53.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:53.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:53.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:52 smithi012 ceph-mon[147926]: pgmap v563: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T10:05:53.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:52 smithi012 ceph-mon[152263]: pgmap v563: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T10:05:53.189 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:53.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:52 smithi110 ceph-mon[136032]: pgmap v563: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-09T10:05:53.224 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:53.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:53.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:53.286 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:05:53.328 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:05:53.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:05:53.366 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:05:53.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:53.397 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:53.414 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:05:53.443 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:53.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:53.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:53.457 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:05:53.498 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:05:53.532 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:05:53.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:53.580 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:53.589 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:05:53.608 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:05:53.608 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:05:53.608 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:05:53.608 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:05:53.608 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:05:53.608 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:05:53.608 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:05:53.608 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:05:53.619 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:53.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:53.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:53.646 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:53.723 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:05:53.750 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:53.794 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:05:53.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:53.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:05:53.900 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:53.934 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:53.981 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:53.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:54.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:54.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1760233641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:54.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1760233641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:54.170 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:54.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1760233641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:05:54.245 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:54.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:54.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:54.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:54.439 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:54.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:54.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:05:54.635 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:54.766 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:54.769 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:05:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:54.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:05:54.874 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:54.911 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:54.951 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:54.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:54.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:55.046 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:54 smithi012 ceph-mon[147926]: pgmap v564: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 728 op/s 2023-10-09T10:05:55.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:54 smithi012 ceph-mon[152263]: pgmap v564: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 728 op/s 2023-10-09T10:05:55.057 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:55.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:55.143 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:55.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:55.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:55.148 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:54 smithi110 ceph-mon[136032]: pgmap v564: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 728 op/s 2023-10-09T10:05:55.249 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:55.277 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:55.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:55.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:55.357 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:05:55.357 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:05:55.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:05:55.491 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:55.787 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:55.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:55.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:55.891 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5] out_osds: [2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T10:05:55.891 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:05:55.891 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T10:05:55.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T10:05:55.898 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:55.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:55.976 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:55.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:55.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:56.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:56.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:05:56.165 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:56.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:05:56.237 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:56.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:56.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:56.511 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:56.661 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:56.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:56.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:56.764 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:56.837 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:56.940 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:56.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:56.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:56.999 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:57.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:57.154 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:56 smithi012 ceph-mon[147926]: pgmap v565: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T10:05:57.154 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:56 smithi012 ceph-mon[152263]: pgmap v565: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T10:05:57.165 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:57.187 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:56 smithi110 ceph-mon[136032]: pgmap v565: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T10:05:57.289 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:05:57.300 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:57.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:57.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:57.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:57.447 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:57.504 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:57.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:57.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:57.610 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:57.642 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:57.695 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:57.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:57.802 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:05:57.880 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:57.927 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:05:57.994 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:05:58.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:58.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:05:58.108 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:05:58.160 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:05:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:05:57] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.33.4" 2023-10-09T10:05:58.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3613601732' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:58.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3613601732' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:58.182 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:58.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3613601732' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:05:58.317 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:05:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:58.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:05:58.421 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:58.465 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:58.520 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:58.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:58.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:05:58.668 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:58.714 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:05:58.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:58.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:05:58.818 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:58.868 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:58.914 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:58.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:59.028 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:59.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:58 smithi110 ceph-mon[136032]: pgmap v566: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T10:05:59.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:58 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:05:59.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:58 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:05:59.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3613601732' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:59.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:58 smithi110 ceph-mon[136032]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T10:05:59.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:58 smithi110 ceph-mon[136032]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:58 smithi110 ceph-mon[136032]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:58 smithi110 ceph-mon[136032]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.071 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:59.114 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:59.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:59.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[152263]: pgmap v566: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3613601732' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[152263]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[152263]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[152263]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[152263]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[147926]: pgmap v566: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:05:59.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:05:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3613601732' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:05:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[147926]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T10:05:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[147926]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[147926]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:58 smithi012 ceph-mon[147926]: failed to encode map e220 with expected crc 2023-10-09T10:05:59.222 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:05:59.254 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:59.297 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:05:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:59.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:05:59.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:05:59.482 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:59.651 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:05:59.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:59.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:05:59.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:05:59.789 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:05:59.829 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:05:59.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:05:59.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:05:59.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:00.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:00.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:05:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:00.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:00.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:05:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:00.015 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:00.127 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:06:00.127 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:06:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:05:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:00.312 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:00.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:00.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:00.454 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:00.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:00.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:00.526 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:06:00.571 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:06:00.604 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:00.606 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:06:00.637 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:00.648 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:06:00.676 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:00.687 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:06:00.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:00.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:00.722 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:06:00.757 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:06:00.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:00.790 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:06:00.796 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:06:00.796 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:06:00.796 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:06:00.796 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:06:00.796 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:06:00.796 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:06:00.797 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:06:00.797 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:06:00.822 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:00.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:06:00.870 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:00.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:00.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:01.073 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:00 smithi012 ceph-mon[147926]: pgmap v568: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T10:06:01.073 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/36168646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:01.073 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:00 smithi012 ceph-mon[152263]: pgmap v568: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T10:06:01.073 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/36168646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:01.081 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:00 smithi110 ceph-mon[136032]: pgmap v568: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T10:06:01.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/36168646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:01.296 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:01.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:01.347 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:01 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:06:01.184+0000 7f85a99d3700 -1 mon.a@0(leader).osd e220 definitely_dead 0 2023-10-09T10:06:01.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:01.487 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:01.549 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:01.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:01.642 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:01.658 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:01.690 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:01.735 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:01.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:01.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:01.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:06:01.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:01.883 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:01.938 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:01.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:01.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:01.966 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.2. 2023-10-09T10:06:02.044 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:02.070 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:02.102 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:02.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:02.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:02.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2428615463' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:06:02.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:06:02.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2428615463' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:06:02.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:06:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2428615463' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:06:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:06:02.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:02.236 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:02.319 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:02.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:02.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:02.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:02.450 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:02.481 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:02.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:02.510 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T10:06:02.585 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:02.611 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:02.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:02.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:02.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:02.745 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:02.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:02.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:02.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:02.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:02.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:02.936 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:03.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[147926]: pgmap v569: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T10:06:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[147926]: failed to encode map e221 with expected crc 2023-10-09T10:06:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:06:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[147926]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T10:06:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[147926]: failed to encode map e221 with expected crc 2023-10-09T10:06:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[147926]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T10:06:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[152263]: pgmap v569: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T10:06:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[152263]: failed to encode map e221 with expected crc 2023-10-09T10:06:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:06:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[152263]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T10:06:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[152263]: failed to encode map e221 with expected crc 2023-10-09T10:06:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:02 smithi012 ceph-mon[152263]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T10:06:03.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:02 smithi110 ceph-mon[136032]: pgmap v569: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T10:06:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:02 smithi110 ceph-mon[136032]: failed to encode map e221 with expected crc 2023-10-09T10:06:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:06:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:02 smithi110 ceph-mon[136032]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T10:06:03.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:02 smithi110 ceph-mon[136032]: failed to encode map e221 with expected crc 2023-10-09T10:06:03.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:02 smithi110 ceph-mon[136032]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T10:06:03.326 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:03.328 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:03.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:03.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:03.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:06:03.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:03.461 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:03.511 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:03.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:03.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:03.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:03.645 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:03.817 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:03.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:03.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:03.865 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:06:03.866 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:06:03.866 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:06:03.866 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:06:03.866 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:06:03.921 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:03.950 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:03.984 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:03.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:03.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:04.088 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:04.114 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:04.135 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:04.146 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:04.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[147926]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[147926]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[147926]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[147926]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[147926]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[147926]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[147926]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[147926]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T10:06:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[152263]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[152263]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[152263]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[152263]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[152263]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[152263]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[152263]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:03 smithi012 ceph-mon[152263]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T10:06:04.250 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:04.303 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:04.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:03 smithi110 ceph-mon[136032]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:03 smithi110 ceph-mon[136032]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:03 smithi110 ceph-mon[136032]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:03 smithi110 ceph-mon[136032]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:03 smithi110 ceph-mon[136032]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:03 smithi110 ceph-mon[136032]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:03 smithi110 ceph-mon[136032]: failed to encode map e222 with expected crc 2023-10-09T10:06:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:03 smithi110 ceph-mon[136032]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T10:06:04.767 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:04.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:04.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:04.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:04.964 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:06:05.013 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:05.081 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: pgmap v572: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: 7.8 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9552 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: 7.3 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9626 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: 7.2 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9832 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: 7.5 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9639 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: 7.a starting backfill to osd.2 from (0'0,0'0] MAX to 220'9665 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.082 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[147926]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: pgmap v572: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T10:06:05.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: 7.8 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9552 2023-10-09T10:06:05.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: 7.3 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9626 2023-10-09T10:06:05.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: 7.2 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9832 2023-10-09T10:06:05.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: 7.5 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9639 2023-10-09T10:06:05.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: 7.a starting backfill to osd.2 from (0'0,0'0] MAX to 220'9665 2023-10-09T10:06:05.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:04 smithi012 ceph-mon[152263]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.116 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:05.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:05.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:05.354 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: pgmap v572: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T10:06:05.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: 7.8 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9552 2023-10-09T10:06:05.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: 7.3 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9626 2023-10-09T10:06:05.466 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: 7.2 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9832 2023-10-09T10:06:05.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: 7.5 starting backfill to osd.2 from (0'0,0'0] MAX to 220'9639 2023-10-09T10:06:05.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: 7.a starting backfill to osd.2 from (0'0,0'0] MAX to 220'9665 2023-10-09T10:06:05.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.467 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:04 smithi110 ceph-mon[136032]: failed to encode map e223 with expected crc 2023-10-09T10:06:05.468 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:05.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:05.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:05.573 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:05.603 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:05.649 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:05.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:05.754 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:05.852 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:06.019 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:06.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:06.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:06.225 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:06.344 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:06.348 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:06.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:06 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:06:06.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:06 smithi012 ceph-mon[147926]: pgmap v574: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:06:06.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:06 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:06:06.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:06 smithi012 ceph-mon[152263]: pgmap v574: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:06:06.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:06.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:06 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:06:06.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:06 smithi110 ceph-mon[136032]: pgmap v574: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:06:06.450 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:06:06.461 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:06.482 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:06:06.494 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:06.528 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:06:06.534 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:06.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:06.571 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:06:06.613 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:06:06.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:06.684 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:06:06.713 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:06:06.737 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:06.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:06:06.789 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:06:06.789 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:06:06.789 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:06:06.789 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:06:06.789 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:06:06.789 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:06:06.789 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:06:06.789 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:06:06.812 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:06.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:06.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:06.924 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:06.956 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:07.000 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:07.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:07.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:07.070 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:06:07.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:07.148 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:07.286 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:07.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:07.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:07.290 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/232212709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:07.290 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/795911465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:07.390 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:07.418 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:07.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/232212709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:07.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/795911465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:07.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/232212709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:07.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/795911465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:07.467 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:07.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:07.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:07.511 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T10:06:07.511 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:06:07.511 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:06:07.512 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-10-09T10:06:07.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-10-09T10:06:07.571 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:07.602 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:07.644 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:07.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:07.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:07.749 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:07.777 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:07.821 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:07.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:07.920 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:06:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:06:07] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T10:06:07.926 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:07.957 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:08.004 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:08.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:08.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:08.108 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:08.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:08.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/232212709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:06:08.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[147926]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T10:06:08.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[147926]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[147926]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[147926]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[147926]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[147926]: pgmap v576: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:06:08.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:08.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/232212709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:06:08.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[152263]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T10:06:08.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[152263]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[152263]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[152263]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[152263]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.183 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:08 smithi012 ceph-mon[152263]: pgmap v576: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:06:08.214 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:08.378 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:08.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:08.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:08 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/232212709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:06:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:08 smithi110 ceph-mon[136032]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T10:06:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:08 smithi110 ceph-mon[136032]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:08 smithi110 ceph-mon[136032]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:08 smithi110 ceph-mon[136032]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:08 smithi110 ceph-mon[136032]: failed to encode map e224 with expected crc 2023-10-09T10:06:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:08 smithi110 ceph-mon[136032]: pgmap v576: 89 pgs: 7 peering, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T10:06:08.489 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:08.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:08.558 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:08.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:08.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:08.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:08.748 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:08.898 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:08.905 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:08.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:08.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:09.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:09.117 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:09.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:09 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 47/181143 objects degraded (0.026%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:09.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:09 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 47/181143 objects degraded (0.026%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:09.538 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:09.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:09.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:09 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 47/181143 objects degraded (0.026%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:09.709 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:09.936 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:06:09.936 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:06:09.938 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:09.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:10.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:10.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:06:10.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:06:10.153 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:10.441 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:10.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:10.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:10.579 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:10 smithi012 ceph-mon[147926]: pgmap v577: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 296 op/s; 47/181143 objects degraded (0.026%); 22446/181143 objects misplaced (12.391%); 606 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T10:06:10.579 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:10 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:10.579 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:10 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:10.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:10 smithi012 ceph-mon[152263]: pgmap v577: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 296 op/s; 47/181143 objects degraded (0.026%); 22446/181143 objects misplaced (12.391%); 606 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T10:06:10.579 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:10 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:10.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:10 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:10.580 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:06:10.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:10 smithi110 ceph-mon[136032]: pgmap v577: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 296 op/s; 47/181143 objects degraded (0.026%); 22446/181143 objects misplaced (12.391%); 606 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T10:06:10.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:10 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:10.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:10 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:10.688 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:10.926 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:10.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:11.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:11.123 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:11.169 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:11.239 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:11.245 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:11.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:11.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:11.389 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:11.488 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:11.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:11.577 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:11.593 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:11.628 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:11.768 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:11.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:11.876 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:12.002 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:12.187 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:12.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:12.306 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:12.349 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.4 primary-affinity to 0 (802) 2023-10-09T10:06:12.360 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:12.401 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:12.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:12.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:12.434 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:06:12.434 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:06:12.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:12 smithi110 ceph-mon[136032]: pgmap v578: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 47/179292 objects degraded (0.026%); 22446/179292 objects misplaced (12.519%); 479 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T10:06:12.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1346889130' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T10:06:12.460 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:12 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T10:06:12.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:12.549 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:12 smithi012 ceph-mon[147926]: pgmap v578: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 47/179292 objects degraded (0.026%); 22446/179292 objects misplaced (12.519%); 479 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T10:06:12.550 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1346889130' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T10:06:12.550 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:12 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T10:06:12.551 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:12 smithi012 ceph-mon[152263]: pgmap v578: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 47/179292 objects degraded (0.026%); 22446/179292 objects misplaced (12.519%); 479 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T10:06:12.552 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1346889130' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T10:06:12.552 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:12 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T10:06:12.557 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:12.609 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:12.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:12.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:12.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:12.754 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:12.859 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:12.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:12.872 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:06:12.968 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:12.998 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:13.035 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:13.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:13.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:13.140 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:13.170 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:13.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[152263]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T10:06:13.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[152263]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T10:06:13.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[152263]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[152263]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[147926]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T10:06:13.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[147926]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T10:06:13.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[147926]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:13 smithi012 ceph-mon[147926]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.504 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:13.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:13.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:13.534 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:13.611 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:13.639 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:13 smithi110 ceph-mon[136032]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.639 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T10:06:13.639 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:13 smithi110 ceph-mon[136032]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T10:06:13.639 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:13 smithi110 ceph-mon[136032]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.639 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:13 smithi110 ceph-mon[136032]: failed to encode map e225 with expected crc 2023-10-09T10:06:13.640 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:13.679 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:13.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:13.784 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:13.921 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:14.136 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:14.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:14.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:14.307 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:06:14.317 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:14.350 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:06:14.388 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:06:14.419 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:06:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: pgmap v580: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 136/173280 objects degraded (0.078%); 21487/173280 objects misplaced (12.400%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T10:06:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T10:06:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2160578077' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/782637084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: pgmap v580: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 136/173280 objects degraded (0.078%); 21487/173280 objects misplaced (12.400%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T10:06:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2160578077' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:14.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/782637084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:14.455 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:14.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:14.460 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:06:14.502 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:06:14.528 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: pgmap v580: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 136/173280 objects degraded (0.078%); 21487/173280 objects misplaced (12.400%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T10:06:14.528 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.528 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.528 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T10:06:14.528 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.528 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: failed to encode map e226 with expected crc 2023-10-09T10:06:14.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:06:14.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:14.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:14.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2160578077' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:14.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:14.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/782637084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:14.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:14.558 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:06:14.597 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:06:14.629 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:06:14.630 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:06:14.630 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:06:14.630 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:06:14.630 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:06:14.630 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:06:14.630 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:06:14.630 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:06:14.664 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:14.737 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:14.737 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:06:14.737 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:06:14.737 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:06:14.737 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:14.928 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:06:15.075 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:06:15.348 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:15 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:15.348 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:15 smithi012 ceph-mon[147926]: failed to encode map e227 with expected crc 2023-10-09T10:06:15.348 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:15.348 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:15 smithi012 ceph-mon[147926]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T10:06:15.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:15 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:15.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:15 smithi012 ceph-mon[152263]: failed to encode map e227 with expected crc 2023-10-09T10:06:15.349 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:15.349 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:15 smithi012 ceph-mon[152263]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T10:06:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:15 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:15 smithi110 ceph-mon[136032]: failed to encode map e227 with expected crc 2023-10-09T10:06:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:15 smithi110 ceph-mon[136032]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T10:06:15.766 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:15.830 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:15.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:15.938 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:16.005 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:16.253 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:16.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:16.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:16.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:16.474 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:16.538 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:16.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:16.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[147926]: failed to encode map e227 with expected crc 2023-10-09T10:06:16.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[147926]: pgmap v583: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 136/171954 objects degraded (0.079%); 21487/171954 objects misplaced (12.496%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:06:16.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:16.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:16.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 136/171954 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:16.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[152263]: failed to encode map e227 with expected crc 2023-10-09T10:06:16.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[152263]: pgmap v583: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 136/171954 objects degraded (0.079%); 21487/171954 objects misplaced (12.496%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:06:16.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:16.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:16.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:16 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 136/171954 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:16.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:16 smithi110 ceph-mon[136032]: failed to encode map e227 with expected crc 2023-10-09T10:06:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:16 smithi110 ceph-mon[136032]: pgmap v583: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 136/171954 objects degraded (0.079%); 21487/171954 objects misplaced (12.496%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:06:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:16 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:16 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 136/171954 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:16.729 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:16.822 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:16.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:16.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:16.931 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:16.979 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:17.022 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:17.028 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:17.028 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:06:17.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:06:17.028 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:06:17.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.028 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96930 6.e 6:710aa3be:::benchmark_data_smithi012_182576_object96929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 363.43108381299999, 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098520888000000001, 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96930 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593685+0000", 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593683+0000", 2023-10-09T10:06:17.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.313e-05 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593746+0000", 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2289999999999992e-06 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593756+0000", 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0159999999999999e-06 2023-10-09T10:06:17.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593761+0000", 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8661999999999998e-05 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.593859+0000", 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146317 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.594006+0000", 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000179419 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.594185+0000", 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081610680000000005 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602346+0000", 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061366659999999998 2023-10-09T10:06:17.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608483+0000", 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079327541000000001 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.687810+0000", 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1600999999999995e-05 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.687872+0000", 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043339989999999998 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692206+0000", 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098520888000000001 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96932 6.e 6:7facdbe4:::benchmark_data_smithi012_182576_object96931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 363.42262756500003, 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094685332999999997, 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96932 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.036 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602141+0000", 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602140+0000", 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8144999999999999e-05 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602168+0000", 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2849999999999997e-06 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602175+0000", 2023-10-09T10:06:17.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7560000000000001e-06 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602179+0000", 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000195152 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602374+0000", 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000206584 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602581+0000", 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000174067 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.602755+0000", 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032482050000000001 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.606003+0000", 2023-10-09T10:06:17.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024921650000000002 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608495+0000", 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083792852000000001 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692288+0000", 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6909000000000006e-05 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.692355+0000", 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044717339999999998 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.696827+0000", 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094685332999999997 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96944 6.e 6:71fd26c2:::benchmark_data_smithi012_182576_object96943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 363.40001314, 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094319584999999997, 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96944 2023-10-09T10:06:17.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624756+0000", 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624753+0000", 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9678999999999994e-05 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624833+0000", 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0849000000000001e-05 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624844+0000", 2023-10-09T10:06:17.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5359999999999998e-06 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.624850+0000", 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012482843 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637333+0000", 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016598700000000001 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637499+0000", 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027263199999999997 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637772+0000", 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033034599999999998 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641075+0000", 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088881500000000005 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.641964+0000", 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072603763000000002 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714568+0000", 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4980000000000005e-05 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.714633+0000", 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044422960000000001 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.719075+0000", 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094319584999999997 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:105483 6.2 6:45b0cc0b:::benchmark_data_smithi012_182576_object105482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 347.30212842399999, 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1741883319999999, 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105483 2023-10-09T10:06:17.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722640+0000", 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5788999999999999e-05 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722676+0000", 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.089e-06 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722684+0000", 2023-10-09T10:06:17.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5950000000000004e-06 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.722690+0000", 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010708354 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733399+0000", 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013741599999999999 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733536+0000", 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017242200000000001 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.733708+0000", 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003714143 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.737423+0000", 2023-10-09T10:06:17.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015821828 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.753244+0000", 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036249568000000003 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789494+0000", 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8799999999999997e-06 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789503+0000", 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1072112519999999 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.896714+0000", 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114647 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.896829+0000", 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1741883319999999 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:105508 6.e 6:77549acc:::benchmark_data_smithi012_182576_object105507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 347.24969205399998, 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1483757859999999, 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:06:17.057 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 105508 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775077+0000", 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775075+0000", 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9004000000000003e-05 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775174+0000", 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.751e-05 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775192+0000", 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6340000000000001e-06 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775199+0000", 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4343000000000003e-05 2023-10-09T10:06:17.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775274+0000", 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000152091 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775426+0000", 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019803600000000001 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.775624+0000", 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057209089999999997 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.781345+0000", 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079732469999999993 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789318+0000", 2023-10-09T10:06:17.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8120000000000002e-06 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.789328+0000", 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.114660743 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.903988+0000", 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000495179 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.904484+0000", 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018860534000000002 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.923344+0000", 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010841900000000001 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:30.923453+0000", 2023-10-09T10:06:17.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1483757859999999 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:178976 6.e 6.4855639e (undecoded) ondisk+retry+write+known_if_redirected e158)", 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 240.05891286599999, 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.74206674699999997, 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:06:17.065 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 178976 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965856+0000", 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965854+0000", 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5659999999999997e-06 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965860+0000", 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9839999999999996e-06 2023-10-09T10:06:17.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965870+0000", 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.72e-06 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.965877+0000", 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020714689999999998 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.967948+0000", 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0054999999999999e-05 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:16.967958+0000", 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.73995176900000004 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.707910+0000", 2023-10-09T10:06:17.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.277e-05 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.707923+0000", 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.74206674699999997 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179004 6.e 6.afd2ea6e (undecoded) ondisk+write+known_if_redirected e158)", 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:06:17.070 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 240.02030324200001, 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.703617838, 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179004 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004466+0000", 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004464+0000", 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9590000000000007e-06 2023-10-09T10:06:17.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004474+0000", 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1376999999999999e-05 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004495+0000", 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4409999999999993e-06 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004504+0000", 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0488000000000002e-05 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004554+0000", 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0353999999999999e-05 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004564+0000", 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70350862300000006 2023-10-09T10:06:17.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708073+0000", 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.04e-05 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708083+0000", 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.703617838 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.075 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:179005 6.e 6.7a62357e (undecoded) ondisk+write+known_if_redirected e158)", 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 240.02019332899999, 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70354121400000003, 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 179005 2023-10-09T10:06:17.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004575+0000", 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8079999999999998e-06 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004579+0000", 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5859999999999999e-06 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004586+0000", 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5590000000000001e-06 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004590+0000", 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6616000000000002e-05 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004637+0000", 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2291e-05 2023-10-09T10:06:17.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.004649+0000", 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70345850600000004 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708107+0000", 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3999999999999998e-06 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:17.708117+0000", 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70354121400000003 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195777 6.12 6:4f83111c:::benchmark_data_smithi012_182576_object90256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 213.13335675900001, 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86944099200000002, 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.081 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195777 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891412+0000", 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891410+0000", 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4070000000000004e-06 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891418+0000", 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4187e-05 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891432+0000", 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6590000000000004e-06 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891438+0000", 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.266e-05 2023-10-09T10:06:17.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891500+0000", 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7346999999999998e-05 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:43.891518+0000", 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.84995838999999995 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741476+0000", 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1224999999999995e-05 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741567+0000", 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034862599999999999 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.741916+0000", 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076180900000000001 2023-10-09T10:06:17.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749534+0000", 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011212183000000001 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760746+0000", 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6701000000000001e-05 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760773+0000", 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.125e-05 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760814+0000", 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8943000000000003e-05 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760853+0000", 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86944099200000002 2023-10-09T10:06:17.088 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195847 6.12 6:4f2f2e62:::benchmark_data_smithi012_182576_object90326:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.90122433100001, 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63759273800000005, 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.089 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195847 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123544+0000", 2023-10-09T10:06:17.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123543+0000", 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.46e-06 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123550+0000", 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3708e-05 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123563+0000", 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8350000000000004e-06 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123571+0000", 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1295000000000002e-05 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123632+0000", 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8449999999999998e-06 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.123642+0000", 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.61961346399999995 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743256+0000", 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8935000000000003e-05 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743325+0000", 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017814799999999999 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.743503+0000", 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060367659999999998 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749540+0000", 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011359289 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760899+0000", 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015118900000000001 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761050+0000", 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.49e-05 2023-10-09T10:06:17.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761095+0000", 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2329000000000001e-05 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761137+0000", 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.63759273800000005 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:195920 6.12 6:4be14f21:::benchmark_data_smithi012_182576_object90399:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.761114991, 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.49761078399999997, 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195920 2023-10-09T10:06:17.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263654+0000", 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:17.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263652+0000", 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6619999999999999e-06 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263659+0000", 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.222e-05 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263671+0000", 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3700000000000003e-06 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263677+0000", 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5055999999999995e-05 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263742+0000", 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.256e-05 2023-10-09T10:06:17.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.263754+0000", 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.48082169600000002 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744576+0000", 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114827 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744691+0000", 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026918100000000001 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.744960+0000", 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096003400000000006 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754560+0000", 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063868149999999997 2023-10-09T10:06:17.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760947+0000", 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024620099999999998 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761193+0000", 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4209000000000003e-05 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761238+0000", 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7093e-05 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761265+0000", 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.49761078399999997 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196022 6.12 6:4bcd6600:::benchmark_data_smithi012_182576_object90501:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.62917220700001, 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.36575843200000002, 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196022 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395597+0000", 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395595+0000", 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2799999999999998e-06 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395602+0000", 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5897999999999999e-05 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395618+0000", 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4340000000000003e-06 2023-10-09T10:06:17.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395626+0000", 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1285999999999995e-05 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395697+0000", 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2662e-05 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.395710+0000", 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35087969000000002 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746589+0000", 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010907400000000001 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746698+0000", 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024344699999999999 2023-10-09T10:06:17.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.746942+0000", 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076297279999999997 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754572+0000", 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063959359999999996 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760967+0000", 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032883000000000001 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761296+0000", 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6942999999999999e-05 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761323+0000", 2023-10-09T10:06:17.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1780999999999999e-05 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.761355+0000", 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.36575843200000002 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196042 6.12 6:4e976c7d:::benchmark_data_smithi012_182576_object90521:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.112 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.59135806800001, 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.330443298, 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196042 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433411+0000", 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433409+0000", 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3280000000000006e-06 2023-10-09T10:06:17.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433417+0000", 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2411e-05 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433430+0000", 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3770000000000004e-06 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433435+0000", 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5327999999999997e-05 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433481+0000", 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3749999999999997e-06 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.433488+0000", 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.31462358499999998 2023-10-09T10:06:17.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748112+0000", 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115394 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748227+0000", 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025816400000000002 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.748485+0000", 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010564301 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759049+0000", 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037354570000000002 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.762785+0000", 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0009886719999999999 2023-10-09T10:06:17.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763774+0000", 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7079000000000001e-05 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763821+0000", 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3420999999999997e-05 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763854+0000", 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.330443298 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196076 6.12 6:49ef5a44:::benchmark_data_smithi012_182576_object90555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.55629928600001, 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30399614400000002, 2023-10-09T10:06:17.120 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196076 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468470+0000", 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468468+0000", 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.251e-06 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468474+0000", 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3427e-05 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468488+0000", 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6200000000000004e-06 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468493+0000", 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3034000000000006e-05 2023-10-09T10:06:17.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468566+0000", 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2557e-05 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.468579+0000", 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.28100584000000001 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749585+0000", 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118539 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749703+0000", 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025233699999999999 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.749955+0000", 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009103547 2023-10-09T10:06:17.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759059+0000", 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048303510000000001 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.763889+0000", 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084814780000000006 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772371+0000", 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2203999999999997e-05 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772423+0000", 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2608000000000003e-05 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772466+0000", 2023-10-09T10:06:17.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30399614400000002 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196085 6.12 6:4d84fb9e:::benchmark_data_smithi012_182576_object90564:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.54561885000001, 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.298003501, 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.128 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196085 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479150+0000", 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479148+0000", 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7549999999999994e-06 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479156+0000", 2023-10-09T10:06:17.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6019e-05 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479172+0000", 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3470000000000004e-06 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479178+0000", 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6717999999999994e-05 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479245+0000", 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2296999999999999e-05 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.479257+0000", 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.27237020299999998 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.751627+0000", 2023-10-09T10:06:17.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000114201 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.751741+0000", 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027028600000000001 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.752012+0000", 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070559289999999998 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.759068+0000", 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013462363 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772530+0000", 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045611979999999998 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777091+0000", 2023-10-09T10:06:17.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3906999999999999e-05 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777125+0000", 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8297e-05 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777153+0000", 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.298003501 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.135 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196149 6.12 6:48b2c020:::benchmark_data_smithi012_182576_object90628:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.45222668599999, 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20477431600000001, 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.136 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196149 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572542+0000", 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:17.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572541+0000", 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572548+0000", 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2437e-05 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572560+0000", 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4049999999999997e-06 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572567+0000", 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6699999999999995e-05 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572633+0000", 2023-10-09T10:06:17.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1873e-05 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.572645+0000", 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.180477941 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753123+0000", 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5816999999999995e-05 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753209+0000", 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000185847 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.753395+0000", 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070544839999999998 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760449+0000", 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012128781 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772578+0000", 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046280150000000001 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777206+0000", 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5373e-05 2023-10-09T10:06:17.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777261+0000", 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5093999999999998e-05 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777316+0000", 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20477431600000001 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196189 6.12 6:4bcf1e87:::benchmark_data_smithi012_182576_object90668:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.39117637499999, 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14382505400000001, 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196189 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.144 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633592+0000", 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633591+0000", 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2080000000000002e-06 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633596+0000", 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9220000000000002e-06 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633602+0000", 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2490000000000001e-06 2023-10-09T10:06:17.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633605+0000", 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8455000000000003e-05 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633653+0000", 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7867999999999999e-05 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633671+0000", 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12094508800000001 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754616+0000", 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2300999999999996e-05 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754709+0000", 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019090900000000001 2023-10-09T10:06:17.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.754899+0000", 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055585510000000001 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760458+0000", 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012168912 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.772627+0000", 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047358959999999999 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777363+0000", 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0698999999999999e-05 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777393+0000", 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4036000000000002e-05 2023-10-09T10:06:17.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777418+0000", 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14382505400000001 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196191 6.12 6:48379c93:::benchmark_data_smithi012_182576_object90670:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:06:17.151 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.39115374799999, 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14395870599999999, 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196191 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:06:17.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633615+0000", 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.013e-06 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633617+0000", 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5360000000000001e-06 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633620+0000", 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3039999999999999e-06 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633623+0000", 2023-10-09T10:06:17.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047045000000000002 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634093+0000", 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0719999999999999e-06 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634098+0000", 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.121978771 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756077+0000", 2023-10-09T10:06:17.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104263 2023-10-09T10:06:17.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756181+0000", 2023-10-09T10:06:17.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000219302 2023-10-09T10:06:17.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.756400+0000", 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004065333 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760466+0000", 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012786838999999999 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.773253+0000", 2023-10-09T10:06:17.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042139990000000004 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777467+0000", 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7116e-05 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777524+0000", 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0078999999999999e-05 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777574+0000", 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14395870599999999 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:196192 6.12 6:4fcc3e62:::benchmark_data_smithi012_182576_object90671:head [delete] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 212.39112597900001, 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14401113700000001, 2023-10-09T10:06:17.160 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196192 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633643+0000", 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7549999999999999e-06 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633644+0000", 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3650000000000003e-06 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633650+0000", 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7549999999999999e-06 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.633652+0000", 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000899926 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634552+0000", 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3590000000000001e-06 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.634557+0000", 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12309893399999999 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757656+0000", 2023-10-09T10:06:17.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1304999999999995e-05 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757737+0000", 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019102100000000001 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.757928+0000", 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027759170000000001 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.760704+0000", 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013396351000000001 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.774100+0000", 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035033410000000001 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777604+0000", 2023-10-09T10:06:17.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7534000000000001e-05 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777631+0000", 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2878e-05 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:44.777654+0000", 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14401113700000001 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.167 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:13667 7.1 7:8e32afd2:::benchmark_data_smithi012_221426_object13666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:03:35.011644+0000", 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 162.013125274, 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11403589, 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:17.168 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13667 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011644+0000", 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011644+0000", 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:17.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011642+0000", 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000147603 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011789+0000", 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6086000000000002e-05 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:17.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011805+0000", 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9779999999999999e-06 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011812+0000", 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2852000000000002e-05 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011865+0000", 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130438 2023-10-09T10:06:17.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.011996+0000", 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000165916 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.012162+0000", 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00091800400000000004 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.013080+0000", 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001552908 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.014633+0000", 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110859579 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.125492+0000", 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7923000000000006e-05 2023-10-09T10:06:17.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.125570+0000", 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109425 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:35.125679+0000", 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11403589 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.174 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.175 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:17.175 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:17.175 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:17.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:17.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:17.368 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:17.438 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:17.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:17.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:06:17.546 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:17.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:17 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:06:17.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:17 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:06:17.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:17 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:17.647 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:17.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:17 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:06:17.649 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:17 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:06:17.649 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:17 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:17 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:06:17.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:17 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:06:17.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:17 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:17.726 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:17.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:17.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:17.910 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:17.987 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:18.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:18.100 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:18.130 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:18.175 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:06:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:06:17] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T10:06:18.178 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:18.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:18.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:18.282 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:18.316 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:06:18.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:06:18.377 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:18.400 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T10:06:18.401 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:06:18.401 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T10:06:18.401 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:06:18.401 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:06:18.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T10:06:18.492 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:18.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:18.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:18.783 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:18.886 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:18.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:18.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:18.999 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:19.081 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:18 smithi012 ceph-mon[147926]: pgmap v584: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 136/171954 objects degraded (0.079%); 21487/171954 objects misplaced (12.496%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:06:19.082 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:18 smithi012 ceph-mon[152263]: pgmap v584: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 136/171954 objects degraded (0.079%); 21487/171954 objects misplaced (12.496%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:06:19.088 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:18 smithi110 ceph-mon[136032]: pgmap v584: 89 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 136/171954 objects degraded (0.079%); 21487/171954 objects misplaced (12.496%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:06:19.218 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:19.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:19.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:19.246 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:19.283 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:19.326 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:19.363 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:19.416 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:19.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:19.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:19.479 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:19.514 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:19.514 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:06:19.514 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:06:19.514 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:19.523 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:19.560 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:19.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:19.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:19.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:19.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:19 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:06:19.922 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:19 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:06:19.960 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:20.081 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:06:20.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:19 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:06:20.224 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:20.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:20.333 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:20.409 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:20.582 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:20.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:20.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:20.730 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:20.775 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:20.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:20.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:20.993 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:21.099 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:21.120 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:21.120 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:06:21.120 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:21.149 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:20 smithi110 ceph-mon[136032]: pgmap v585: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 162/166065 objects degraded (0.098%); 20586/166065 objects misplaced (12.396%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T10:06:21.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:20 smithi012 ceph-mon[147926]: pgmap v585: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 162/166065 objects degraded (0.098%); 20586/166065 objects misplaced (12.396%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T10:06:21.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:20 smithi012 ceph-mon[152263]: pgmap v585: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 162/166065 objects degraded (0.098%); 20586/166065 objects misplaced (12.396%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T10:06:22.040 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:22.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:22.157 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:22.169 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/147608980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:22.169 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/147608980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:22.187 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/147608980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:22.189 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:22.503 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:06:22.503 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:06:22.503 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:06:22.503 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:06:22.503 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:06:22.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:06:22.505 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:06:23.159 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:23.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:23.217 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:22 smithi012 ceph-mon[147926]: pgmap v586: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 162/164517 objects degraded (0.098%); 20586/164517 objects misplaced (12.513%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T10:06:23.217 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:22 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 162/164517 objects degraded (0.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:23.217 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:22 smithi012 ceph-mon[152263]: pgmap v586: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 162/164517 objects degraded (0.098%); 20586/164517 objects misplaced (12.513%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T10:06:23.217 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:22 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 162/164517 objects degraded (0.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:23.264 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:23.293 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:22 smithi110 ceph-mon[136032]: pgmap v586: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 162/164517 objects degraded (0.098%); 20586/164517 objects misplaced (12.513%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T10:06:23.293 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:22 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 162/164517 objects degraded (0.098%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:23.298 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:24.010 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:06:24.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:24.317 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:23 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1393021231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:06:24.317 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:06:24.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:24.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:24.419 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:23 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1393021231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:06:24.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:06:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:23 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1393021231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:06:24.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:06:24.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:24.498 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:24.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:24.605 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:24.675 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:06:24.675 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:06:24.677 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:24.769 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:24.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:24.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:24.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:24.964 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:25.058 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:25.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:06:25.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:25.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:25.067 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[147926]: pgmap v587: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 214/157569 objects degraded (0.136%); 19516/157569 objects misplaced (12.386%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:25.067 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:25.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[147926]: failed to encode map e228 with expected crc 2023-10-09T10:06:25.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:06:25.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[147926]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T10:06:25.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[152263]: pgmap v587: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 214/157569 objects degraded (0.136%); 19516/157569 objects misplaced (12.386%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:25.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:25.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[152263]: failed to encode map e228 with expected crc 2023-10-09T10:06:25.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:06:25.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:24 smithi012 ceph-mon[152263]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T10:06:25.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:25.203 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:25.248 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:25.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:25.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:25.254 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:24 smithi110 ceph-mon[136032]: pgmap v587: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 214/157569 objects degraded (0.136%); 19516/157569 objects misplaced (12.386%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:25.254 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:24 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:25.254 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:24 smithi110 ceph-mon[136032]: failed to encode map e228 with expected crc 2023-10-09T10:06:25.254 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:06:25.254 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:24 smithi110 ceph-mon[136032]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T10:06:25.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:25.442 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:25.529 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:25.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:25.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:25.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:06:25.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:25.758 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:25.762 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:25.832 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:25.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:25.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:06:26.041 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:26.134 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:06:26.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:26.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:06:26.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:26.335 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:26.445 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:26.458 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:26.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:26.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:26.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:06:26.633 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:26.790 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:26.796 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T10:06:26.796 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:06:26.796 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T10:06:26.796 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T10:06:26.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T10:06:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:26.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:26.900 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:06:26.900 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:06:26.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:26.943 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:27.187 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:26 smithi012 ceph-mon[152263]: pgmap v589: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 214/155967 objects degraded (0.137%); 19516/155967 objects misplaced (12.513%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:27.193 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:26 smithi012 ceph-mon[147926]: pgmap v589: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 214/155967 objects degraded (0.137%); 19516/155967 objects misplaced (12.513%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:27.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:26 smithi110 ceph-mon[136032]: pgmap v589: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 214/155967 objects degraded (0.137%); 19516/155967 objects misplaced (12.513%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:27.193 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:06:26 smithi012 systemd[1]: Stopping Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:06:27.246 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:27.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:27.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:27.302 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:06:27.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:27.415 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:27.515 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:27.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:27.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:27.626 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:27.659 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:27.704 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:27.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:27.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:27.751 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:27.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:27.823 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[170172]: 2023-10-09T10:06:27.818+0000 7fac1d356700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:06:27.823 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[170172]: 2023-10-09T10:06:27.818+0000 7fac1d356700 -1 osd.3 228 *** Got signal Terminated *** 2023-10-09T10:06:27.823 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[170172]: 2023-10-09T10:06:27.818+0000 7fac1d356700 -1 osd.3 228 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:06:27.924 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:27.976 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:28.022 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:28.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:28.132 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:28.158 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:28.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-mon[147926]: pgmap v590: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 214/155967 objects degraded (0.137%); 19516/155967 objects misplaced (12.513%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:28.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-mon[147926]: osd.3 marked itself down and dead 2023-10-09T10:06:28.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 214/155967 objects degraded (0.137%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:06:27] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T10:06:28.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-mon[152263]: pgmap v590: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 214/155967 objects degraded (0.137%); 19516/155967 objects misplaced (12.513%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:28.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-mon[152263]: osd.3 marked itself down and dead 2023-10-09T10:06:28.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:27 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 214/155967 objects degraded (0.137%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:28.193 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:28.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:28.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:28.214 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:06:28.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:06:28.252 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:06:28.271 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:06:28.290 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:06:28.297 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:28.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:06:28.320 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:27 smithi110 ceph-mon[136032]: pgmap v590: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 214/155967 objects degraded (0.137%); 19516/155967 objects misplaced (12.513%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:28.321 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:27 smithi110 ceph-mon[136032]: osd.3 marked itself down and dead 2023-10-09T10:06:28.321 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:27 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 214/155967 objects degraded (0.137%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:06:28.324 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:28.328 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:06:28.328 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:06:28.328 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:06:28.328 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:06:28.328 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:06:28.329 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:06:28.329 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:06:28.329 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:06:28.329 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:06:28.356 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:28.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:28.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:28.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:28.494 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:28.534 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:28.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:28.637 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:28.665 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:28.699 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:28.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:28.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:28.802 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:28.829 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:28.868 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:28.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:28.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:28.971 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:29.001 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:29.088 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:29.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:29.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[147926]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:06:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[147926]: failed to encode map e229 with expected crc 2023-10-09T10:06:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[147926]: osdmap e229: 8 total, 7 up, 8 in 2023-10-09T10:06:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[147926]: failed to encode map e229 with expected crc 2023-10-09T10:06:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2396830236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:29.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:29.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[152263]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:06:29.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[152263]: failed to encode map e229 with expected crc 2023-10-09T10:06:29.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[152263]: osdmap e229: 8 total, 7 up, 8 in 2023-10-09T10:06:29.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[152263]: failed to encode map e229 with expected crc 2023-10-09T10:06:29.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2396830236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:29.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:29.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:28 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:29.203 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:29.266 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:28 smithi110 ceph-mon[136032]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:06:29.266 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:28 smithi110 ceph-mon[136032]: failed to encode map e229 with expected crc 2023-10-09T10:06:29.266 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:28 smithi110 ceph-mon[136032]: osdmap e229: 8 total, 7 up, 8 in 2023-10-09T10:06:29.266 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:28 smithi110 ceph-mon[136032]: failed to encode map e229 with expected crc 2023-10-09T10:06:29.267 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2396830236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:29.267 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:28 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:06:29.267 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:28 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:29.279 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:29.342 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:29.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:29.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:29.354 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:29.354 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:06:29.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.354 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35157.0:166728 7.2 e230/229 7:44e54330:::benchmark_data_smithi012_221426_object58950:head v 230'10616, mlcod=230'10614)", 2023-10-09T10:06:29.354 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:06:29.328176+0000", 2023-10-09T10:06:29.354 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.011977689999999999, 2023-10-09T10:06:29.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012009733999999999, 2023-10-09T10:06:29.354 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328176+0000", 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328176+0000", 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328175+0000", 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8930000000000005e-06 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328183+0000", 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3969e-05 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:29.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328197+0000", 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4159999999999998e-06 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328204+0000", 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1581e-05 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328256+0000", 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3209000000000001e-05 2023-10-09T10:06:29.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328279+0000", 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103115 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.35157.0:166733 7.2 e230/229 7:4030a59e:::benchmark_data_smithi012_221426_object58955:head v 230'10617, mlcod=230'10614)", 2023-10-09T10:06:29.358 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:06:29.328969+0000", 2023-10-09T10:06:29.359 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.011184126000000001, 2023-10-09T10:06:29.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011285933999999999, 2023-10-09T10:06:29.359 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:29.359 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:06:29.359 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:29.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328969+0000", 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328969+0000", 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328969+0000", 2023-10-09T10:06:29.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.708e-06 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328975+0000", 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1504e-05 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:29.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328987+0000", 2023-10-09T10:06:29.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9910000000000002e-06 2023-10-09T10:06:29.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:29.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.328992+0000", 2023-10-09T10:06:29.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021890270000000001 2023-10-09T10:06:29.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.331181+0000", 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2785999999999998e-05 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.331204+0000", 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022343879999999999 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:29.363 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:166731 7.c 7:3937f697:::benchmark_data_smithi012_221426_object58953:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:06:29.332338+0000", 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0078150600000000004, 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079868309999999998, 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 166731 2023-10-09T10:06:29.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332338+0000", 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332338+0000", 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332337+0000", 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0829999999999993e-06 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332346+0000", 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3495e-05 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:29.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332359+0000", 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1040000000000001e-06 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332366+0000", 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7435000000000007e-05 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332444+0000", 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012497300000000001 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332569+0000", 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138104 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.332707+0000", 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036852100000000002 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:166739 7.c 7:3e5ff44d:::benchmark_data_smithi012_221426_object58961:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:06:29.340275+0000", 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4294967295.9998789, 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139467, 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 166739 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.369 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.340275+0000", 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.340275+0000", 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.340272+0000", 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.008e-05 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.340283+0000", 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1371e-05 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.340294+0000", 2023-10-09T10:06:29.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5669999999999997e-06 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.340302+0000", 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7466999999999998e-05 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:06:29.340349+0000", 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4337000000000003e-05 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.372 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:29.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.373 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:29.373 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:06:29.373 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 4 2023-10-09T10:06:29.373 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:29.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:29.511 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:29.705 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:06:29.755 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:29.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:29.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:29.859 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:29.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:30.010 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:30.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:30.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:30.199 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:30.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: osdmap e230: 8 total, 7 up, 8 in 2023-10-09T10:06:30.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: 7.3 continuing backfill to osd.2 from (196'3200,228'10411] 7:c00dd601:::benchmark_data_smithi012_221426_object92342:head to 228'10411 2023-10-09T10:06:30.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.313 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: 7.2 continuing backfill to osd.2 from (197'3400,228'10607] MIN to 228'10607 2023-10-09T10:06:30.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: 7.a continuing backfill to osd.2 from (196'3200,228'10433] MIN to 228'10433 2023-10-09T10:06:30.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: 7.8 continuing backfill to osd.2 from (196'3110,228'10316] MIN to 228'10316 2023-10-09T10:06:30.314 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[147926]: pgmap v593: 89 pgs: 4 stale+active+remapped+backfill_wait, 18 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 65 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 243/148938 objects degraded (0.163%); 18430/148938 objects misplaced (12.374%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:06:30.315 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.315 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: osdmap e230: 8 total, 7 up, 8 in 2023-10-09T10:06:30.315 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.315 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: 7.3 continuing backfill to osd.2 from (196'3200,228'10411] 7:c00dd601:::benchmark_data_smithi012_221426_object92342:head to 228'10411 2023-10-09T10:06:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: 7.2 continuing backfill to osd.2 from (197'3400,228'10607] MIN to 228'10607 2023-10-09T10:06:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: 7.a continuing backfill to osd.2 from (196'3200,228'10433] MIN to 228'10433 2023-10-09T10:06:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: 7.8 continuing backfill to osd.2 from (196'3110,228'10316] MIN to 228'10316 2023-10-09T10:06:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:30 smithi012 ceph-mon[152263]: pgmap v593: 89 pgs: 4 stale+active+remapped+backfill_wait, 18 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 65 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 243/148938 objects degraded (0.163%); 18430/148938 objects misplaced (12.374%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:06:30.364 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:06:30.365 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:06:30.383 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:30.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:30.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:30.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: osdmap e230: 8 total, 7 up, 8 in 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: 7.3 continuing backfill to osd.2 from (196'3200,228'10411] 7:c00dd601:::benchmark_data_smithi012_221426_object92342:head to 228'10411 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: 7.2 continuing backfill to osd.2 from (197'3400,228'10607] MIN to 228'10607 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: failed to encode map e230 with expected crc 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: 7.a continuing backfill to osd.2 from (196'3200,228'10433] MIN to 228'10433 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: 7.8 continuing backfill to osd.2 from (196'3110,228'10316] MIN to 228'10316 2023-10-09T10:06:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:30 smithi110 ceph-mon[136032]: pgmap v593: 89 pgs: 4 stale+active+remapped+backfill_wait, 18 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 65 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 243/148938 objects degraded (0.163%); 18430/148938 objects misplaced (12.374%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:06:30.463 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:30.494 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:30.548 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:30.634 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:06:30 smithi012 bash[247471]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T10:06:30.641 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:30.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:30.747 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:30.792 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:30.843 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:30.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:30.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:31.060 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:31.149 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:31.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:31.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:31.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:31.330 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:31.402 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:31.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:31.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:31.623 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:31.760 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:31.760 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:06:31.760 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:06:31.760 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:06:31.761 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:31.767 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:31.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:31.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:31.929 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:32.057 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:32.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:32.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:32.159 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:06:32.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:32.270 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:06:32.279 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:32.330 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:32.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:32.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:32.339 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:32 smithi110 ceph-mon[136032]: pgmap v594: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 4 stale+active+clean, 4 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 54 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 21462/146787 objects degraded (14.621%); 15230/146787 objects misplaced (10.376%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T10:06:32.339 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4054593793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:32.339 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:32 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:32.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:06:32.440 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:32.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:32 smithi012 ceph-mon[147926]: pgmap v594: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 4 stale+active+clean, 4 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 54 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 21462/146787 objects degraded (14.621%); 15230/146787 objects misplaced (10.376%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T10:06:32.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4054593793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:32.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:32 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:32.523 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:32 smithi012 ceph-mon[152263]: pgmap v594: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 4 stale+active+clean, 4 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 54 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 21462/146787 objects degraded (14.621%); 15230/146787 objects misplaced (10.376%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T10:06:32.523 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4054593793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:32.523 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:32 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:32.535 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:32.679 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:32.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:32.787 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:32.797 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:32.821 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:32.990 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:32.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:33.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:33.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T10:06:33.142 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:33.195 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:33.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:33.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:33.214 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:06:33.215 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T10:06:33.303 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:33.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:33 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:06:33.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:33 smithi110 ceph-mon[136032]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:33 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:33.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:33 smithi110 ceph-mon[136032]: osdmap e231: 8 total, 7 up, 8 in 2023-10-09T10:06:33.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:33 smithi110 ceph-mon[136032]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:33 smithi110 ceph-mon[136032]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 21462/146787 objects degraded (14.621%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:33.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:33.411 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:33.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:33.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:33.420 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:33.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:06:33.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[147926]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:33.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[147926]: osdmap e231: 8 total, 7 up, 8 in 2023-10-09T10:06:33.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[147926]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[147926]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 21462/146787 objects degraded (14.621%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:33.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:06:33.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[152263]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:33.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[152263]: osdmap e231: 8 total, 7 up, 8 in 2023-10-09T10:06:33.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[152263]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[152263]: failed to encode map e231 with expected crc 2023-10-09T10:06:33.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 21462/146787 objects degraded (14.621%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:33.517 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:33.550 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:33.660 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:33.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:33.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:33.765 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:33.806 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:33.851 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:33.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:33.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:33.955 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:34.006 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:34.063 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:34.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:34.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:34.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13895 6.3 6:cb4c122f:::benchmark_data_smithi012_182576_object13894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.641039879, 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.111765246, 2023-10-09T10:06:34.189 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.190 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.190 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.190 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.190 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.190 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13895 2023-10-09T10:06:34.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.190 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544303+0000", 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7165999999999999e-05 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544380+0000", 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1870000000000003e-06 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544389+0000", 2023-10-09T10:06:34.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6619999999999996e-06 2023-10-09T10:06:34.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544394+0000", 2023-10-09T10:06:34.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2271999999999998e-05 2023-10-09T10:06:34.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544436+0000", 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138376 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544575+0000", 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000172697 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:06:34.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544747+0000", 2023-10-09T10:06:34.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019732679999999998 2023-10-09T10:06:34.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546721+0000", 2023-10-09T10:06:34.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00053507399999999997 2023-10-09T10:06:34.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.547256+0000", 2023-10-09T10:06:34.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10869904900000001 2023-10-09T10:06:34.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655955+0000", 2023-10-09T10:06:34.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1564999999999998e-05 2023-10-09T10:06:34.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656016+0000", 2023-10-09T10:06:34.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2889999999999997e-05 2023-10-09T10:06:34.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656069+0000", 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.111765246 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13902 6.5 6:a6447830:::benchmark_data_smithi012_182576_object13901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.199 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.64093919700002, 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.112868624, 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13902 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544404+0000", 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103757 2023-10-09T10:06:34.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544508+0000", 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3230999999999999e-05 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544521+0000", 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6330000000000004e-06 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544526+0000", 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4892000000000003e-05 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544570+0000", 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107476 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544678+0000", 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010674400000000001 2023-10-09T10:06:34.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544785+0000", 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022269719999999998 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.547012+0000", 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001912513 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.548924+0000", 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108253376 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657178+0000", 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4778000000000003e-05 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657232+0000", 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0898e-05 2023-10-09T10:06:34.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657273+0000", 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.112868624 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.206 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13907 6.3 6:c0d71b15:::benchmark_data_smithi012_182576_object13906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.207 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:06:34.207 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.635626817, 2023-10-09T10:06:34.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108893135, 2023-10-09T10:06:34.207 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.207 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.207 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.207 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.207 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13907 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:06:34.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549716+0000", 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9034999999999999e-05 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549745+0000", 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1456e-05 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549756+0000", 2023-10-09T10:06:34.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0289999999999996e-06 2023-10-09T10:06:34.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549761+0000", 2023-10-09T10:06:34.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6547000000000001e-05 2023-10-09T10:06:34.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549798+0000", 2023-10-09T10:06:34.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012402 2023-10-09T10:06:34.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549922+0000", 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021892400000000001 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550141+0000", 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017843049999999999 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551925+0000", 2023-10-09T10:06:34.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031652469999999999 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555090+0000", 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10346385299999999 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658554+0000", 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1267999999999999e-05 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658585+0000", 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4669e-05 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658610+0000", 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108893135 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13908 6.5 6:aed1463a:::benchmark_data_smithi012_182576_object13907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.63557423499998, 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.107611733, 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13908 2023-10-09T10:06:34.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549769+0000", 2023-10-09T10:06:34.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7673000000000001e-05 2023-10-09T10:06:34.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549797+0000", 2023-10-09T10:06:34.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4170000000000003e-06 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549804+0000", 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.826e-06 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549809+0000", 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9627999999999999e-05 2023-10-09T10:06:34.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549869+0000", 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000173229 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550042+0000", 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019173700000000001 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550234+0000", 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099897300000000001 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551233+0000", 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021422310000000001 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.553375+0000", 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.103950349 2023-10-09T10:06:34.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657325+0000", 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2319e-05 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657358+0000", 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3770000000000001e-05 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657381+0000", 2023-10-09T10:06:34.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.107611733 2023-10-09T10:06:34.222 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.222 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.222 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.222 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13911 6.3 6:c7c65153:::benchmark_data_smithi012_182576_object13910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.63552313600002, 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108882675, 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13911 2023-10-09T10:06:34.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549820+0000", 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3742000000000003e-05 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549854+0000", 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7170000000000002e-06 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549860+0000", 2023-10-09T10:06:34.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5469999999999998e-06 2023-10-09T10:06:34.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549865+0000", 2023-10-09T10:06:34.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00089393100000000004 2023-10-09T10:06:34.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550759+0000", 2023-10-09T10:06:34.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0737999999999998e-05 2023-10-09T10:06:34.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550850+0000", 2023-10-09T10:06:34.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012563299999999999 2023-10-09T10:06:34.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550975+0000", 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00095688000000000001 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551932+0000", 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032028109999999999 2023-10-09T10:06:34.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555135+0000", 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10350596099999999 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658641+0000", 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2475000000000003e-05 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658673+0000", 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9924999999999999e-05 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658703+0000", 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108882675 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13918 6.5 6:a8d8ff46:::benchmark_data_smithi012_182576_object13917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.631339207, 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104220092, 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13918 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554003+0000", 2023-10-09T10:06:34.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4011000000000006e-05 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554087+0000", 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8400000000000001e-06 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554096+0000", 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4819999999999996e-06 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554100+0000", 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3441e-05 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554144+0000", 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126305 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554270+0000", 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016421900000000001 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554434+0000", 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000721037 2023-10-09T10:06:34.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555155+0000", 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078462779999999999 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563002+0000", 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095134755000000001 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658136+0000", 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0183999999999999e-05 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658187+0000", 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8050000000000003e-05 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658225+0000", 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104220092 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13922 6.3 6:cfff9caa:::benchmark_data_smithi012_182576_object13921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.62674357600002, 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10019183399999999, 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.239 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13922 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558599+0000", 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7776999999999997e-05 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558687+0000", 2023-10-09T10:06:34.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3254999999999999e-05 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558700+0000", 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.977e-06 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558705+0000", 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0094e-05 2023-10-09T10:06:34.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558735+0000", 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8771999999999997e-05 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558814+0000", 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9957999999999995e-05 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:06:34.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558904+0000", 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070560600000000001 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.559610+0000", 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002049763 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.561659+0000", 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.097076735999999997 2023-10-09T10:06:34.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658736+0000", 2023-10-09T10:06:34.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.726e-05 2023-10-09T10:06:34.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658763+0000", 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8696000000000001e-05 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658792+0000", 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10019183399999999 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13930 6.3 6:cba38fc5:::benchmark_data_smithi012_182576_object13929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.620561931, 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094088516999999997, 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.247 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13930 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564781+0000", 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0343000000000002e-05 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564831+0000", 2023-10-09T10:06:34.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7879999999999995e-06 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564840+0000", 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4070000000000003e-06 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564844+0000", 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0728999999999997e-05 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564875+0000", 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9303999999999997e-05 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564964+0000", 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107612 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.565072+0000", 2023-10-09T10:06:34.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050480800000000004 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.565577+0000", 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090698299999999999 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566484+0000", 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092340803999999999 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658824+0000", 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6945e-05 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658851+0000", 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9034000000000001e-05 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658870+0000", 2023-10-09T10:06:34.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094088516999999997 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13932 6.3 6:cf3c9e78:::benchmark_data_smithi012_182576_object13931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.61889358399998, 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092527188999999996, 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.254 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13932 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566449+0000", 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6433000000000003e-05 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566546+0000", 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2155e-05 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566558+0000", 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2870000000000001e-06 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566564+0000", 2023-10-09T10:06:34.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1313000000000002e-05 2023-10-09T10:06:34.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566606+0000", 2023-10-09T10:06:34.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2416000000000005e-05 2023-10-09T10:06:34.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566688+0000", 2023-10-09T10:06:34.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9309999999999996e-05 2023-10-09T10:06:34.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:06:34.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566787+0000", 2023-10-09T10:06:34.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043576999999999998 2023-10-09T10:06:34.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.567223+0000", 2023-10-09T10:06:34.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00077269100000000005 2023-10-09T10:06:34.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.567996+0000", 2023-10-09T10:06:34.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.090905342 2023-10-09T10:06:34.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658901+0000", 2023-10-09T10:06:34.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3823000000000003e-05 2023-10-09T10:06:34.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658955+0000", 2023-10-09T10:06:34.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2500000000000001e-05 2023-10-09T10:06:34.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658977+0000", 2023-10-09T10:06:34.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092527188999999996 2023-10-09T10:06:34.263 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.264 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.264 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.264 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13936 6.5 6:acd4ecb9:::benchmark_data_smithi012_182576_object13935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:06:34.264 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.575103+0000", 2023-10-09T10:06:34.265 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 448.610240508, 2023-10-09T10:06:34.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083239002000000006, 2023-10-09T10:06:34.265 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.265 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.265 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.265 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.265 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.265 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13936 2023-10-09T10:06:34.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.266 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575103+0000", 2023-10-09T10:06:34.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575103+0000", 2023-10-09T10:06:34.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:06:34.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575103+0000", 2023-10-09T10:06:34.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4441000000000003e-05 2023-10-09T10:06:34.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575157+0000", 2023-10-09T10:06:34.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0647e-05 2023-10-09T10:06:34.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575168+0000", 2023-10-09T10:06:34.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0260000000000004e-06 2023-10-09T10:06:34.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575174+0000", 2023-10-09T10:06:34.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2377999999999999e-05 2023-10-09T10:06:34.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575226+0000", 2023-10-09T10:06:34.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000135566 2023-10-09T10:06:34.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575362+0000", 2023-10-09T10:06:34.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016899800000000001 2023-10-09T10:06:34.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:06:34.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.575531+0000", 2023-10-09T10:06:34.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00061649199999999995 2023-10-09T10:06:34.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.576147+0000", 2023-10-09T10:06:34.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000487092 2023-10-09T10:06:34.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.576634+0000", 2023-10-09T10:06:34.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081631312999999997 2023-10-09T10:06:34.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658266+0000", 2023-10-09T10:06:34.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3992999999999997e-05 2023-10-09T10:06:34.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658310+0000", 2023-10-09T10:06:34.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2771999999999997e-05 2023-10-09T10:06:34.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658342+0000", 2023-10-09T10:06:34.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083239002000000006 2023-10-09T10:06:34.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.276 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.277 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:36563 6.8 6:10f4a19b:::benchmark_data_smithi012_182576_object36562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-10-09T10:06:34.277 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:17.197546+0000", 2023-10-09T10:06:34.277 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 436.98779769499998, 2023-10-09T10:06:34.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079528249999999995, 2023-10-09T10:06:34.277 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.277 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.278 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 36563 2023-10-09T10:06:34.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.278 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197546+0000", 2023-10-09T10:06:34.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197546+0000", 2023-10-09T10:06:34.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:34.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197545+0000", 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9709000000000001e-05 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197594+0000", 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3861e-05 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197608+0000", 2023-10-09T10:06:34.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8610000000000003e-06 2023-10-09T10:06:34.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197614+0000", 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2370000000000003e-05 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197656+0000", 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019276800000000001 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.197849+0000", 2023-10-09T10:06:34.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017279 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.198022+0000", 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028725869999999998 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.200895+0000", 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085708599999999996 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.201752+0000", 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.075220750000000003 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.276972+0000", 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9431000000000001e-05 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.277032+0000", 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2469999999999998e-05 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:17.277074+0000", 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079528249999999995 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.287 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96742 6.3 6:ccfab877:::benchmark_data_smithi012_182576_object96741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:06:34.287 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:06:34.287 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 380.90609004800001, 2023-10-09T10:06:34.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11727006399999999, 2023-10-09T10:06:34.287 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96742 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:06:34.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:06:34.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:06:34.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279251+0000", 2023-10-09T10:06:34.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6714e-05 2023-10-09T10:06:34.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279298+0000", 2023-10-09T10:06:34.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3504999999999999e-05 2023-10-09T10:06:34.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279321+0000", 2023-10-09T10:06:34.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0139999999999999e-05 2023-10-09T10:06:34.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279331+0000", 2023-10-09T10:06:34.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018866076999999998 2023-10-09T10:06:34.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298197+0000", 2023-10-09T10:06:34.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018085900000000001 2023-10-09T10:06:34.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298378+0000", 2023-10-09T10:06:34.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022614600000000001 2023-10-09T10:06:34.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298604+0000", 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033326643000000003 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.331931+0000", 2023-10-09T10:06:34.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5298e-05 2023-10-09T10:06:34.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.331966+0000", 2023-10-09T10:06:34.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060961738000000001 2023-10-09T10:06:34.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.392928+0000", 2023-10-09T10:06:34.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4974000000000005e-05 2023-10-09T10:06:34.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.392993+0000", 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035309400000000002 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396524+0000", 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11727006399999999 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.299 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96748 6.3 6:ce4562bc:::benchmark_data_smithi012_182576_object96747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:06:34.299 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:06:34.299 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 380.89757302599998, 2023-10-09T10:06:34.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11285076300000001, 2023-10-09T10:06:34.299 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.299 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.300 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96748 2023-10-09T10:06:34.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.300 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:06:34.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:06:34.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:06:34.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287768+0000", 2023-10-09T10:06:34.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8236999999999999e-05 2023-10-09T10:06:34.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287816+0000", 2023-10-09T10:06:34.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2195999999999999e-05 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287838+0000", 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0397000000000001e-05 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287849+0000", 2023-10-09T10:06:34.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014132728000000001 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.301981+0000", 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021475899999999999 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.302196+0000", 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024190600000000001 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.302438+0000", 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029561812 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332000+0000", 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2532e-05 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332032+0000", 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.064502819000000003 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396535+0000", 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4147999999999998e-05 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396579+0000", 2023-10-09T10:06:34.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040421540000000001 2023-10-09T10:06:34.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400622+0000", 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11285076300000001 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96749 6.3 6:c755d6fb:::benchmark_data_smithi012_182576_object96748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 380.89533530800003, 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11442171800000001, 2023-10-09T10:06:34.309 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96749 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:06:34.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290007+0000", 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9950000000000001e-05 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290056+0000", 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9181000000000002e-05 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290076+0000", 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8350000000000004e-06 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290083+0000", 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015899986000000001 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.305983+0000", 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000202692 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.306186+0000", 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023536100000000001 2023-10-09T10:06:34.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.306422+0000", 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025644121999999998 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332066+0000", 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068449019999999999 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338911+0000", 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.061723756999999997 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400634+0000", 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9935e-05 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400704+0000", 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003725983 2023-10-09T10:06:34.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.404430+0000", 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11442171800000001 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.34797.0:96749 6.3 e119/117 ondisk, result = 0)", 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.317299+0000", 2023-10-09T10:06:34.316 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 380.868045102, 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.087135486999999998, 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317299+0000", 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317299+0000", 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317298+0000", 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.54e-06 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317302+0000", 2023-10-09T10:06:34.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1517e-05 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317313+0000", 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8250000000000001e-06 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.317319+0000", 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021573518999999999 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338893+0000", 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9680000000000001e-06 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338903+0000", 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.065531639000000003 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.404434+0000", 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.087135486999999998 2023-10-09T10:06:34.320 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96765 6.b e119/97 6:d2083b06:::benchmark_data_smithi012_182576_object96764:head v 119'6152, mlcod=119'6148)", 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.325246+0000", 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 380.860098155, 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081827742999999994, 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325246+0000", 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325246+0000", 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325245+0000", 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2183e-05 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325277+0000", 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.363e-05 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325291+0000", 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8780000000000003e-06 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.325297+0000", 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067885280000000003 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332085+0000", 2023-10-09T10:06:34.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6765999999999998e-05 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332102+0000", 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.074909911999999995 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.407012+0000", 2023-10-09T10:06:34.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1492e-05 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.407073+0000", 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081827742999999994 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:97524 6.3 6:c7f23124:::benchmark_data_smithi012_182576_object97523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:14.603054+0000", 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 379.58229007099999, 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081403381999999996, 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.327 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97524 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603054+0000", 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603054+0000", 2023-10-09T10:06:34.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:34.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603053+0000", 2023-10-09T10:06:34.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7908000000000001e-05 2023-10-09T10:06:34.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603081+0000", 2023-10-09T10:06:34.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0090000000000004e-06 2023-10-09T10:06:34.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603087+0000", 2023-10-09T10:06:34.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1999999999999996e-06 2023-10-09T10:06:34.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.603091+0000", 2023-10-09T10:06:34.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018937381 2023-10-09T10:06:34.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.622029+0000", 2023-10-09T10:06:34.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017725299999999999 2023-10-09T10:06:34.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.622206+0000", 2023-10-09T10:06:34.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000194873 2023-10-09T10:06:34.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:06:34.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.622401+0000", 2023-10-09T10:06:34.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017724859999999999 2023-10-09T10:06:34.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.640126+0000", 2023-10-09T10:06:34.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6192999999999997e-05 2023-10-09T10:06:34.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.640162+0000", 2023-10-09T10:06:34.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040276379000000001 2023-10-09T10:06:34.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.680438+0000", 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8034999999999995e-05 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.680506+0000", 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039506890000000003 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.684457+0000", 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081403381999999996 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:97526 6.3 6:cd3b62b4:::benchmark_data_smithi012_182576_object97525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:14.604333+0000", 2023-10-09T10:06:34.338 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 379.581011139, 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084183229999999998, 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 97526 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604333+0000", 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604333+0000", 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604331+0000", 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6183999999999998e-05 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604367+0000", 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6690000000000001e-05 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604384+0000", 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.604390+0000", 2023-10-09T10:06:34.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021184410000000001 2023-10-09T10:06:34.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.625575+0000", 2023-10-09T10:06:34.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012946799999999999 2023-10-09T10:06:34.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.625704+0000", 2023-10-09T10:06:34.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015452700000000001 2023-10-09T10:06:34.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:06:34.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.625859+0000", 2023-10-09T10:06:34.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014272895000000001 2023-10-09T10:06:34.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.640132+0000", 2023-10-09T10:06:34.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9092000000000003e-05 2023-10-09T10:06:34.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.640191+0000", 2023-10-09T10:06:34.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044416529000000003 2023-10-09T10:06:34.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.684607+0000", 2023-10-09T10:06:34.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4169000000000002e-05 2023-10-09T10:06:34.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.684672+0000", 2023-10-09T10:06:34.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038443459999999998 2023-10-09T10:06:34.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:14.688516+0000", 2023-10-09T10:06:34.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084183229999999998 2023-10-09T10:06:34.348 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.348 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.348 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.348 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:142238 6.5 6:a70ceffa:::benchmark_data_smithi012_182576_object36717:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:01:29.828808+0000", 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 304.35653581000003, 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086854495000000004, 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 142238 2023-10-09T10:06:34.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.828808+0000", 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.828808+0000", 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.828807+0000", 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5529999999999998e-06 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.828813+0000", 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.876e-06 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.828823+0000", 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7200000000000003e-06 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.828829+0000", 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3162000000000002e-05 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.828902+0000", 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000125845 2023-10-09T10:06:34.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.829028+0000", 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032203700000000001 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.829350+0000", 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032362799999999998 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.832586+0000", 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7160000000000003e-06 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.832594+0000", 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082968843 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.915563+0000", 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3483000000000003e-05 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.915616+0000", 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6254000000000001e-05 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:29.915663+0000", 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086854495000000004 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:208258 6.13 6:cf08a223:::benchmark_data_smithi012_182576_object102737:head [delete] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:03:03.857848+0000", 2023-10-09T10:06:34.356 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.327496008, 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082501582000000004, 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 208258 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.857848+0000", 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.857848+0000", 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:06:34.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.857847+0000", 2023-10-09T10:06:34.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.879e-06 2023-10-09T10:06:34.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:06:34.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.857853+0000", 2023-10-09T10:06:34.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1416999999999999e-05 2023-10-09T10:06:34.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:06:34.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.857864+0000", 2023-10-09T10:06:34.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4860000000000002e-06 2023-10-09T10:06:34.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:06:34.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.857870+0000", 2023-10-09T10:06:34.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2299000000000002e-05 2023-10-09T10:06:34.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:06:34.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.857923+0000", 2023-10-09T10:06:34.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000140457 2023-10-09T10:06:34.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:06:34.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.858063+0000", 2023-10-09T10:06:34.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058371900000000001 2023-10-09T10:06:34.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:06:34.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.858647+0000", 2023-10-09T10:06:34.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025862300000000001 2023-10-09T10:06:34.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:06:34.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.861233+0000", 2023-10-09T10:06:34.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000140679 2023-10-09T10:06:34.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.861374+0000", 2023-10-09T10:06:34.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.078879960999999998 2023-10-09T10:06:34.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:06:34.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.940254+0000", 2023-10-09T10:06:34.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3841999999999997e-05 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.940308+0000", 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1801000000000002e-05 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:03:03.940349+0000", 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082501582000000004 2023-10-09T10:06:34.367 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.368 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:06:34.368 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:06:34.368 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:34.369 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:34.369 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:34.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:34.470 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:34.516 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:34.570 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:34.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:34.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:06:34.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:34.797 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:34.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:34 smithi012 ceph-mon[147926]: pgmap v596: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 33164/143151 objects degraded (23.167%); 12025/143151 objects misplaced (8.400%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T10:06:34.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:34 smithi012 ceph-mon[152263]: pgmap v596: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 33164/143151 objects degraded (23.167%); 12025/143151 objects misplaced (8.400%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T10:06:34.945 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:34.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:34.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:35.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:06:35.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:35.124 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:06:35.132 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:35.174 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:06:35.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:34 smithi110 ceph-mon[136032]: pgmap v596: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 33164/143151 objects degraded (23.167%); 12025/143151 objects misplaced (8.400%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T10:06:35.224 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:06:35.244 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:35.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:35.273 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:06:35.309 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:06:35.347 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:06:35.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:35.380 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:06:35.380 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:06:35.380 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:06:35.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:06:35.381 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:06:35.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:06:35.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:06:35.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:06:35.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:06:35.422 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:35.423 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:35.508 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:35.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:35.618 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:35.678 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:35.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:35.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:35.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:35.888 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2555900337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:35.892 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2555900337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:35.894 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:35.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2555900337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:35.944 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:35.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:36.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:36.122 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:36.758 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:36.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:36.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:36.868 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:36.909 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:36.962 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:36.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:36.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:37.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:37.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:37.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:36 smithi012 ceph-mon[152263]: pgmap v597: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 31582/136965 objects degraded (23.058%); 11350/136965 objects misplaced (8.287%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:37.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:36 smithi012 ceph-mon[147926]: pgmap v597: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 31582/136965 objects degraded (23.058%); 11350/136965 objects misplaced (8.287%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:37.183 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:37.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:36 smithi110 ceph-mon[136032]: pgmap v597: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 31582/136965 objects degraded (23.058%); 11350/136965 objects misplaced (8.287%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:37.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:37.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:37.293 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:37.329 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:37.392 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:37.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:37.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:37.543 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:37.589 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:37.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:37.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:37.699 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:37.751 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:37.801 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:37.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:37.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:37.910 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:37.941 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:37.989 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:37.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:38.006 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:06:38.006 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:06:38.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:38.123 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:37 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 31582/136965 objects degraded (23.058%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:38.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:38.139 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:06:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:06:37] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T10:06:38.139 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:37 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 31582/136965 objects degraded (23.058%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:38.140 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:37 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 31582/136965 objects degraded (23.058%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:38.185 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:38.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:38.216 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [3] live_osds: [0, 1, 4, 5, 6, 7, 2] 2023-10-09T10:06:38.216 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:06:38.216 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T10:06:38.216 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:06:38.216 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:06:38.217 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T10:06:38.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:38.327 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:38.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:38.512 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:38.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:38.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:38.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:38.664 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:38.730 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:38.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:38.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:38.839 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:38.875 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:38.914 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:38.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:38.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:38.921 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:38 smithi110 ceph-mon[136032]: pgmap v598: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 31582/136965 objects degraded (23.058%); 11350/136965 objects misplaced (8.287%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:06:38.976 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:39.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:39.105 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:38 smithi012 ceph-mon[147926]: pgmap v598: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 31582/136965 objects degraded (23.058%); 11350/136965 objects misplaced (8.287%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:06:39.105 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:38 smithi012 ceph-mon[152263]: pgmap v598: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 31582/136965 objects degraded (23.058%); 11350/136965 objects misplaced (8.287%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:06:39.110 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:39.178 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:06:39.178 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:06:39.188 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:39.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:39.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:39.378 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:39.665 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:39.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:39.772 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:06:39.777 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:39.841 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:39.921 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:39.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:40.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:06:40.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:40.085 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:40.179 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:40.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:40.256 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:40.257 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:06:40.257 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:40.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:40.322 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:40.624 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:40.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:40.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:40.710 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:40.728 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:40.766 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:40.805 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:40.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:40.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:40.960 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:06:41.036 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:41.106 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:41.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:41.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:41.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:41.248 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:40 smithi110 ceph-mon[136032]: pgmap v599: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 30803/132165 objects degraded (23.306%); 11149/132165 objects misplaced (8.436%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:06:41.249 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4138799307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:41.249 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:41.251 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:41.305 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:41.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:41.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:40 smithi012 ceph-mon[152263]: pgmap v599: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 30803/132165 objects degraded (23.306%); 11149/132165 objects misplaced (8.436%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:06:41.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4138799307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:41.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:41.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:40 smithi012 ceph-mon[147926]: pgmap v599: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 30803/132165 objects degraded (23.306%); 11149/132165 objects misplaced (8.436%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:06:41.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4138799307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:41.399 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:06:41.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:41.414 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:41.510 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:41.804 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:41.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:41.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:41.910 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:41.958 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:42.005 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:42.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:42.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:42.151 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:42.205 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:42.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:41 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:42.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:41 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:06:42.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:41 smithi110 ceph-mon[136032]: osdmap e232: 8 total, 7 up, 8 in 2023-10-09T10:06:42.209 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:41 smithi110 ceph-mon[136032]: failed to encode map e232 with expected crc 2023-10-09T10:06:42.210 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:41 smithi110 ceph-mon[136032]: failed to encode map e232 with expected crc 2023-10-09T10:06:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:42.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:42.278 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:42.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:06:42.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[147926]: osdmap e232: 8 total, 7 up, 8 in 2023-10-09T10:06:42.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[147926]: failed to encode map e232 with expected crc 2023-10-09T10:06:42.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[147926]: failed to encode map e232 with expected crc 2023-10-09T10:06:42.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:42.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:06:42.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[152263]: osdmap e232: 8 total, 7 up, 8 in 2023-10-09T10:06:42.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[152263]: failed to encode map e232 with expected crc 2023-10-09T10:06:42.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:41 smithi012 ceph-mon[152263]: failed to encode map e232 with expected crc 2023-10-09T10:06:42.311 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:42.396 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:42.535 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:42.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:42.641 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:42.678 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:42.717 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:42.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:42.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:42.894 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:43.076 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:43.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:43.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:43.155 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:42 smithi012 ceph-mon[152263]: pgmap v601: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 29173/125847 objects degraded (23.181%); 10434/125847 objects misplaced (8.291%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:06:43.155 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:42 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 29173/125847 objects degraded (23.181%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:43.155 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:42 smithi012 ceph-mon[147926]: pgmap v601: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 29173/125847 objects degraded (23.181%); 10434/125847 objects misplaced (8.291%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:06:43.155 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:42 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 29173/125847 objects degraded (23.181%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:43.189 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:43.251 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:43.342 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:06:43.342 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:06:43.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:42 smithi110 ceph-mon[136032]: pgmap v601: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 29173/125847 objects degraded (23.181%); 10434/125847 objects misplaced (8.291%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:06:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:42 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 29173/125847 objects degraded (23.181%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:43.473 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:43.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:43.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:06:43.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:43.635 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:06:43.670 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:06:43.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:43.713 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:06:43.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:06:43.824 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:06:43.875 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:06:43.876 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:43.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:43.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:43.931 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:06:43.931 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:06:43.931 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:06:43.931 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:06:43.931 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:06:43.931 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:06:43.931 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:06:43.932 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:06:43.932 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:06:43.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:06:43.987 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:44.023 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:44.065 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:44.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:44.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:44.173 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:44.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1864550227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:44.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:44.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1864550227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:44.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:44.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1864550227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:44.235 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:44.237 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:44.292 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:44.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:44.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:44.325 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:06:44.326 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:06:44.398 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:44.448 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:44.507 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:44.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:44.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:44.691 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:44.723 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:44.767 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:44.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:44.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:44.877 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:44.923 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:44.961 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:44 smithi012 ceph-mon[147926]: pgmap v602: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 28328/121038 objects degraded (23.404%); 10174/121038 objects misplaced (8.406%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:44.966 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:44.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:44.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:45.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:45.144 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:44 smithi110 ceph-mon[136032]: pgmap v602: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 28328/121038 objects degraded (23.404%); 10174/121038 objects misplaced (8.406%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:44 smithi012 ceph-mon[152263]: pgmap v602: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 28328/121038 objects degraded (23.404%); 10174/121038 objects misplaced (8.406%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:45.653 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [3] live_osds: [0, 1, 4, 5, 6, 7, 2] 2023-10-09T10:06:45.654 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:06:45.654 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T10:06:45.654 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:06:45.654 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:06:45.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T10:06:45.923 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:45.929 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:45.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:46.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:46.099 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:46.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:06:46.408 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:06:46.624 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:46.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:46.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:46.766 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:47.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:46 smithi110 ceph-mon[136032]: pgmap v603: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 26786/114996 objects degraded (23.293%); 9511/114996 objects misplaced (8.271%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:47.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:46 smithi012 ceph-mon[147926]: pgmap v603: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 26786/114996 objects degraded (23.293%); 9511/114996 objects misplaced (8.271%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:47.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:46 smithi012 ceph-mon[152263]: pgmap v603: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 26786/114996 objects degraded (23.293%); 9511/114996 objects misplaced (8.271%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:47.709 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:06:47.710 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-10-09T10:06:47.710 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:06:47.987 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:47 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 26786/114996 objects degraded (23.293%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:06:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:06:47] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T10:06:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:47 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 26786/114996 objects degraded (23.293%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:48.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:47 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 26786/114996 objects degraded (23.293%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:48.271 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:48.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:48.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:48.417 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:48.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:48.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:48.566 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:48.608 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:48.654 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:48.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:48.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:48.759 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:48.800 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:48.853 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:48.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:48.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:48.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:49.004 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:49.007 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:49.050 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:49.054 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:48 smithi110 ceph-mon[136032]: pgmap v604: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 26786/114996 objects degraded (23.293%); 9511/114996 objects misplaced (8.271%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:49.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:49.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:49.242 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:48 smithi012 ceph-mon[147926]: pgmap v604: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 26786/114996 objects degraded (23.293%); 9511/114996 objects misplaced (8.271%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:49.243 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:49 smithi012 ceph-mon[152263]: pgmap v604: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 26786/114996 objects degraded (23.293%); 9511/114996 objects misplaced (8.271%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:06:49.245 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:49.329 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:49.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:49.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:49.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:49.562 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:49.760 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:49.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:49.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:49.799 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:06:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:06:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:06:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:06:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154065 7.8 7.6bf60448 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:06:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:06:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.823351406, 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86727346999999999, 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154065 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969553+0000", 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969561+0000", 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7109999999999995e-06 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969572+0000", 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.115e-05 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969582+0000", 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5100000000000004e-06 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969631+0000", 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9395999999999999e-05 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969640+0000", 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8540000000000003e-06 2023-10-09T10:06:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836819+0000", 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86717872399999996 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836828+0000", 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6709999999999994e-06 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154067 7.e 7.7d5b394e (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.823309113999997, 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86689926799999995, 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154067 2023-10-09T10:06:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969600+0000", 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8399999999999999e-06 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969607+0000", 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.922e-06 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969612+0000", 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7830000000000004e-06 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976392+0000", 2023-10-09T10:06:49.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067794639999999998 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976404+0000", 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1769e-05 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836486+0000", 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008206399999998 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836497+0000", 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0947e-05 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154074 7.e 7.21f6315e (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.823279206000002, 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86689908999999998, 2023-10-09T10:06:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154074 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:06:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969629+0000", 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4509999999999999e-06 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969636+0000", 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.216e-06 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969642+0000", 2023-10-09T10:06:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0129999999999999e-06 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976425+0000", 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006782649 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976431+0000", 2023-10-09T10:06:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8719999999999999e-06 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836518+0000", 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008733299999995 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836526+0000", 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9610000000000005e-06 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154080 7.5 7.d54c8f95 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.823258543999998, 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.868647155, 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154080 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:06:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969647+0000", 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969650+0000", 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9440000000000001e-06 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969656+0000", 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2160000000000003e-06 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969663+0000", 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5780000000000004e-06 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969744+0000", 2023-10-09T10:06:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0678000000000002e-05 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969751+0000", 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3719999999999999e-06 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838286+0000", 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.868534837 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838295+0000", 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0669999999999998e-06 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154083 7.e 7.33ccdeae (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.823230062, 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86687670800000005, 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154083 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969679+0000", 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.892e-06 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969686+0000", 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0500000000000003e-06 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969692+0000", 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7699999999999998e-06 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976451+0000", 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067598859999999997 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976457+0000", 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1880000000000001e-06 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836546+0000", 2023-10-09T10:06:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008883199999997 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836553+0000", 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5619999999999998e-06 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154085 7.5 7.5fb2e7e5 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:06:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.823210037000003, 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86864703600000004, 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154085 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969698+0000", 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2050000000000001e-06 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969704+0000", 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6799999999999998e-06 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969709+0000", 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5639999999999996e-06 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969770+0000", 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0999000000000003e-05 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969775+0000", 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.95e-06 2023-10-09T10:06:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838335+0000", 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86855936700000003 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838343+0000", 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6770000000000001e-06 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154089 7.8 7.cb7923e8 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.813821509, 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.8577726, 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154089 2023-10-09T10:06:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979083+0000", 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979093+0000", 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5319999999999995e-06 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979109+0000", 2023-10-09T10:06:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.677e-05 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979118+0000", 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979202+0000", 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3975999999999997e-05 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979214+0000", 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1324e-05 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836849+0000", 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85763573500000001 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836858+0000", 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0099999999999995e-06 2023-10-09T10:06:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154090 7.5 7.72a407a5 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.813772555, 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85923808499999998, 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154090 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979133+0000", 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979137+0000", 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.332e-06 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979144+0000", 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0809999999999996e-06 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979149+0000", 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5770000000000001e-06 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979256+0000", 2023-10-09T10:06:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106482 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979263+0000", 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2509999999999997e-06 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838364+0000", 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85910122099999997 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838372+0000", 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7430000000000002e-06 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154096 7.8 7.af393d68 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.798245659000003, 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84222455699999998, 2023-10-09T10:06:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154096 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994659+0000", 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994668+0000", 2023-10-09T10:06:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0710000000000001e-06 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994692+0000", 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3887999999999999e-05 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994702+0000", 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0111e-05 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994765+0000", 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3428000000000003e-05 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994779+0000", 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3946e-05 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836878+0000", 2023-10-09T10:06:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84209816000000004 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836885+0000", 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7600000000000002e-06 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154097 7.e 7.c78fc5fe (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:06:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.797467343999998, 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84114410299999998, 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154097 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995438+0000", 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995444+0000", 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2299999999999999e-06 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995459+0000", 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5116999999999999e-05 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995467+0000", 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5329999999999994e-06 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995527+0000", 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0043000000000001e-05 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995535+0000", 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9219999999999997e-06 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836576+0000", 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84104053400000001 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836583+0000", 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4970000000000004e-06 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160622 7.e e227/222 7:7d366766:::benchmark_data_smithi012_221426_object52844:head v 227'9935, mlcod=196'3292)", 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.112292588000003, 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069261389000000007, 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680613+0000", 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680622+0000", 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2410000000000001e-06 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680638+0000", 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5410999999999999e-05 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680646+0000", 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0849999999999997e-06 2023-10-09T10:06:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680752+0000", 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105919 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680779+0000", 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7885000000000002e-05 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749829+0000", 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069049252000000005 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749875+0000", 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6548000000000002e-05 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160656 7.e e227/222 7:75914ec4:::benchmark_data_smithi012_221426_object52878:head v 227'9936, mlcod=196'3292)", 2023-10-09T10:06:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.689198+0000", 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.103708304000001, 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060697886, 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.689198+0000", 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.689198+0000", 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.689197+0000", 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.689206+0000", 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2820000000000002e-06 2023-10-09T10:06:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.689218+0000", 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1270999999999999e-05 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.689225+0000", 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3490000000000003e-06 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.693105+0000", 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038796429999999999 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.693138+0000", 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3929e-05 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749878+0000", 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.056739174000000003 2023-10-09T10:06:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749896+0000", 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8343999999999999e-05 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160660 7.e e227/222 7:71abe4cf:::benchmark_data_smithi012_221426_object52882:head v 227'9937, mlcod=196'3292)", 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.691161+0000", 2023-10-09T10:06:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.101745106000003, 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058756317000000002, 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691161+0000", 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691161+0000", 2023-10-09T10:06:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691160+0000", 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691169+0000", 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9460000000000004e-06 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691183+0000", 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.366e-05 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691190+0000", 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6190000000000002e-06 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.700275+0000", 2023-10-09T10:06:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090844240000000007 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.700307+0000", 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2289000000000002e-05 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749901+0000", 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049593657999999999 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749918+0000", 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6762999999999999e-05 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160664 7.e e227/222 7:711e5991:::benchmark_data_smithi012_221426_object52886:head v 227'9938, mlcod=196'3292)", 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.691557+0000", 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.101349149000001, 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058378409999999999, 2023-10-09T10:06:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691557+0000", 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691557+0000", 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691556+0000", 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691564+0000", 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7640000000000005e-06 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691574+0000", 2023-10-09T10:06:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.009e-05 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.691581+0000", 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.956e-06 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.708990+0000", 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017408347000000001 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.709019+0000", 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9022000000000001e-05 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749919+0000", 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.040900539999999999 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749936+0000", 2023-10-09T10:06:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6475000000000001e-05 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160778 7.e e227/222 7:7dd31466:::benchmark_data_smithi012_221426_object53000:head v 227'9946, mlcod=196'3305)", 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.896003+0000", 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 31.896903408, 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055132926999999998, 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896003+0000", 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896003+0000", 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896002+0000", 2023-10-09T10:06:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896012+0000", 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5710000000000004e-06 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896032+0000", 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0797000000000001e-05 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896041+0000", 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.721e-06 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896092+0000", 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1412999999999997e-05 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896121+0000", 2023-10-09T10:06:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.864e-05 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.951109+0000", 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054987873 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.951136+0000", 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6962e-05 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160784 7.e e227/222 7:7888284a:::benchmark_data_smithi012_221426_object53006:head v 227'9947, mlcod=196'3305)", 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.896085+0000", 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 31.896821204999998, 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055073877, 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896085+0000", 2023-10-09T10:06:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896085+0000", 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896085+0000", 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896101+0000", 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5566999999999999e-05 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896107+0000", 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2400000000000004e-06 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.896112+0000", 2023-10-09T10:06:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3460000000000001e-06 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.906850+0000", 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010737391000000001 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.906869+0000", 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9279999999999998e-05 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.951138+0000", 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044269571000000001 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.951159+0000", 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0650999999999999e-05 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160791 7.e e227/222 7:7a10b96b:::benchmark_data_smithi012_221426_object53013:head v 227'9949, mlcod=196'3305)", 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.907549+0000", 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 31.885357765999998, 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055238817000000003, 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.907549+0000", 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.907549+0000", 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.907548+0000", 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.907553+0000", 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1559999999999997e-06 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.907558+0000", 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.434e-06 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.907562+0000", 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6969999999999999e-06 2023-10-09T10:06:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.923720+0000", 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016157891000000001 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.923745+0000", 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5896000000000001e-05 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.962757+0000", 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.039011128999999999 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.962787+0000", 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0877999999999998e-05 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160793 7.e e227/222 7:7fde3033:::benchmark_data_smithi012_221426_object53015:head v 227'9950, mlcod=196'3305)", 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.911742+0000", 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 31.881164743999999, 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.051083227000000002, 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.911742+0000", 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.911742+0000", 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.911741+0000", 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.911751+0000", 2023-10-09T10:06:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0105999999999999e-05 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.911765+0000", 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4323e-05 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.911775+0000", 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7319999999999993e-06 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.932137+0000", 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020362333999999999 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.932162+0000", 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4505999999999999e-05 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.962802+0000", 2023-10-09T10:06:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030640311999999999 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.962825+0000", 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2972999999999998e-05 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:166692 7.e e230/222 7:7fd46aa8:::benchmark_data_smithi012_221426_object58914:head v 230'10288, mlcod=199'3705)", 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:29.288693+0000", 2023-10-09T10:06:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 20.504213249999999, 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054971163000000003, 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.288693+0000", 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.288693+0000", 2023-10-09T10:06:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.288692+0000", 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.288700+0000", 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4379999999999998e-06 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.288711+0000", 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0900000000000001e-05 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.288718+0000", 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1969999999999999e-06 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.304848+0000", 2023-10-09T10:06:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016130541000000002 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.304874+0000", 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5921000000000001e-05 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.343636+0000", 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.038761950000000003 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:29.343664+0000", 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7996000000000001e-05 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(7.e 232/222 [PushOp(7:7d9f0791:::benchmark_data_smithi012_221426_object72605:head, version: 201'4494, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:7d9f0791:::benchmark_data_smithi012_221426_object72605:head@201'4494, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:47.943023+0000", 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.8498838279999998, 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.051978096000000001, 2023-10-09T10:06:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.943023+0000", 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.943023+0000", 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.943022+0000", 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.943050+0000", 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8464999999999999e-05 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.943065+0000", 2023-10-09T10:06:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4704e-05 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.943077+0000", 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2146e-05 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.943156+0000", 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8912000000000006e-05 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.943164+0000", 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5979999999999997e-06 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:47.995001+0000", 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.051836210000000001 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:06:49.900 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:06:49.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:49.981 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:50.070 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:06:50.109 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2507810727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:50.109 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2507810727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:50.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:50.273 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:50.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:50.374 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:50.425 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2507810727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:06:50.427 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:50.482 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:50.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:50.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:50.587 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:50.659 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:50.689 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:06:50.932 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:50.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:51.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:51.142 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:51.274 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:51.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[147926]: pgmap v605: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 25939/110115 objects degraded (23.556%); 9267/110115 objects misplaced (8.416%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:51.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:51.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[147926]: failed to encode map e233 with expected crc 2023-10-09T10:06:51.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[147926]: failed to encode map e233 with expected crc 2023-10-09T10:06:51.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2507810727' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:51.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[147926]: osdmap e233: 8 total, 7 up, 8 in 2023-10-09T10:06:51.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[152263]: pgmap v605: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 25939/110115 objects degraded (23.556%); 9267/110115 objects misplaced (8.416%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:51.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:51.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[152263]: failed to encode map e233 with expected crc 2023-10-09T10:06:51.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[152263]: failed to encode map e233 with expected crc 2023-10-09T10:06:51.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2507810727' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:51.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:51 smithi012 ceph-mon[152263]: osdmap e233: 8 total, 7 up, 8 in 2023-10-09T10:06:51.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:51.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:51.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:51.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:51 smithi110 ceph-mon[136032]: pgmap v605: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 25939/110115 objects degraded (23.556%); 9267/110115 objects misplaced (8.416%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:51.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:51 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:06:51.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:51 smithi110 ceph-mon[136032]: failed to encode map e233 with expected crc 2023-10-09T10:06:51.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:51 smithi110 ceph-mon[136032]: failed to encode map e233 with expected crc 2023-10-09T10:06:51.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2507810727' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:06:51.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:51 smithi110 ceph-mon[136032]: osdmap e233: 8 total, 7 up, 8 in 2023-10-09T10:06:51.418 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:51.470 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:51.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:51.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:51.578 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:51.630 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:51.657 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:06:51.679 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:51.688 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:51.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:51.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:51.696 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:06:51.743 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:06:51.785 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:06:51.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:51.834 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:51.835 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:06:51.877 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:06:51.892 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:51.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:51.910 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:06:51.911 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:06:51.911 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:06:51.911 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:06:51.911 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:06:51.911 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:06:51.911 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:06:51.912 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:06:51.912 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:06:52.004 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:52.046 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:52.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:52.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:52.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:52.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:06:52.235 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:52 smithi012 ceph-mon[147926]: pgmap v607: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 24308/103956 objects degraded (23.383%); 8549/103956 objects misplaced (8.224%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:52.235 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3180131276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:52.235 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:52 smithi012 ceph-mon[152263]: pgmap v607: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 24308/103956 objects degraded (23.383%); 8549/103956 objects misplaced (8.224%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:52.235 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3180131276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:52.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:52.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:06:52.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:52.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:06:52.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:06:52.426 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:52 smithi110 ceph-mon[136032]: pgmap v607: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 24308/103956 objects degraded (23.383%); 8549/103956 objects misplaced (8.224%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:52.426 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3180131276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:06:52.433 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:52.445 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:06:52.445 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:06:52.484 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:06:52.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:52.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:06:52.589 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:52.665 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:52.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:06:52.787 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:52.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:52.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:52.903 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:52.937 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:52.991 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:52.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:52.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:53.063 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:53 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 24308/103956 objects degraded (23.383%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:53.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:53.159 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [3] live_osds: [0, 1, 4, 5, 6, 7, 2] 2023-10-09T10:06:53.159 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:06:53.159 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:06:53.159 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-10-09T10:06:53.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-10-09T10:06:53.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:53.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:53 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 24308/103956 objects degraded (23.383%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:53.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:53 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 24308/103956 objects degraded (23.383%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:53.729 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:53.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:53.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:53.895 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:53.963 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:53.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:53.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:54.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:54.136 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:54.372 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:54.379 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:54 smithi012 ceph-mon[147926]: pgmap v608: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 23513/99219 objects degraded (23.698%); 8323/99219 objects misplaced (8.389%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:54.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:54.385 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:54 smithi012 ceph-mon[152263]: pgmap v608: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 23513/99219 objects degraded (23.698%); 8323/99219 objects misplaced (8.389%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:54.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:54 smithi110 ceph-mon[136032]: pgmap v608: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 23513/99219 objects degraded (23.698%); 8323/99219 objects misplaced (8.389%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:54.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:06:54.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:54.499 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:06:54.499 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:06:54.583 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:54.698 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:54.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:54.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:54.713 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:54.809 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:54.918 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:55.083 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:55.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:55.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:55.114 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:55.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:06:55.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:55.321 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:55.506 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:55.546 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:55.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:55.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:55.720 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:55.846 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:55.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:55.953 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:55.991 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:56.035 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:56.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:56.143 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:56.198 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:56.249 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:56.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:56.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:56.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:06:56.426 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:56.661 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:06:56.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:56 smithi110 ceph-mon[136032]: pgmap v609: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 21928/93114 objects degraded (23.550%); 7641/93114 objects misplaced (8.206%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:56.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:56.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:06:56.741 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:56 smithi012 ceph-mon[147926]: pgmap v609: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 21928/93114 objects degraded (23.550%); 7641/93114 objects misplaced (8.206%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:56.741 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:56 smithi012 ceph-mon[152263]: pgmap v609: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 21928/93114 objects degraded (23.550%); 7641/93114 objects misplaced (8.206%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:56.842 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:06:56.887 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:56.943 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:06:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:56.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:06:57.046 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:06:57.111 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:57.368 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:06:57.427 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:06:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:57.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:06:57.535 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:57.621 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:58.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:06:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:06:57] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T10:06:58.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:58 smithi012 ceph-mon[147926]: pgmap v610: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 21928/93114 objects degraded (23.550%); 7641/93114 objects misplaced (8.206%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:58.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 21928/93114 objects degraded (23.550%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:58.683 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:58 smithi012 ceph-mon[152263]: pgmap v610: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 21928/93114 objects degraded (23.550%); 7641/93114 objects misplaced (8.206%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:58.683 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 21928/93114 objects degraded (23.550%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:58 smithi110 ceph-mon[136032]: pgmap v610: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 21928/93114 objects degraded (23.550%); 7641/93114 objects misplaced (8.206%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:06:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 21928/93114 objects degraded (23.550%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:06:58.780 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:06:58.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:58.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:06:58.885 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:06:58.911 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:58.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:06:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:06:58.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:06:59.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:06:59.133 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:06:59.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:06:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:59.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:06:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:06:59.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:06:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:00.549 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:00.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:00.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:00.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:00 smithi012 ceph-mon[147926]: pgmap v611: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 21083/88356 objects degraded (23.861%); 7427/88356 objects misplaced (8.406%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:07:00.593 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:00 smithi012 ceph-mon[152263]: pgmap v611: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 21083/88356 objects degraded (23.861%); 7427/88356 objects misplaced (8.406%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:07:00.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:00 smithi110 ceph-mon[136032]: pgmap v611: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 21083/88356 objects degraded (23.861%); 7427/88356 objects misplaced (8.406%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:07:00.759 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:00.888 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:00.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:00.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:00.997 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:01.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:01.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:01.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:01.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:01.282 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:01.338 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/438537291' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:07:01.338 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/651964045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:01.338 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:01.354 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.2 primary-affinity to 0 (802) 2023-10-09T10:07:01.368 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:07:01.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/438537291' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:07:01.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/651964045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:01.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/438537291' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:07:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/651964045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:01.532 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:01.554 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:01.577 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:01.600 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:07:01.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:01.645 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:01.667 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:07:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:07:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:07:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:07:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:07:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:07:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:07:01.668 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:07:02.045 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:02.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:02.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:02.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:02.184 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:02.424 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:02.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:02.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:02.452 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:07:02.452 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:07:02.537 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:02.568 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:02.610 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:02.616 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: pgmap v612: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 19492/82359 objects degraded (23.667%); 6780/82359 objects misplaced (8.232%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:07:02.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:02.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/438537291' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T10:07:02.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:07:02.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: osdmap e234: 8 total, 7 up, 8 in 2023-10-09T10:07:02.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1809492064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:02.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:02.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: pgmap v612: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 19492/82359 objects degraded (23.667%); 6780/82359 objects misplaced (8.232%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:07:02.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:02.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/438537291' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T10:07:02.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:07:02.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: osdmap e234: 8 total, 7 up, 8 in 2023-10-09T10:07:02.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1809492064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:02.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: pgmap v612: 89 pgs: 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+undersized+degraded, 46 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 19492/82359 objects degraded (23.667%); 6780/82359 objects misplaced (8.232%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:07:02.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:02.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/438537291' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T10:07:02.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:07:02.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: osdmap e234: 8 total, 7 up, 8 in 2023-10-09T10:07:02.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: failed to encode map e234 with expected crc 2023-10-09T10:07:02.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1809492064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:02.711 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:07:02.711 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:07:02.711 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:07:02.712 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:07:02.712 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:07:02.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:02.763 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:03.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:07:03.068 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:03.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:03.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:03.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:03.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:03.235 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:03.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:03.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:03.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:03.367 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:03.401 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:03.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:03.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:03.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:03.533 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:03.569 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:03.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:03.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:03.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: osdmap e235: 8 total, 7 up, 8 in 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: 7.8 continuing backfill to osd.2 from (199'3410,233'11782] MIN to 233'11782 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: 7.a continuing backfill to osd.2 from (196'3200,233'11909] MIN to 233'11909 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: 7.5 starting backfill to osd.2 from (196'3000,233'11865] MIN to 233'11866 2023-10-09T10:07:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 19492/82359 objects degraded (23.667%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: osdmap e235: 8 total, 7 up, 8 in 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: 7.8 continuing backfill to osd.2 from (199'3410,233'11782] MIN to 233'11782 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: 7.a continuing backfill to osd.2 from (196'3200,233'11909] MIN to 233'11909 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: 7.5 starting backfill to osd.2 from (196'3000,233'11865] MIN to 233'11866 2023-10-09T10:07:03.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 19492/82359 objects degraded (23.667%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:03.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: osdmap e235: 8 total, 7 up, 8 in 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: failed to encode map e235 with expected crc 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: 7.8 continuing backfill to osd.2 from (199'3410,233'11782] MIN to 233'11782 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: 7.a continuing backfill to osd.2 from (196'3200,233'11909] MIN to 233'11909 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: 7.5 starting backfill to osd.2 from (196'3000,233'11865] MIN to 233'11866 2023-10-09T10:07:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 19492/82359 objects degraded (23.667%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:03.702 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:03.702 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:03.737 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:07:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:03.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:07:03.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:03.868 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:03.904 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:03.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:03.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:04.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:07:04.035 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:04.071 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:07:04.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:04.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:07:04.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:07:04.176 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:07:04.203 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:04.239 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:07:04.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:04.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:07:04.343 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:04.354 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:04 smithi110 ceph-mon[136032]: pgmap v615: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 18998/79440 objects degraded (23.915%); 6621/79440 objects misplaced (8.335%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:07:04.354 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:04 smithi110 ceph-mon[136032]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.374 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:04.417 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:04.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:04.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:04.454 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:07:04.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[147926]: pgmap v615: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 18998/79440 objects degraded (23.915%); 6621/79440 objects misplaced (8.335%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:07:04.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[147926]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[147926]: osdmap e236: 8 total, 7 up, 8 in 2023-10-09T10:07:04.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[147926]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.491 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[147926]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[152263]: pgmap v615: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 18998/79440 objects degraded (23.915%); 6621/79440 objects misplaced (8.335%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:07:04.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[152263]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[152263]: osdmap e236: 8 total, 7 up, 8 in 2023-10-09T10:07:04.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[152263]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:04 smithi012 ceph-mon[152263]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:04.553 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:04.590 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:04.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:04.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:04.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:04 smithi110 ceph-mon[136032]: osdmap e236: 8 total, 7 up, 8 in 2023-10-09T10:07:04.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:04 smithi110 ceph-mon[136032]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:04 smithi110 ceph-mon[136032]: failed to encode map e236 with expected crc 2023-10-09T10:07:04.694 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:04.721 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:04.758 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:04.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:04.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:04.862 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191677 6.19 6:9839b5ab:::benchmark_data_smithi012_182576_object86156:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e165)", 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.71015117100001, 2023-10-09T10:07:04.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5138246070000001, 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191677 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:07:04.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167168+0000", 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167175+0000", 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4989999999999999e-06 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167184+0000", 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7329999999999998e-06 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167193+0000", 2023-10-09T10:07:04.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1599999999999998e-06 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174376+0000", 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071834530000000002 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174387+0000", 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1457999999999999e-05 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.658706+0000", 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4843186290000001 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.658834+0000", 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012832699999999999 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680928+0000", 2023-10-09T10:07:04.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022093278000000001 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680994+0000", 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6519000000000005e-05 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191684 6.19 6:9f873d67:::benchmark_data_smithi012_182576_object86163:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e165)", 2023-10-09T10:07:04.887 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.710122098, 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5032064950000001, 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191684 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167198+0000", 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:04.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167201+0000", 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3e-06 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167206+0000", 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8479999999999997e-06 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167212+0000", 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1420000000000001e-06 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174497+0000", 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072854699999999996 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174507+0000", 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0071000000000001e-05 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659145+0000", 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4846380749999999 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659343+0000", 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019807000000000001 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659860+0000", 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051712700000000004 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670116+0000", 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010255966 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670223+0000", 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010616 2023-10-09T10:07:04.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670251+0000", 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8269999999999999e-05 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670302+0000", 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1674e-05 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670405+0000", 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102691 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191706 6.19 6:9a3f0857:::benchmark_data_smithi012_182576_object86185:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.68995099400001, 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.483309049, 2023-10-09T10:07:04.895 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191706 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187368+0000", 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187373+0000", 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.087e-06 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187381+0000", 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6139999999999992e-06 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187388+0000", 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9029999999999998e-06 2023-10-09T10:07:04.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191191+0000", 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038029449999999998 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191201+0000", 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0463e-05 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.662490+0000", 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.471288079 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.662641+0000", 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015126899999999999 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.663089+0000", 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00044850599999999999 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670456+0000", 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007366956 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670469+0000", 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2931e-05 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670520+0000", 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1273000000000002e-05 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670579+0000", 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8789999999999998e-05 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670679+0000", 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9545999999999999e-05 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191710 6.19 6:9b680818:::benchmark_data_smithi012_182576_object86189:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.68992752999998, 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4835924069999999, 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191710 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.903 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187395+0000", 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.238e-06 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187400+0000", 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.074e-06 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187405+0000", 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5569999999999997e-06 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191222+0000", 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038174189999999998 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191229+0000", 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.665559+0000", 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4743292960000001 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.665709+0000", 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015078499999999999 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.666141+0000", 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043125999999999999 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670478+0000", 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043378319999999998 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670789+0000", 2023-10-09T10:07:04.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031025900000000001 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670831+0000", 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2349999999999999e-05 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670880+0000", 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9397999999999997e-05 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670986+0000", 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105147 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191721 6.19 6:98cdd829:::benchmark_data_smithi012_182576_object86200:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.67189571599999, 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4866997529999999, 2023-10-09T10:07:04.910 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191721 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205423+0000", 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205432+0000", 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1829999999999994e-06 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205449+0000", 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7813e-05 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205460+0000", 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0966e-05 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205526+0000", 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5363000000000004e-05 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205537+0000", 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.148e-05 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671019+0000", 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.465481982 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671151+0000", 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013229600000000001 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671734+0000", 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058208800000000003 2023-10-09T10:07:04.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680757+0000", 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090230250000000005 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681028+0000", 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027170299999999998 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.691971+0000", 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010942708000000001 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692026+0000", 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5075999999999999e-05 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692125+0000", 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8728999999999996e-05 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.917 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191734 6.19 6:9ad5bbea:::benchmark_data_smithi012_182576_object86213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.66170783699999, 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.476698235, 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.918 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191734 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215612+0000", 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215617+0000", 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1109999999999999e-06 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215630+0000", 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.347e-05 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215639+0000", 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2190000000000004e-06 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215696+0000", 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7519000000000003e-05 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215707+0000", 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1391e-05 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678298+0000", 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.462590391 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678424+0000", 2023-10-09T10:07:04.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012647799999999999 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678879+0000", 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045511999999999999 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689119+0000", 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010239244999999999 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692138+0000", 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030192180000000002 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692181+0000", 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2972999999999997e-05 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692224+0000", 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3328999999999998e-05 2023-10-09T10:07:04.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692311+0000", 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6976000000000002e-05 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191762 6.19 6:98c0cf1b:::benchmark_data_smithi012_182576_object86241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.629571892, 2023-10-09T10:07:04.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.470776208, 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191762 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:07:04.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247747+0000", 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247756+0000", 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.003e-06 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247771+0000", 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4816e-05 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247779+0000", 2023-10-09T10:07:04.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.551e-06 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247839+0000", 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9710999999999998e-05 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247851+0000", 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.221e-05 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681049+0000", 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4331976559999999 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681140+0000", 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1250999999999998e-05 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681494+0000", 2023-10-09T10:07:04.930 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035398299999999999 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689149+0000", 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076551509999999998 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692143+0000", 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029938090000000001 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.931 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718366+0000", 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026223026999999999 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718421+0000", 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4262999999999999e-05 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718525+0000", 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104549 2023-10-09T10:07:04.932 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191773 6.19 6:9b9ae800:::benchmark_data_smithi012_182576_object86252:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.61797137899998, 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.45934027, 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.933 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191773 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:07:04.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259348+0000", 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259355+0000", 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8310000000000003e-06 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259368+0000", 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3388e-05 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259378+0000", 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4809999999999995e-06 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.262058+0000", 2023-10-09T10:07:04.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026803130000000001 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.262070+0000", 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1612e-05 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.683593+0000", 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.421522669 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.683700+0000", 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107054 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.684074+0000", 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037416800000000003 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689177+0000", 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051031940000000001 2023-10-09T10:07:04.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692342+0000", 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003165026 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718562+0000", 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026219606999999999 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718600+0000", 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8834999999999997e-05 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718690+0000", 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9263000000000002e-05 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.940 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191842 6.19 6:9f5df5a7:::benchmark_data_smithi012_182576_object86321:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.53237692300002, 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.373904045, 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191842 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344942+0000", 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:04.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344950+0000", 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2900000000000002e-06 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344964+0000", 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3373000000000001e-05 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344973+0000", 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8200000000000003e-06 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.345026+0000", 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3804999999999997e-05 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.345038+0000", 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1443999999999999e-05 2023-10-09T10:07:04.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686361+0000", 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3413228020000001 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686486+0000", 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012562000000000001 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686879+0000", 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039255099999999999 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689204+0000", 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002325503 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692347+0000", 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031431340000000001 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718723+0000", 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026375604 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718760+0000", 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7398999999999999e-05 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718848+0000", 2023-10-09T10:07:04.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7406999999999998e-05 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191919 6.19 6:9f034d99:::benchmark_data_smithi012_182576_object86398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 268.42781607299997, 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2695003030000001, 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.948 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191919 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:07:04.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449503+0000", 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449510+0000", 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.08e-06 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449527+0000", 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6714999999999998e-05 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449537+0000", 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5950000000000005e-06 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449603+0000", 2023-10-09T10:07:04.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5975000000000004e-05 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449617+0000", 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4163e-05 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689226+0000", 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.239609637 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689349+0000", 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012278500000000001 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689741+0000", 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039135500000000002 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.697237+0000", 2023-10-09T10:07:04.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074966369999999996 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.697277+0000", 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9959000000000001e-05 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718881+0000", 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021603832999999999 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718918+0000", 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7262999999999999e-05 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.719005+0000", 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6716000000000001e-05 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195708 6.a 6:5cc4cade:::benchmark_data_smithi012_182576_object90187:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 262.60592602000003, 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.548119284, 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195708 2023-10-09T10:07:04.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271393+0000", 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271398+0000", 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9450000000000001e-06 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271408+0000", 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0183000000000001e-05 2023-10-09T10:07:04.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271416+0000", 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7559999999999999e-06 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271479+0000", 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2669000000000006e-05 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271486+0000", 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5900000000000002e-06 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717349+0000", 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.44586224400000002 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717362+0000", 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3382e-05 2023-10-09T10:07:04.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.803595+0000", 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0862334440000001 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.806135+0000", 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025396630000000002 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:07:04.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.806223+0000", 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7700000000000004e-05 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.813674+0000", 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074509019999999997 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819055+0000", 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053811429999999997 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819227+0000", 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017252200000000001 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819303+0000", 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5587000000000007e-05 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819381+0000", 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7582000000000004e-05 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819514+0000", 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013344400000000001 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195730 6.a 6:51d763b1:::benchmark_data_smithi012_182576_object90209:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:04.964 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 262.547956715, 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4903754600000001, 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195730 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329363+0000", 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:04.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329371+0000", 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1990000000000006e-06 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329394+0000", 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2909999999999999e-05 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329404+0000", 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9809999999999998e-06 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329466+0000", 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2092000000000001e-05 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329479+0000", 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.359e-05 2023-10-09T10:07:04.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717394+0000", 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38791446600000001 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717405+0000", 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1208e-05 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809765+0000", 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0923597 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809844+0000", 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8974999999999995e-05 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809918+0000", 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4696000000000004e-05 2023-10-09T10:07:04.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819132+0000", 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092136140000000002 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819148+0000", 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6092000000000001e-05 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819267+0000", 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119096 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819562+0000", 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029480100000000003 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819615+0000", 2023-10-09T10:07:04.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2502e-05 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819740+0000", 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012496799999999999 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195764 6.a 6:515d6e24:::benchmark_data_smithi012_182576_object90243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:07:04.973 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 262.47756802599997, 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.426156247, 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195764 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399751+0000", 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399758+0000", 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3080000000000002e-06 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399771+0000", 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2680000000000001e-05 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399779+0000", 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1049999999999995e-06 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399843+0000", 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3441999999999993e-05 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399856+0000", 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3008e-05 2023-10-09T10:07:04.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717436+0000", 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.317580268 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717445+0000", 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7660000000000004e-06 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814690+0000", 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0972454300000001 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814827+0000", 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013737299999999999 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814985+0000", 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015749700000000001 2023-10-09T10:07:04.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819155+0000", 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041702220000000003 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.823975+0000", 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048196610000000003 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825698+0000", 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017233140000000001 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825737+0000", 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8433000000000003e-05 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825794+0000", 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7871000000000002e-05 2023-10-09T10:07:04.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825909+0000", 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114583 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154067 7.e 7:729cdabe:::benchmark_data_smithi012_221426_object46289:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.044175019999997, 2023-10-09T10:07:04.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.711856684, 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154067 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:07:04.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833144+0000", 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833150+0000", 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0079999999999999e-06 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833167+0000", 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6198999999999999e-05 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833176+0000", 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4490000000000005e-06 2023-10-09T10:07:04.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833926+0000", 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00074946799999999999 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833934+0000", 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1589999999999993e-06 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841253+0000", 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073188610000000003 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841264+0000", 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1069e-05 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497295+0000", 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6560310870000001 2023-10-09T10:07:04.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497435+0000", 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013998400000000001 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.544884+0000", 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.047449005000000002 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.545002+0000", 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118678 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154074 7.e 7:7a8c6f84:::benchmark_data_smithi012_221426_object46296:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.043441344999998, 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7164543729999999, 2023-10-09T10:07:04.989 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154074 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833884+0000", 2023-10-09T10:07:04.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1059999999999999e-06 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833895+0000", 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1639e-05 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833906+0000", 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1104000000000001e-05 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833999+0000", 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2387999999999997e-05 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841286+0000", 2023-10-09T10:07:04.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072812739999999999 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841293+0000", 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7290000000000003e-06 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500331+0000", 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.659038185 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500464+0000", 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013265799999999999 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550212+0000", 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049747711999999999 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550334+0000", 2023-10-09T10:07:04.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012227200000000001 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154080 7.5 7:a9f132ab:::benchmark_data_smithi012_221426_object46302:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.043404885000001, 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5768529529999999, 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:04.996 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154080 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:04.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833915+0000", 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833918+0000", 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6740000000000001e-06 2023-10-09T10:07:04.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833924+0000", 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2179999999999996e-06 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833929+0000", 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2009999999999998e-06 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:04.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834017+0000", 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7529999999999997e-05 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834024+0000", 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2350000000000002e-06 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837176+0000", 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031517139999999999 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837194+0000", 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7634000000000001e-05 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395551+0000", 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5583572540000001 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395681+0000", 2023-10-09T10:07:05.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130059 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395763+0000", 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1928999999999994e-05 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400135+0000", 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043726360000000001 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409675+0000", 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095396509999999997 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409717+0000", 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2036000000000002e-05 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410592+0000", 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00087471800000000002 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410648+0000", 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6589000000000003e-05 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410769+0000", 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120383 2023-10-09T10:07:05.004 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154083 7.e 7:757b33cc:::benchmark_data_smithi012_221426_object46305:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.043376946000002, 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7033852390000002, 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:05.005 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154083 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:05.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833943+0000", 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833946+0000", 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0000000000000001e-06 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833952+0000", 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9000000000000003e-06 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833957+0000", 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9629999999999997e-06 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834051+0000", 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4141999999999994e-05 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834060+0000", 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8079999999999994e-06 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841319+0000", 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072592009999999998 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841327+0000", 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.571e-06 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500500+0000", 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6591724889999999 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:05.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500595+0000", 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5142000000000005e-05 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.501295+0000", 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00069987000000000001 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510695+0000", 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094008289999999994 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520701+0000", 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010005623 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534431+0000", 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013730086000000001 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:05.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534503+0000", 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2304000000000004e-05 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537329+0000", 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028256510000000002 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154085 7.5 7:a7e74dfa:::benchmark_data_smithi012_221426_object46307:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:07:05.013 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.043359043999999, 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5985369870000001, 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154085 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833961+0000", 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:05.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833964+0000", 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.294e-06 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833970+0000", 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8540000000000002e-06 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833975+0000", 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9710000000000003e-06 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834061+0000", 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6417999999999998e-05 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834067+0000", 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.421e-06 2023-10-09T10:07:05.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837215+0000", 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031475790000000002 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837222+0000", 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4370000000000001e-06 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400164+0000", 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5629411879999999 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400262+0000", 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8307000000000006e-05 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400392+0000", 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013009100000000001 2023-10-09T10:07:05.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409725+0000", 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093329940000000007 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409759+0000", 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4034e-05 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.425006+0000", 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015246994 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432361+0000", 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073547029999999998 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432413+0000", 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2305999999999997e-05 2023-10-09T10:07:05.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432499+0000", 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5711000000000007e-05 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154090 7.5 7:a5e0254e:::benchmark_data_smithi012_221426_object46312:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:07:05.022 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.043339789000001, 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.598732759, 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154090 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:07:05.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833983+0000", 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.26e-06 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833988+0000", 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4190000000000001e-06 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833993+0000", 2023-10-09T10:07:05.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5419999999999999e-06 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834104+0000", 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011161499999999999 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834111+0000", 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0650000000000004e-06 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837242+0000", 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031316959999999998 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837250+0000", 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1799999999999996e-06 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.404888+0000", 2023-10-09T10:07:05.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5676373880000001 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.405039+0000", 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151543 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.405181+0000", 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014114600000000001 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409999+0000", 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048189210000000003 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410086+0000", 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6949999999999999e-05 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.425038+0000", 2023-10-09T10:07:05.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014951341999999999 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432561+0000", 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0075235340000000001 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432611+0000", 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9985000000000003e-05 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432714+0000", 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102497 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154097 7.e 7:7fa3f1e3:::benchmark_data_smithi012_221426_object46319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.043322705000001, 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.706036967, 2023-10-09T10:07:05.031 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154097 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:05.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:05.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834000+0000", 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.384e-06 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2499999999999997e-06 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834010+0000", 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8359999999999999e-06 2023-10-09T10:07:05.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834090+0000", 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9957999999999996e-05 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834096+0000", 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3350000000000003e-06 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841347+0000", 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072508479999999998 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841354+0000", 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7159999999999997e-06 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510724+0000", 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.669369753 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510903+0000", 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017934200000000001 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.511077+0000", 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017371899999999999 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520762+0000", 2023-10-09T10:07:05.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0096847530000000008 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.525146+0000", 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043842450000000002 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537388+0000", 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012242494 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537446+0000", 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7565999999999997e-05 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.540035+0000", 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025890919999999999 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:05.039 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:05.040 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:07:05.040 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:05.041 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:05.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:05.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:05.142 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:05.170 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:05.204 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:05.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:05.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:05.308 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:05.333 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:05.367 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:05.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:05.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:05.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:05.505 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:05.841 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:05.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:05.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:05.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:05.977 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:06.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:07:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:06 smithi012 ceph-mon[147926]: pgmap v617: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 17998/77529 objects degraded (23.215%); 6432/77529 objects misplaced (8.296%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:07:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:06 smithi012 ceph-mon[147926]: failed to encode map e237 with expected crc 2023-10-09T10:07:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:06 smithi012 ceph-mon[147926]: failed to encode map e237 with expected crc 2023-10-09T10:07:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:06 smithi012 ceph-mon[147926]: osdmap e237: 8 total, 7 up, 8 in 2023-10-09T10:07:06.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:06 smithi012 ceph-mon[152263]: pgmap v617: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 17998/77529 objects degraded (23.215%); 6432/77529 objects misplaced (8.296%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:07:06.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:06 smithi012 ceph-mon[152263]: failed to encode map e237 with expected crc 2023-10-09T10:07:06.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:06 smithi012 ceph-mon[152263]: failed to encode map e237 with expected crc 2023-10-09T10:07:06.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:06 smithi012 ceph-mon[152263]: osdmap e237: 8 total, 7 up, 8 in 2023-10-09T10:07:06.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:06 smithi110 ceph-mon[136032]: pgmap v617: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 17998/77529 objects degraded (23.215%); 6432/77529 objects misplaced (8.296%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:07:06.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:06 smithi110 ceph-mon[136032]: failed to encode map e237 with expected crc 2023-10-09T10:07:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:06 smithi110 ceph-mon[136032]: failed to encode map e237 with expected crc 2023-10-09T10:07:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:06 smithi110 ceph-mon[136032]: osdmap e237: 8 total, 7 up, 8 in 2023-10-09T10:07:07.374 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [3] live_osds: [0, 1, 4, 5, 6, 7, 2] 2023-10-09T10:07:07.374 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:07:07.375 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 1, 4, 5, 6, 7, 2] 2023-10-09T10:07:07.375 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T10:07:07.375 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T10:07:07.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:07 smithi012 ceph-mon[147926]: failed to encode map e237 with expected crc 2023-10-09T10:07:07.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:07 smithi012 ceph-mon[147926]: failed to encode map e237 with expected crc 2023-10-09T10:07:07.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:07 smithi012 ceph-mon[152263]: failed to encode map e237 with expected crc 2023-10-09T10:07:07.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:07 smithi012 ceph-mon[152263]: failed to encode map e237 with expected crc 2023-10-09T10:07:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:07 smithi110 ceph-mon[136032]: failed to encode map e237 with expected crc 2023-10-09T10:07:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:07 smithi110 ceph-mon[136032]: failed to encode map e237 with expected crc 2023-10-09T10:07:07.694 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:07:07 smithi110 systemd[1]: Stopping Ceph osd.4 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:07:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:07:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:07:07] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-09T10:07:08.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:07:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[101265]: debug 2023-10-09T10:07:07.749+0000 7fd30b989700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:07:08.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:07:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[101265]: debug 2023-10-09T10:07:07.749+0000 7fd30b989700 -1 osd.4 238 *** Got signal Terminated *** 2023-10-09T10:07:08.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:07:07 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[101265]: debug 2023-10-09T10:07:07.749+0000 7fd30b989700 -1 osd.4 238 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:07:08.488 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:08.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: failed to encode map e237 with expected crc 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: pgmap v619: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 17998/77529 objects degraded (23.215%); 6432/77529 objects misplaced (8.296%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: osdmap e238: 8 total, 7 up, 8 in 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: osd.4 marked itself down and dead 2023-10-09T10:07:08.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17998/77529 objects degraded (23.215%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:07:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: failed to encode map e237 with expected crc 2023-10-09T10:07:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: pgmap v619: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 17998/77529 objects degraded (23.215%); 6432/77529 objects misplaced (8.296%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:07:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: osdmap e238: 8 total, 7 up, 8 in 2023-10-09T10:07:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: osd.4 marked itself down and dead 2023-10-09T10:07:08.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17998/77529 objects degraded (23.215%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:07:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: failed to encode map e237 with expected crc 2023-10-09T10:07:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: pgmap v619: 89 pgs: 1 activating+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 28 active+undersized, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 17998/77529 objects degraded (23.215%); 6432/77529 objects misplaced (8.296%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T10:07:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: osdmap e238: 8 total, 7 up, 8 in 2023-10-09T10:07:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: failed to encode map e238 with expected crc 2023-10-09T10:07:08.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: osd.4 marked itself down and dead 2023-10-09T10:07:08.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17998/77529 objects degraded (23.215%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:07:08.786 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:08.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:08.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:08.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:08.922 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:08.942 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:07:08 smithi110 bash[180597]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T10:07:09.563 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:09.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:09.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[147926]: failed to encode map e238 with expected crc 2023-10-09T10:07:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[147926]: failed to encode map e238 with expected crc 2023-10-09T10:07:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[147926]: Health check update: 2 osds down (OSD_DOWN) 2023-10-09T10:07:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[147926]: osdmap e239: 8 total, 6 up, 8 in 2023-10-09T10:07:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[147926]: failed to encode map e239 with expected crc 2023-10-09T10:07:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[147926]: failed to encode map e239 with expected crc 2023-10-09T10:07:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[152263]: failed to encode map e238 with expected crc 2023-10-09T10:07:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[152263]: failed to encode map e238 with expected crc 2023-10-09T10:07:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[152263]: Health check update: 2 osds down (OSD_DOWN) 2023-10-09T10:07:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[152263]: osdmap e239: 8 total, 6 up, 8 in 2023-10-09T10:07:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[152263]: failed to encode map e239 with expected crc 2023-10-09T10:07:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:09 smithi012 ceph-mon[152263]: failed to encode map e239 with expected crc 2023-10-09T10:07:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:09 smithi110 ceph-mon[136032]: failed to encode map e238 with expected crc 2023-10-09T10:07:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:09 smithi110 ceph-mon[136032]: failed to encode map e238 with expected crc 2023-10-09T10:07:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:09 smithi110 ceph-mon[136032]: Health check update: 2 osds down (OSD_DOWN) 2023-10-09T10:07:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:09 smithi110 ceph-mon[136032]: osdmap e239: 8 total, 6 up, 8 in 2023-10-09T10:07:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:09 smithi110 ceph-mon[136032]: failed to encode map e239 with expected crc 2023-10-09T10:07:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:09 smithi110 ceph-mon[136032]: failed to encode map e239 with expected crc 2023-10-09T10:07:09.699 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:09.853 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:09.855 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:09.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:09.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:09.991 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:10.032 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:10.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:10.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:10.113 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T10:07:10.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:10.166 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:10.171 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:07:10.171 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T10:07:10.405 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:10.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:10.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:10.423 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: pgmap v622: 89 pgs: 1 activating+degraded, 1 remapped+peering, 1 stale+active+undersized+degraded, 1 stale+active+undersized+degraded+remapped+backfill_wait, 1 stale+active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 18902/76236 objects degraded (24.794%); 4737/76236 objects misplaced (6.214%); 24 MiB/s, 391 objects/s recovering 2023-10-09T10:07:10.423 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: osdmap e240: 8 total, 6 up, 8 in 2023-10-09T10:07:10.423 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: 7.3 continuing backfill to osd.2 from (199'3900,235'11871] 7:c01241cc:::benchmark_data_smithi012_221426_object106716:head to 235'11871 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[147926]: 7.8 continuing backfill to osd.2 from (199'3410,235'11816] 7:195f487c:::benchmark_data_smithi012_221426_object95029:head to 235'11816 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: pgmap v622: 89 pgs: 1 activating+degraded, 1 remapped+peering, 1 stale+active+undersized+degraded, 1 stale+active+undersized+degraded+remapped+backfill_wait, 1 stale+active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 18902/76236 objects degraded (24.794%); 4737/76236 objects misplaced (6.214%); 24 MiB/s, 391 objects/s recovering 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: osdmap e240: 8 total, 6 up, 8 in 2023-10-09T10:07:10.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: 7.3 continuing backfill to osd.2 from (199'3900,235'11871] 7:c01241cc:::benchmark_data_smithi012_221426_object106716:head to 235'11871 2023-10-09T10:07:10.425 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:10 smithi012 ceph-mon[152263]: 7.8 continuing backfill to osd.2 from (199'3410,235'11816] 7:195f487c:::benchmark_data_smithi012_221426_object95029:head to 235'11816 2023-10-09T10:07:10.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:10.542 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: pgmap v622: 89 pgs: 1 activating+degraded, 1 remapped+peering, 1 stale+active+undersized+degraded, 1 stale+active+undersized+degraded+remapped+backfill_wait, 1 stale+active+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 27 active+undersized, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 18902/76236 objects degraded (24.794%); 4737/76236 objects misplaced (6.214%); 24 MiB/s, 391 objects/s recovering 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: osdmap e240: 8 total, 6 up, 8 in 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: failed to encode map e240 with expected crc 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: 7.3 continuing backfill to osd.2 from (199'3900,235'11871] 7:c01241cc:::benchmark_data_smithi012_221426_object106716:head to 235'11871 2023-10-09T10:07:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:10 smithi110 ceph-mon[136032]: 7.8 continuing backfill to osd.2 from (199'3410,235'11816] 7:195f487c:::benchmark_data_smithi012_221426_object95029:head to 235'11816 2023-10-09T10:07:10.810 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:10.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:10.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:10.916 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:10.943 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:10.978 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:10.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:10.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:11.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:11.117 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:11.562 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:11.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:11.637 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:11 smithi012 ceph-mon[147926]: failed to encode map e241 with expected crc 2023-10-09T10:07:11.637 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:11 smithi012 ceph-mon[147926]: osdmap e241: 8 total, 6 up, 8 in 2023-10-09T10:07:11.638 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:11 smithi012 ceph-mon[147926]: failed to encode map e241 with expected crc 2023-10-09T10:07:11.638 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:11 smithi012 ceph-mon[152263]: failed to encode map e241 with expected crc 2023-10-09T10:07:11.638 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:11 smithi012 ceph-mon[152263]: osdmap e241: 8 total, 6 up, 8 in 2023-10-09T10:07:11.638 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:11 smithi012 ceph-mon[152263]: failed to encode map e241 with expected crc 2023-10-09T10:07:11.671 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:11.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:11 smithi110 ceph-mon[136032]: failed to encode map e241 with expected crc 2023-10-09T10:07:11.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:11 smithi110 ceph-mon[136032]: osdmap e241: 8 total, 6 up, 8 in 2023-10-09T10:07:11.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:11 smithi110 ceph-mon[136032]: failed to encode map e241 with expected crc 2023-10-09T10:07:11.698 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:11.733 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:11.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:11.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:11.869 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:11.914 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:11.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:11.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:12.019 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:12.048 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:12.085 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:12.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:12.189 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:12.216 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:12.245 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:12.250 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:12.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:12.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:12.267 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:12.291 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:12.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:12.332 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:12.353 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:12.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:12.355 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:07:12.355 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:07:12.355 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:07:12.355 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:07:12.355 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:07:12.355 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:07:12.356 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:07:12.356 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:07:12.382 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:12.417 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:07:12.418 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:12.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:12.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:12.497 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:12.497 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:12.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:12.534 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[147926]: pgmap v625: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 active+clean+remapped, 1 activating+degraded, 1 remapped+peering, 1 undersized+degraded+remapped+backfill_wait+peered, 6 undersized+peered, 34 active+undersized, 2 undersized+degraded+peered, 1 active+remapped+backfill_wait, 11 active+undersized+degraded, 30 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 19815/76233 objects degraded (25.993%); 6280/76233 objects misplaced (8.238%); 63 MiB/s, 1k objects/s recovering 2023-10-09T10:07:12.535 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T10:07:12.535 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[147926]: osdmap e242: 8 total, 6 up, 8 in 2023-10-09T10:07:12.535 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[147926]: failed to encode map e242 with expected crc 2023-10-09T10:07:12.535 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[147926]: failed to encode map e242 with expected crc 2023-10-09T10:07:12.535 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/627767324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:12.535 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:12.535 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2752195626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:12.535 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[152263]: pgmap v625: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 active+clean+remapped, 1 activating+degraded, 1 remapped+peering, 1 undersized+degraded+remapped+backfill_wait+peered, 6 undersized+peered, 34 active+undersized, 2 undersized+degraded+peered, 1 active+remapped+backfill_wait, 11 active+undersized+degraded, 30 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 19815/76233 objects degraded (25.993%); 6280/76233 objects misplaced (8.238%); 63 MiB/s, 1k objects/s recovering 2023-10-09T10:07:12.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T10:07:12.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[152263]: osdmap e242: 8 total, 6 up, 8 in 2023-10-09T10:07:12.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[152263]: failed to encode map e242 with expected crc 2023-10-09T10:07:12.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[152263]: failed to encode map e242 with expected crc 2023-10-09T10:07:12.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/627767324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:12.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:12.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2752195626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:12.554 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:12 smithi110 ceph-mon[136032]: pgmap v625: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 active+clean+remapped, 1 activating+degraded, 1 remapped+peering, 1 undersized+degraded+remapped+backfill_wait+peered, 6 undersized+peered, 34 active+undersized, 2 undersized+degraded+peered, 1 active+remapped+backfill_wait, 11 active+undersized+degraded, 30 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 19815/76233 objects degraded (25.993%); 6280/76233 objects misplaced (8.238%); 63 MiB/s, 1k objects/s recovering 2023-10-09T10:07:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:12 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T10:07:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:12 smithi110 ceph-mon[136032]: osdmap e242: 8 total, 6 up, 8 in 2023-10-09T10:07:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:12 smithi110 ceph-mon[136032]: failed to encode map e242 with expected crc 2023-10-09T10:07:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:12 smithi110 ceph-mon[136032]: failed to encode map e242 with expected crc 2023-10-09T10:07:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/627767324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:12 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2752195626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:12.746 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:12.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:12.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:12.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:12.887 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:13.022 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:13.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:07:13.110 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:07:13.126 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:13.152 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:13.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:13.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:13.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:13.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:13.325 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:13.359 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:13.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:13.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:13.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:07:13.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:13.509 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:13.544 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:13.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:13.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:13.675 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:07:13.675 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[152263]: failed to encode map e243 with expected crc 2023-10-09T10:07:13.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:13.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[152263]: osdmap e243: 8 total, 6 up, 8 in 2023-10-09T10:07:13.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 19815/76233 objects degraded (25.993%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:13.676 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[152263]: osdmap e244: 8 total, 6 up, 8 in 2023-10-09T10:07:13.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:07:13.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[147926]: failed to encode map e243 with expected crc 2023-10-09T10:07:13.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:13.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[147926]: osdmap e243: 8 total, 6 up, 8 in 2023-10-09T10:07:13.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 19815/76233 objects degraded (25.993%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:13.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:13 smithi012 ceph-mon[147926]: osdmap e244: 8 total, 6 up, 8 in 2023-10-09T10:07:13.678 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:13.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:13 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:07:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:13 smithi110 ceph-mon[136032]: failed to encode map e243 with expected crc 2023-10-09T10:07:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:13 smithi110 ceph-mon[136032]: osdmap e243: 8 total, 6 up, 8 in 2023-10-09T10:07:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 19815/76233 objects degraded (25.993%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:13 smithi110 ceph-mon[136032]: osdmap e244: 8 total, 6 up, 8 in 2023-10-09T10:07:13.712 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:13.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:13.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:13.816 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:13.842 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:13.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:13.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:13.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:14.010 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:14.058 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:14.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:14.088 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:14.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:14.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:14.228 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:14.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:14.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:14.333 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:14.359 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:14.396 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:14.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:14.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:14.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:14 smithi110 ceph-mon[136032]: failed to encode map e243 with expected crc 2023-10-09T10:07:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:14 smithi110 ceph-mon[136032]: failed to encode map e243 with expected crc 2023-10-09T10:07:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:14 smithi110 ceph-mon[136032]: failed to encode map e244 with expected crc 2023-10-09T10:07:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:14 smithi110 ceph-mon[136032]: pgmap v629: 89 pgs: 1 active+recovering+degraded, 1 undersized+degraded+remapped+backfilling+peered, 2 active+clean+remapped, 1 undersized+degraded+remapped+backfill_wait+peered, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 15 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 19275/75948 objects degraded (25.379%); 7334/75948 objects misplaced (9.657%); 47 MiB/s, 751 objects/s recovering 2023-10-09T10:07:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:14 smithi110 ceph-mon[136032]: osdmap e245: 8 total, 6 up, 8 in 2023-10-09T10:07:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:14.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:14.530 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:14.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[147926]: failed to encode map e243 with expected crc 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[147926]: failed to encode map e243 with expected crc 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[147926]: failed to encode map e244 with expected crc 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[147926]: pgmap v629: 89 pgs: 1 active+recovering+degraded, 1 undersized+degraded+remapped+backfilling+peered, 2 active+clean+remapped, 1 undersized+degraded+remapped+backfill_wait+peered, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 15 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 19275/75948 objects degraded (25.379%); 7334/75948 objects misplaced (9.657%); 47 MiB/s, 751 objects/s recovering 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[147926]: osdmap e245: 8 total, 6 up, 8 in 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[152263]: failed to encode map e243 with expected crc 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[152263]: failed to encode map e243 with expected crc 2023-10-09T10:07:14.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[152263]: failed to encode map e244 with expected crc 2023-10-09T10:07:14.619 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[152263]: pgmap v629: 89 pgs: 1 active+recovering+degraded, 1 undersized+degraded+remapped+backfilling+peered, 2 active+clean+remapped, 1 undersized+degraded+remapped+backfill_wait+peered, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 15 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 19275/75948 objects degraded (25.379%); 7334/75948 objects misplaced (9.657%); 47 MiB/s, 751 objects/s recovering 2023-10-09T10:07:14.619 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[152263]: osdmap e245: 8 total, 6 up, 8 in 2023-10-09T10:07:14.619 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:14.619 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:14.810 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:14.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:14.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:07:14.914 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:14.940 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:14.976 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:14.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:14.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:15.080 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:15.107 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:15.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:07:15.144 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:15.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:15.173 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [3, 4] live_osds: [0, 1, 5, 6, 7, 2] 2023-10-09T10:07:15.173 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:07:15.173 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T10:07:15.173 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:07:15.173 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:07:15.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T10:07:15.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:15.277 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:15.277 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:15.291 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:15.340 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:15.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:15.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:15.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:15.470 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:15 smithi012 ceph-mon[147926]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.471 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:15 smithi012 ceph-mon[147926]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.471 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:15 smithi012 ceph-mon[147926]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.471 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:15 smithi012 ceph-mon[152263]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.471 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:15 smithi012 ceph-mon[152263]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.471 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:15 smithi012 ceph-mon[152263]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.473 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:15.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:07:15.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:15 smithi110 ceph-mon[136032]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:15 smithi110 ceph-mon[136032]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:15 smithi110 ceph-mon[136032]: failed to encode map e245 with expected crc 2023-10-09T10:07:15.725 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:15.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:15.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:15.778 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:15.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:15.857 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:15.907 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:15.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:15.974 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:16.012 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:16.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:16.073 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:16.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:16.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:16.177 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:16.203 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:16.236 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:16.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:16.328 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:16.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:16.374 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:16.410 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:16.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:16.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:16.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:16.540 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:16.573 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:16.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:16.676 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:16.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:16 smithi012 ceph-mon[147926]: pgmap v631: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 17388/75579 objects degraded (23.006%); 7277/75579 objects misplaced (9.628%); 38 MiB/s, 610 objects/s recovering 2023-10-09T10:07:16.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:16 smithi012 ceph-mon[152263]: pgmap v631: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 17388/75579 objects degraded (23.006%); 7277/75579 objects misplaced (9.628%); 38 MiB/s, 610 objects/s recovering 2023-10-09T10:07:16.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:16 smithi110 ceph-mon[136032]: pgmap v631: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 17388/75579 objects degraded (23.006%); 7277/75579 objects misplaced (9.628%); 38 MiB/s, 610 objects/s recovering 2023-10-09T10:07:16.703 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:16.737 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:16.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:16.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:16.839 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:16.865 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:16.898 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:16.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:16.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:17.002 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:17.032 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:17.225 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:17.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:17.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:17.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:17.359 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:17.441 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:17.658 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:17 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:07:17.658 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:17 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:07:17.682 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:17.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:17 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:07:17.786 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:17.815 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:17.861 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:17.882 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:17.898 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:17.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:17.902 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:17.914 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:07:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:07:17] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T10:07:17.925 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:17.944 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:17.965 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:17.965 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:07:17.966 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:07:17.966 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:07:17.966 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:07:17.966 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:07:17.966 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:07:17.966 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:07:17.966 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:07:18.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:18.032 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:18.120 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:18.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:18.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:18.224 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:18.256 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:18.296 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:18.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:18.403 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:18.422 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:18 smithi110 ceph-mon[136032]: pgmap v632: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 17388/75579 objects degraded (23.006%); 7277/75579 objects misplaced (9.628%); 26 MiB/s, 420 objects/s recovering 2023-10-09T10:07:18.422 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3384851358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:18.422 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17388/75579 objects degraded (23.006%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:18.422 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/17858993' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:18.431 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:18.471 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:18.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:18.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:18.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:18 smithi012 ceph-mon[147926]: pgmap v632: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 17388/75579 objects degraded (23.006%); 7277/75579 objects misplaced (9.628%); 26 MiB/s, 420 objects/s recovering 2023-10-09T10:07:18.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3384851358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:18.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17388/75579 objects degraded (23.006%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:18.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/17858993' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:18.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:18 smithi012 ceph-mon[152263]: pgmap v632: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 17388/75579 objects degraded (23.006%); 7277/75579 objects misplaced (9.628%); 26 MiB/s, 420 objects/s recovering 2023-10-09T10:07:18.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3384851358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:18.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17388/75579 objects degraded (23.006%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:18.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/17858993' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:18.575 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:18.603 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:18.643 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:18.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:07:18.734 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:07:18.747 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:18.775 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:18.808 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:18.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:18.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:18.879 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:07:18.912 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:18.946 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:19.446 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:19.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:19.513 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:07:19.513 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "" 2023-10-09T10:07:19.513 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:07:19.541 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:19 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:19.541 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/17858993' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:07:19.541 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:19 smithi012 ceph-mon[147926]: osdmap e246: 8 total, 6 up, 8 in 2023-10-09T10:07:19.541 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:19 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:19.541 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/17858993' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:07:19.542 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:19 smithi012 ceph-mon[152263]: osdmap e246: 8 total, 6 up, 8 in 2023-10-09T10:07:19.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:19.584 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:19.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:19 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/17858993' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:07:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:19 smithi110 ceph-mon[136032]: osdmap e246: 8 total, 6 up, 8 in 2023-10-09T10:07:19.749 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:07:19.749 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:07:19.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:07:19.749 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:07:19.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.749 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:105493 6.1 6:8f9efc74:::benchmark_data_smithi012_182576_object105492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:07:19.749 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:29.738009+0000", 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 409.84551929000003, 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1782003400000001, 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 105493 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738009+0000", 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738009+0000", 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738007+0000", 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738033+0000", 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6239999999999999e-05 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738064+0000", 2023-10-09T10:07:19.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0701999999999998e-05 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738075+0000", 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1086e-05 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738140+0000", 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5228000000000006e-05 2023-10-09T10:07:19.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738344+0000", 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020381300000000001 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738536+0000", 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019225700000000001 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.745389+0000", 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0068533930000000002 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.749820+0000", 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044309930000000003 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.794250+0000", 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044429767000000002 2023-10-09T10:07:19.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.794259+0000", 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9190000000000005e-06 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.916108+0000", 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1218486379999999 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.916209+0000", 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101322 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:105495 6.1 6:82b7d31c:::benchmark_data_smithi012_182576_object105494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:29.738219+0000", 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 409.84530877100002, 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1787103320000001, 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.757 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 105495 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738219+0000", 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738219+0000", 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738218+0000", 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738241+0000", 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2305000000000001e-05 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738254+0000", 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3628999999999999e-05 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.738262+0000", 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0339999999999997e-06 2023-10-09T10:07:19.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.742029+0000", 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037663169999999999 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.742199+0000", 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017052199999999999 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.742397+0000", 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000197989 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.745543+0000", 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031454550000000001 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.765076+0000", 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019533937000000001 2023-10-09T10:07:19.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.795086+0000", 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030009621 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.795093+0000", 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8310000000000003e-06 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.916827+0000", 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.121734215 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.916929+0000", 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102383 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:105504 6.1 6:8b1a10f0:::benchmark_data_smithi012_182576_object105503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:29.759244+0000", 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 409.82428441799999, 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.158141388, 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 105504 2023-10-09T10:07:19.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.759244+0000", 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.759244+0000", 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.759242+0000", 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.759268+0000", 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6313e-05 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.759279+0000", 2023-10-09T10:07:19.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0732e-05 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.759286+0000", 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8580000000000002e-06 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.765107+0000", 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058214770000000002 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.765284+0000", 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000176684 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.765512+0000", 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000227906 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.770180+0000", 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046685959999999997 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.795124+0000", 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024943318999999999 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.795132+0000", 2023-10-09T10:07:19.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5490000000000006e-06 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.917294+0000", 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1221619309999999 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.917385+0000", 2023-10-09T10:07:19.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0755999999999996e-05 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:105518 6.1 6:8a22ffc9:::benchmark_data_smithi012_182576_object105517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:29.817614+0000", 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 409.765913759, 2023-10-09T10:07:19.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1117486640000001, 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 105518 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.817614+0000", 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.817614+0000", 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.817613+0000", 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.817644+0000", 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1037000000000001e-05 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.817655+0000", 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1371e-05 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.817666+0000", 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1411e-05 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.817801+0000", 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013433 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:29.817813+0000", 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1995e-05 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.917422+0000", 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.099609404 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.917598+0000", 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017620400000000001 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.917811+0000", 2023-10-09T10:07:19.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021242099999999999 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.925273+0000", 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074622400000000002 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.927856+0000", 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025830990000000002 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.928613+0000", 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00075657100000000004 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.928693+0000", 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9868000000000002e-05 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:30.929363+0000", 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00067023799999999998 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:124353 6.e 6:79973428:::benchmark_data_smithi012_182576_object18832:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:59.962929+0000", 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 379.62059909800001, 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1067407619999998, 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.780 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124353 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962929+0000", 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962929+0000", 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962928+0000", 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962932+0000", 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9859999999999996e-06 2023-10-09T10:07:19.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962938+0000", 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1580000000000002e-06 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962942+0000", 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8020000000000002e-06 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962991+0000", 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9018999999999999e-05 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962998+0000", 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0489999999999998e-06 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:00.921490+0000", 2023-10-09T10:07:19.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95849217399999997 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:00.921503+0000", 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2435e-05 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.044012+0000", 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1225092349999999 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.044162+0000", 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014947499999999999 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.044707+0000", 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00054556799999999996 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.066186+0000", 2023-10-09T10:07:19.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021478902000000001 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.067240+0000", 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001054148 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.068394+0000", 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011534270000000001 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.069529+0000", 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001135547 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.069584+0000", 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4979999999999999e-05 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.069670+0000", 2023-10-09T10:07:19.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5443000000000002e-05 2023-10-09T10:07:19.789 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.789 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.789 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.789 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:178976 6.e 6:79c6aa12:::benchmark_data_smithi012_182576_object73455:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e159)", 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.711636+0000", 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 301.87189221699998, 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.151123753, 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178976 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.790 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711636+0000", 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711636+0000", 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711634+0000", 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711640+0000", 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5849999999999999e-06 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711651+0000", 2023-10-09T10:07:19.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1365e-05 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711662+0000", 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0293e-05 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713443+0000", 2023-10-09T10:07:19.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001780986 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713457+0000", 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4611e-05 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.849523+0000", 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.13606574 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.849757+0000", 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023429500000000001 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862706+0000", 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012948382 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862759+0000", 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3903999999999998e-05 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179004 6.e 6:76574bf5:::benchmark_data_smithi012_182576_object73483:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e159)", 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.713468+0000", 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 301.87005996900001, 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.180355152, 2023-10-09T10:07:19.796 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179004 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713468+0000", 2023-10-09T10:07:19.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713468+0000", 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713467+0000", 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:07:19.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713473+0000", 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4030000000000004e-06 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713483+0000", 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.73e-06 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713492+0000", 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3340000000000008e-06 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713588+0000", 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6204000000000003e-05 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713598+0000", 2023-10-09T10:07:19.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1910000000000006e-06 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.850004+0000", 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1364062589999999 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.850114+0000", 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109714 2023-10-09T10:07:19.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.850241+0000", 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012723899999999999 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.859149+0000", 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089080449999999999 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862489+0000", 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033404979999999999 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862510+0000", 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0551e-05 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893616+0000", 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.031106249999999998 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893683+0000", 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6798999999999995e-05 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893823+0000", 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014010499999999999 2023-10-09T10:07:19.804 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179005 6.e 6:7eac465e:::benchmark_data_smithi012_182576_object73484:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e159)", 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.713499+0000", 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 301.87002851900002, 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.180540097, 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.805 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179005 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713499+0000", 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713499+0000", 2023-10-09T10:07:19.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713499+0000", 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713502+0000", 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5119999999999998e-06 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713507+0000", 2023-10-09T10:07:19.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6210000000000001e-06 2023-10-09T10:07:19.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713512+0000", 2023-10-09T10:07:19.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8520000000000003e-06 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713622+0000", 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011010699999999999 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713630+0000", 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2810000000000008e-06 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.852723+0000", 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1390927390000001 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.852856+0000", 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013302100000000001 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.852955+0000", 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8459e-05 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862454+0000", 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094993660000000004 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862530+0000", 2023-10-09T10:07:19.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5827000000000004e-05 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875797+0000", 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01326738 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893869+0000", 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018071595999999999 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893923+0000", 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4400000000000001e-05 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894040+0000", 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011633600000000001 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.815 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179023 6.e 6:7174819a:::benchmark_data_smithi012_182576_object73502:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-09T10:07:19.815 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.834938+0000", 2023-10-09T10:07:19.815 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 301.748589858, 2023-10-09T10:07:19.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0592905880000001, 2023-10-09T10:07:19.815 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.815 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.815 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.815 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179023 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834938+0000", 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834938+0000", 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834936+0000", 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834945+0000", 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2689999999999999e-06 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834965+0000", 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0072e-05 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834975+0000", 2023-10-09T10:07:19.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0453e-05 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.835050+0000", 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5319000000000002e-05 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.835065+0000", 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4058e-05 2023-10-09T10:07:19.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.855671+0000", 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.020606715 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.855802+0000", 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013044 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.855936+0000", 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000134544 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862781+0000", 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0068451839999999998 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.868766+0000", 2023-10-09T10:07:19.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059849960000000002 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875941+0000", 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071743689999999999 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894101+0000", 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018160477000000001 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894138+0000", 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7097000000000001e-05 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894229+0000", 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0295000000000002e-05 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179038 6.e 6:70e5e476:::benchmark_data_smithi012_182576_object73517:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.876255+0000", 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 301.70727258300002, 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0181381389999999, 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179038 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.824 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876255+0000", 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876255+0000", 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876253+0000", 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876265+0000", 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1569999999999999e-05 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876285+0000", 2023-10-09T10:07:19.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0534e-05 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876297+0000", 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2055999999999999e-05 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876381+0000", 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4178000000000006e-05 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876395+0000", 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3522e-05 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.859167+0000", 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98277201000000003 2023-10-09T10:07:19.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.859382+0000", 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021509900000000001 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.859526+0000", 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000143683 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.868813+0000", 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092870709999999992 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875816+0000", 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070034199999999998 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875984+0000", 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016793600000000001 2023-10-09T10:07:19.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894266+0000", 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018281822 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894295+0000", 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9096000000000001e-05 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894393+0000", 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8420999999999998e-05 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179055 6.e 6:7b811f8d:::benchmark_data_smithi012_182576_object73534:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.895097+0000", 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 301.688431154, 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99952659899999996, 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.832 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179055 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895097+0000", 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895097+0000", 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895095+0000", 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895102+0000", 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1790000000000002e-06 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895119+0000", 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6711e-05 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895129+0000", 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9899999999999992e-06 2023-10-09T10:07:19.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895191+0000", 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2129000000000001e-05 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895201+0000", 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.026e-05 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862795+0000", 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96759368599999995 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862878+0000", 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3028000000000005e-05 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.863006+0000", 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012810499999999999 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.868853+0000", 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058470140000000002 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875826+0000", 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006972677 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.876204+0000", 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037816400000000001 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894454+0000", 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018249656 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894506+0000", 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2283000000000001e-05 2023-10-09T10:07:19.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894623+0000", 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011736 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179059 6.e 6:791d2037:::benchmark_data_smithi012_182576_object73538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.901344+0000", 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 301.68218430399997, 2023-10-09T10:07:19.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99352547499999999, 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179059 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901344+0000", 2023-10-09T10:07:19.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901344+0000", 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901342+0000", 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:19.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901349+0000", 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5409999999999997e-06 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901365+0000", 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5889e-05 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901373+0000", 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9929999999999995e-06 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901442+0000", 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8917000000000004e-05 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901453+0000", 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1832e-05 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.865783+0000", 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96432974800000004 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.865913+0000", 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012940900000000001 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:07:19.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.866043+0000", 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013029 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875833+0000", 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0097904150000000002 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.876030+0000", 2023-10-09T10:07:19.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019691000000000001 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.876273+0000", 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00024249800000000001 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894688+0000", 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018415177000000001 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894747+0000", 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9382999999999997e-05 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894869+0000", 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121925 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195715 6.14 6:2f73fb93:::benchmark_data_smithi012_182576_object90194:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.28631684300001, 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.606497794, 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195715 2023-10-09T10:07:19.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297210+0000", 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297215+0000", 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2869999999999998e-06 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297225+0000", 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9709999999999999e-06 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297233+0000", 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1200000000000002e-06 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.301089+0000", 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038566210000000002 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.301096+0000", 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0010000000000004e-06 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.715860+0000", 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.41476375300000001 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.715875+0000", 2023-10-09T10:07:19.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5322e-05 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.886573+0000", 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1706976149999999 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.886704+0000", 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000131271 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.887173+0000", 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00046823600000000002 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.899644+0000", 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012471365 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.899817+0000", 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000172919 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900021+0000", 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000204556 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903542+0000", 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035206030000000002 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903591+0000", 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8568e-05 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903709+0000", 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118261 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195727 6.4 6:27a88943:::benchmark_data_smithi012_182576_object90206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.25467159200002, 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5877643209999999, 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195727 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328855+0000", 2023-10-09T10:07:19.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328861+0000", 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0430000000000004e-06 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328874+0000", 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3360000000000001e-05 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328882+0000", 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328994+0000", 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111865 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329006+0000", 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.255e-05 2023-10-09T10:07:19.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716358+0000", 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38735175199999999 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716368+0000", 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8479999999999996e-06 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900045+0000", 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1836774349999999 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900183+0000", 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013796500000000001 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900820+0000", 2023-10-09T10:07:19.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00063694999999999995 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.907584+0000", 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067633459999999999 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916106+0000", 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085223179999999992 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916306+0000", 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020026700000000001 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916459+0000", 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000153323 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916508+0000", 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8109999999999998e-05 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916621+0000", 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000113142 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.866 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195736 6.d 6:b34830fd:::benchmark_data_smithi012_182576_object90215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.246135573, 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6770504659999999, 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195736 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337391+0000", 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:19.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337398+0000", 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2910000000000002e-06 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337410+0000", 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2217e-05 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337419+0000", 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6989999999999992e-06 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339092+0000", 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016734510000000001 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339102+0000", 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7520000000000008e-06 2023-10-09T10:07:19.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718181+0000", 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37907882999999998 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718193+0000", 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1955e-05 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.972506+0000", 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2543128530000001 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.975318+0000", 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028124920000000002 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.975658+0000", 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000340136 2023-10-09T10:07:19.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999408+0000", 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023749816 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999451+0000", 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3038999999999999e-05 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999490+0000", 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8520999999999999e-05 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014261+0000", 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014770873 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014326+0000", 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5018999999999995e-05 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014443+0000", 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117192 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195737 6.d 6:b631f0af:::benchmark_data_smithi012_182576_object90216:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.24608373299998, 2023-10-09T10:07:19.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6917381869999999, 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195737 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337447+0000", 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5310000000000002e-06 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337455+0000", 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8790000000000003e-06 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337462+0000", 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0650000000000001e-06 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339139+0000", 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016767990000000001 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339151+0000", 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1959e-05 2023-10-09T10:07:19.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718219+0000", 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37906844499999998 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718228+0000", 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6910000000000003e-06 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999515+0000", 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2812874459999999 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999630+0000", 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114138 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999802+0000", 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000172535 2023-10-09T10:07:19.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014252+0000", 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014449500000000001 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014454+0000", 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020259100000000001 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014691+0000", 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023644399999999999 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029029+0000", 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014338379 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029086+0000", 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.694e-05 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029182+0000", 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6429999999999997e-05 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195738 6.d 6:b7beca9a:::benchmark_data_smithi012_182576_object90217:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:19.884 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.24023855399997, 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.686112541, 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195738 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343288+0000", 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:19.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343296+0000", 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6650000000000003e-06 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343334+0000", 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8760999999999997e-05 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343345+0000", 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0872e-05 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343426+0000", 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0876000000000003e-05 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343442+0000", 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6049000000000002e-05 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718250+0000", 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37480775999999999 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718258+0000", 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4360000000000002e-06 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.002824+0000", 2023-10-09T10:07:19.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.284566216 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.002952+0000", 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127133 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.003293+0000", 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00034169000000000002 2023-10-09T10:07:19.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014460+0000", 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011166931999999999 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014643+0000", 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018320000000000001 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014731+0000", 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7837999999999995e-05 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029238+0000", 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014506865000000001 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029290+0000", 2023-10-09T10:07:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2261e-05 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029402+0000", 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111514 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195740 6.d 6:ba67ed77:::benchmark_data_smithi012_182576_object90219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:07:19.893 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.23971404999998, 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.685801323, 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195740 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343813+0000", 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:07:19.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343819+0000", 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.049e-06 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343834+0000", 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5458999999999999e-05 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343843+0000", 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7250000000000003e-06 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343908+0000", 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4981999999999995e-05 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343924+0000", 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5843999999999999e-05 2023-10-09T10:07:19.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718282+0000", 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37435805300000002 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718299+0000", 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6651999999999999e-05 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006544+0000", 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2882459150000001 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006671+0000", 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012675699999999999 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006813+0000", 2023-10-09T10:07:19.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141285 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014467+0000", 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076547940000000004 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014774+0000", 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00030705499999999998 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.028866+0000", 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014091942 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029464+0000", 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00059783999999999998 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029512+0000", 2023-10-09T10:07:19.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7426000000000003e-05 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029615+0000", 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103649 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195753 6.14 6:2fd4c659:::benchmark_data_smithi012_182576_object90232:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:19.902 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.21906317200001, 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5530476879999999, 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195753 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364464+0000", 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:19.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364468+0000", 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.433e-06 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364478+0000", 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0186e-05 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364484+0000", 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9349999999999999e-06 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364551+0000", 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7554999999999999e-05 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364562+0000", 2023-10-09T10:07:19.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0964999999999999e-05 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717712+0000", 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35315011400000001 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.721170+0000", 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003457702 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900344+0000", 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.179173823 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900467+0000", 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000122846 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900983+0000", 2023-10-09T10:07:19.908 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051649900000000002 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.907573+0000", 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0065898750000000002 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.908808+0000", 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012348979999999999 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916252+0000", 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074441109999999998 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916394+0000", 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014227099999999999 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.910 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916845+0000", 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045041999999999999 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.917512+0000", 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00066757000000000003 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.911 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195755 6.4 6:21d202b2:::benchmark_data_smithi012_182576_object90234:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.21515256999999, 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5484353739999999, 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195755 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:19.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:07:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368378+0000", 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7379999999999999e-06 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368392+0000", 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3913e-05 2023-10-09T10:07:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368399+0000", 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6340000000000001e-06 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368462+0000", 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3113000000000004e-05 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368476+0000", 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3366e-05 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716390+0000", 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.347914734 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716399+0000", 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6989999999999992e-06 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904175+0000", 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.187776342 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904334+0000", 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015860900000000001 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:07:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904813+0000", 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00047921500000000001 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916182+0000", 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011368645 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916200+0000", 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7609e-05 2023-10-09T10:07:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916359+0000", 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015949700000000001 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916659+0000", 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029949099999999998 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916705+0000", 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.693e-05 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916811+0000", 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105287 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:19.921 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:07:20.189 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:20.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:20.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:20.323 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:20.362 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:20.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:20.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:20.465 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:20.475 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:07:20.497 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:20.634 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:20.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:20.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:20 smithi012 ceph-mon[152263]: failed to encode map e246 with expected crc 2023-10-09T10:07:20.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:20 smithi012 ceph-mon[152263]: pgmap v634: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 16753/75228 objects degraded (22.270%); 7259/75228 objects misplaced (9.649%); 24 MiB/s, 378 objects/s recovering 2023-10-09T10:07:20.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:20 smithi012 ceph-mon[147926]: failed to encode map e246 with expected crc 2023-10-09T10:07:20.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:20 smithi012 ceph-mon[147926]: pgmap v634: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 16753/75228 objects degraded (22.270%); 7259/75228 objects misplaced (9.649%); 24 MiB/s, 378 objects/s recovering 2023-10-09T10:07:20.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:20 smithi110 ceph-mon[136032]: failed to encode map e246 with expected crc 2023-10-09T10:07:20.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:20 smithi110 ceph-mon[136032]: pgmap v634: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 16753/75228 objects degraded (22.270%); 7259/75228 objects misplaced (9.649%); 24 MiB/s, 378 objects/s recovering 2023-10-09T10:07:20.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:07:20.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:20.768 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:20.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:20.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:20.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:20.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:20.936 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:20.976 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:20.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:20.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:21.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:21.112 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:21.153 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:21.155 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:21.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:21.257 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:21.283 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:21.320 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:21.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:21.371 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:21.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:21.454 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:21.562 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:21 smithi110 ceph-mon[136032]: failed to encode map e246 with expected crc 2023-10-09T10:07:21.748 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:21.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:21.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:21.784 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:21 smithi012 ceph-mon[147926]: failed to encode map e246 with expected crc 2023-10-09T10:07:21.784 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:21 smithi012 ceph-mon[152263]: failed to encode map e246 with expected crc 2023-10-09T10:07:21.853 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:21.883 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:21.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:21.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:22.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:22.051 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:22.084 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:22.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:22.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:07:22.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:22.236 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:22.290 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:22.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:22.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:22.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:22.428 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:22.517 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:22.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:22.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:22.621 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:22.622 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:07:22.622 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:07:22.622 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:07:22.622 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:07:22.652 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:22.718 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:22.739 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:22.761 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:22.784 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:22.803 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:22.822 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:22.823 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:07:22.823 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:07:22.823 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:07:22.823 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:07:22.823 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:07:22.823 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:07:22.823 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:07:22.823 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:07:22.874 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:22.986 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:22.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:23.077 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:07:23.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:23.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[147926]: pgmap v635: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 29 op/s; 16753/75228 objects degraded (22.270%); 7259/75228 objects misplaced (9.649%); 19 MiB/s, 300 objects/s recovering 2023-10-09T10:07:23.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:23.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:23.117 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3530934013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:23.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 16753/75228 objects degraded (22.270%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:23.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[152263]: pgmap v635: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 29 op/s; 16753/75228 objects degraded (22.270%); 7259/75228 objects misplaced (9.649%); 19 MiB/s, 300 objects/s recovering 2023-10-09T10:07:23.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:23.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:23.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3530934013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:23.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:22 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 16753/75228 objects degraded (22.270%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:23.122 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:23.155 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:23.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:23.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:23.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:23.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:22 smithi110 ceph-mon[136032]: pgmap v635: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 29 op/s; 16753/75228 objects degraded (22.270%); 7259/75228 objects misplaced (9.649%); 19 MiB/s, 300 objects/s recovering 2023-10-09T10:07:23.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:22 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:23.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:22 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:23.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3530934013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:23.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:22 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 16753/75228 objects degraded (22.270%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:23.284 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:07:23.284 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:07:23.286 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:23.326 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:23.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:23.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:23.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:23.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:23.495 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:23.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:23.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:23.599 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:23.629 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:23.708 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:23.940 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:23.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:24.044 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:24.074 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:24.213 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:24.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:24.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:24.318 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:24.347 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:24.382 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:24.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:24.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:24.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:24.518 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:24.560 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:24.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:24.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:24.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:24.693 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:24.728 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:24.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:24.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:24.833 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:24.862 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:24.897 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:24.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:24.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:25.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:25.014 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:07:25.031 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:25.037 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:25.037 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:25.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:24 smithi110 ceph-mon[136032]: pgmap v636: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 21 MiB/s, 334 objects/s recovering 2023-10-09T10:07:25.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/171472861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:25.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:25.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:24 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:25.070 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:24 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:25.204 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [3, 4] live_osds: [0, 1, 5, 6, 7, 2] 2023-10-09T10:07:25.204 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:07:25.204 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2023-10-09T10:07:25.205 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2023-10-09T10:07:25.205 DEBUG:teuthology.orchestra.run.smithi012:osd.3> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T10:07:25.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl start ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T10:07:25.241 INFO:journalctl@ceph.osd.3.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T10:07:25.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:25 smithi012 ceph-mon[152263]: pgmap v636: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 21 MiB/s, 334 objects/s recovering 2023-10-09T10:07:25.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/171472861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:25.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:25.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:25 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:25.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:25 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:25.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:24 smithi012 ceph-mon[147926]: pgmap v636: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 21 MiB/s, 334 objects/s recovering 2023-10-09T10:07:25.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/171472861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:25.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:25.266 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:24 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:25.266 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:24 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:25.285 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:07:25.361 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:25.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:25.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:07:25.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:25.501 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:25.537 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:25.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:25.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:25.640 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:25.670 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:25.677 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:25 smithi012 systemd[1]: Starting Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:07:25.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:25.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:25.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:25.840 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:25.879 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:25.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:25.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:25.973 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:25.983 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:26.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:26.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:26.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[152263]: osdmap e247: 8 total, 6 up, 8 in 2023-10-09T10:07:26.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[152263]: failed to encode map e247 with expected crc 2023-10-09T10:07:26.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:07:26.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:07:26.001 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:26.011 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:26.045 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:26.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:26.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:26.062 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:26.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:26.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:26.225 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:26.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:26.251 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:26 smithi012 bash[256178]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T10:07:26.251 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:26 smithi012 bash[256178]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T10:07:26.251 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:26 smithi012 bash[256178]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T10:07:26.252 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:26 smithi012 bash[256178]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T10:07:26.252 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:26 smithi012 bash[256178]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T10:07:26.252 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:26 smithi012 bash[256178]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T10:07:26.252 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:26 smithi012 bash[256178]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T10:07:26.252 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:26 smithi012 bash[256178]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T10:07:26.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:26.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:26.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[147926]: osdmap e247: 8 total, 6 up, 8 in 2023-10-09T10:07:26.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[147926]: failed to encode map e247 with expected crc 2023-10-09T10:07:26.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:07:26.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:07:26.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:26 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:26.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:26 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:26.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:26.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:26 smithi110 ceph-mon[136032]: osdmap e247: 8 total, 6 up, 8 in 2023-10-09T10:07:26.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:26 smithi110 ceph-mon[136032]: failed to encode map e247 with expected crc 2023-10-09T10:07:26.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:26 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:07:26.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:26 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:07:26.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:26 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:26.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:26.360 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:26.395 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:26.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:26.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:26.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:26.529 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:27.235 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:27.235 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:27.259 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:27 smithi110 ceph-mon[136032]: pgmap v638: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 10 MiB/s, 159 objects/s recovering 2023-10-09T10:07:27.260 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:27 smithi110 ceph-mon[136032]: failed to encode map e247 with expected crc 2023-10-09T10:07:27.260 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:27 smithi110 ceph-mon[136032]: failed to encode map e247 with expected crc 2023-10-09T10:07:27.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:07:27.445 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:27 smithi012 ceph-mon[147926]: pgmap v638: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 10 MiB/s, 159 objects/s recovering 2023-10-09T10:07:27.445 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:27 smithi012 ceph-mon[147926]: failed to encode map e247 with expected crc 2023-10-09T10:07:27.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:27 smithi012 ceph-mon[147926]: failed to encode map e247 with expected crc 2023-10-09T10:07:27.446 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:27 smithi012 ceph-mon[152263]: pgmap v638: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 10 MiB/s, 159 objects/s recovering 2023-10-09T10:07:27.446 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:27 smithi012 ceph-mon[152263]: failed to encode map e247 with expected crc 2023-10-09T10:07:27.446 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:27 smithi012 ceph-mon[152263]: failed to encode map e247 with expected crc 2023-10-09T10:07:27.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:07:28.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 16029/75228 objects degraded (21.307%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:07:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:07:27] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T10:07:28.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 16029/75228 objects degraded (21.307%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:28.290 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:28.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 16029/75228 objects degraded (21.307%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:28.579 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:28.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:28.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:28.711 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:28.799 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:28.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:28.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:28.902 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:28.929 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:29.017 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:29.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:29.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:29.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:29 smithi110 ceph-mon[136032]: pgmap v639: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:07:29.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:29.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:29.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:29.151 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:29 smithi012 ceph-mon[147926]: pgmap v639: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:07:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:29.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:29 smithi012 ceph-mon[152263]: pgmap v639: 89 pgs: 1 active+recovering+degraded, 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 16029/75228 objects degraded (21.307%); 7259/75228 objects misplaced (9.649%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:07:29.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:29.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:29.474 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:29.475 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:29.807 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:07:30.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:30 smithi110 ceph-mon[136032]: pgmap v640: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 6.0 MiB/s, 148 objects/s recovering 2023-10-09T10:07:30.346 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:30.348 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:30 smithi012 ceph-mon[147926]: pgmap v640: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 6.0 MiB/s, 148 objects/s recovering 2023-10-09T10:07:30.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:30 smithi012 ceph-mon[152263]: pgmap v640: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 6.0 MiB/s, 148 objects/s recovering 2023-10-09T10:07:30.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:30.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:30.418 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:30.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:30.478 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:31.281 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:31.506 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:31.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:31.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:31.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:31.636 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:31.667 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:31.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:31.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:31.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:31.797 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:31.831 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:31.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:31.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:31.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:31.965 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:32.359 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:32.380 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:32.403 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:32.408 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:32.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:32.425 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:32.443 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:32.461 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:32.461 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:07:32.461 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:07:32.461 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:07:32.461 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:07:32.462 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:07:32.462 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:07:32.463 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:07:32.463 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:07:32.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:32.539 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:32 smithi110 ceph-mon[136032]: pgmap v641: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 6.0 MiB/s, 148 objects/s recovering 2023-10-09T10:07:32.545 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:32.551 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:32 smithi012 ceph-mon[152263]: pgmap v641: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 6.0 MiB/s, 148 objects/s recovering 2023-10-09T10:07:32.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:32 smithi012 ceph-mon[147926]: pgmap v641: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 6.0 MiB/s, 148 objects/s recovering 2023-10-09T10:07:32.579 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:32.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:32.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:32.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:32.722 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:32.920 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:32.921 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:32 smithi012 podman[256831]: 2023-10-09T10:07:32.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:32.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:32.941 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:07:32.942 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:07:33.024 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:33.054 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:33.111 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:33.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:33.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:33.177 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:33 smithi012 ceph-osd[256998]: -- 172.21.15.12:0/1300072446 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fcce1744e0 con 0x55fcce145400 2023-10-09T10:07:33.177 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:33 smithi012 bash[256831]: 6b4849f47412020b2934aa93dd9705e11ec6d8f2e8c1baaa7521c648a4201612 2023-10-09T10:07:33.209 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:07:33.214 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:33.244 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:33.379 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:33.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:33.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:33.443 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:07:33.443 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:07:33.443 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:07:33.443 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:07:33.444 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:07:33.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2532838789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:33.463 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 15793/75228 objects degraded (20.994%), 19 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T10:07:33.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:33.514 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:33 smithi012 systemd[1]: Started Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T10:07:33.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2532838789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:33.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 15793/75228 objects degraded (20.994%), 19 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T10:07:33.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2532838789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:33.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 15793/75228 objects degraded (20.994%), 19 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T10:07:33.517 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:33.581 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:33.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:33.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:33.718 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:33.851 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:07:33.854 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:33.856 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[256974]: 2023-10-09T10:07:33.672+0000 7fe121cbe640 -1 Falling back to public interface 2023-10-09T10:07:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:33.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:33.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:33.964 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:33.993 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:34.092 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:34.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:34.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:34.228 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:34.266 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:34.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:34.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:34.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:34.431 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:34.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:34.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:34.456 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:34.535 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:34.565 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:34.670 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:34.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:34.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:34.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:34.805 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:34.855 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:34.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:34.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:34.887 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:07:34.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:34 smithi110 ceph-mon[136032]: pgmap v642: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:34 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3915463381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:34 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:34.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:07:34.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:35.024 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:35.115 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:35.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:34 smithi012 ceph-mon[147926]: pgmap v642: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:35.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:34 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3915463381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:35.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:34 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:35.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:35.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:35.120 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:34 smithi012 ceph-mon[152263]: pgmap v642: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:35.120 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:34 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3915463381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:35.120 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:34 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:no valid command found; 10 closest matches: 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:0 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:1 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:2 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:abort 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:assert 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:config diff 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:config diff get 2023-10-09T10:07:35.204 INFO:teuthology.orchestra.run.smithi012.stderr:config get 2023-10-09T10:07:35.205 INFO:teuthology.orchestra.run.smithi012.stderr:config help [] 2023-10-09T10:07:35.205 INFO:teuthology.orchestra.run.smithi012.stderr:config set ... 2023-10-09T10:07:35.205 INFO:teuthology.orchestra.run.smithi012.stderr:admin_socket: invalid command 2023-10-09T10:07:35.220 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:35.253 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:35.353 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:35.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:35.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:35.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:35.487 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:35.651 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13899 6.c e103/97 6:39dbd6a1:::benchmark_data_smithi012_182576_object13898:head v 103'862, mlcod=103'860)", 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.10369447699998, 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109510709, 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.652 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545773+0000", 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545772+0000", 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4119999999999999e-05 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545806+0000", 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4049000000000001e-05 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545820+0000", 2023-10-09T10:07:35.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1470000000000004e-06 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545827+0000", 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5004000000000002e-05 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545902+0000", 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3674999999999999e-05 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.545926+0000", 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109217777 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655143+0000", 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014009299999999999 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655284+0000", 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.109510709 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13895 6.3 e103/97 6:cb4c122f:::benchmark_data_smithi012_182576_object13894:head v 103'855, mlcod=103'853)", 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.10316570399999, 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108952985, 2023-10-09T10:07:35.657 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546302+0000", 2023-10-09T10:07:35.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546301+0000", 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2656000000000001e-05 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546324+0000", 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6180000000000001e-05 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546340+0000", 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9619999999999998e-06 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546346+0000", 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5816000000000003e-05 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546391+0000", 2023-10-09T10:07:35.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0018e-05 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546421+0000", 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108744009 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655165+0000", 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9089e-05 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655255+0000", 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108952985 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13902 6.5 e103/97 6:a6447830:::benchmark_data_smithi012_182576_object13901:head v 103'874, mlcod=103'872)", 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.103107341, 2023-10-09T10:07:35.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110437017, 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546360+0000", 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0539000000000001e-05 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546380+0000", 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8390000000000001e-06 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546387+0000", 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2400000000000001e-06 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546391+0000", 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1564999999999998e-05 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546443+0000", 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1464999999999999e-05 2023-10-09T10:07:35.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546464+0000", 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110295141 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656759+0000", 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7586999999999998e-05 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656797+0000", 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.110437017 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13914 6.a e103/97 6:527234fb:::benchmark_data_smithi012_182576_object13913:head v 103'853, mlcod=103'851)", 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.09834244400002, 2023-10-09T10:07:35.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.105616578, 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551125+0000", 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551124+0000", 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4997000000000001e-05 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551149+0000", 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104127211 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655276+0000", 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4699999999999999e-06 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655283+0000", 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9079000000000003e-05 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655322+0000", 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1195000000000002e-05 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655353+0000", 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013534339999999999 2023-10-09T10:07:35.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656706+0000", 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5194000000000002e-05 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656741+0000", 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.105616578 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.672 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13926 6.1 e103/97 6:8de8341f:::benchmark_data_smithi012_182576_object13925:head v 103'823, mlcod=103'821)", 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.085640447, 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092614976000000002, 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563827+0000", 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563825+0000", 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014807399999999999 2023-10-09T10:07:35.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563974+0000", 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.091111844999999997 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655085+0000", 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0784000000000002e-05 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655116+0000", 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017427 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655290+0000", 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4281999999999999e-05 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655315+0000", 2023-10-09T10:07:35.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010930600000000001 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656408+0000", 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4076999999999999e-05 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656442+0000", 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092614976000000002 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.677 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:13929 6.c e103/97 6:39e98511:::benchmark_data_smithi012_182576_object13928:head v 103'863, mlcod=103'860)", 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 510.08548270799997, 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092504991999999994, 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563985+0000", 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563984+0000", 2023-10-09T10:07:35.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2965000000000004e-05 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564047+0000", 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.091099323999999995 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655146+0000", 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0924000000000002e-05 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655167+0000", 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016929299999999999 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655336+0000", 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0599999999999999e-05 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655357+0000", 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011074889999999999 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656464+0000", 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5162999999999999e-05 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656490+0000", 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092504991999999994 2023-10-09T10:07:35.682 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96028 6.f e118/117 6:f25f2574:::benchmark_data_smithi012_182576_object96027:head v 118'5972, mlcod=0'0)", 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 443.490118698, 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041325972000000002, 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159349+0000", 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159348+0000", 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0521e-05 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159388+0000", 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9960000000000005e-06 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159398+0000", 2023-10-09T10:07:35.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.981e-06 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159404+0000", 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0812e-05 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159465+0000", 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2714999999999999e-05 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.159488+0000", 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041139283999999998 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.200627+0000", 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7336000000000002e-05 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:12.200675+0000", 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041325972000000002 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96906 6.8 e119/117 6:1349a12f:::benchmark_data_smithi012_182576_object96905:head v 119'6178, mlcod=0'0)", 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 442.08582090099998, 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043713813999999997, 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.688 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563646+0000", 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4641e-05 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563671+0000", 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3489999999999997e-06 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563677+0000", 2023-10-09T10:07:35.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0620000000000002e-06 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563681+0000", 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024566689999999999 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.566138+0000", 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8526000000000001e-05 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.566166+0000", 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041156198999999997 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.607322+0000", 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7769999999999999e-05 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.607360+0000", 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043713813999999997 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96908 6.8 e119/117 6:16fa2f2f:::benchmark_data_smithi012_182576_object96907:head v 119'6179, mlcod=0'0)", 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 442.085775732, 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052663933000000003, 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.693 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563692+0000", 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563691+0000", 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3872000000000001e-05 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563715+0000", 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0090000000000004e-06 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563721+0000", 2023-10-09T10:07:35.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9999999999999998e-06 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.563725+0000", 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010034111 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.573759+0000", 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.565e-05 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.573785+0000", 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042522310000000001 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616307+0000", 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8340000000000001e-05 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616356+0000", 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052663933000000003 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96912 6.8 e119/117 6:11471bcc:::benchmark_data_smithi012_182576_object96911:head v 119'6180, mlcod=0'0)", 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 442.078599575, 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04550949, 2023-10-09T10:07:35.698 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570868+0000", 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570867+0000", 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5475000000000002e-05 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570903+0000", 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1657999999999999e-05 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570914+0000", 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8560000000000001e-06 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.570921+0000", 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027926853000000001 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.598848+0000", 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8767000000000001e-05 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.598877+0000", 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017483862999999999 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616361+0000", 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6659000000000001e-05 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616377+0000", 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04550949 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.703 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96916 6.8 e119/117 6:11cbd0a6:::benchmark_data_smithi012_182576_object96915:head v 119'6181, mlcod=0'0)", 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 442.05407564500001, 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042488638000000002, 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595392+0000", 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595390+0000", 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6316e-05 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595416+0000", 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.643e-05 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595433+0000", 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0700000000000007e-06 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595441+0000", 2023-10-09T10:07:35.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013293442000000001 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608734+0000", 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4462000000000003e-05 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.608769+0000", 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029077441999999998 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637846+0000", 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3998999999999998e-05 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637880+0000", 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042488638000000002 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:96919 6.8 e119/117 6:16cabaa4:::benchmark_data_smithi012_182576_object96918:head v 119'6182, mlcod=0'0)", 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 442.05401866900002, 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042453923999999997, 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:07:35.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595449+0000", 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595448+0000", 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9534e-05 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595468+0000", 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1179999999999997e-06 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595474+0000", 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2780000000000004e-06 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.595478+0000", 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020955610999999999 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616434+0000", 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5219000000000003e-05 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.616469+0000", 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021415979000000002 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637885+0000", 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7509000000000001e-05 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.637903+0000", 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042453923999999997 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:98780 6.2 e119/117 6:46aa6df3:::benchmark_data_smithi012_182576_object98779:head v 119'6253, mlcod=0'0)", 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 438.97181568299999, 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053554971999999999, 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:07:35.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677652+0000", 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677650+0000", 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0443e-05 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677681+0000", 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8221999999999999e-05 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677699+0000", 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1400000000000002e-06 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.677706+0000", 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070735980000000004 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.684780+0000", 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4288000000000001e-05 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.684814+0000", 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046346799000000001 2023-10-09T10:07:35.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731161+0000", 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5666e-05 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731207+0000", 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053554971999999999 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.718 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:98784 6.2 e119/117 6:44e3e91a:::benchmark_data_smithi012_182576_object98783:head v 119'6254, mlcod=0'0)", 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 438.97045982700001, 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052221194999999998, 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679007+0000", 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6214e-05 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679033+0000", 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6022999999999999e-05 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679049+0000", 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7379999999999997e-06 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.679056+0000", 2023-10-09T10:07:35.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013390713 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.692446+0000", 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8688e-05 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.692475+0000", 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038735865000000001 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731211+0000", 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.783e-05 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:16.731229+0000", 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.052221194999999998 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103547 6.8 e122/121 6:1259b0ac:::benchmark_data_smithi012_182576_object103546:head v 122'6626, mlcod=0'0)", 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 429.16634511400002, 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040879891000000002, 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483122+0000", 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483121+0000", 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105214 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483226+0000", 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9423999999999999e-05 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483246+0000", 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4619999999999999e-06 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.483253+0000", 2023-10-09T10:07:35.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038037510000000002 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.487057+0000", 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.73e-05 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.487084+0000", 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036875568999999997 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.523960+0000", 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2120000000000003e-05 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.524002+0000", 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040879891000000002 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103568 6.f e122/117 6:f1ce72d6:::benchmark_data_smithi012_182576_object103567:head v 122'6429, mlcod=0'0)", 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 429.12967794799999, 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047394631, 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519789+0000", 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7826e-05 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519827+0000", 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.409e-06 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519833+0000", 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1130000000000003e-06 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.519837+0000", 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.008335238 2023-10-09T10:07:35.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.528173+0000", 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7257e-05 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.528200+0000", 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038942002000000003 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.567142+0000", 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2160999999999998e-05 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.567184+0000", 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047394631 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.733 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:103834 6.2 e122/117 6:481c5bb1:::benchmark_data_smithi012_182576_object103833:head v 122'6597, mlcod=0'0)", 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 428.69596588399997, 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040357787999999999, 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953501+0000", 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953500+0000", 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8389999999999997e-05 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953538+0000", 2023-10-09T10:07:35.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6200000000000001e-05 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953554+0000", 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.303e-06 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.953563+0000", 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011028718999999999 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.964591+0000", 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2540000000000001e-05 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.964614+0000", 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029196083000000001 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.993810+0000", 2023-10-09T10:07:35.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9212999999999998e-05 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:26.993859+0000", 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040357787999999999 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:104205 6.f e122/117 6:f3405025:::benchmark_data_smithi012_182576_object104204:head v 122'6474, mlcod=0'0)", 2023-10-09T10:07:35.738 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 428.07023624700003, 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048731662000000002, 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579231+0000", 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1229999999999997e-05 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579262+0000", 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1770000000000001e-05 2023-10-09T10:07:35.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579274+0000", 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8019999999999999e-06 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.579278+0000", 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079426949999999996 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.587221+0000", 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8532000000000001e-05 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.587250+0000", 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040668007999999999 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.627918+0000", 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5040999999999998e-05 2023-10-09T10:07:35.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:27.627963+0000", 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048731662000000002 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.34797.0:105353 6.2 e123/117 6:4730719a:::benchmark_data_smithi012_182576_object105352:head v 123'6695, mlcod=0'0)", 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 426.12888641000001, 2023-10-09T10:07:35.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041308386000000002, 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:35.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520581+0000", 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1875e-05 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520602+0000", 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4280000000000001e-06 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520609+0000", 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0799999999999999e-06 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.520613+0000", 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078419579999999996 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.528455+0000", 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0639999999999998e-05 2023-10-09T10:07:35.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.528486+0000", 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033364444 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.561850+0000", 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9289999999999998e-05 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:29.561889+0000", 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041308386000000002 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(6.2 144/138 [PushOp(6:4011f362:::benchmark_data_smithi012_182576_object40319:head, version: 105'2578, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:4011f362:::benchmark_data_smithi012_182576_object40319:head@105'2578, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 368.68299229399997, 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048028721000000003, 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:35.748 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966475+0000", 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966474+0000", 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8900000000000002e-05 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966533+0000", 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7698000000000001e-05 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966561+0000", 2023-10-09T10:07:35.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2663000000000001e-05 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:26.966573+0000", 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.039901351000000002 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.006475+0000", 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4449999999999996e-06 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.006483+0000", 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080206500000000007 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:27.014504+0000", 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048028721000000003 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:35.752 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:07:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:35.754 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:35.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:35.799 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:07:35.799 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2023-10-09T10:07:35.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:35.885 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:35 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:07:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:35 smithi110 ceph-mon[136032]: osdmap e248: 8 total, 6 up, 8 in 2023-10-09T10:07:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:35 smithi110 ceph-mon[136032]: failed to encode map e248 with expected crc 2023-10-09T10:07:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:35 smithi110 ceph-mon[136032]: failed to encode map e248 with expected crc 2023-10-09T10:07:36.004 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:36.023 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:36.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:36.084 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:07:36.126 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:36.153 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:07:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[147926]: osdmap e248: 8 total, 6 up, 8 in 2023-10-09T10:07:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[147926]: failed to encode map e248 with expected crc 2023-10-09T10:07:36.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[147926]: failed to encode map e248 with expected crc 2023-10-09T10:07:36.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:36.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:07:36.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[152263]: osdmap e248: 8 total, 6 up, 8 in 2023-10-09T10:07:36.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[152263]: failed to encode map e248 with expected crc 2023-10-09T10:07:36.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:35 smithi012 ceph-mon[152263]: failed to encode map e248 with expected crc 2023-10-09T10:07:36.188 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:36.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:36.292 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:36.321 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:36.353 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:36.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:36.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:36.483 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:36.514 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:36.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:36.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:36.651 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:36.696 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:36.823 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:36.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:36.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:36.961 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:37.006 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:37.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:37.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:36 smithi012 ceph-mon[147926]: pgmap v644: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:37.096 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:36 smithi012 ceph-mon[152263]: pgmap v644: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:37.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:37.136 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:36 smithi110 ceph-mon[136032]: pgmap v644: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:37.139 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:37.179 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:37.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:37.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:37.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:37.309 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:37.341 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:37.368 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:37.387 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:37.408 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:37.429 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:37.453 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:37.453 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:07:37.453 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:07:37.453 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:07:37.453 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:07:37.453 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:07:37.453 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:07:37.453 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:07:37.454 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:07:37.454 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:37.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:37.557 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:37.584 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:37.668 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:37.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:37.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:37.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:07:37.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:37.802 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:37.835 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:37.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:07:37.846 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:07:37.882 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:37.882 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:37.940 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:37.974 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:38.008 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:38.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:38.112 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:38.136 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/123499084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:38.139 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:38.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/123499084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:07:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:07:37] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T10:07:38.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/123499084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:38.266 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:38.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:38.323 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:07:38.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:38.405 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:38.410 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:38.446 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:38.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:38.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:38.579 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:38.669 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:38.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:38.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:38.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:38.805 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:38.848 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:38.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:38.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:38.923 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:38 smithi012 ceph-mon[152263]: pgmap v645: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:38 smithi012 ceph-mon[147926]: pgmap v645: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:38.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:38.984 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:39.139 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:39.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:39.160 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:38 smithi110 ceph-mon[136032]: pgmap v645: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 1.5 MiB/s, 175 objects/s recovering 2023-10-09T10:07:39.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:39.283 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:39.431 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:39.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:39.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:39.571 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:39.617 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:39.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:39.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:39.721 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:39.749 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:39.787 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:39.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:39.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:39.847 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:07:39.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:39.919 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:40.004 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:40.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:40.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:40.108 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:40.136 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:40.170 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:40.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:40.203 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:40.203 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:40.274 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:40.308 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:40.508 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:40.516 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:40.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:40.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:07:40.620 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:40.662 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:40.695 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:40.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:40.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:40.799 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:40.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:07:40.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:40.860 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:40.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:40.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:40.915 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:07:40.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:40.996 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[256974]: 2023-10-09T10:07:40.706+0000 7fe121cbe640 -1 osd.3 228 log_to_monitors true 2023-10-09T10:07:40.997 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[256974]: 2023-10-09T10:07:40.937+0000 7fe10f122700 -1 osd.3 228 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:07:40.997 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-mon[147926]: pgmap v646: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 0 B/s, 208 objects/s recovering 2023-10-09T10:07:40.997 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/783470728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:40.997 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:40.998 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-mon[147926]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:07:40.998 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-mon[152263]: pgmap v646: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 0 B/s, 208 objects/s recovering 2023-10-09T10:07:40.998 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/783470728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:40.998 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:40.998 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:40 smithi012 ceph-mon[152263]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:07:41.011 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:41.102 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:41.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:40 smithi110 ceph-mon[136032]: pgmap v646: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 0 B/s, 208 objects/s recovering 2023-10-09T10:07:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/783470728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:40 smithi110 ceph-mon[136032]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:07:41.203 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:07:41.207 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:41.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:41.326 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:41.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:41.429 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:41.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:41.494 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:41.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:41.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:41.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:41.632 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:41.673 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:41.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:41.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:41.778 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:41.806 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:41.937 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:41.971 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:41.995 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:41.996 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:41.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:41 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:07:41.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:41 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:41.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:41 smithi110 ceph-mon[136032]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:07:41.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:41 smithi110 ceph-mon[136032]: osdmap e249: 8 total, 6 up, 8 in 2023-10-09T10:07:41.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:41 smithi110 ceph-mon[136032]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:07:41.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:41 smithi110 ceph-mon[136032]: failed to encode map e249 with expected crc 2023-10-09T10:07:41.998 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:41 smithi110 ceph-mon[136032]: failed to encode map e249 with expected crc 2023-10-09T10:07:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:41.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:42.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:07:42.038 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:42.057 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:42.070 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:42.080 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:42.080 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:07:42.080 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:07:42.081 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:07:42.081 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:07:42.081 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:07:42.081 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:07:42.081 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:07:42.081 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:07:42.100 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:42.127 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:42.161 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:42.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:07:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[147926]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:07:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[147926]: osdmap e249: 8 total, 6 up, 8 in 2023-10-09T10:07:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[147926]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:07:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[147926]: failed to encode map e249 with expected crc 2023-10-09T10:07:42.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[147926]: failed to encode map e249 with expected crc 2023-10-09T10:07:42.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:07:42.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:42.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[152263]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:07:42.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[152263]: osdmap e249: 8 total, 6 up, 8 in 2023-10-09T10:07:42.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[152263]: from='osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:07:42.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[152263]: failed to encode map e249 with expected crc 2023-10-09T10:07:42.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:41 smithi012 ceph-mon[152263]: failed to encode map e249 with expected crc 2023-10-09T10:07:42.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:42.299 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:42.468 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13895 6.3 6:cb4c122f:::benchmark_data_smithi012_182576_object13894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.92163189500002, 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.111765246, 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13895 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544304+0000", 2023-10-09T10:07:42.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:42.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544303+0000", 2023-10-09T10:07:42.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7165999999999999e-05 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544380+0000", 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1870000000000003e-06 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544389+0000", 2023-10-09T10:07:42.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6619999999999996e-06 2023-10-09T10:07:42.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544394+0000", 2023-10-09T10:07:42.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2271999999999998e-05 2023-10-09T10:07:42.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544436+0000", 2023-10-09T10:07:42.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138376 2023-10-09T10:07:42.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544575+0000", 2023-10-09T10:07:42.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000172697 2023-10-09T10:07:42.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:07:42.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544747+0000", 2023-10-09T10:07:42.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019732679999999998 2023-10-09T10:07:42.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.546721+0000", 2023-10-09T10:07:42.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00053507399999999997 2023-10-09T10:07:42.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.547256+0000", 2023-10-09T10:07:42.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10869904900000001 2023-10-09T10:07:42.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.655955+0000", 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1564999999999998e-05 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656016+0000", 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2889999999999997e-05 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.656069+0000", 2023-10-09T10:07:42.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.111765246 2023-10-09T10:07:42.479 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.479 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.479 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.480 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13902 6.5 6:a6447830:::benchmark_data_smithi012_182576_object13901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.480 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:07:42.480 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.92153121299998, 2023-10-09T10:07:42.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.112868624, 2023-10-09T10:07:42.480 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.480 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.480 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.480 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.481 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.481 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13902 2023-10-09T10:07:42.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.481 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544405+0000", 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544404+0000", 2023-10-09T10:07:42.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103757 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544508+0000", 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3230999999999999e-05 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544521+0000", 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6330000000000004e-06 2023-10-09T10:07:42.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544526+0000", 2023-10-09T10:07:42.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4892000000000003e-05 2023-10-09T10:07:42.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544570+0000", 2023-10-09T10:07:42.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107476 2023-10-09T10:07:42.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544678+0000", 2023-10-09T10:07:42.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010674400000000001 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.544785+0000", 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022269719999999998 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.547012+0000", 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001912513 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.548924+0000", 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108253376 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657178+0000", 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4778000000000003e-05 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657232+0000", 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0898e-05 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657273+0000", 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.112868624 2023-10-09T10:07:42.488 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13907 6.3 6:c0d71b15:::benchmark_data_smithi012_182576_object13906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.91621883300002, 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108893135, 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.489 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13907 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549717+0000", 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549716+0000", 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9034999999999999e-05 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549745+0000", 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1456e-05 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549756+0000", 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0289999999999996e-06 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549761+0000", 2023-10-09T10:07:42.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6547000000000001e-05 2023-10-09T10:07:42.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549798+0000", 2023-10-09T10:07:42.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012402 2023-10-09T10:07:42.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549922+0000", 2023-10-09T10:07:42.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021892400000000001 2023-10-09T10:07:42.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550141+0000", 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017843049999999999 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551925+0000", 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031652469999999999 2023-10-09T10:07:42.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555090+0000", 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10346385299999999 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658554+0000", 2023-10-09T10:07:42.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1267999999999999e-05 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658585+0000", 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4669e-05 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658610+0000", 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108893135 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.497 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13908 6.5 6:aed1463a:::benchmark_data_smithi012_182576_object13907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.91616625100005, 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.107611733, 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.498 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13908 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549770+0000", 2023-10-09T10:07:42.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549769+0000", 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7673000000000001e-05 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549797+0000", 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4170000000000003e-06 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549804+0000", 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.826e-06 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549809+0000", 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9627999999999999e-05 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549869+0000", 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000173229 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550042+0000", 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019173700000000001 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550234+0000", 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099897300000000001 2023-10-09T10:07:42.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551233+0000", 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021422310000000001 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.553375+0000", 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.103950349 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657325+0000", 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2319e-05 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657358+0000", 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3770000000000001e-05 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.657381+0000", 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.107611733 2023-10-09T10:07:42.504 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13911 6.3 6:c7c65153:::benchmark_data_smithi012_182576_object13910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.91611515199997, 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108882675, 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.505 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13911 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549821+0000", 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549820+0000", 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3742000000000003e-05 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549854+0000", 2023-10-09T10:07:42.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7170000000000002e-06 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549860+0000", 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5469999999999998e-06 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.549865+0000", 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00089393100000000004 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550759+0000", 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0737999999999998e-05 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550850+0000", 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012563299999999999 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.550975+0000", 2023-10-09T10:07:42.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00095688000000000001 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.551932+0000", 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032028109999999999 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555135+0000", 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10350596099999999 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658641+0000", 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2475000000000003e-05 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658673+0000", 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9924999999999999e-05 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658703+0000", 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.108882675 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13918 6.5 6:a8d8ff46:::benchmark_data_smithi012_182576_object13917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.91193122300001, 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104220092, 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.512 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13918 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554005+0000", 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554003+0000", 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4011000000000006e-05 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554087+0000", 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8400000000000001e-06 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554096+0000", 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4819999999999996e-06 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554100+0000", 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3441e-05 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554144+0000", 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126305 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554270+0000", 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016421900000000001 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:07:42.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.554434+0000", 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000721037 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.555155+0000", 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078462779999999999 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.563002+0000", 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095134755000000001 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658136+0000", 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0183999999999999e-05 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658187+0000", 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8050000000000003e-05 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658225+0000", 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104220092 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13922 6.3 6:cfff9caa:::benchmark_data_smithi012_182576_object13921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.90733559199998, 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10019183399999999, 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.519 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13922 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558600+0000", 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558599+0000", 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7776999999999997e-05 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558687+0000", 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3254999999999999e-05 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558700+0000", 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.977e-06 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558705+0000", 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0094e-05 2023-10-09T10:07:42.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558735+0000", 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8771999999999997e-05 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558814+0000", 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9957999999999995e-05 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.558904+0000", 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070560600000000001 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.559610+0000", 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002049763 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.561659+0000", 2023-10-09T10:07:42.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.097076735999999997 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658736+0000", 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.726e-05 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658763+0000", 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8696000000000001e-05 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658792+0000", 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10019183399999999 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13930 6.3 6:cba38fc5:::benchmark_data_smithi012_182576_object13929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.90115394700001, 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094088516999999997, 2023-10-09T10:07:42.526 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13930 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564782+0000", 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564781+0000", 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0343000000000002e-05 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564831+0000", 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7879999999999995e-06 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564840+0000", 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4070000000000003e-06 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564844+0000", 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0728999999999997e-05 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564875+0000", 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9303999999999997e-05 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.564964+0000", 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107612 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:07:42.530 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.565072+0000", 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050480800000000004 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.565577+0000", 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090698299999999999 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566484+0000", 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092340803999999999 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658824+0000", 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6945e-05 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658851+0000", 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9034000000000001e-05 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658870+0000", 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094088516999999997 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:13932 6.3 6:cf3c9e78:::benchmark_data_smithi012_182576_object13931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e103)", 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 516.89948560000005, 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092527188999999996, 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.533 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 13932 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566450+0000", 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566449+0000", 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6433000000000003e-05 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566546+0000", 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2155e-05 2023-10-09T10:07:42.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566558+0000", 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2870000000000001e-06 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566564+0000", 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1313000000000002e-05 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566606+0000", 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2416000000000005e-05 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566688+0000", 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9309999999999996e-05 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.566787+0000", 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043576999999999998 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.567223+0000", 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00077269100000000005 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.567996+0000", 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.090905342 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658901+0000", 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3823000000000003e-05 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658955+0000", 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2500000000000001e-05 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T09:59:05.658977+0000", 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.092527188999999996 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.539 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96742 6.3 6:ccfab877:::benchmark_data_smithi012_182576_object96741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 449.18668206400002, 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11727006399999999, 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.540 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96742 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279254+0000", 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:07:42.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279251+0000", 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6714e-05 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279298+0000", 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3504999999999999e-05 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279321+0000", 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0139999999999999e-05 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.279331+0000", 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018866076999999998 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298197+0000", 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018085900000000001 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298378+0000", 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022614600000000001 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.298604+0000", 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033326643000000003 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.331931+0000", 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5298e-05 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.331966+0000", 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060961738000000001 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.392928+0000", 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4974000000000005e-05 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.392993+0000", 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035309400000000002 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396524+0000", 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11727006399999999 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96748 6.3 6:ce4562bc:::benchmark_data_smithi012_182576_object96747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 449.17816504199999, 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11285076300000001, 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.547 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96748 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287771+0000", 2023-10-09T10:07:42.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287768+0000", 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8236999999999999e-05 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287816+0000", 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2195999999999999e-05 2023-10-09T10:07:42.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287838+0000", 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0397000000000001e-05 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.287849+0000", 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014132728000000001 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.301981+0000", 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021475899999999999 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.302196+0000", 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024190600000000001 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.302438+0000", 2023-10-09T10:07:42.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029561812 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332000+0000", 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2532e-05 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332032+0000", 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.064502819000000003 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396535+0000", 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4147999999999998e-05 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.396579+0000", 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040421540000000001 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400622+0000", 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11285076300000001 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:96749 6.3 6:c755d6fb:::benchmark_data_smithi012_182576_object96748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 449.17592732399999, 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11442171800000001, 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.554 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 96749 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290009+0000", 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290007+0000", 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9950000000000001e-05 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290056+0000", 2023-10-09T10:07:42.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9181000000000002e-05 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290076+0000", 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8350000000000004e-06 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.290083+0000", 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015899986000000001 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.305983+0000", 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000202692 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.306186+0000", 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023536100000000001 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.306422+0000", 2023-10-09T10:07:42.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025644121999999998 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.332066+0000", 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068449019999999999 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.338911+0000", 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.061723756999999997 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400634+0000", 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9935e-05 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.400704+0000", 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003725983 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:13.404430+0000", 2023-10-09T10:07:42.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11442171800000001 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190193 7.8 7:1a8121c5:::benchmark_data_smithi012_221426_object82415:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e240)", 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.068278786, 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.051354506, 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.561 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190193 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397656+0000", 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7679999999999997e-06 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397662+0000", 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0439999999999996e-06 2023-10-09T10:07:42.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397670+0000", 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0260000000000004e-06 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397676+0000", 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00094881100000000003 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398625+0000", 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2380000000000003e-06 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398634+0000", 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0429888860000001 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441623+0000", 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011069399999999999 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441733+0000", 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4486000000000003e-05 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441828+0000", 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033881190000000002 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445216+0000", 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036284030000000001 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.448844+0000", 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.923e-05 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.448904+0000", 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010786 2023-10-09T10:07:42.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449012+0000", 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.051354506 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190203 7.8 7:1d4612d0:::benchmark_data_smithi012_221426_object82425:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:07:42.568 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.068255706000002, 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0571886909999999, 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190203 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0150000000000002e-06 2023-10-09T10:07:42.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397682+0000", 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.647e-06 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397685+0000", 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7410000000000001e-06 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397688+0000", 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099381899999999995 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398682+0000", 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5570000000000003e-06 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398688+0000", 2023-10-09T10:07:42.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.046572745 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445260+0000", 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012410700000000001 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445384+0000", 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000154804 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445539+0000", 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058752240000000001 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451414+0000", 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033171709999999998 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454732+0000", 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5908000000000001e-05 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454788+0000", 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1254999999999994e-05 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454869+0000", 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0571886909999999 2023-10-09T10:07:42.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190210 7.8 7:1b3a4cee:::benchmark_data_smithi012_221426_object82432:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.068244425000003, 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0644897120000001, 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:07:42.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190210 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:07:42.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7489999999999999e-06 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397693+0000", 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0630000000000002e-06 2023-10-09T10:07:42.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397696+0000", 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2269999999999996e-06 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397703+0000", 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001020782 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398724+0000", 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3509999999999996e-06 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398731+0000", 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0503301599999999 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449062+0000", 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013289700000000001 2023-10-09T10:07:42.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449194+0000", 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018222 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449377+0000", 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012586909 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.461964+0000", 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4403e-05 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462028+0000", 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3278e-05 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462081+0000", 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9912000000000002e-05 2023-10-09T10:07:42.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462181+0000", 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0644897120000001 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190212 7.8 7:1e06d44a:::benchmark_data_smithi012_221426_object82434:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:07:42.583 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.068230090999997, 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.071536394, 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190212 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:07:42.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397705+0000", 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7659999999999999e-06 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397707+0000", 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8750000000000002e-06 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397711+0000", 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4600000000000002e-06 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397714+0000", 2023-10-09T10:07:42.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010809909999999999 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398795+0000", 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5300000000000004e-06 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398800+0000", 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0526496919999999 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451450+0000", 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013340300000000001 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451583+0000", 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017290599999999999 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451756+0000", 2023-10-09T10:07:42.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010213186000000001 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.461969+0000", 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071399239999999997 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469109+0000", 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9132000000000003e-05 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469158+0000", 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3792e-05 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469242+0000", 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.071536394 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190219 7.8 7:155dd6d1:::benchmark_data_smithi012_221426_object82441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.068219094, 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0716708749999999, 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190219 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.591 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397716+0000", 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6100000000000002e-06 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397723+0000", 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5250000000000001e-06 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397727+0000", 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.711e-06 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397729+0000", 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001098705 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398828+0000", 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5500000000000002e-06 2023-10-09T10:07:42.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398834+0000", 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0560779309999999 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454912+0000", 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012238900000000001 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.455034+0000", 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015768999999999999 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.455192+0000", 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010377422000000001 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465569+0000", 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037032409999999999 2023-10-09T10:07:42.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469272+0000", 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1624e-05 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469314+0000", 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3724000000000001e-05 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469388+0000", 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0716708749999999 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190227 7.8 7:129c86af:::benchmark_data_smithi012_221426_object82449:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.068204137000002, 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0717717900000001, 2023-10-09T10:07:42.598 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190227 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397731+0000", 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5820000000000001e-06 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397733+0000", 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8680000000000001e-06 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397737+0000", 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4420000000000001e-06 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397739+0000", 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011216900000000001 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398861+0000", 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4060000000000004e-06 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398866+0000", 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0599802039999999 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.458847+0000", 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9543000000000006e-05 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.458946+0000", 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5651999999999999e-05 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.459032+0000", 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010030136 2023-10-09T10:07:42.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469062+0000", 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036131999999999999 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469423+0000", 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0482000000000001e-05 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469454+0000", 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9725000000000002e-05 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469503+0000", 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0717717900000001 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190233 7.8 7:13f66138:::benchmark_data_smithi012_221426_object82455:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.068194351999999, 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.076501538, 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190233 2023-10-09T10:07:42.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4050000000000001e-06 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397743+0000", 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.63e-06 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397746+0000", 2023-10-09T10:07:42.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.255e-06 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397749+0000", 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011455700000000001 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398894+0000", 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3720000000000004e-06 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398899+0000", 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0633114340000001 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462211+0000", 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8008000000000001e-05 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462309+0000", 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013975599999999999 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462449+0000", 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068008950000000004 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469250+0000", 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048359240000000001 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474086+0000", 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7553999999999998e-05 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474143+0000", 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9965999999999998e-05 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474243+0000", 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.076501538 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190240 7.8 7:1bb20cb0:::benchmark_data_smithi012_221426_object82462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.068185362999998, 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0788701549999999, 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:07:42.613 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190240 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.508e-06 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:07:42.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397752+0000", 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5329999999999999e-06 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397755+0000", 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1100000000000001e-06 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397757+0000", 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001159413 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398917+0000", 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4650000000000002e-06 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398922+0000", 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.066675222 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:07:42.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465597+0000", 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5288000000000001e-05 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465683+0000", 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107876 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465791+0000", 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036041200000000002 2023-10-09T10:07:42.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469395+0000", 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071257839999999996 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476521+0000", 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2398999999999998e-05 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:07:42.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476563+0000", 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7669e-05 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476621+0000", 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0788701549999999 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:07:42.620 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:07:42.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:07:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: pgmap v648: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 0 B/s, 137 objects/s recovering 2023-10-09T10:07:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: failed to encode map e249 with expected crc 2023-10-09T10:07:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: Health check update: 1 osds down (OSD_DOWN) 2023-10-09T10:07:43.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3294378440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:43.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: failed to encode map e250 with expected crc 2023-10-09T10:07:43.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568] boot 2023-10-09T10:07:43.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: osdmap e250: 8 total, 7 up, 8 in 2023-10-09T10:07:43.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:07:43.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[152263]: osdmap e251: 8 total, 7 up, 8 in 2023-10-09T10:07:43.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: pgmap v648: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 0 B/s, 137 objects/s recovering 2023-10-09T10:07:43.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: failed to encode map e249 with expected crc 2023-10-09T10:07:43.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: Health check update: 1 osds down (OSD_DOWN) 2023-10-09T10:07:43.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3294378440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:43.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: failed to encode map e250 with expected crc 2023-10-09T10:07:43.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568] boot 2023-10-09T10:07:43.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: osdmap e250: 8 total, 7 up, 8 in 2023-10-09T10:07:43.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:07:43.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:42 smithi012 ceph-mon[147926]: osdmap e251: 8 total, 7 up, 8 in 2023-10-09T10:07:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: pgmap v648: 89 pgs: 2 active+clean+remapped, 8 undersized+peered, 36 active+undersized, 1 active+undersized+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 undersized+degraded+peered, 16 active+undersized+degraded, 21 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 15793/75228 objects degraded (20.994%); 7259/75228 objects misplaced (9.649%); 0 B/s, 137 objects/s recovering 2023-10-09T10:07:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: failed to encode map e249 with expected crc 2023-10-09T10:07:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: Health check update: 1 osds down (OSD_DOWN) 2023-10-09T10:07:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3294378440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: failed to encode map e250 with expected crc 2023-10-09T10:07:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: osd.3 [v2:172.21.15.12:6826/268556568,v1:172.21.15.12:6827/268556568] boot 2023-10-09T10:07:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: osdmap e250: 8 total, 7 up, 8 in 2023-10-09T10:07:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:07:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:42 smithi110 ceph-mon[136032]: osdmap e251: 8 total, 7 up, 8 in 2023-10-09T10:07:43.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:07:43.521 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:43.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:43.549 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:07:43.549 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:07:43.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:43.664 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:43.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:43.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:43.807 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:43.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:43.985 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:43.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:44.088 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[147926]: osdmap e252: 8 total, 7 up, 8 in 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.101 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.101 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:43 smithi012 ceph-mon[152263]: osdmap e252: 8 total, 7 up, 8 in 2023-10-09T10:07:44.117 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:44.154 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:44.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:44.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: failed to encode map e251 with expected crc 2023-10-09T10:07:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:43 smithi110 ceph-mon[136032]: osdmap e252: 8 total, 7 up, 8 in 2023-10-09T10:07:44.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:44.268 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:44.355 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:44.382 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:44.478 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:44.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:44.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:44.641 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:44.734 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:44.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:44.786 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:07:44.786 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:07:44.786 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:07:44.786 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:07:44.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:44.874 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: pgmap v651: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 1 active+clean+remapped, 7 undersized+peered, 29 active+undersized, 1 active+undersized+remapped, 3 undersized+degraded+peered, 12 active+undersized+degraded, 22 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 9484/75228 objects degraded (12.607%); 5177/75228 objects misplaced (6.882%); 5.2 MiB/s, 434 objects/s recovering 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 9484/75228 objects degraded (12.607%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:07:45.145 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:45.145 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:45.145 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[152263]: osdmap e253: 8 total, 7 up, 8 in 2023-10-09T10:07:45.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: pgmap v651: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 1 active+clean+remapped, 7 undersized+peered, 29 active+undersized, 1 active+undersized+remapped, 3 undersized+degraded+peered, 12 active+undersized+degraded, 22 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 9484/75228 objects degraded (12.607%); 5177/75228 objects misplaced (6.882%); 5.2 MiB/s, 434 objects/s recovering 2023-10-09T10:07:45.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 9484/75228 objects degraded (12.607%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:07:45.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:45.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:45.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:44 smithi012 ceph-mon[147926]: osdmap e253: 8 total, 7 up, 8 in 2023-10-09T10:07:45.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: pgmap v651: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 1 active+clean+remapped, 7 undersized+peered, 29 active+undersized, 1 active+undersized+remapped, 3 undersized+degraded+peered, 12 active+undersized+degraded, 22 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 9484/75228 objects degraded (12.607%); 5177/75228 objects misplaced (6.882%); 5.2 MiB/s, 434 objects/s recovering 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: failed to encode map e252 with expected crc 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 9484/75228 objects degraded (12.607%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:45.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:44 smithi110 ceph-mon[136032]: osdmap e253: 8 total, 7 up, 8 in 2023-10-09T10:07:45.550 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:07:46.006 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:46.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:46.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:46.194 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:46.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:45 smithi012 ceph-mon[147926]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:45 smithi012 ceph-mon[147926]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:45 smithi012 ceph-mon[147926]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.303 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:45 smithi012 ceph-mon[152263]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.303 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:45 smithi012 ceph-mon[152263]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:45 smithi012 ceph-mon[152263]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:45 smithi110 ceph-mon[136032]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:45 smithi110 ceph-mon[136032]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:45 smithi110 ceph-mon[136032]: failed to encode map e253 with expected crc 2023-10-09T10:07:46.703 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:46.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:46.764 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:46.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:46.779 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:07:46.779 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:07:46.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.779 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:191870 7.2 e253/251 7:47a56061:::benchmark_data_smithi012_221426_object84092:head v 253'12224, mlcod=228'10607)", 2023-10-09T10:07:46.779 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:46.743412+0000", 2023-10-09T10:07:46.779 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.025374066000000001, 2023-10-09T10:07:46.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025406499999999999, 2023-10-09T10:07:46.779 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.743412+0000", 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:46.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.743412+0000", 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.743412+0000", 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.743416+0000", 2023-10-09T10:07:46.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3830000000000002e-06 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.743422+0000", 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9179999999999999e-06 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:46.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.743426+0000", 2023-10-09T10:07:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3900000000000003e-06 2023-10-09T10:07:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.745918+0000", 2023-10-09T10:07:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024917519999999999 2023-10-09T10:07:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.745944+0000", 2023-10-09T10:07:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6043000000000001e-05 2023-10-09T10:07:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:191882 7.2 e253/251 7:49dd48c3:::benchmark_data_smithi012_221426_object84104:head v 253'12225, mlcod=228'10607)", 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:46.765089+0000", 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0036972760000000002, 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037995009999999998, 2023-10-09T10:07:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:07:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.765089+0000", 2023-10-09T10:07:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.765089+0000", 2023-10-09T10:07:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.765087+0000", 2023-10-09T10:07:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:07:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.765114+0000", 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6387e-05 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.765140+0000", 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6329000000000002e-05 2023-10-09T10:07:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.765150+0000", 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9380000000000004e-06 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766549+0000", 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013993440000000001 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766571+0000", 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1268999999999999e-05 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:191884 7.5 e253/252 7:a2c4640e:::benchmark_data_smithi012_221426_object84106:head v 253'11985, mlcod=253'11983)", 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:46.766377+0000", 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0024087739999999998, 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025739280000000001, 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:07:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766377+0000", 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766377+0000", 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766376+0000", 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766385+0000", 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9509999999999995e-06 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766404+0000", 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8963000000000001e-05 2023-10-09T10:07:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766413+0000", 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9630000000000004e-06 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766510+0000", 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6353000000000004e-05 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:07:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:46.766538+0000", 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8804999999999999e-05 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 3 2023-10-09T10:07:46.796 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:07:46.865 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:46.957 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:47.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:46 smithi012 ceph-mon[147926]: pgmap v654: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 4 undersized+peered, 30 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 24 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 29 op/s; 10613/74874 objects degraded (14.174%); 2082/74874 objects misplaced (2.781%); 7.8 MiB/s, 432 objects/s recovering 2023-10-09T10:07:47.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:46 smithi012 ceph-mon[152263]: pgmap v654: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 4 undersized+peered, 30 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 24 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 29 op/s; 10613/74874 objects degraded (14.174%); 2082/74874 objects misplaced (2.781%); 7.8 MiB/s, 432 objects/s recovering 2023-10-09T10:07:47.328 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:47.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:47.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:47.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:07:47.433 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:47.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:46 smithi110 ceph-mon[136032]: pgmap v654: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 4 undersized+peered, 30 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 24 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 29 op/s; 10613/74874 objects degraded (14.174%); 2082/74874 objects misplaced (2.781%); 7.8 MiB/s, 432 objects/s recovering 2023-10-09T10:07:47.547 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:47.642 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:47.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:47.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:47.864 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:47.938 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:47.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:48.037 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:07:48.056 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:48.084 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:48.124 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:48.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:48.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:48.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1897688850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:48.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:47 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:48.179 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:07:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:07:47] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.33.4" 2023-10-09T10:07:48.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1897688850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:48.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:47 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:48.230 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:48.266 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1897688850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:48.266 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:47 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:07:48.268 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:48.308 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:48.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:48.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:48.413 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:48.449 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:48.484 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:48.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:48.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:48.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:07:48.695 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[147926]: pgmap v655: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 23 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 36 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 10611/74874 objects degraded (14.172%); 2082/74874 objects misplaced (2.781%); 5.9 MiB/s, 325 objects/s recovering 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[147926]: osdmap e254: 8 total, 7 up, 8 in 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[147926]: failed to encode map e254 with expected crc 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[147926]: failed to encode map e254 with expected crc 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[152263]: pgmap v655: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 23 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 36 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 10611/74874 objects degraded (14.172%); 2082/74874 objects misplaced (2.781%); 5.9 MiB/s, 325 objects/s recovering 2023-10-09T10:07:48.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T10:07:48.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:48.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:07:48.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[152263]: osdmap e254: 8 total, 7 up, 8 in 2023-10-09T10:07:48.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[152263]: failed to encode map e254 with expected crc 2023-10-09T10:07:48.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:48 smithi012 ceph-mon[152263]: failed to encode map e254 with expected crc 2023-10-09T10:07:49.083 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:07:49.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:49.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:07:49.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:49.292 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:49.364 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:49.392 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:49.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:49.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:49.437 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:49.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:48 smithi110 ceph-mon[136032]: pgmap v655: 89 pgs: 1 remapped+peering, 2 unknown, 1 active+remapped+backfilling, 10 peering, 23 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 36 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 10611/74874 objects degraded (14.172%); 2082/74874 objects misplaced (2.781%); 5.9 MiB/s, 325 objects/s recovering 2023-10-09T10:07:49.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:48 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T10:07:49.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:48 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:48 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:07:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:48 smithi110 ceph-mon[136032]: osdmap e254: 8 total, 7 up, 8 in 2023-10-09T10:07:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:48 smithi110 ceph-mon[136032]: failed to encode map e254 with expected crc 2023-10-09T10:07:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:48 smithi110 ceph-mon[136032]: failed to encode map e254 with expected crc 2023-10-09T10:07:49.477 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:49.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:07:49.518 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:07:49.547 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:49.554 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:49.601 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:49.640 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:49.654 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:49.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:07:49.655 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:07:49.655 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:07:49.655 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:07:49.655 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:07:49.655 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:07:49.655 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:07:49.655 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:07:49.746 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:07:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:49.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:07:49.858 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:49.932 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:50.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3930463698' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:50.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3930463698' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:50.240 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:50.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:50.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:50.253 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:07:50.254 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:07:50.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:50.371 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3930463698' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:50.373 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:50.411 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:50.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:50.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:50.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:50.620 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:50.821 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:50.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:50.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:50.930 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:50.958 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:50.996 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:51.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:51.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:51.051 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [4] live_osds: [0, 1, 5, 6, 7, 2, 3] 2023-10-09T10:07:51.052 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:07:51.052 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T10:07:51.052 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:07:51.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:07:51.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:07:51.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:51 smithi012 ceph-mon[147926]: pgmap v657: 89 pgs: 24 active+undersized, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 189 op/s; 17993/71814 objects degraded (25.055%); 1514/71814 objects misplaced (2.108%); 5.9 MiB/s, 130 objects/s recovering 2023-10-09T10:07:51.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:51 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17993/71814 objects degraded (25.055%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:51.164 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:51 smithi012 ceph-mon[152263]: pgmap v657: 89 pgs: 24 active+undersized, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 189 op/s; 17993/71814 objects degraded (25.055%); 1514/71814 objects misplaced (2.108%); 5.9 MiB/s, 130 objects/s recovering 2023-10-09T10:07:51.164 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:51 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17993/71814 objects degraded (25.055%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:51.219 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:51.396 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:07:51.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:51.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:07:51.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:51 smithi110 ceph-mon[136032]: pgmap v657: 89 pgs: 24 active+undersized, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 189 op/s; 17993/71814 objects degraded (25.055%); 1514/71814 objects misplaced (2.108%); 5.9 MiB/s, 130 objects/s recovering 2023-10-09T10:07:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:51 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17993/71814 objects degraded (25.055%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:51.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:51.541 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:51.582 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:51.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:51.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:51.715 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:51.716 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:51.716 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:51.751 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:51.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:51.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:51.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:51.882 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:51.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:51.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:51.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:52.021 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:52.053 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:52.097 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:52.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:52.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:52.203 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:52.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:07:52.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:07:52.267 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:52.318 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:52.458 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:52.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:52.502 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:52.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:52.692 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:52.841 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:52.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:52.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:52.946 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:52.975 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:53.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:53.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:53.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:53.117 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:53.147 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:53.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:53 smithi012 ceph-mon[147926]: pgmap v658: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 17621/68082 objects degraded (25.882%); 1341/68082 objects misplaced (1.970%); 4.8 MiB/s, 146 objects/s recovering 2023-10-09T10:07:53.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:53 smithi012 ceph-mon[152263]: pgmap v658: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 17621/68082 objects degraded (25.882%); 1341/68082 objects misplaced (1.970%); 4.8 MiB/s, 146 objects/s recovering 2023-10-09T10:07:53.193 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:53.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:53.298 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:53.328 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:53 smithi110 ceph-mon[136032]: pgmap v658: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 17621/68082 objects degraded (25.882%); 1341/68082 objects misplaced (1.970%); 4.8 MiB/s, 146 objects/s recovering 2023-10-09T10:07:53.331 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:53.368 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:53.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:53.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:53.472 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:53.503 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:53.543 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:53.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:53.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:53.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:53.772 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:53.815 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:53.818 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:53.967 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:53.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:54.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:54.193 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:54.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:54 smithi012 ceph-mon[147926]: pgmap v659: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 17209/63015 objects degraded (27.309%); 1341/63015 objects misplaced (2.128%); 4.2 MiB/s, 103 objects/s recovering 2023-10-09T10:07:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:54 smithi012 ceph-mon[152263]: pgmap v659: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 17209/63015 objects degraded (27.309%); 1341/63015 objects misplaced (2.128%); 4.2 MiB/s, 103 objects/s recovering 2023-10-09T10:07:54.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:54 smithi110 ceph-mon[136032]: pgmap v659: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 17209/63015 objects degraded (27.309%); 1341/63015 objects misplaced (2.128%); 4.2 MiB/s, 103 objects/s recovering 2023-10-09T10:07:54.923 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:54.933 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:07:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:54.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:55.036 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:55.061 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:55.063 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:07:55.107 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:55.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:55.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:55.211 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:55.243 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:55.288 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:55.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:55.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:55.394 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:55.422 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:55 smithi110 ceph-mon[136032]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:07:55.425 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:55.426 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:55 smithi012 ceph-mon[152263]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:07:55.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:55 smithi012 ceph-mon[147926]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:07:55.466 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:55.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:55.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:55.569 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:55.599 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:55.649 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:55.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:55.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:55.753 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:55.784 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:55.826 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:55.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:55.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:56.039 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:56.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:56 smithi012 ceph-mon[152263]: pgmap v660: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 16733/59256 objects degraded (28.238%); 1314/59256 objects misplaced (2.217%); 3.5 MiB/s, 113 objects/s recovering 2023-10-09T10:07:56.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:56 smithi012 ceph-mon[147926]: pgmap v660: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 16733/59256 objects degraded (28.238%); 1314/59256 objects misplaced (2.217%); 3.5 MiB/s, 113 objects/s recovering 2023-10-09T10:07:56.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:56 smithi110 ceph-mon[136032]: pgmap v660: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 16733/59256 objects degraded (28.238%); 1314/59256 objects misplaced (2.217%); 3.5 MiB/s, 113 objects/s recovering 2023-10-09T10:07:56.848 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:56.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:56.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:57.065 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:57.250 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:57 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 16733/59256 objects degraded (28.238%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:57.251 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:57 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 16733/59256 objects degraded (28.238%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:57.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:57 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 16733/59256 objects degraded (28.238%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:07:57.481 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:57.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:57.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:57.588 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:07:57.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:57.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:07:57.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:57.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:07:57.762 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:57.789 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:57.810 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":659,"stamp":"2023-10-09T10:07:53.226739+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1375541176,"num_objects":21005,"num_object_clones":0,"num_object_copies":63015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17209,"num_objects_misplaced":1341,"num_objects_unfound":0,"num_objects_dirty":21005,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":194831,"num_write_kb":6900046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15249,"num_bytes_recovered":724575864,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137170,"ondisk_log_size":137170,"up":229,"acting":223,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7873108,"kb_used_data":5860820,"kb_used_omap":131,"kb_used_meta":2011708,"kb_avail":742088108,"statfs":{"total":767960285184,"available":759898222592,"internally_reserved":0,"allocated":6001479680,"data_stored":5995746390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":134805,"internal_metadata":2059989355},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,5,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-260702208,"num_objects":-3978,"num_object_clones":0,"num_object_copies":-11934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6569,"num_objects_misplaced":-741,"num_objects_unfound":0,"num_objects_dirty":-3978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3978,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":860,"num_bytes_recovered":37158912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.342802"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":85,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.894686+0000","last_change":"2023-10-09T10:07:09.393757+0000","last_active":"2023-10-09T10:07:44.894686+0000","last_peered":"2023-10-09T10:07:44.894686+0000","last_clean":"2023-10-09T10:07:07.387003+0000","last_became_active":"2023-10-09T10:07:09.393757+0000","last_became_peered":"2023-10-09T10:07:09.393757+0000","last_unstale":"2023-10-09T10:07:44.894686+0000","last_undegraded":"2023-10-09T10:07:44.894686+0000","last_fullsized":"2023-10-09T10:07:09.389827+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:25:07.677678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":201,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.884762+0000","last_change":"2023-10-09T10:07:42.902228+0000","last_active":"2023-10-09T10:07:44.884762+0000","last_peered":"2023-10-09T10:07:44.884762+0000","last_clean":"2023-10-09T10:02:14.927734+0000","last_became_active":"2023-10-09T10:07:42.902228+0000","last_became_peered":"2023-10-09T10:07:42.902228+0000","last_unstale":"2023-10-09T10:07:44.884762+0000","last_undegraded":"2023-10-09T10:07:44.884762+0000","last_fullsized":"2023-10-09T10:07:42.873504+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:51.844208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":246,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.894124+0000","last_change":"2023-10-09T10:07:42.918989+0000","last_active":"2023-10-09T10:07:44.894124+0000","last_peered":"2023-10-09T10:07:44.894124+0000","last_clean":"2023-10-09T10:06:24.355522+0000","last_became_active":"2023-10-09T10:07:42.918989+0000","last_became_peered":"2023-10-09T10:07:42.918989+0000","last_unstale":"2023-10-09T10:07:44.894124+0000","last_undegraded":"2023-10-09T10:07:44.894124+0000","last_fullsized":"2023-10-09T10:07:42.878012+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:43.530544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":118,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887824+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:07:44.887824+0000","last_peered":"2023-10-09T10:07:44.887824+0000","last_clean":"2023-10-09T10:07:44.887824+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:07:44.887824+0000","last_undegraded":"2023-10-09T10:07:44.887824+0000","last_fullsized":"2023-10-09T10:07:44.887824+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":224,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.883927+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:07:44.883927+0000","last_peered":"2023-10-09T10:07:44.883927+0000","last_clean":"2023-10-09T10:07:44.883927+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:07:44.883927+0000","last_undegraded":"2023-10-09T10:07:44.883927+0000","last_fullsized":"2023-10-09T10:07:44.883927+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887740+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:07:44.887740+0000","last_peered":"2023-10-09T10:07:44.887740+0000","last_clean":"2023-10-09T10:07:44.887740+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:07:44.887740+0000","last_undegraded":"2023-10-09T10:07:44.887740+0000","last_fullsized":"2023-10-09T10:07:44.887740+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":136,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.888818+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:07:44.888818+0000","last_peered":"2023-10-09T10:07:44.888818+0000","last_clean":"2023-10-09T10:07:44.888818+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:07:44.888818+0000","last_undegraded":"2023-10-09T10:07:44.888818+0000","last_fullsized":"2023-10-09T10:07:44.888818+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":205,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.883960+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:07:44.883960+0000","last_peered":"2023-10-09T10:07:44.883960+0000","last_clean":"2023-10-09T10:07:44.883960+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:07:44.883960+0000","last_undegraded":"2023-10-09T10:07:44.883960+0000","last_fullsized":"2023-10-09T10:07:44.883960+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887670+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:07:44.887670+0000","last_peered":"2023-10-09T10:07:44.887670+0000","last_clean":"2023-10-09T10:07:44.887670+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:07:44.887670+0000","last_undegraded":"2023-10-09T10:07:44.887670+0000","last_fullsized":"2023-10-09T10:07:44.887670+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":166,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.884852+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:07:44.884852+0000","last_peered":"2023-10-09T10:07:44.884852+0000","last_clean":"2023-10-09T10:07:44.884852+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:07:44.884852+0000","last_undegraded":"2023-10-09T10:07:44.884852+0000","last_fullsized":"2023-10-09T10:07:44.884852+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":283,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.891722+0000","last_change":"2023-10-09T10:07:42.909605+0000","last_active":"2023-10-09T10:07:44.891722+0000","last_peered":"2023-10-09T10:07:44.891722+0000","last_clean":"2023-10-09T10:06:24.003173+0000","last_became_active":"2023-10-09T10:07:42.909605+0000","last_became_peered":"2023-10-09T10:07:42.909605+0000","last_unstale":"2023-10-09T10:07:44.891722+0000","last_undegraded":"2023-10-09T10:07:42.886200+0000","last_fullsized":"2023-10-09T10:07:42.886089+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":247,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.894374+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:07:44.894374+0000","last_peered":"2023-10-09T10:07:44.894374+0000","last_clean":"2023-10-09T10:07:44.894374+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:07:44.894374+0000","last_undegraded":"2023-10-09T10:07:44.894374+0000","last_fullsized":"2023-10-09T10:07:44.894374+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":132,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.891486+0000","last_change":"2023-10-09T10:07:09.411171+0000","last_active":"2023-10-09T10:07:44.891486+0000","last_peered":"2023-10-09T10:07:44.891486+0000","last_clean":"2023-10-09T10:07:07.393013+0000","last_became_active":"2023-10-09T10:07:09.411171+0000","last_became_peered":"2023-10-09T10:07:09.411171+0000","last_unstale":"2023-10-09T10:07:44.891486+0000","last_undegraded":"2023-10-09T10:07:09.399156+0000","last_fullsized":"2023-10-09T10:07:09.399031+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.895010+0000","last_change":"2023-10-09T10:07:09.397735+0000","last_active":"2023-10-09T10:07:44.895010+0000","last_peered":"2023-10-09T10:07:44.895010+0000","last_clean":"2023-10-09T10:07:07.387050+0000","last_became_active":"2023-10-09T10:07:09.397735+0000","last_became_peered":"2023-10-09T10:07:09.397735+0000","last_unstale":"2023-10-09T10:07:44.895010+0000","last_undegraded":"2023-10-09T10:07:44.895010+0000","last_fullsized":"2023-10-09T10:07:09.389622+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:48.230014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":316,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.891234+0000","last_change":"2023-10-09T10:07:09.402936+0000","last_active":"2023-10-09T10:07:44.891234+0000","last_peered":"2023-10-09T10:07:44.891234+0000","last_clean":"2023-10-09T10:07:07.392168+0000","last_became_active":"2023-10-09T10:07:09.402936+0000","last_became_peered":"2023-10-09T10:07:09.402936+0000","last_unstale":"2023-10-09T10:07:44.891234+0000","last_undegraded":"2023-10-09T10:07:09.399715+0000","last_fullsized":"2023-10-09T10:07:09.399574+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":192,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.884391+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:07:44.884391+0000","last_peered":"2023-10-09T10:07:44.884391+0000","last_clean":"2023-10-09T10:07:44.884391+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:07:44.884391+0000","last_undegraded":"2023-10-09T10:07:44.884391+0000","last_fullsized":"2023-10-09T10:07:44.884391+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":118,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887180+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:07:44.887180+0000","last_peered":"2023-10-09T10:07:44.887180+0000","last_clean":"2023-10-09T10:07:44.887180+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:07:44.887180+0000","last_undegraded":"2023-10-09T10:07:44.887180+0000","last_fullsized":"2023-10-09T10:07:44.887180+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":87,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887605+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:07:44.887605+0000","last_peered":"2023-10-09T10:07:44.887605+0000","last_clean":"2023-10-09T10:07:44.887605+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:07:44.887605+0000","last_undegraded":"2023-10-09T10:07:44.887605+0000","last_fullsized":"2023-10-09T10:07:44.887605+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.891095+0000","last_change":"2023-10-09T10:07:09.436438+0000","last_active":"2023-10-09T10:07:44.891095+0000","last_peered":"2023-10-09T10:07:44.891095+0000","last_clean":"2023-10-09T10:07:07.392118+0000","last_became_active":"2023-10-09T10:07:09.436438+0000","last_became_peered":"2023-10-09T10:07:09.436438+0000","last_unstale":"2023-10-09T10:07:44.891095+0000","last_undegraded":"2023-10-09T10:07:44.891095+0000","last_fullsized":"2023-10-09T10:07:09.398990+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":118,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887672+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:07:44.887672+0000","last_peered":"2023-10-09T10:07:44.887672+0000","last_clean":"2023-10-09T10:07:44.887672+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:07:44.887672+0000","last_undegraded":"2023-10-09T10:07:44.887672+0000","last_fullsized":"2023-10-09T10:07:44.887672+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":141,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887166+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:07:44.887166+0000","last_peered":"2023-10-09T10:07:44.887166+0000","last_clean":"2023-10-09T10:07:44.887166+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:07:44.887166+0000","last_undegraded":"2023-10-09T10:07:44.887166+0000","last_fullsized":"2023-10-09T10:07:44.887166+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":205,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887963+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:07:44.887963+0000","last_peered":"2023-10-09T10:07:44.887963+0000","last_clean":"2023-10-09T10:07:44.887963+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:07:44.887963+0000","last_undegraded":"2023-10-09T10:07:44.887963+0000","last_fullsized":"2023-10-09T10:07:44.887963+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"254'12150","reported_seq":12306,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T10:07:49.341434+0000","last_change":"2023-10-09T10:03:30.261125+0000","last_active":"2023-10-09T10:07:49.341434+0000","last_peered":"2023-10-09T10:07:49.341434+0000","last_clean":"2023-10-09T10:07:49.341434+0000","last_became_active":"2023-10-09T10:03:30.095093+0000","last_became_peered":"2023-10-09T10:03:30.095093+0000","last_unstale":"2023-10-09T10:07:49.341434+0000","last_undegraded":"2023-10-09T10:07:49.341434+0000","last_fullsized":"2023-10-09T10:07:49.341434+0000","mapping_epoch":191,"log_start":"202'4800","ondisk_log_start":"202'4800","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":7350,"log_dups_size":0,"ondisk_log_size":7350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:57.462339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91095040,"num_objects":1390,"num_object_clones":0,"num_object_copies":4170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12150,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":2228224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":277,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.891593+0000","last_change":"2023-10-09T10:07:09.429113+0000","last_active":"2023-10-09T10:07:44.891593+0000","last_peered":"2023-10-09T10:07:44.891593+0000","last_clean":"2023-10-09T10:07:07.393372+0000","last_became_active":"2023-10-09T10:07:09.429113+0000","last_became_peered":"2023-10-09T10:07:09.429113+0000","last_unstale":"2023-10-09T10:07:44.891593+0000","last_undegraded":"2023-10-09T10:07:09.406381+0000","last_fullsized":"2023-10-09T10:07:09.406267+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":72,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887212+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:07:44.887212+0000","last_peered":"2023-10-09T10:07:44.887212+0000","last_clean":"2023-10-09T10:07:44.887212+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:07:44.887212+0000","last_undegraded":"2023-10-09T10:07:44.887212+0000","last_fullsized":"2023-10-09T10:07:44.887212+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"254'12256","reported_seq":15672,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:52.740359+0000","last_change":"2023-10-09T10:07:44.078776+0000","last_active":"2023-10-09T10:07:52.740359+0000","last_peered":"2023-10-09T10:07:52.740359+0000","last_clean":"2023-10-09T10:07:40.910518+0000","last_became_active":"2023-10-09T10:07:44.037558+0000","last_became_peered":"2023-10-09T10:07:44.037558+0000","last_unstale":"2023-10-09T10:07:52.740359+0000","last_undegraded":"2023-10-09T10:07:43.908119+0000","last_fullsized":"2023-10-09T10:07:43.885276+0000","mapping_epoch":251,"log_start":"196'3200","ondisk_log_start":"196'3200","created":187,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":9056,"log_dups_size":0,"ondisk_log_size":9056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1520,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12256,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1640,"num_bytes_recovered":107347968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"2,5","objects":1278}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":265,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.891391+0000","last_change":"2023-10-09T10:07:09.403669+0000","last_active":"2023-10-09T10:07:44.891391+0000","last_peered":"2023-10-09T10:07:44.891391+0000","last_clean":"2023-10-09T10:07:07.393162+0000","last_became_active":"2023-10-09T10:07:09.403669+0000","last_became_peered":"2023-10-09T10:07:09.403669+0000","last_unstale":"2023-10-09T10:07:44.891391+0000","last_undegraded":"2023-10-09T10:07:09.400223+0000","last_fullsized":"2023-10-09T10:07:09.400104+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":154,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.893990+0000","last_change":"2023-10-09T10:07:42.919300+0000","last_active":"2023-10-09T10:07:44.893990+0000","last_peered":"2023-10-09T10:07:44.893990+0000","last_clean":"2023-10-09T10:06:23.987360+0000","last_became_active":"2023-10-09T10:07:42.919300+0000","last_became_peered":"2023-10-09T10:07:42.919300+0000","last_unstale":"2023-10-09T10:07:44.893990+0000","last_undegraded":"2023-10-09T10:07:44.893990+0000","last_fullsized":"2023-10-09T10:07:42.878368+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:13.972983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"254'11979","reported_seq":12062,"reported_epoch":254,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:52.514968+0000","last_change":"2023-10-09T10:07:09.426130+0000","last_active":"2023-10-09T10:07:52.514968+0000","last_peered":"2023-10-09T10:07:52.514968+0000","last_clean":"2023-10-09T10:07:08.045160+0000","last_became_active":"2023-10-09T10:07:09.426130+0000","last_became_peered":"2023-10-09T10:07:09.426130+0000","last_unstale":"2023-10-09T10:07:52.514968+0000","last_undegraded":"2023-10-09T10:07:09.391836+0000","last_fullsized":"2023-10-09T10:07:09.391713+0000","mapping_epoch":239,"log_start":"196'3100","ondisk_log_start":"196'3100","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8879,"log_dups_size":0,"ondisk_log_size":8879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:09.468027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1257,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11979,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1257}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":76,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887665+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:07:44.887665+0000","last_peered":"2023-10-09T10:07:44.887665+0000","last_clean":"2023-10-09T10:07:44.887665+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:07:44.887665+0000","last_undegraded":"2023-10-09T10:07:44.887665+0000","last_fullsized":"2023-10-09T10:07:44.887665+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":160,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.894577+0000","last_change":"2023-10-09T10:01:36.025782+0000","last_active":"2023-10-09T10:07:44.894577+0000","last_peered":"2023-10-09T10:07:44.894577+0000","last_clean":"2023-10-09T10:07:44.894577+0000","last_became_active":"2023-10-09T10:01:36.025196+0000","last_became_peered":"2023-10-09T10:01:36.025196+0000","last_unstale":"2023-10-09T10:07:44.894577+0000","last_undegraded":"2023-10-09T10:07:44.894577+0000","last_fullsized":"2023-10-09T10:07:44.894577+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:04.486165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"254'12073","reported_seq":15339,"reported_epoch":254,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:07:49.942489+0000","last_change":"2023-10-09T10:07:44.088004+0000","last_active":"2023-10-09T10:07:49.942489+0000","last_peered":"2023-10-09T10:07:49.942489+0000","last_clean":"2023-10-09T10:06:01.928095+0000","last_became_active":"2023-10-09T10:07:44.043489+0000","last_became_peered":"2023-10-09T10:07:44.043489+0000","last_unstale":"2023-10-09T10:07:49.942489+0000","last_undegraded":"2023-10-09T10:07:43.899664+0000","last_fullsized":"2023-10-09T10:07:49.942489+0000","mapping_epoch":251,"log_start":"199'3710","ondisk_log_start":"199'3710","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8363,"log_dups_size":0,"ondisk_log_size":8363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12073,"num_write_kb":428288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1570,"num_bytes_recovered":102760448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,3],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,3,7","objects":1311}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887518+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:07:44.887518+0000","last_peered":"2023-10-09T10:07:44.887518+0000","last_clean":"2023-10-09T10:07:44.887518+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:07:44.887518+0000","last_undegraded":"2023-10-09T10:07:44.887518+0000","last_fullsized":"2023-10-09T10:07:44.887518+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":110,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.887530+0000","last_change":"2023-10-09T10:07:09.425767+0000","last_active":"2023-10-09T10:07:44.887530+0000","last_peered":"2023-10-09T10:07:44.887530+0000","last_clean":"2023-10-09T10:07:08.045540+0000","last_became_active":"2023-10-09T10:07:09.425767+0000","last_became_peered":"2023-10-09T10:07:09.425767+0000","last_unstale":"2023-10-09T10:07:44.887530+0000","last_undegraded":"2023-10-09T10:07:44.887530+0000","last_fullsized":"2023-10-09T10:07:09.391712+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:10.652258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"254'12195","reported_seq":12289,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:52.530445+0000","last_change":"2023-10-09T10:07:44.083753+0000","last_active":"2023-10-09T10:07:52.530445+0000","last_peered":"2023-10-09T10:07:52.530445+0000","last_clean":"2023-10-09T10:06:28.008285+0000","last_became_active":"2023-10-09T10:07:44.050959+0000","last_became_peered":"2023-10-09T10:07:44.050959+0000","last_unstale":"2023-10-09T10:07:52.530445+0000","last_undegraded":"2023-10-09T10:07:43.905318+0000","last_fullsized":"2023-10-09T10:07:43.894043+0000","mapping_epoch":251,"log_start":"196'3300","ondisk_log_start":"196'3300","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8895,"log_dups_size":0,"ondisk_log_size":8895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:53.936060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86048768,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1441,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12195,"num_write_kb":432256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1313}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":237,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.884577+0000","last_change":"2023-10-09T10:07:42.892015+0000","last_active":"2023-10-09T10:07:44.884577+0000","last_peered":"2023-10-09T10:07:44.884577+0000","last_clean":"2023-10-09T10:02:25.793729+0000","last_became_active":"2023-10-09T10:07:42.892015+0000","last_became_peered":"2023-10-09T10:07:42.892015+0000","last_unstale":"2023-10-09T10:07:44.884577+0000","last_undegraded":"2023-10-09T10:07:42.874036+0000","last_fullsized":"2023-10-09T10:07:42.873933+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:48.359868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":107,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.884567+0000","last_change":"2023-10-09T10:07:42.914892+0000","last_active":"2023-10-09T10:07:44.884567+0000","last_peered":"2023-10-09T10:07:44.884567+0000","last_clean":"2023-10-09T09:57:18.838926+0000","last_became_active":"2023-10-09T10:07:42.914892+0000","last_became_peered":"2023-10-09T10:07:42.914892+0000","last_unstale":"2023-10-09T10:07:44.884567+0000","last_undegraded":"2023-10-09T10:07:44.884567+0000","last_fullsized":"2023-10-09T10:07:42.874570+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:54.003754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"254'12050","reported_seq":25450,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T10:07:52.788411+0000","last_change":"2023-10-09T10:07:41.583207+0000","last_active":"2023-10-09T10:07:52.788411+0000","last_peered":"2023-10-09T10:07:52.788411+0000","last_clean":"2023-10-09T10:07:52.788411+0000","last_became_active":"2023-10-09T10:07:07.828343+0000","last_became_peered":"2023-10-09T10:07:07.828343+0000","last_unstale":"2023-10-09T10:07:52.788411+0000","last_undegraded":"2023-10-09T10:07:52.788411+0000","last_fullsized":"2023-10-09T10:07:52.788411+0000","mapping_epoch":235,"log_start":"196'2854","ondisk_log_start":"196'2854","created":187,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":9196,"log_dups_size":0,"ondisk_log_size":9196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80609280,"num_objects":1230,"num_object_clones":0,"num_object_copies":3690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12050,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6666,"num_bytes_recovered":199163904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.893928+0000","last_change":"2023-10-09T10:07:09.397808+0000","last_active":"2023-10-09T10:07:44.893928+0000","last_peered":"2023-10-09T10:07:44.893928+0000","last_clean":"2023-10-09T10:07:07.386593+0000","last_became_active":"2023-10-09T10:07:09.397808+0000","last_became_peered":"2023-10-09T10:07:09.397808+0000","last_unstale":"2023-10-09T10:07:44.893928+0000","last_undegraded":"2023-10-09T10:07:44.893928+0000","last_fullsized":"2023-10-09T10:07:09.389273+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:54.658288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":154,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.895929+0000","last_change":"2023-10-09T10:07:09.393727+0000","last_active":"2023-10-09T10:07:44.895929+0000","last_peered":"2023-10-09T10:07:44.895929+0000","last_clean":"2023-10-09T10:07:07.387694+0000","last_became_active":"2023-10-09T10:07:09.393727+0000","last_became_peered":"2023-10-09T10:07:09.393727+0000","last_unstale":"2023-10-09T10:07:44.895929+0000","last_undegraded":"2023-10-09T10:07:44.895929+0000","last_fullsized":"2023-10-09T10:07:09.389268+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:53.797374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"254'12040","reported_seq":12118,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T10:07:49.359749+0000","last_change":"2023-10-09T10:03:22.356577+0000","last_active":"2023-10-09T10:07:49.359749+0000","last_peered":"2023-10-09T10:07:49.359749+0000","last_clean":"2023-10-09T10:07:49.359749+0000","last_became_active":"2023-10-09T10:03:22.356253+0000","last_became_peered":"2023-10-09T10:03:22.356253+0000","last_unstale":"2023-10-09T10:07:49.359749+0000","last_undegraded":"2023-10-09T10:07:49.359749+0000","last_fullsized":"2023-10-09T10:07:49.359749+0000","mapping_epoch":187,"log_start":"202'4700","ondisk_log_start":"202'4700","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":7340,"log_dups_size":0,"ondisk_log_size":7340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:14.903436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88277016,"num_objects":1348,"num_object_clones":0,"num_object_copies":4044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1348,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12040,"num_write_kb":428353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":139,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.891061+0000","last_change":"2023-10-09T10:07:09.402891+0000","last_active":"2023-10-09T10:07:44.891061+0000","last_peered":"2023-10-09T10:07:44.891061+0000","last_clean":"2023-10-09T10:07:07.392089+0000","last_became_active":"2023-10-09T10:07:09.402891+0000","last_became_peered":"2023-10-09T10:07:09.402891+0000","last_unstale":"2023-10-09T10:07:44.891061+0000","last_undegraded":"2023-10-09T10:07:44.891061+0000","last_fullsized":"2023-10-09T10:07:09.399389+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.894811+0000","last_change":"2023-10-09T10:07:09.397638+0000","last_active":"2023-10-09T10:07:44.894811+0000","last_peered":"2023-10-09T10:07:44.894811+0000","last_clean":"2023-10-09T10:07:07.387413+0000","last_became_active":"2023-10-09T10:07:09.397638+0000","last_became_peered":"2023-10-09T10:07:09.397638+0000","last_unstale":"2023-10-09T10:07:44.894811+0000","last_undegraded":"2023-10-09T10:07:44.894811+0000","last_fullsized":"2023-10-09T10:07:09.389190+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:40.804777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.c","version":"254'12101","reported_seq":12184,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:49.801184+0000","last_change":"2023-10-09T10:07:44.101430+0000","last_active":"2023-10-09T10:07:49.801184+0000","last_peered":"2023-10-09T10:07:49.801184+0000","last_clean":"2023-10-09T10:06:27.947522+0000","last_became_active":"2023-10-09T10:07:44.051006+0000","last_became_peered":"2023-10-09T10:07:44.051006+0000","last_unstale":"2023-10-09T10:07:49.801184+0000","last_undegraded":"2023-10-09T10:07:43.905846+0000","last_fullsized":"2023-10-09T10:07:43.887979+0000","mapping_epoch":251,"log_start":"199'3500","ondisk_log_start":"199'3500","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8601,"log_dups_size":0,"ondisk_log_size":8601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87752704,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1502,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12101,"num_write_kb":430080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"6,7","objects":1339}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":206,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.884014+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:07:44.884014+0000","last_peered":"2023-10-09T10:07:44.884014+0000","last_clean":"2023-10-09T10:07:44.884014+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:07:44.884014+0000","last_undegraded":"2023-10-09T10:07:44.884014+0000","last_fullsized":"2023-10-09T10:07:44.884014+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":246,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.895028+0000","last_change":"2023-10-09T10:07:42.921152+0000","last_active":"2023-10-09T10:07:44.895028+0000","last_peered":"2023-10-09T10:07:44.895028+0000","last_clean":"2023-10-09T10:06:23.997188+0000","last_became_active":"2023-10-09T10:07:42.921152+0000","last_became_peered":"2023-10-09T10:07:42.921152+0000","last_unstale":"2023-10-09T10:07:44.895028+0000","last_undegraded":"2023-10-09T10:07:44.895028+0000","last_fullsized":"2023-10-09T10:07:42.897813+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:12.449092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"254'12071","reported_seq":12858,"reported_epoch":254,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:49.805465+0000","last_change":"2023-10-09T10:07:44.126913+0000","last_active":"2023-10-09T10:07:49.805465+0000","last_peered":"2023-10-09T10:07:49.805465+0000","last_clean":"2023-10-09T10:06:27.961965+0000","last_became_active":"2023-10-09T10:07:44.059164+0000","last_became_peered":"2023-10-09T10:07:44.059164+0000","last_unstale":"2023-10-09T10:07:49.805465+0000","last_undegraded":"2023-10-09T10:07:43.899799+0000","last_fullsized":"2023-10-09T10:07:43.887927+0000","mapping_epoch":251,"log_start":"199'3931","ondisk_log_start":"199'3931","created":187,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92340224,"num_objects":1409,"num_object_clones":0,"num_object_copies":4227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1409,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12071,"num_write_kb":431360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":346,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[6,3],"avail_no_missing":["6"],"object_location_counts":[{"shards":"3,6","objects":1409}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":255,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.894648+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:07:44.894648+0000","last_peered":"2023-10-09T10:07:44.894648+0000","last_clean":"2023-10-09T10:07:44.894648+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:07:44.894648+0000","last_undegraded":"2023-10-09T10:07:44.894648+0000","last_fullsized":"2023-10-09T10:07:44.894648+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":96,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887542+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:07:44.887542+0000","last_peered":"2023-10-09T10:07:44.887542+0000","last_clean":"2023-10-09T10:07:44.887542+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:07:44.887542+0000","last_undegraded":"2023-10-09T10:07:44.887542+0000","last_fullsized":"2023-10-09T10:07:44.887542+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":202,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.894879+0000","last_change":"2023-10-09T10:07:09.396005+0000","last_active":"2023-10-09T10:07:44.894879+0000","last_peered":"2023-10-09T10:07:44.894879+0000","last_clean":"2023-10-09T10:07:07.387078+0000","last_became_active":"2023-10-09T10:07:09.396005+0000","last_became_peered":"2023-10-09T10:07:09.396005+0000","last_unstale":"2023-10-09T10:07:44.894879+0000","last_undegraded":"2023-10-09T10:07:44.894879+0000","last_fullsized":"2023-10-09T10:07:09.390160+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:47.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"254'12103","reported_seq":12195,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:52.463136+0000","last_change":"2023-10-09T10:07:44.083745+0000","last_active":"2023-10-09T10:07:52.463136+0000","last_peered":"2023-10-09T10:07:52.463136+0000","last_clean":"2023-10-09T10:06:28.012041+0000","last_became_active":"2023-10-09T10:07:44.051088+0000","last_became_peered":"2023-10-09T10:07:44.051088+0000","last_unstale":"2023-10-09T10:07:52.463136+0000","last_undegraded":"2023-10-09T10:07:43.899758+0000","last_fullsized":"2023-10-09T10:07:43.890625+0000","mapping_epoch":251,"log_start":"196'3200","ondisk_log_start":"196'3200","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8903,"log_dups_size":0,"ondisk_log_size":8903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:18.646652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79626240,"num_objects":1215,"num_object_clones":0,"num_object_copies":3645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1556,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12103,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":1215}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":72,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887283+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:07:44.887283+0000","last_peered":"2023-10-09T10:07:44.887283+0000","last_clean":"2023-10-09T10:07:44.887283+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:07:44.887283+0000","last_undegraded":"2023-10-09T10:07:44.887283+0000","last_fullsized":"2023-10-09T10:07:44.887283+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.887293+0000","last_change":"2023-10-09T10:07:09.427463+0000","last_active":"2023-10-09T10:07:44.887293+0000","last_peered":"2023-10-09T10:07:44.887293+0000","last_clean":"2023-10-09T10:07:08.045430+0000","last_became_active":"2023-10-09T10:07:09.427463+0000","last_became_peered":"2023-10-09T10:07:09.427463+0000","last_unstale":"2023-10-09T10:07:44.887293+0000","last_undegraded":"2023-10-09T10:07:44.887293+0000","last_fullsized":"2023-10-09T10:07:09.392460+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:40:45.061108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":278,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.894566+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:07:44.894566+0000","last_peered":"2023-10-09T10:07:44.894566+0000","last_clean":"2023-10-09T10:07:44.894566+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:07:44.894566+0000","last_undegraded":"2023-10-09T10:07:44.894566+0000","last_fullsized":"2023-10-09T10:07:44.894566+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"254'12112","reported_seq":12195,"reported_epoch":254,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:52.441780+0000","last_change":"2023-10-09T10:07:09.425995+0000","last_active":"2023-10-09T10:07:52.441780+0000","last_peered":"2023-10-09T10:07:52.441780+0000","last_clean":"2023-10-09T10:07:08.045468+0000","last_became_active":"2023-10-09T10:07:09.425995+0000","last_became_peered":"2023-10-09T10:07:09.425995+0000","last_unstale":"2023-10-09T10:07:52.441780+0000","last_undegraded":"2023-10-09T10:07:09.391875+0000","last_fullsized":"2023-10-09T10:07:09.391753+0000","mapping_epoch":239,"log_start":"196'3200","ondisk_log_start":"196'3200","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8912,"log_dups_size":0,"ondisk_log_size":8912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:45.326680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83623936,"num_objects":1276,"num_object_clones":0,"num_object_copies":3828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12112,"num_write_kb":428416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1276}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":154,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:45.943932+0000","last_change":"2023-10-09T10:07:42.894058+0000","last_active":"2023-10-09T10:07:45.943932+0000","last_peered":"2023-10-09T10:07:45.943932+0000","last_clean":"2023-10-09T10:07:45.943932+0000","last_became_active":"2023-10-09T10:07:42.892616+0000","last_became_peered":"2023-10-09T10:07:42.892616+0000","last_unstale":"2023-10-09T10:07:45.943932+0000","last_undegraded":"2023-10-09T10:07:45.943932+0000","last_fullsized":"2023-10-09T10:07:45.943932+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:49.869459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":247,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.893370+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:07:44.893370+0000","last_peered":"2023-10-09T10:07:44.893370+0000","last_clean":"2023-10-09T10:07:44.893370+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:07:44.893370+0000","last_undegraded":"2023-10-09T10:07:44.893370+0000","last_fullsized":"2023-10-09T10:07:44.893370+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":268,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.894775+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:07:44.894775+0000","last_peered":"2023-10-09T10:07:44.894775+0000","last_clean":"2023-10-09T10:07:44.894775+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:07:44.894775+0000","last_undegraded":"2023-10-09T10:07:44.894775+0000","last_fullsized":"2023-10-09T10:07:44.894775+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"254'12311","reported_seq":12408,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:52.557510+0000","last_change":"2023-10-09T10:07:44.089503+0000","last_active":"2023-10-09T10:07:52.557510+0000","last_peered":"2023-10-09T10:07:52.557510+0000","last_clean":"2023-10-09T10:06:27.990211+0000","last_became_active":"2023-10-09T10:07:44.043635+0000","last_became_peered":"2023-10-09T10:07:44.043635+0000","last_unstale":"2023-10-09T10:07:52.557510+0000","last_undegraded":"2023-10-09T10:07:43.896776+0000","last_fullsized":"2023-10-09T10:07:43.885233+0000","mapping_epoch":251,"log_start":"198'3400","ondisk_log_start":"198'3400","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8911,"log_dups_size":0,"ondisk_log_size":8911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:57:34.048839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86835200,"num_objects":1325,"num_object_clones":0,"num_object_copies":3975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1512,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12311,"num_write_kb":436352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1325}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":81,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887428+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:07:44.887428+0000","last_peered":"2023-10-09T10:07:44.887428+0000","last_clean":"2023-10-09T10:07:44.887428+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:07:44.887428+0000","last_undegraded":"2023-10-09T10:07:44.887428+0000","last_fullsized":"2023-10-09T10:07:44.887428+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":500,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.891330+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:07:44.891330+0000","last_peered":"2023-10-09T10:07:44.891330+0000","last_clean":"2023-10-09T10:07:44.891330+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:07:44.891330+0000","last_undegraded":"2023-10-09T10:07:44.891330+0000","last_fullsized":"2023-10-09T10:07:44.891330+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":156,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.884124+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:07:44.884124+0000","last_peered":"2023-10-09T10:07:44.884124+0000","last_clean":"2023-10-09T10:07:44.884124+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:07:44.884124+0000","last_undegraded":"2023-10-09T10:07:44.884124+0000","last_fullsized":"2023-10-09T10:07:44.884124+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":160,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.891778+0000","last_change":"2023-10-09T10:07:09.409329+0000","last_active":"2023-10-09T10:07:44.891778+0000","last_peered":"2023-10-09T10:07:44.891778+0000","last_clean":"2023-10-09T10:07:07.393101+0000","last_became_active":"2023-10-09T10:07:09.409329+0000","last_became_peered":"2023-10-09T10:07:09.409329+0000","last_unstale":"2023-10-09T10:07:44.891778+0000","last_undegraded":"2023-10-09T10:07:44.891778+0000","last_fullsized":"2023-10-09T10:07:09.400016+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"254'12197","reported_seq":15523,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T10:07:52.511496+0000","last_change":"2023-10-09T10:07:44.939508+0000","last_active":"2023-10-09T10:07:52.511496+0000","last_peered":"2023-10-09T10:07:52.511496+0000","last_clean":"2023-10-09T10:07:52.511496+0000","last_became_active":"2023-10-09T10:07:44.892253+0000","last_became_peered":"2023-10-09T10:07:44.892253+0000","last_unstale":"2023-10-09T10:07:52.511496+0000","last_undegraded":"2023-10-09T10:07:52.511496+0000","last_fullsized":"2023-10-09T10:07:52.511496+0000","mapping_epoch":252,"log_start":"196'3300","ondisk_log_start":"196'3300","created":187,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8897,"log_dups_size":0,"ondisk_log_size":8897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:19.545031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85393408,"num_objects":1303,"num_object_clones":0,"num_object_copies":3909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12197,"num_write_kb":432000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1605,"num_bytes_recovered":105119744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":227,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.893893+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:07:44.893893+0000","last_peered":"2023-10-09T10:07:44.893893+0000","last_clean":"2023-10-09T10:07:44.893893+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:07:44.893893+0000","last_undegraded":"2023-10-09T10:07:44.893893+0000","last_fullsized":"2023-10-09T10:07:44.893893+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.887226+0000","last_change":"2023-10-09T10:07:42.922601+0000","last_active":"2023-10-09T10:07:44.887226+0000","last_peered":"2023-10-09T10:07:44.887226+0000","last_clean":"2023-10-09T10:06:23.997655+0000","last_became_active":"2023-10-09T10:07:42.922601+0000","last_became_peered":"2023-10-09T10:07:42.922601+0000","last_unstale":"2023-10-09T10:07:44.887226+0000","last_undegraded":"2023-10-09T10:07:44.887226+0000","last_fullsized":"2023-10-09T10:07:42.882695+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:15.362915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":139,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887819+0000","last_change":"2023-10-09T10:06:02.847914+0000","last_active":"2023-10-09T10:07:44.887819+0000","last_peered":"2023-10-09T10:07:44.887819+0000","last_clean":"2023-10-09T10:07:44.887819+0000","last_became_active":"2023-10-09T10:06:02.847614+0000","last_became_peered":"2023-10-09T10:06:02.847614+0000","last_unstale":"2023-10-09T10:07:44.887819+0000","last_undegraded":"2023-10-09T10:07:44.887819+0000","last_fullsized":"2023-10-09T10:07:44.887819+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:12.492752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"254'12541","reported_seq":12653,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:52.724010+0000","last_change":"2023-10-09T10:07:44.096960+0000","last_active":"2023-10-09T10:07:52.724010+0000","last_peered":"2023-10-09T10:07:52.724010+0000","last_clean":"2023-10-09T10:06:27.977268+0000","last_became_active":"2023-10-09T10:07:44.050114+0000","last_became_peered":"2023-10-09T10:07:44.050114+0000","last_unstale":"2023-10-09T10:07:52.724010+0000","last_undegraded":"2023-10-09T10:07:43.908038+0000","last_fullsized":"2023-10-09T10:07:43.885286+0000","mapping_epoch":251,"log_start":"194'2500","ondisk_log_start":"194'2500","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":10041,"log_dups_size":0,"ondisk_log_size":10041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83427328,"num_objects":1273,"num_object_clones":0,"num_object_copies":3819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1571,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12541,"num_write_kb":442048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":1376256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1273}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":264,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.891681+0000","last_change":"2023-10-09T10:07:09.428383+0000","last_active":"2023-10-09T10:07:44.891681+0000","last_peered":"2023-10-09T10:07:44.891681+0000","last_clean":"2023-10-09T10:07:07.392931+0000","last_became_active":"2023-10-09T10:07:09.428383+0000","last_became_peered":"2023-10-09T10:07:09.428383+0000","last_unstale":"2023-10-09T10:07:44.891681+0000","last_undegraded":"2023-10-09T10:07:44.891681+0000","last_fullsized":"2023-10-09T10:07:09.405689+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":266,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.894679+0000","last_change":"2023-10-09T10:06:02.844891+0000","last_active":"2023-10-09T10:07:44.894679+0000","last_peered":"2023-10-09T10:07:44.894679+0000","last_clean":"2023-10-09T10:07:44.894679+0000","last_became_active":"2023-10-09T10:06:02.844551+0000","last_became_peered":"2023-10-09T10:06:02.844551+0000","last_unstale":"2023-10-09T10:07:44.894679+0000","last_undegraded":"2023-10-09T10:07:44.894679+0000","last_fullsized":"2023-10-09T10:07:44.894679+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:22.872596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":121,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.887409+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:07:44.887409+0000","last_peered":"2023-10-09T10:07:44.887409+0000","last_clean":"2023-10-09T10:07:44.887409+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:07:44.887409+0000","last_undegraded":"2023-10-09T10:07:44.887409+0000","last_fullsized":"2023-10-09T10:07:44.887409+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"254'12073","reported_seq":15624,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:49.482300+0000","last_change":"2023-10-09T10:07:44.124523+0000","last_active":"2023-10-09T10:07:49.482300+0000","last_peered":"2023-10-09T10:07:49.482300+0000","last_clean":"2023-10-09T10:06:01.908229+0000","last_became_active":"2023-10-09T10:07:42.970659+0000","last_became_peered":"2023-10-09T10:07:42.970659+0000","last_unstale":"2023-10-09T10:07:49.482300+0000","last_undegraded":"2023-10-09T10:07:42.895706+0000","last_fullsized":"2023-10-09T10:07:42.884261+0000","mapping_epoch":250,"log_start":"199'3900","ondisk_log_start":"199'3900","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":8173,"log_dups_size":0,"ondisk_log_size":8173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:19.582459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87883776,"num_objects":1341,"num_object_clones":0,"num_object_copies":4023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1314,"num_objects_misplaced":1341,"num_objects_unfound":0,"num_objects_dirty":1341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12073,"num_write_kb":429248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1714,"num_bytes_recovered":101580800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":1341}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":215,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.884562+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:07:44.884562+0000","last_peered":"2023-10-09T10:07:44.884562+0000","last_clean":"2023-10-09T10:07:44.884562+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:07:44.884562+0000","last_undegraded":"2023-10-09T10:07:44.884562+0000","last_fullsized":"2023-10-09T10:07:44.884562+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":220,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.895242+0000","last_change":"2023-10-09T10:07:09.397565+0000","last_active":"2023-10-09T10:07:44.895242+0000","last_peered":"2023-10-09T10:07:44.895242+0000","last_clean":"2023-10-09T10:07:07.387521+0000","last_became_active":"2023-10-09T10:07:09.397565+0000","last_became_peered":"2023-10-09T10:07:09.397565+0000","last_unstale":"2023-10-09T10:07:44.895242+0000","last_undegraded":"2023-10-09T10:07:44.895242+0000","last_fullsized":"2023-10-09T10:07:09.389532+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:09.202106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":204,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.884042+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:07:44.884042+0000","last_peered":"2023-10-09T10:07:44.884042+0000","last_clean":"2023-10-09T10:07:44.884042+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:07:44.884042+0000","last_undegraded":"2023-10-09T10:07:44.884042+0000","last_fullsized":"2023-10-09T10:07:44.884042+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"254'12319","reported_seq":15611,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:07:49.407915+0000","last_change":"2023-10-09T10:07:44.071790+0000","last_active":"2023-10-09T10:07:49.407915+0000","last_peered":"2023-10-09T10:07:49.407915+0000","last_clean":"2023-10-09T10:07:40.909279+0000","last_became_active":"2023-10-09T10:07:44.037130+0000","last_became_peered":"2023-10-09T10:07:44.037130+0000","last_unstale":"2023-10-09T10:07:49.407915+0000","last_undegraded":"2023-10-09T10:07:43.893033+0000","last_fullsized":"2023-10-09T10:07:43.883028+0000","mapping_epoch":251,"log_start":"202'5000","ondisk_log_start":"202'5000","created":187,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:03:21.013158+0000","last_clean_scrub_stamp":"2023-10-09T10:03:21.013158+0000","objects_scrubbed":0,"log_size":7319,"log_dups_size":0,"ondisk_log_size":7319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:40.022767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89980928,"num_objects":1373,"num_object_clones":0,"num_object_copies":4119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1526,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12319,"num_write_kb":438144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1585,"num_bytes_recovered":103809024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":1373}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":162,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.893182+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:07:44.893182+0000","last_peered":"2023-10-09T10:07:44.893182+0000","last_clean":"2023-10-09T10:07:44.893182+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:07:44.893182+0000","last_undegraded":"2023-10-09T10:07:44.893182+0000","last_fullsized":"2023-10-09T10:07:44.893182+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":114,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.887933+0000","last_change":"2023-10-09T10:07:09.425880+0000","last_active":"2023-10-09T10:07:44.887933+0000","last_peered":"2023-10-09T10:07:44.887933+0000","last_clean":"2023-10-09T10:07:08.046018+0000","last_became_active":"2023-10-09T10:07:09.425880+0000","last_became_peered":"2023-10-09T10:07:09.425880+0000","last_unstale":"2023-10-09T10:07:44.887933+0000","last_undegraded":"2023-10-09T10:07:09.391905+0000","last_fullsized":"2023-10-09T10:07:09.391775+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:00.692636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":98,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.887269+0000","last_change":"2023-10-09T10:07:09.396554+0000","last_active":"2023-10-09T10:07:44.887269+0000","last_peered":"2023-10-09T10:07:44.887269+0000","last_clean":"2023-10-09T10:07:07.811442+0000","last_became_active":"2023-10-09T10:07:09.396554+0000","last_became_peered":"2023-10-09T10:07:09.396554+0000","last_unstale":"2023-10-09T10:07:44.887269+0000","last_undegraded":"2023-10-09T10:07:44.887269+0000","last_fullsized":"2023-10-09T10:07:09.394202+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":109,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.887187+0000","last_change":"2023-10-09T10:07:09.436176+0000","last_active":"2023-10-09T10:07:44.887187+0000","last_peered":"2023-10-09T10:07:44.887187+0000","last_clean":"2023-10-09T10:07:08.045223+0000","last_became_active":"2023-10-09T10:07:09.436176+0000","last_became_peered":"2023-10-09T10:07:09.436176+0000","last_unstale":"2023-10-09T10:07:44.887187+0000","last_undegraded":"2023-10-09T10:07:44.887187+0000","last_fullsized":"2023-10-09T10:07:09.392768+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:40.866936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":172,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.894744+0000","last_change":"2023-10-09T10:07:42.918751+0000","last_active":"2023-10-09T10:07:44.894744+0000","last_peered":"2023-10-09T10:07:44.894744+0000","last_clean":"2023-10-09T10:07:44.894744+0000","last_became_active":"2023-10-09T10:07:42.918411+0000","last_became_peered":"2023-10-09T10:07:42.918411+0000","last_unstale":"2023-10-09T10:07:44.894744+0000","last_undegraded":"2023-10-09T10:07:44.894744+0000","last_fullsized":"2023-10-09T10:07:44.894744+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:25.316704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":82,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.887482+0000","last_change":"2023-10-09T10:07:09.395988+0000","last_active":"2023-10-09T10:07:44.887482+0000","last_peered":"2023-10-09T10:07:44.887482+0000","last_clean":"2023-10-09T10:07:07.811715+0000","last_became_active":"2023-10-09T10:07:09.395988+0000","last_became_peered":"2023-10-09T10:07:09.395988+0000","last_unstale":"2023-10-09T10:07:44.887482+0000","last_undegraded":"2023-10-09T10:07:09.393624+0000","last_fullsized":"2023-10-09T10:07:09.393488+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":122,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.883996+0000","last_change":"2023-10-09T10:07:42.901799+0000","last_active":"2023-10-09T10:07:44.883996+0000","last_peered":"2023-10-09T10:07:44.883996+0000","last_clean":"2023-10-09T09:57:18.839774+0000","last_became_active":"2023-10-09T10:07:42.901799+0000","last_became_peered":"2023-10-09T10:07:42.901799+0000","last_unstale":"2023-10-09T10:07:44.883996+0000","last_undegraded":"2023-10-09T10:07:44.883996+0000","last_fullsized":"2023-10-09T10:07:42.873507+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.037795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":82,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:07:44.887763+0000","last_change":"2023-10-09T10:07:09.397569+0000","last_active":"2023-10-09T10:07:44.887763+0000","last_peered":"2023-10-09T10:07:44.887763+0000","last_clean":"2023-10-09T10:07:07.811597+0000","last_became_active":"2023-10-09T10:07:09.397569+0000","last_became_peered":"2023-10-09T10:07:09.397569+0000","last_unstale":"2023-10-09T10:07:44.887763+0000","last_undegraded":"2023-10-09T10:07:09.394453+0000","last_fullsized":"2023-10-09T10:07:09.393868+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":131,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.884480+0000","last_change":"2023-10-09T10:07:42.898283+0000","last_active":"2023-10-09T10:07:44.884480+0000","last_peered":"2023-10-09T10:07:44.884480+0000","last_clean":"2023-10-09T10:02:25.781720+0000","last_became_active":"2023-10-09T10:07:42.898283+0000","last_became_peered":"2023-10-09T10:07:42.898283+0000","last_unstale":"2023-10-09T10:07:44.884480+0000","last_undegraded":"2023-10-09T10:07:44.884480+0000","last_fullsized":"2023-10-09T10:07:42.876681+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:41.992676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":251,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.884730+0000","last_change":"2023-10-09T10:07:42.911606+0000","last_active":"2023-10-09T10:07:44.884730+0000","last_peered":"2023-10-09T10:07:44.884730+0000","last_clean":"2023-10-09T10:07:44.884730+0000","last_became_active":"2023-10-09T10:07:42.910643+0000","last_became_peered":"2023-10-09T10:07:42.910643+0000","last_unstale":"2023-10-09T10:07:44.884730+0000","last_undegraded":"2023-10-09T10:07:44.884730+0000","last_fullsized":"2023-10-09T10:07:44.884730+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:53:23.090704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":290,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.884986+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:07:44.884986+0000","last_peered":"2023-10-09T10:07:44.884986+0000","last_clean":"2023-10-09T10:07:44.884986+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:07:44.884986+0000","last_undegraded":"2023-10-09T10:07:44.884986+0000","last_fullsized":"2023-10-09T10:07:44.884986+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":169,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T10:07:44.888687+0000","last_change":"2023-10-09T10:06:02.854755+0000","last_active":"2023-10-09T10:07:44.888687+0000","last_peered":"2023-10-09T10:07:44.888687+0000","last_clean":"2023-10-09T10:07:44.888687+0000","last_became_active":"2023-10-09T10:06:02.847740+0000","last_became_peered":"2023-10-09T10:06:02.847740+0000","last_unstale":"2023-10-09T10:07:44.888687+0000","last_undegraded":"2023-10-09T10:07:44.888687+0000","last_fullsized":"2023-10-09T10:07:44.888687+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:44.016487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-10-09T10:07:44.893187+0000","last_change":"2023-10-09T10:07:09.397643+0000","last_active":"2023-10-09T10:07:44.893187+0000","last_peered":"2023-10-09T10:07:44.893187+0000","last_clean":"2023-10-09T10:07:07.386433+0000","last_became_active":"2023-10-09T10:07:09.397643+0000","last_became_peered":"2023-10-09T10:07:09.397643+0000","last_unstale":"2023-10-09T10:07:44.893187+0000","last_undegraded":"2023-10-09T10:07:44.893187+0000","last_fullsized":"2023-10-09T10:07:09.388598+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:03:16.144782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1374945304,"num_objects":20981,"num_object_clones":0,"num_object_copies":62943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17195,"num_objects_misplaced":1341,"num_objects_unfound":0,"num_objects_dirty":20981,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":194571,"num_write_kb":6897601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15184,"num_bytes_recovered":723386368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5933379584,"data_stored":5933367368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136976,"ondisk_log_size":136976,"up":43,"acting":37,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":82,"acting":82,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":8760,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9967,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":79,"acting":79,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430458,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846680,"kb_used_data":549456,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92898472,"statfs":{"total":95995035648,"available":95128035328,"internally_reserved":0,"allocated":562642944,"data_stored":562122122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":2.419,"15min":1.4810000000000001},"min":{"1min":0.316,"5min":0.25,"15min":0.20599999999999999},"max":{"1min":4.1929999999999996,"5min":50.591999999999999,"15min":50.591999999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.3440000000000001,"5min":2.5270000000000001,"15min":1.4279999999999999},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":6.5410000000000004,"5min":50.747,"15min":50.747},"last":0.28899999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:07:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":2.2010000000000001,"15min":1.351},"min":{"1min":0.33700000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":10.045,"5min":25.338000000000001,"15min":25.338000000000001},"last":0.317},{"interface":"front","average":{"1min":2.8799999999999999,"5min":1.538,"15min":1.143},"min":{"1min":0.31,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.417999999999999,"5min":11.417999999999999,"15min":12.996},"last":0.375}]},{"osd":2,"last update":"Mon Oct 9 10:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":2.1859999999999999,"15min":2.4220000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.218,"5min":7.952,"15min":7.952},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.583,"5min":2.1019999999999999,"15min":2.1880000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.0879999999999992,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.41799999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:06:03 2023","interfaces":[{"interface":"back","average":{"1min":1.599,"5min":3.742,"15min":1.8109999999999999},"min":{"1min":0.32600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9260000000000002,"5min":59.302999999999997,"15min":59.302999999999997},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.0310000000000001,"5min":3.1019999999999999,"15min":1.5920000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.8289999999999997,"5min":50.904000000000003,"15min":50.904000000000003},"last":2.5579999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.94499999999999995,"15min":0.72299999999999998},"min":{"1min":0.246,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.2679999999999998,"5min":5.7009999999999996,"15min":6.1689999999999996},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.103,"15min":0.76000000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.141,"5min":13.332000000000001,"15min":13.332000000000001},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.0369999999999999,"15min":0.82899999999999996},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.1699999999999999,"5min":6.0510000000000002,"15min":12.958},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.048,"15min":0.83199999999999996},"min":{"1min":0.34699999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.3559999999999999,"5min":5.8470000000000004,"15min":6.6120000000000001},"last":0.33000000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.1739999999999999,"15min":0.83899999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.3129999999999997,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.764,"5min":1.139,"15min":0.81899999999999995},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.1920000000000002,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":0.34000000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626686,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":827084,"kb_used_data":521668,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92918068,"statfs":{"total":95995035648,"available":95148101632,"internally_reserved":0,"allocated":534188032,"data_stored":533679061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:07:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.764,"15min":1.1970000000000001},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.222},"max":{"1min":7.4569999999999999,"5min":11.756,"15min":11.756},"last":0.255},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.722,"15min":1.1639999999999999},"min":{"1min":0.28899999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.2290000000000001,"5min":9.5760000000000005,"15min":12.439},"last":0.29599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:06:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6320000000000001,"5min":1.796,"15min":1.2949999999999999},"min":{"1min":0.249,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.9909999999999997,"5min":10.664,"15min":18.853000000000002},"last":3.4100000000000001},{"interface":"front","average":{"1min":2.407,"5min":2.2909999999999999,"15min":1.3480000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.7059999999999995,"5min":21.561,"15min":21.561},"last":0.41199999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.6619999999999999,"15min":1.8009999999999999},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.9140000000000001,"5min":7.6760000000000002,"15min":7.6760000000000002},"last":3.1640000000000001},{"interface":"front","average":{"1min":0.91100000000000003,"5min":2.9710000000000001,"15min":3.3149999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.117,"5min":10.26,"15min":10.26},"last":3.4470000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:06:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4710000000000001,"5min":2.048,"15min":1.3100000000000001},"min":{"1min":0.36399999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":11.744,"5min":11.744,"15min":12.337},"last":0.35299999999999998},{"interface":"front","average":{"1min":3.3769999999999998,"5min":1.9810000000000001,"15min":1.2509999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":11.789999999999999,"5min":11.789999999999999,"15min":11.789999999999999},"last":8.202}]},{"osd":4,"last update":"Mon Oct 9 10:06:27 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":0.80000000000000004,"15min":0.82699999999999996},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.23599999999999999},"max":{"1min":5.1269999999999998,"5min":6.6929999999999996,"15min":23.446000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.70099999999999996,"15min":0.69299999999999995},"min":{"1min":0.28299999999999997,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":2.5880000000000001,"5min":6.4269999999999996,"15min":8.2379999999999995},"last":0.32000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.78700000000000003,"15min":0.73299999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.7080000000000002,"5min":7.7080000000000002,"15min":14.09},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.4350000000000001,"5min":0.874,"15min":0.76300000000000001},"min":{"1min":0.27900000000000003,"5min":0.254,"15min":0.254},"max":{"1min":7.508,"5min":7.508,"15min":7.508},"last":0.33900000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.82599999999999996,"15min":0.76300000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.4849999999999999,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.375},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.86799999999999999,"15min":0.81899999999999995},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.5409999999999999,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.41699999999999998}]}]},{"osd":1,"up_from":84,"seq":360777252999,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1045808,"kb_used_data":738344,"kb_used_omap":9,"kb_used_meta":307382,"kb_avail":92699344,"statfs":{"total":95995035648,"available":94924128256,"internally_reserved":0,"allocated":756064256,"data_stored":755167009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":314760169},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:07:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4660000000000002,"5min":3.2040000000000002,"15min":1.7050000000000001},"min":{"1min":0.21199999999999999,"5min":0.192,"15min":0.185},"max":{"1min":14.135999999999999,"5min":14.140000000000001,"15min":16.324999999999999},"last":4.319},{"interface":"front","average":{"1min":2.7919999999999998,"5min":2.7639999999999998,"15min":1.587},"min":{"1min":0.28599999999999998,"5min":0.192,"15min":0.192},"max":{"1min":14.532999999999999,"5min":16.629000000000001,"15min":16.629000000000001},"last":4.1349999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:07:48 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":3.8759999999999999,"15min":4.0899999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.369,"5min":17.369,"15min":17.369},"last":16.442},{"interface":"front","average":{"1min":1.373,"5min":3.7610000000000001,"15min":4.1589999999999998},"min":{"1min":0.27300000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":7.8230000000000004,"5min":16.023,"15min":16.023},"last":3.774}]},{"osd":3,"last update":"Mon Oct 9 10:06:07 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":3.2130000000000001,"15min":1.792},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":16.949000000000002,"5min":19.225999999999999,"15min":25.129999999999999},"last":4.117},{"interface":"front","average":{"1min":3.1619999999999999,"5min":3.4500000000000002,"15min":1.829},"min":{"1min":0.26000000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":17.029,"5min":17.029,"15min":22.847999999999999},"last":16.507000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:06:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":2.3700000000000001,"15min":1.431},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":15.615,"5min":15.615,"15min":15.615},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.5219999999999998,"5min":2.1320000000000001,"15min":1.2749999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.864000000000001,"5min":16.053000000000001,"15min":16.053000000000001},"last":0.32500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:07:09 2023","interfaces":[{"interface":"back","average":{"1min":2.1230000000000002,"5min":2.0640000000000001,"15min":1.458},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":14.416,"5min":14.416,"15min":15.531000000000001},"last":4.056},{"interface":"front","average":{"1min":2.9990000000000001,"5min":2.867,"15min":1.7549999999999999},"min":{"1min":0.36699999999999999,"5min":0.308,"15min":0.30299999999999999},"max":{"1min":12.183,"5min":15.605,"15min":19.706},"last":16.277999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:07:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.3239999999999998,"15min":1.5389999999999999},"min":{"1min":0.35799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":14.215,"5min":14.246,"15min":16.193999999999999},"last":1.478},{"interface":"front","average":{"1min":2.1280000000000001,"5min":2.0960000000000001,"15min":1.5249999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":14.471,"5min":14.471,"15min":15.481999999999999},"last":4.0830000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:07:09 2023","interfaces":[{"interface":"back","average":{"1min":3.1920000000000002,"5min":2.7949999999999999,"15min":1.752},"min":{"1min":0.39400000000000002,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":12.148999999999999,"5min":15.638999999999999,"15min":15.638999999999999},"last":16.331},{"interface":"front","average":{"1min":2.9300000000000002,"5min":2.5049999999999999,"15min":1.5960000000000001},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.34799999999999998},"max":{"1min":14.242000000000001,"5min":14.281000000000001,"15min":16.100999999999999},"last":1.514}]}]},{"osd":0,"up_from":79,"seq":339302416526,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906044,"kb_used_data":669684,"kb_used_omap":28,"kb_used_meta":236323,"kb_avail":92839108,"statfs":{"total":95995035648,"available":95067246592,"internally_reserved":0,"allocated":685756416,"data_stored":684846572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":241995600},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:07:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":1.9350000000000001,"15min":1.252},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":6.8150000000000004,"5min":13.084,"15min":13.084},"last":0.184},{"interface":"front","average":{"1min":4.2750000000000004,"5min":3.5179999999999998,"15min":1.7649999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":22.178999999999998,"5min":22.178999999999998,"15min":22.178999999999998},"last":0.439}]},{"osd":2,"last update":"Mon Oct 9 10:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.0739999999999998,"15min":2.238},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.0270000000000001,"5min":6.516,"15min":6.516},"last":0.25},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.9260000000000002,"15min":3.3039999999999998},"min":{"1min":0.28499999999999998,"5min":0.248,"15min":0.248},"max":{"1min":4.2060000000000004,"5min":14.693,"15min":14.693},"last":0.28599999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:06:04 2023","interfaces":[{"interface":"back","average":{"1min":4.0270000000000001,"5min":4.6820000000000004,"15min":2.0899999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":16.649999999999999,"5min":59.524999999999999,"15min":59.524999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":5.5099999999999998,"5min":4.9560000000000004,"15min":2.2559999999999998},"min":{"1min":0.34499999999999997,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":16.437999999999999,"5min":56.609999999999999,"15min":56.609999999999999},"last":0.71799999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:06:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":1.26,"15min":1.0449999999999999},"min":{"1min":0.377,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.8659999999999997,"5min":9.8659999999999997,"15min":11.587999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":3.282,"5min":2.6200000000000001,"15min":1.5209999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.8789999999999996,"5min":14.653,"15min":14.653},"last":0.36599999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":2.6379999999999999,"15min":1.6100000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.2850000000000001,"5min":14.558,"15min":14.558},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.409,"15min":1.1919999999999999},"min":{"1min":0.314,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.7789999999999999,"5min":12.478,"15min":12.478},"last":0.41699999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":1.4870000000000001,"15min":1.306},"min":{"1min":0.33100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.6429999999999998,"5min":9.7260000000000009,"15min":11.699},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.526,"5min":2.6680000000000001,"15min":1.76},"min":{"1min":0.376,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.6050000000000004,"5min":14.384,"15min":15.108000000000001},"last":0.46400000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":2.9049999999999998,"15min":1.7549999999999999},"min":{"1min":0.37,"5min":0.34300000000000003,"15min":0.33900000000000002},"max":{"1min":5.3239999999999998,"5min":14.631,"15min":26.978000000000002},"last":0.624},{"interface":"front","average":{"1min":1.46,"5min":1.6319999999999999,"15min":1.4350000000000001},"min":{"1min":0.41899999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.8979999999999997,"5min":12.428000000000001,"15min":26.768000000000001},"last":0.48099999999999998}]}]},{"osd":2,"up_from":217,"seq":932007903259,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649444,"kb_used_data":536604,"kb_used_omap":83,"kb_used_meta":112620,"kb_avail":93095708,"statfs":{"total":95995035648,"available":95330004992,"internally_reserved":0,"allocated":549482496,"data_stored":548596275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":115323375},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.579,"15min":1.7050000000000001},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":6.8460000000000001,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":6.444},{"interface":"front","average":{"1min":1.6359999999999999,"5min":2.681,"15min":2.855},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.218,"5min":11.974,"15min":11.974},"last":0.42599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":2.2989999999999999,"15min":2.5579999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.234,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":6.1379999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":2.2919999999999998,"15min":2.5619999999999998},"min":{"1min":0.29399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.2210000000000001,"5min":12.016999999999999,"15min":12.016999999999999},"last":3.3290000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":6.6749999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:06:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.4670000000000001,"15min":1.4670000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.298,"5min":4.298,"15min":4.298},"last":0.439},{"interface":"front","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":0.48199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:07:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.3859999999999999,"15min":1.444},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.9180000000000001,"5min":6.9180000000000001,"15min":6.9180000000000001},"last":3.2869999999999999},{"interface":"front","average":{"1min":0.88,"5min":1.5840000000000001,"15min":1.702},"min":{"1min":0.33400000000000002,"5min":0.318,"15min":0.318},"max":{"1min":6.0170000000000003,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":0.48199999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.90500000000000003,"15min":0.93000000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.5049999999999999,"5min":4.5499999999999998,"15min":4.5499999999999998},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.613,"15min":1.7},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.992,"5min":6.992,"15min":6.992},"last":3.3780000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.88800000000000001,"15min":0.94499999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.367,"5min":4.5819999999999999,"15min":4.5819999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.4670000000000001,"15min":1.5289999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":6.96,"5min":6.96,"15min":6.96},"last":3.3439999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824003,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2148068,"kb_used_data":2005980,"kb_used_omap":10,"kb_used_meta":141941,"kb_avail":91597084,"statfs":{"total":95995035648,"available":93795414016,"internally_reserved":0,"allocated":2054123520,"data_stored":2053120882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":145347651},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":6.9790000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":13.18}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":13.411}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.1369999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.0970000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019133,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":809852,"kb_used_data":508532,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92935300,"statfs":{"total":95995035648,"available":95165747200,"internally_reserved":0,"allocated":520736768,"data_stored":520230337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:06:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":1.6639999999999999,"15min":1.0409999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":10.500999999999999,"5min":31.059000000000001,"15min":31.059000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.141,"5min":1.9419999999999999,"15min":1.1990000000000001},"min":{"1min":0.27100000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.2469999999999999,"5min":31.279,"15min":31.279},"last":0.39000000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.573,"15min":1.046},"min":{"1min":0.23899999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.9880000000000004,"5min":13.898999999999999,"15min":13.898999999999999},"last":0.316},{"interface":"front","average":{"1min":2.335,"5min":2.2210000000000001,"15min":1.383},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":5.9569999999999999,"5min":15.353,"15min":15.353},"last":0.42999999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:06:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":2.0550000000000002,"15min":2.0550000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":5.2430000000000003},"last":0.437},{"interface":"front","average":{"1min":2.21,"5min":2.21,"15min":2.21},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.9239999999999995,"5min":8.9239999999999995,"15min":8.9239999999999995},"last":0.33800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:05:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4820000000000002,"5min":3.46,"15min":1.6599999999999999},"min":{"1min":0.34699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.3829999999999991,"5min":67.156999999999996,"15min":67.156999999999996},"last":3.069},{"interface":"front","average":{"1min":2.0230000000000001,"5min":2.7490000000000001,"15min":1.4330000000000001},"min":{"1min":0.30599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.5780000000000003,"5min":70.238,"15min":70.238},"last":2.8860000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:06:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.018,"15min":0.71999999999999997},"min":{"1min":0.247,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":9.0129999999999999,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.2170000000000001,"5min":0.98499999999999999,"15min":0.752},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.581,"5min":9.3109999999999999,"15min":14.773},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:06:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.877,"15min":0.63700000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":2.831,"5min":9.4079999999999995,"15min":9.4079999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.0920000000000001,"15min":0.79900000000000004},"min":{"1min":0.32300000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.0999999999999996,"5min":8.9849999999999994,"15min":9.0109999999999992},"last":0.50700000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.999,"15min":0.82099999999999995},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":4.5170000000000003,"5min":4.5170000000000003,"15min":14.81},"last":0.495},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.83499999999999996,"15min":0.66600000000000004},"min":{"1min":0.36099999999999999,"5min":0.32800000000000001,"15min":0.32200000000000001},"max":{"1min":1.863,"5min":4.0350000000000001,"15min":4.0350000000000001},"last":0.46300000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822914,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640128,"kb_used_data":330552,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93105024,"statfs":{"total":95995035648,"available":95339544576,"internally_reserved":0,"allocated":338485248,"data_stored":337984132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:07:28 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.722,"15min":1.2889999999999999},"min":{"1min":0.248,"5min":0.218,"15min":0.218},"max":{"1min":6.3760000000000003,"5min":10.92,"15min":15.773999999999999},"last":4.1509999999999998},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.748,"15min":1.333},"min":{"1min":0.26800000000000002,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":6.8719999999999999,"5min":13.552,"15min":15.699},"last":1.8200000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:07:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.3999999999999999,"15min":1.2330000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.248},"max":{"1min":5.6970000000000001,"5min":5.6970000000000001,"15min":15.689},"last":3.3959999999999999},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.9370000000000001,"15min":1.359},"min":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.6379999999999999,"5min":7.6379999999999999,"15min":15.896000000000001},"last":9.0039999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:07:47 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":2.4769999999999999,"15min":2.6469999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.5589999999999993,"5min":15.640000000000001,"15min":15.640000000000001},"last":2.859},{"interface":"front","average":{"1min":1.0860000000000001,"5min":2.242,"15min":2.4350000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.1329999999999991,"5min":10.656000000000001,"15min":10.656000000000001},"last":5.2910000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:06:08 2023","interfaces":[{"interface":"back","average":{"1min":3.8180000000000001,"5min":2.5489999999999999,"15min":1.506},"min":{"1min":0.30299999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":12.693,"5min":39.345999999999997,"15min":39.345999999999997},"last":3.6949999999999998},{"interface":"front","average":{"1min":2.9820000000000002,"5min":3.3199999999999998,"15min":1.774},"min":{"1min":0.36099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":15.843,"5min":38.906999999999996,"15min":38.906999999999996},"last":3.8260000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:06:08 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.92900000000000005,"15min":0.73499999999999999},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":4.2359999999999998,"5min":9.3040000000000003,"15min":12.304},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.754,"15min":0.77600000000000002},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.3210000000000002,"5min":6.9219999999999997,"15min":27.768000000000001},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:07:28 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.0600000000000001,"15min":0.90300000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":13.483000000000001,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.448,"5min":1.048,"15min":0.80900000000000005},"min":{"1min":0.28299999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":18.600999999999999,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.35099999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:06:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.0109999999999999,"15min":0.85099999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.5469999999999997,"5min":7.4290000000000003,"15min":10.334},"last":0.317},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.98699999999999999,"15min":0.88100000000000001},"min":{"1min":0.39700000000000002,"5min":0.32800000000000001,"15min":0.318},"max":{"1min":3.6190000000000002,"5min":8.0210000000000008,"15min":12.356},"last":0.33500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":665391104,"data_stored":665387032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":748949504,"data_stored":748945432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":539361280,"data_stored":539361280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2048655360,"data_stored":2048655360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":518586368,"data_stored":518586368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":329318400,"data_stored":329318400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":529731584,"data_stored":529727512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":553385984,"data_stored":553385984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:07:57.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-09T10:07:57.827 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:57.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:57.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:57.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:07:58.066 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:58.067 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:07:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:07:57] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.33.4" 2023-10-09T10:07:58.229 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:07:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:58.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:07:58.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:07:58.430 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:58.689 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:07:58.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:58.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:07:58.821 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:07:58.821 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:07:58.833 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:07:58.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:58.908 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:07:58.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:58.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:07:58.995 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:07:59.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:07:59.034 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:58 smithi012 ceph-mon[147926]: pgmap v661: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 16733/59256 objects degraded (28.238%); 1314/59256 objects misplaced (2.217%); 3.5 MiB/s, 112 objects/s recovering 2023-10-09T10:07:59.035 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/227719872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:59.035 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:59.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:58 smithi012 ceph-mon[152263]: pgmap v661: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 16733/59256 objects degraded (28.238%); 1314/59256 objects misplaced (2.217%); 3.5 MiB/s, 112 objects/s recovering 2023-10-09T10:07:59.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/227719872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:59.035 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:59.049 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:07:59.129 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:59.162 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:07:59.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:58 smithi110 ceph-mon[136032]: pgmap v661: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 16733/59256 objects degraded (28.238%); 1314/59256 objects misplaced (2.217%); 3.5 MiB/s, 112 objects/s recovering 2023-10-09T10:07:59.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/227719872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:59.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:07:59.209 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:07:59.266 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:07:59.334 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:07:59.357 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:07:59.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:59.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:07:59.387 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:07:59.423 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:07:59.459 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:07:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:07:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:07:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:07:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:07:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:07:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:07:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:07:59.460 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:07:59.468 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:07:59.542 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:59.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:07:59.724 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:07:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:59.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:07:59.829 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:07:59.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:07:59.895 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:07:59.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:07:59.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:07:59.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:07:59.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:07:59.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[147926]: osdmap e255: 8 total, 7 up, 8 in 2023-10-09T10:07:59.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:07:59.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[147926]: failed to encode map e255 with expected crc 2023-10-09T10:07:59.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[147926]: failed to encode map e255 with expected crc 2023-10-09T10:07:59.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:07:59.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1792631687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:07:59.999 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:00.029 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:00.069 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:00.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:00.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:00.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:00.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:00.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:00.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[152263]: osdmap e255: 8 total, 7 up, 8 in 2023-10-09T10:08:00.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:00.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[152263]: failed to encode map e255 with expected crc 2023-10-09T10:08:00.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[152263]: failed to encode map e255 with expected crc 2023-10-09T10:08:00.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:00.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:07:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1792631687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:59 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:59 smithi110 ceph-mon[136032]: osdmap e255: 8 total, 7 up, 8 in 2023-10-09T10:08:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:59 smithi110 ceph-mon[136032]: failed to encode map e255 with expected crc 2023-10-09T10:08:00.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:59 smithi110 ceph-mon[136032]: failed to encode map e255 with expected crc 2023-10-09T10:08:00.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:00.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:07:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1792631687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:00.269 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:00.353 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:00.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:00.424 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:08:00.424 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:08:00.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:00.581 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:00.665 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:00.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:00.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:00.787 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:00.814 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:00.852 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:00.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:00.956 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:00.957 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:00.985 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:01.028 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:01.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:01.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:01.075 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:00 smithi012 ceph-mon[147926]: pgmap v663: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 16388/54084 objects degraded (30.301%); 1314/54084 objects misplaced (2.430%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:01.075 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:00 smithi012 ceph-mon[152263]: pgmap v663: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 16388/54084 objects degraded (30.301%); 1314/54084 objects misplaced (2.430%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:01.132 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:01.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:00 smithi110 ceph-mon[136032]: pgmap v663: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 16388/54084 objects degraded (30.301%); 1314/54084 objects misplaced (2.430%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:01.162 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:01.201 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:01.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:01.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:01.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:01.419 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:01.537 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:01.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:01.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:01.654 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:01.683 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:01.722 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:01.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:01.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:01.826 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:01.858 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:01.858 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T10:08:01.898 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:01.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:01.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:02.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/969338470' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:08:02.085 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:08:02.085 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/969338470' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:08:02.085 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:08:02.099 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:02.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:08:02.426 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:08:02.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/969338470' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:08:02.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:08:02.453 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:02.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:02.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:02.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:02.641 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T10:08:02.785 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:02.880 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:02.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:02.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:02.994 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:03.007 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:08:03.007 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:08:03.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.007 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201276 7.8 e255/251 7:1cd7026b:::benchmark_data_smithi012_221426_object93498:head v 255'12512, mlcod=228'10316)", 2023-10-09T10:08:03.007 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.987731+0000", 2023-10-09T10:08:03.007 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014152201999999999, 2023-10-09T10:08:03.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014183664, 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987731+0000", 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987731+0000", 2023-10-09T10:08:03.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987731+0000", 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987734+0000", 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6669999999999998e-06 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987740+0000", 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0190000000000003e-06 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987744+0000", 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1500000000000001e-06 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.991367+0000", 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003622372 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.991384+0000", 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7136e-05 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.011 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201282 7.c e255/251 7:3b86f144:::benchmark_data_smithi012_221426_object93504:head v 255'12574, mlcod=228'10405)", 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.987785+0000", 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014098489000000001, 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014199994000000001, 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987785+0000", 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987785+0000", 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987784+0000", 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987789+0000", 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1410000000000004e-06 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987796+0000", 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8669999999999996e-06 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987802+0000", 2023-10-09T10:08:03.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.181e-06 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987872+0000", 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0685999999999994e-05 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987898+0000", 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5358999999999999e-05 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.015 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201277 7.8 e255/251 7:19d76e64:::benchmark_data_smithi012_221426_object93499:head v 255'12513, mlcod=228'10316)", 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.989197+0000", 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.012686485000000001, 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012851710000000001, 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989197+0000", 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989197+0000", 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989196+0000", 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989201+0000", 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3079999999999998e-06 2023-10-09T10:08:03.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989212+0000", 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0648999999999999e-05 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989219+0000", 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5069999999999997e-06 2023-10-09T10:08:03.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.994847+0000", 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056283219999999998 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.994869+0000", 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2237000000000001e-05 2023-10-09T10:08:03.020 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201274 7.a e255/251 7:5f07f590:::benchmark_data_smithi012_221426_object93496:head v 255'12630, mlcod=228'10433)", 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.989373+0000", 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.012509928, 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012736768000000001, 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.021 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989373+0000", 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989373+0000", 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989373+0000", 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989378+0000", 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7699999999999998e-06 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989387+0000", 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4370000000000007e-06 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989392+0000", 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1599999999999997e-06 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.992661+0000", 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032695939999999998 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.992687+0000", 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5045000000000002e-05 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.025 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201275 7.a e255/251 7:54228d75:::benchmark_data_smithi012_221426_object93497:head v 255'12631, mlcod=228'10433)", 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.989397+0000", 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.012485794, 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012775824, 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.026 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989397+0000", 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989397+0000", 2023-10-09T10:08:03.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989397+0000", 2023-10-09T10:08:03.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:08:03.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989400+0000", 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7460000000000001e-06 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989403+0000", 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2940000000000001e-06 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.989406+0000", 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6699999999999998e-06 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.997953+0000", 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085476230000000007 2023-10-09T10:08:03.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.997974+0000", 2023-10-09T10:08:03.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0324000000000001e-05 2023-10-09T10:08:03.031 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.031 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.032 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.032 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201278 7.8 e255/251 7:13f5d4e3:::benchmark_data_smithi012_221426_object93500:head v 255'12514, mlcod=228'10316)", 2023-10-09T10:08:03.032 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.992392+0000", 2023-10-09T10:08:03.032 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0094915659999999999, 2023-10-09T10:08:03.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009842913, 2023-10-09T10:08:03.033 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.033 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.033 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.992392+0000", 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.992392+0000", 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.992390+0000", 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:08:03.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.992401+0000", 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0453e-05 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.992413+0000", 2023-10-09T10:08:03.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.259e-05 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.992423+0000", 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.234e-06 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.999401+0000", 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0069788250000000001 2023-10-09T10:08:03.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.999423+0000", 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1242000000000001e-05 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201267 7.7 e255/251 7:e6c6ff53:::benchmark_data_smithi012_221426_object93489:head v 255'12472, mlcod=228'10277)", 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.972863+0000", 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.029019764999999999, 2023-10-09T10:08:03.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029433282000000002, 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.972863+0000", 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.972863+0000", 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.972863+0000", 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.972869+0000", 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4579999999999998e-06 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.972883+0000", 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4143e-05 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.972890+0000", 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1500000000000002e-06 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.972958+0000", 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7674000000000003e-05 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.972981+0000", 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3113e-05 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.042 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201270 7.2 e255/251 7:4159b2e9:::benchmark_data_smithi012_221426_object93492:head v 255'12826, mlcod=228'10607)", 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.982729+0000", 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.019154423, 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019722284999999999, 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982729+0000", 2023-10-09T10:08:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982729+0000", 2023-10-09T10:08:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982728+0000", 2023-10-09T10:08:03.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:08:03.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982738+0000", 2023-10-09T10:08:03.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8379999999999997e-06 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982770+0000", 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2416999999999998e-05 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982781+0000", 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0983e-05 2023-10-09T10:08:03.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.983922+0000", 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011407310000000001 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.983943+0000", 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1619000000000001e-05 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.047 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201268 7.7 e255/251 7:efe36006:::benchmark_data_smithi012_221426_object93490:head v 255'12473, mlcod=228'10277)", 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.982852+0000", 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.019031501999999999, 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019659210999999999, 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982852+0000", 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982852+0000", 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982851+0000", 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982859+0000", 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8609999999999998e-06 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982871+0000", 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2401e-05 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982879+0000", 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5379999999999997e-06 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982950+0000", 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1134e-05 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.982978+0000", 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8357e-05 2023-10-09T10:08:03.051 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201273 7.7 e255/251 7:e6bd87bc:::benchmark_data_smithi012_221426_object93495:head v 255'12474, mlcod=228'10277)", 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.987659+0000", 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014223875, 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01490879, 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987659+0000", 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987659+0000", 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987658+0000", 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987669+0000", 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0637e-05 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987683+0000", 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3740000000000001e-05 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987692+0000", 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1430000000000004e-06 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.988178+0000", 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00048609100000000002 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.988199+0000", 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1132e-05 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.056 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:201272 7.6 e255/251 7:6607d8e1:::benchmark_data_smithi012_221426_object93494:head v 255'12693, mlcod=228'10474)", 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:02.987700+0000", 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014183408, 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014924876, 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:03.057 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987700+0000", 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987700+0000", 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987699+0000", 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987703+0000", 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9389999999999999e-06 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987709+0000", 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6590000000000004e-06 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.987713+0000", 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8990000000000003e-06 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:03.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.996844+0000", 2023-10-09T10:08:03.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091315170000000005 2023-10-09T10:08:03.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:03.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:03.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:03.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:02.996870+0000", 2023-10-09T10:08:03.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5221000000000001e-05 2023-10-09T10:08:03.061 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.062 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:03.062 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.062 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:03.062 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:08:03.062 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 11 2023-10-09T10:08:03.062 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:08:03.138 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:03 smithi012 ceph-mon[147926]: pgmap v664: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 15877/50217 objects degraded (31.617%); 1314/50217 objects misplaced (2.617%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 15877/50217 objects degraded (31.617%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:08:03.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:03 smithi012 ceph-mon[152263]: pgmap v664: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 15877/50217 objects degraded (31.617%); 1314/50217 objects misplaced (2.617%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:03.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 15877/50217 objects degraded (31.617%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:08:03.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:03 smithi110 ceph-mon[136032]: pgmap v664: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 15877/50217 objects degraded (31.617%); 1314/50217 objects misplaced (2.617%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:03.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 15877/50217 objects degraded (31.617%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:08:03.492 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:03.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:08:03.539 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:03.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:03.602 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:03.658 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:03.687 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:03.722 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:03.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:03.830 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:03.858 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:03.893 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:03.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:03.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:04.127 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:04.406 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:04.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:04.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:04.669 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:04.853 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:04.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:04.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:04.951 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:04.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:05.094 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:05.273 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:05.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:05.292 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:05 smithi012 ceph-mon[147926]: pgmap v665: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 948 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 15516/45552 objects degraded (34.062%); 1314/45552 objects misplaced (2.885%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:05.293 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:05 smithi012 ceph-mon[152263]: pgmap v665: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 948 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 15516/45552 objects degraded (34.062%); 1314/45552 objects misplaced (2.885%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:05.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:05 smithi110 ceph-mon[136032]: pgmap v665: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 948 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 15516/45552 objects degraded (34.062%); 1314/45552 objects misplaced (2.885%); 0 B/s, 56 objects/s recovering 2023-10-09T10:08:05.484 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:05.580 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:05.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:05.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:05.796 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:06.063 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:08:06.068 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:06.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:06.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:06.173 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:06.265 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:08:06.290 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:06.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/256415973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:06.300 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/256415973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:06.319 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:08:06.372 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:08:06.411 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:08:06.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/256415973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:06.481 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:08:06.531 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:08:06.589 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:08:06.590 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:08:06.590 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:08:06.590 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:08:06.590 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:08:06.590 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:08:06.590 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:08:06.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:08:06.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:08:06.611 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:06.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:06.718 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:06.746 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:06.784 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:06.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:06.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:06.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:06.955 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:07.049 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:07 smithi110 ceph-mon[136032]: pgmap v666: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 872 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 14945/41910 objects degraded (35.660%); 1314/41910 objects misplaced (3.135%); 0 B/s, 58 objects/s recovering 2023-10-09T10:08:07.049 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:07 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:07.049 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/256415973' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:08:07.049 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:07 smithi110 ceph-mon[136032]: osdmap e256: 8 total, 7 up, 8 in 2023-10-09T10:08:07.049 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:07 smithi110 ceph-mon[136032]: failed to encode map e256 with expected crc 2023-10-09T10:08:07.049 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:07 smithi110 ceph-mon[136032]: failed to encode map e256 with expected crc 2023-10-09T10:08:07.049 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/539558500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:07.125 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:08:07.128 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:08:07.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:07.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:07.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:07.257 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[152263]: pgmap v666: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 872 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 14945/41910 objects degraded (35.660%); 1314/41910 objects misplaced (3.135%); 0 B/s, 58 objects/s recovering 2023-10-09T10:08:07.257 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:07.257 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/256415973' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:08:07.257 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[152263]: osdmap e256: 8 total, 7 up, 8 in 2023-10-09T10:08:07.258 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[152263]: failed to encode map e256 with expected crc 2023-10-09T10:08:07.258 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[152263]: failed to encode map e256 with expected crc 2023-10-09T10:08:07.258 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/539558500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:07.258 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[147926]: pgmap v666: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 872 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 14945/41910 objects degraded (35.660%); 1314/41910 objects misplaced (3.135%); 0 B/s, 58 objects/s recovering 2023-10-09T10:08:07.258 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:07.258 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/256415973' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:08:07.258 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[147926]: osdmap e256: 8 total, 7 up, 8 in 2023-10-09T10:08:07.258 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[147926]: failed to encode map e256 with expected crc 2023-10-09T10:08:07.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[147926]: failed to encode map e256 with expected crc 2023-10-09T10:08:07.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/539558500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:07.339 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:07.523 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:07.526 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:07.526 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:07.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:07.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:07.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:07.642 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [4] live_osds: [0, 1, 5, 6, 7, 2, 3] 2023-10-09T10:08:07.642 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:08:07.642 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.5538844368872213 2023-10-09T10:08:07.642 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 0.5538844368872213 2023-10-09T10:08:07.699 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:07.794 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:07.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:07.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:07.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:08.002 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:08:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:08:07] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.33.4" 2023-10-09T10:08:08.018 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:08.249 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:08.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:08 smithi012 ceph-mon[152263]: pgmap v668: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 872 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 14945/41910 objects degraded (35.660%); 1314/41910 objects misplaced (3.135%); 0 B/s, 70 objects/s recovering 2023-10-09T10:08:08.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 14945/41910 objects degraded (35.660%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:08:08.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:08.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:08.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:08 smithi012 ceph-mon[147926]: pgmap v668: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 872 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 14945/41910 objects degraded (35.660%); 1314/41910 objects misplaced (3.135%); 0 B/s, 70 objects/s recovering 2023-10-09T10:08:08.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 14945/41910 objects degraded (35.660%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:08:08.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:08:08.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:08 smithi110 ceph-mon[136032]: pgmap v668: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 872 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 14945/41910 objects degraded (35.660%); 1314/41910 objects misplaced (3.135%); 0 B/s, 70 objects/s recovering 2023-10-09T10:08:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 14945/41910 objects degraded (35.660%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:08:08.507 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:08.624 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:08.624 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:08.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:08.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:08.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:08.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:08:08.840 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:09.025 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:09.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:09.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:09.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:08:09.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:09.169 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:09.210 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:09.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:09.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:09.314 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:09.340 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:09.376 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:09.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:09.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:09.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:09.535 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:09.597 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:09.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:09.658 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:09.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:09.739 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:09.791 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:09.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:09.870 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:09.896 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:09.921 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:09.953 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:09.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:09.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:10.056 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:10.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:10.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[147926]: osdmap e257: 8 total, 7 up, 8 in 2023-10-09T10:08:10.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[147926]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[147926]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[147926]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[147926]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[147926]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[147926]: pgmap v670: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 771 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 14624/37074 objects degraded (39.445%); 1314/37074 objects misplaced (3.544%); 0 B/s, 35 objects/s recovering 2023-10-09T10:08:10.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[152263]: osdmap e257: 8 total, 7 up, 8 in 2023-10-09T10:08:10.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[152263]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[152263]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[152263]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[152263]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[152263]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:10 smithi012 ceph-mon[152263]: pgmap v670: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 771 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 14624/37074 objects degraded (39.445%); 1314/37074 objects misplaced (3.544%); 0 B/s, 35 objects/s recovering 2023-10-09T10:08:10.118 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:10.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:10.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:10.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:10 smithi110 ceph-mon[136032]: osdmap e257: 8 total, 7 up, 8 in 2023-10-09T10:08:10.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:10 smithi110 ceph-mon[136032]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:10 smithi110 ceph-mon[136032]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:10 smithi110 ceph-mon[136032]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:10 smithi110 ceph-mon[136032]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:10 smithi110 ceph-mon[136032]: failed to encode map e257 with expected crc 2023-10-09T10:08:10.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:10 smithi110 ceph-mon[136032]: pgmap v670: 89 pgs: 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 771 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 14624/37074 objects degraded (39.445%); 1314/37074 objects misplaced (3.544%); 0 B/s, 35 objects/s recovering 2023-10-09T10:08:10.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:10.335 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:10.486 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:10.517 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:10.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:10.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:10.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:10.697 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:10.818 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:10.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:10.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:11.015 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:11.144 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.2 to 0.553884 (8dcb) 2023-10-09T10:08:11.346 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:11.349 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:11 smithi012 ceph-mon[147926]: failed to encode map e258 with expected crc 2023-10-09T10:08:11.349 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:11 smithi012 ceph-mon[147926]: failed to encode map e258 with expected crc 2023-10-09T10:08:11.349 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:11 smithi012 ceph-mon[147926]: osdmap e258: 8 total, 7 up, 8 in 2023-10-09T10:08:11.349 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/233783337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5538844368872213}]: dispatch 2023-10-09T10:08:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:11.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:11.351 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:11 smithi012 ceph-mon[152263]: failed to encode map e258 with expected crc 2023-10-09T10:08:11.352 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:11 smithi012 ceph-mon[152263]: failed to encode map e258 with expected crc 2023-10-09T10:08:11.352 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:11 smithi012 ceph-mon[152263]: osdmap e258: 8 total, 7 up, 8 in 2023-10-09T10:08:11.352 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/233783337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5538844368872213}]: dispatch 2023-10-09T10:08:11.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:11 smithi110 ceph-mon[136032]: failed to encode map e258 with expected crc 2023-10-09T10:08:11.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:11 smithi110 ceph-mon[136032]: failed to encode map e258 with expected crc 2023-10-09T10:08:11.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:11 smithi110 ceph-mon[136032]: osdmap e258: 8 total, 7 up, 8 in 2023-10-09T10:08:11.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/233783337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5538844368872213}]: dispatch 2023-10-09T10:08:11.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:11.535 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:11.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:11.923 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:11.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:12.003 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:12.003 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:12.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:12.174 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/233783337' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5538844368872213}]': finished 2023-10-09T10:08:12.174 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: osdmap e259: 8 total, 7 up, 8 in 2023-10-09T10:08:12.174 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.174 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.175 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.175 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.175 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.175 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: pgmap v673: 89 pgs: 24 active+undersized, 8 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 696 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 14084/33474 objects degraded (42.074%); 1314/33474 objects misplaced (3.925%); 0 B/s, 48 objects/s recovering 2023-10-09T10:08:12.175 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[147926]: osdmap e260: 8 total, 7 up, 8 in 2023-10-09T10:08:12.179 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:12.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/233783337' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5538844368872213}]': finished 2023-10-09T10:08:12.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: osdmap e259: 8 total, 7 up, 8 in 2023-10-09T10:08:12.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: pgmap v673: 89 pgs: 24 active+undersized, 8 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 696 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 14084/33474 objects degraded (42.074%); 1314/33474 objects misplaced (3.925%); 0 B/s, 48 objects/s recovering 2023-10-09T10:08:12.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:12 smithi012 ceph-mon[152263]: osdmap e260: 8 total, 7 up, 8 in 2023-10-09T10:08:12.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/233783337' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5538844368872213}]': finished 2023-10-09T10:08:12.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: osdmap e259: 8 total, 7 up, 8 in 2023-10-09T10:08:12.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: failed to encode map e259 with expected crc 2023-10-09T10:08:12.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: pgmap v673: 89 pgs: 24 active+undersized, 8 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 696 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 14084/33474 objects degraded (42.074%); 1314/33474 objects misplaced (3.925%); 0 B/s, 48 objects/s recovering 2023-10-09T10:08:12.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:12 smithi110 ceph-mon[136032]: osdmap e260: 8 total, 7 up, 8 in 2023-10-09T10:08:12.941 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:12.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:12.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:08:13.052 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:13.084 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:13.127 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:13.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:13.228 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:08:13.234 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:13.320 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 14084/33474 objects degraded (42.074%), 21 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/980363673' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:13 smithi110 ceph-mon[136032]: osdmap e261: 8 total, 7 up, 8 in 2023-10-09T10:08:13.468 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:13.477 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.477 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.477 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 14084/33474 objects degraded (42.074%), 21 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/980363673' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[147926]: osdmap e261: 8 total, 7 up, 8 in 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: failed to encode map e260 with expected crc 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 14084/33474 objects degraded (42.074%), 21 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/980363673' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:13.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:08:13.481 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:13.481 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:13 smithi012 ceph-mon[152263]: osdmap e261: 8 total, 7 up, 8 in 2023-10-09T10:08:13.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:13.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:13.650 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[147926]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[147926]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[147926]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[147926]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[147926]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[147926]: pgmap v676: 89 pgs: 1 active+recovering+undersized+remapped, 24 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 610 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 12226/29337 objects degraded (41.674%); 1314/29337 objects misplaced (4.479%); 138 B/s, 131 objects/s recovering 2023-10-09T10:08:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[152263]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[152263]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[152263]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[152263]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[152263]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[152263]: pgmap v676: 89 pgs: 1 active+recovering+undersized+remapped, 24 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 610 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 12226/29337 objects degraded (41.674%); 1314/29337 objects misplaced (4.479%); 138 B/s, 131 objects/s recovering 2023-10-09T10:08:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:14.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:14 smithi110 ceph-mon[136032]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:14 smithi110 ceph-mon[136032]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:14 smithi110 ceph-mon[136032]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:14 smithi110 ceph-mon[136032]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:14 smithi110 ceph-mon[136032]: failed to encode map e261 with expected crc 2023-10-09T10:08:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:14 smithi110 ceph-mon[136032]: pgmap v676: 89 pgs: 1 active+recovering+undersized+remapped, 24 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 610 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 12226/29337 objects degraded (41.674%); 1314/29337 objects misplaced (4.479%); 138 B/s, 131 objects/s recovering 2023-10-09T10:08:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:14.899 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:14.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:14.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:15.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:15.102 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:15.188 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:15.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:15.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:15.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:15.347 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:08:15.388 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:15.396 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:08:15.456 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:15.460 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:08:15.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:15.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:15.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1140696432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:15.472 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1140696432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:15.512 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:08:15.559 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:08:15.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:15.591 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:08:15.624 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:08:15.624 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:08:15.624 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:08:15.624 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:08:15.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:08:15.625 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:08:15.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:08:15.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:08:15.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:08:15.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:15.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1140696432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:15.722 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:16.037 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:16.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:16.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:16.067 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:08:16.067 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:08:16.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:16.241 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:16.360 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:16.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:16.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:16.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:16.569 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:16.652 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:16 smithi012 ceph-mon[152263]: pgmap v677: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 548 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 11890/26358 objects degraded (45.110%); 1314/26358 objects misplaced (4.985%); 107 B/s, 102 objects/s recovering 2023-10-09T10:08:16.657 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:16.658 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:16 smithi012 ceph-mon[147926]: pgmap v677: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 548 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 11890/26358 objects degraded (45.110%); 1314/26358 objects misplaced (4.985%); 107 B/s, 102 objects/s recovering 2023-10-09T10:08:16.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:16.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:16 smithi110 ceph-mon[136032]: pgmap v677: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 548 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 11890/26358 objects degraded (45.110%); 1314/26358 objects misplaced (4.985%); 107 B/s, 102 objects/s recovering 2023-10-09T10:08:16.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:16.803 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:16.848 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:16.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:16.951 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:16.955 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [4] live_osds: [0, 1, 5, 6, 7, 2, 3] 2023-10-09T10:08:16.955 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:08:16.957 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T10:08:16.957 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:08:16.957 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:08:16.957 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T10:08:16.980 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:17.023 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:17.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:17.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:17.214 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:17.310 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:17.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:17.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:17.444 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:17.482 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:17.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:17.512 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:08:17.589 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:17.650 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:17.656 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:08:17.656 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:08:17.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.656 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:209511 7.4 7:2e670c8e:::benchmark_data_smithi012_221426_object101733:head [delete] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:17.620102+0000", 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.019095814999999999, 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019133171000000001, 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 209511 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:17.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620102+0000", 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620102+0000", 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:17.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620100+0000", 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620111+0000", 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0754000000000001e-05 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620134+0000", 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3776000000000001e-05 2023-10-09T10:08:17.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620145+0000", 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0961999999999999e-05 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620209+0000", 2023-10-09T10:08:17.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3839999999999999e-05 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620391+0000", 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018167300000000001 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620541+0000", 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000150054 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625195+0000", 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046543629999999999 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:209513 7.f 7:f9c5a3c9:::benchmark_data_smithi012_221426_object101735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:17.620165+0000", 2023-10-09T10:08:17.662 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.019032731000000001, 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019184591000000001, 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 209513 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620165+0000", 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620165+0000", 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620164+0000", 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620169+0000", 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0350000000000003e-06 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:17.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620176+0000", 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3660000000000003e-06 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620181+0000", 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4790000000000004e-06 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620222+0000", 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0707999999999997e-05 2023-10-09T10:08:17.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620391+0000", 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016869699999999999 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.620541+0000", 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000150051 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:17.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625152+0000", 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046110480000000004 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632339+0000", 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071870889999999998 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:209510 7.e e261/235 7:72e65395:::benchmark_data_smithi012_221426_object101732:head v 261'12912, mlcod=261'12910)", 2023-10-09T10:08:17.668 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:17.625027+0000", 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014170708000000001, 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014424298, 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625027+0000", 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:17.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625027+0000", 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625027+0000", 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625036+0000", 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1689999999999998e-06 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625044+0000", 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1790000000000008e-06 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625050+0000", 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6910000000000002e-06 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:17.671 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625231+0000", 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000181058 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.625253+0000", 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2586999999999999e-05 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.672 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:209519 7.d e261/187 7:b444da69:::benchmark_data_smithi012_221426_object101741:head v 261'13032, mlcod=261'13030)", 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:17.632182+0000", 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0070158319999999996, 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073397979999999998, 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632182+0000", 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632182+0000", 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632181+0000", 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632188+0000", 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6100000000000002e-06 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632197+0000", 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8219999999999996e-06 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632203+0000", 2023-10-09T10:08:17.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3169999999999998e-06 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632477+0000", 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027366000000000001 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632495+0000", 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8213999999999999e-05 2023-10-09T10:08:17.676 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:209515 7.e e261/235 7:720cd7ea:::benchmark_data_smithi012_221426_object101737:head v 261'12913, mlcod=261'12910)", 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:17.632247+0000", 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0069502740000000002, 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007341981, 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:17.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632247+0000", 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632247+0000", 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:17.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632247+0000", 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632251+0000", 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9809999999999997e-06 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632260+0000", 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5939999999999994e-06 2023-10-09T10:08:17.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.632264+0000", 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6399999999999996e-06 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.637305+0000", 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050403380000000001 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:17.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.637330+0000", 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5242999999999999e-05 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:209516 7.f 7:fe86e8b4:::benchmark_data_smithi012_221426_object101738:head [delete] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:17.634421+0000", 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0047765070000000001, 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052391549999999997, 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:08:17.681 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 209516 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634421+0000", 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.682 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634421+0000", 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634419+0000", 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634428+0000", 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8510000000000005e-06 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634445+0000", 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6443000000000002e-05 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634454+0000", 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4080000000000004e-06 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634532+0000", 2023-10-09T10:08:17.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8009000000000005e-05 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634684+0000", 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151736 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634830+0000", 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000145711 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:17.685 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:209518 7.9 7:9bf7d13d:::benchmark_data_smithi012_221426_object101740:head [delete] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:17.634471+0000", 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0047266390000000004, 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052784820000000001, 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:17.686 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 209518 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634471+0000", 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634471+0000", 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634471+0000", 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634473+0000", 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8990000000000001e-06 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634479+0000", 2023-10-09T10:08:17.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9839999999999998e-06 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.634484+0000", 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8670000000000002e-06 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.636381+0000", 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018962009999999999 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.636527+0000", 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014695600000000001 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1", 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:17.636653+0000", 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012588399999999999 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:08:17.690 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:08:17.691 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 7 2023-10-09T10:08:17.691 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:08:17.691 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:18.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:08:18.119 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:18.130 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:08:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:08:17] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.33.4" 2023-10-09T10:08:18.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:18.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:18.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:08:18.232 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:18.273 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:18.422 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:18.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:18.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:18.664 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:18.678 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:18.747 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:18.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:18.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:18.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:18 smithi012 ceph-mon[147926]: pgmap v678: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 537 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 11704/25800 objects degraded (45.364%); 1314/25800 objects misplaced (5.093%); 90 B/s, 44 objects/s recovering 2023-10-09T10:08:18.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 11704/25800 objects degraded (45.364%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:18.948 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:19.012 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:08:19.013 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-10-09T10:08:19.013 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:08:19.017 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:19.125 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:18 smithi110 ceph-mon[136032]: pgmap v678: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 537 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 11704/25800 objects degraded (45.364%); 1314/25800 objects misplaced (5.093%); 90 B/s, 44 objects/s recovering 2023-10-09T10:08:19.125 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 11704/25800 objects degraded (45.364%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:19.175 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:18 smithi012 ceph-mon[152263]: pgmap v678: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 537 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 11704/25800 objects degraded (45.364%); 1314/25800 objects misplaced (5.093%); 90 B/s, 44 objects/s recovering 2023-10-09T10:08:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 11704/25800 objects degraded (45.364%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:19.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:19.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:19.436 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:19.877 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:19.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:19.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:19.979 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:19.997 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:20.029 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:20.078 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:20.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:20.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:20.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:20.268 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:20.400 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:20.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:20.509 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:20.542 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:20.581 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:20.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:20.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:20.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:20.717 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:20.758 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:20.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:20.862 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:20.892 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:20.929 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:20.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:21.034 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:21.065 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:21.102 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:21.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:21.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:21.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:20 smithi012 ceph-mon[147926]: pgmap v679: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 425 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11313/20442 objects degraded (55.342%); 1314/20442 objects misplaced (6.428%); 69 B/s, 64 objects/s recovering 2023-10-09T10:08:21.184 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:20 smithi012 ceph-mon[152263]: pgmap v679: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 425 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11313/20442 objects degraded (55.342%); 1314/20442 objects misplaced (6.428%); 69 B/s, 64 objects/s recovering 2023-10-09T10:08:21.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:20 smithi110 ceph-mon[136032]: pgmap v679: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 425 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11313/20442 objects degraded (55.342%); 1314/20442 objects misplaced (6.428%); 69 B/s, 64 objects/s recovering 2023-10-09T10:08:21.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:21.314 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:08:21.320 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:21.351 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:08:21.413 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:08:21.478 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:08:21.505 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:21.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:21.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:21.527 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:08:21.577 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:08:21.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:21.634 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:08:21.634 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:08:21.634 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:08:21.634 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:08:21.634 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:08:21.634 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:08:21.635 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:08:21.635 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:08:21.635 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:08:21.724 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:21.799 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:21.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:21.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:21.938 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:21.979 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:21.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:21.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:22.008 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:08:22.084 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:22.113 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:22.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:22.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:22.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2134053031' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2164715561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:21 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2134053031' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2164715561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:21 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:22.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2134053031' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:22.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2164715561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:22.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:21 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:22.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:22.282 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:08:22.283 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:08:22.286 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:22.329 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:22.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:22.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:22.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:22.510 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:22.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:22.616 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:22.645 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:22.681 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:22.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:22.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:22.711 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:22.711 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:22.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:22.901 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:23.154 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:23.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:23.238 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:08:23.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: pgmap v680: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 342 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 10762/16443 objects degraded (65.450%); 1314/16443 objects misplaced (7.991%); 60 B/s, 56 objects/s recovering 2023-10-09T10:08:23.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:23.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:08:23.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: osdmap e262: 8 total, 7 up, 8 in 2023-10-09T10:08:23.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 10762/16443 objects degraded (65.450%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: pgmap v680: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 342 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 10762/16443 objects degraded (65.450%); 1314/16443 objects misplaced (7.991%); 60 B/s, 56 objects/s recovering 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: osdmap e262: 8 total, 7 up, 8 in 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:22 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 10762/16443 objects degraded (65.450%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:23.272 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:23.298 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: pgmap v680: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 342 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 10762/16443 objects degraded (65.450%); 1314/16443 objects misplaced (7.991%); 60 B/s, 56 objects/s recovering 2023-10-09T10:08:23.298 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:23.298 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:08:23.299 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: osdmap e262: 8 total, 7 up, 8 in 2023-10-09T10:08:23.299 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.299 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.299 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.299 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: failed to encode map e262 with expected crc 2023-10-09T10:08:23.299 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:22 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 10762/16443 objects degraded (65.450%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:23.301 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:23.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:23.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:23.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:23.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:23.551 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:23.660 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:23.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:23.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:23.880 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:23.967 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:23.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:23.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:24.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:24.100 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:24.178 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:24.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:08:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:24.389 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:24.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:24.400 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [4] live_osds: [0, 1, 5, 6, 7, 2, 3] 2023-10-09T10:08:24.400 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:08:24.400 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-10-09T10:08:24.400 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-10-09T10:08:24.401 DEBUG:teuthology.orchestra.run.smithi110:osd.4> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T10:08:24.403 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl start ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T10:08:24.435 INFO:journalctl@ceph.osd.4.smithi110.stdout:-- Logs begin at Mon 2023-10-09 09:27:04 UTC. -- 2023-10-09T10:08:24.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:24.644 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:24.692 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:24 smithi110 systemd[1]: Starting Ceph osd.4 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:08:24.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:08:24.746 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:24.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:24.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:24.859 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:24.895 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:25.025 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:24 smithi012 ceph-mon[147926]: pgmap v682: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 239 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 10395/11502 objects degraded (90.376%); 1314/11502 objects misplaced (11.424%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:25.025 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:24 smithi012 ceph-mon[152263]: pgmap v682: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 239 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 10395/11502 objects degraded (90.376%); 1314/11502 objects misplaced (11.424%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:25.049 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:25.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:25.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:25.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:24 smithi110 ceph-mon[136032]: pgmap v682: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 239 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 10395/11502 objects degraded (90.376%); 1314/11502 objects misplaced (11.424%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:25.268 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:25.351 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:25.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:25.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:25.462 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:25.498 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:25.607 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:25.692 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:25 smithi110 bash[187519]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-09T10:08:25.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:25 smithi110 bash[187519]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-10-09T10:08:25.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:25 smithi110 bash[187519]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-10-09T10:08:25.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:25 smithi110 bash[187519]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T10:08:25.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:25 smithi110 bash[187519]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2023-10-09T10:08:25.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:25 smithi110 bash[187519]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-09T10:08:25.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:25 smithi110 bash[187519]: --> ceph-volume raw activate successful for osd ID: 4 2023-10-09T10:08:25.807 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:25.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:25.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:25.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:25.955 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:26.003 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:26.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:26.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:26.110 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:26.141 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:26 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:08:26.144 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:26.242 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:26.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:26.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:26 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:08:26.343 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:26 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:08:26.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:26.428 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:26 smithi110 podman[187949]: 2023-10-09T10:08:26.445 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:26.445 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:26.453 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:26.482 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:26.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:08:26.692 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:08:26.426+0000 7feb90c36700 1 -- 172.21.15.110:0/3355264836 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564e25e36680 con 0x564e25dfd400 2023-10-09T10:08:26.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:26 smithi110 bash[187949]: b460c3754de5c01ba28e0ed252abb697bd67e593abcef700509f25ce861a646c 2023-10-09T10:08:26.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:26 smithi110 systemd[1]: Started Ceph osd.4 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T10:08:27.215 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:27.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:27.227 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:27 smithi012 ceph-mon[147926]: pgmap v683: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 169 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 10018/8136 objects degraded (123.132%); 1314/8136 objects misplaced (16.150%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:27.227 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:27 smithi012 ceph-mon[152263]: pgmap v683: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 169 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 10018/8136 objects degraded (123.132%); 1314/8136 objects misplaced (16.150%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:27.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:27.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:08:27.317 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:27.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:27.443 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:08:26.947+0000 7feb986b1200 -1 Falling back to public interface 2023-10-09T10:08:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:27 smithi110 ceph-mon[136032]: pgmap v683: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 169 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 10018/8136 objects degraded (123.132%); 1314/8136 objects misplaced (16.150%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:27.529 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:27.671 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:27.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:27.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:27.784 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:27.900 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:27.952 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Removed 107775 objects 2023-10-09T10:08:27.952 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Clean up completed and total clean up time :208.959 2023-10-09T10:08:28.024 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:08:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:08:27] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.33.4" 2023-10-09T10:08:28.217 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:08:28.223 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:28.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:28.238 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:08:28.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:08:28.284 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:08:28.309 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:08:28.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:28.330 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:08:28.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 10018/8136 objects degraded (123.132%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:28.334 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 10018/8136 objects degraded (123.132%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:28.359 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:08:28.359 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:08:28.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:08:28.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:08:28.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:08:28.360 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T10:08:28.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:08:28.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:08:28.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:08:28.360 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:28.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 10018/8136 objects degraded (123.132%), 20 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:28.930 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:08:28.933 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:08:29.072 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:29.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:29.177 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:29.208 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:29 smithi012 ceph-mon[147926]: pgmap v684: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 156 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 9820/7542 objects degraded (130.204%); 1314/7542 objects misplaced (17.422%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:29.208 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/998427453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:29.208 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:29.208 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:29.209 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:29 smithi012 ceph-mon[152263]: pgmap v684: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 156 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 9820/7542 objects degraded (130.204%); 1314/7542 objects misplaced (17.422%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:29.209 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/998427453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:29.209 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:29.209 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:29.211 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:29.301 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:29.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:29.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:29.438 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:29.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:29 smithi110 ceph-mon[136032]: pgmap v684: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 156 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 9820/7542 objects degraded (130.204%); 1314/7542 objects misplaced (17.422%); 0 B/s, 46 objects/s recovering 2023-10-09T10:08:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/998427453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:29.775 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:29.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:29.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:29.910 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:30.050 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:08:30.052 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:30.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:30.162 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:30.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3717426316' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:30.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:30.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3717426316' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:30.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:30.224 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:30.283 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:08:30.283 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:08:30.284 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:08:30.284 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:08:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3717426316' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:30.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:30.670 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:30.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:30.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:30.752 INFO:teuthology.orchestra.run.smithi110.stderr:no valid command found; 10 closest matches: 2023-10-09T10:08:30.752 INFO:teuthology.orchestra.run.smithi110.stderr:0 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:1 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:2 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:abort 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:assert 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:bluefs debug_inject_read_zeros 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:bluefs files list 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:bluefs stats 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:bluestore allocator dump block 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:bluestore allocator fragmentation block 2023-10-09T10:08:30.753 INFO:teuthology.orchestra.run.smithi110.stderr:admin_socket: invalid command 2023-10-09T10:08:30.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:30.810 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:30.863 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:30.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:30.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:30.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:08:30.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:08:30.968 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:30.997 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:31.120 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:31.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:31.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:31 smithi110 ceph-mon[136032]: pgmap v685: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 50 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 9434/2436 objects degraded (387.274%); 1314/2436 objects misplaced (53.941%); 0 B/s, 45 objects/s recovering 2023-10-09T10:08:31.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:31 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:31.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:31 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:31.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:31 smithi110 ceph-mon[136032]: osdmap e263: 8 total, 7 up, 8 in 2023-10-09T10:08:31.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:31 smithi110 ceph-mon[136032]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:31 smithi110 ceph-mon[136032]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:31 smithi110 ceph-mon[136032]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:31 smithi110 ceph-mon[136032]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.170 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:08:31.170 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-10-09T10:08:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[147926]: pgmap v685: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 50 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 9434/2436 objects degraded (387.274%); 1314/2436 objects misplaced (53.941%); 0 B/s, 45 objects/s recovering 2023-10-09T10:08:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[147926]: osdmap e263: 8 total, 7 up, 8 in 2023-10-09T10:08:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[147926]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[147926]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[147926]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[147926]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[152263]: pgmap v685: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 23 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 50 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 9434/2436 objects degraded (387.274%); 1314/2436 objects misplaced (53.941%); 0 B/s, 45 objects/s recovering 2023-10-09T10:08:31.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:31.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:31.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[152263]: osdmap e263: 8 total, 7 up, 8 in 2023-10-09T10:08:31.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[152263]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[152263]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[152263]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:31 smithi012 ceph-mon[152263]: failed to encode map e263 with expected crc 2023-10-09T10:08:31.224 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:31.252 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:31.287 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:31.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:31.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:31.391 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:31.418 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:31.603 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:31.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:31.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:31.745 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:31.789 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:31.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:31.892 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:31.920 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:32.009 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:32.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:32.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:32.100 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:32.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:32.155 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:32 smithi110 ceph-mon[136032]: osdmap e264: 8 total, 7 up, 8 in 2023-10-09T10:08:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:32 smithi110 ceph-mon[136032]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:32 smithi110 ceph-mon[136032]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:32 smithi110 ceph-mon[136032]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:32 smithi110 ceph-mon[136032]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.242 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:32.260 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:32.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:32.325 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[147926]: osdmap e264: 8 total, 7 up, 8 in 2023-10-09T10:08:32.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[147926]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[147926]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[147926]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[147926]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[152263]: osdmap e264: 8 total, 7 up, 8 in 2023-10-09T10:08:32.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[152263]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[152263]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[152263]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.327 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:32 smithi012 ceph-mon[152263]: failed to encode map e264 with expected crc 2023-10-09T10:08:32.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:32.403 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:32.555 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:32.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:32.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:32.694 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:32.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:08:32.942 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:08:32.603+0000 7feb986b1200 -1 osd.4 238 log_to_monitors {default=true} 2023-10-09T10:08:32.987 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:32.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:32.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:33.091 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:33.120 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:33.154 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:33.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:33.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:33.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:33 smithi012 ceph-mon[147926]: pgmap v688: 89 pgs: 1 peering, 1 active+clean+remapped, 24 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 9146/264 objects degraded (3464.394%); 1314/264 objects misplaced (497.727%); 0 B/s, 29 objects/s recovering 2023-10-09T10:08:33.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:33 smithi012 ceph-mon[147926]: from='osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:08:33.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:33 smithi012 ceph-mon[147926]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:08:33.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 9146/264 objects degraded (3464.394%), 19 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:33.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:33 smithi012 ceph-mon[152263]: pgmap v688: 89 pgs: 1 peering, 1 active+clean+remapped, 24 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 9146/264 objects degraded (3464.394%); 1314/264 objects misplaced (497.727%); 0 B/s, 29 objects/s recovering 2023-10-09T10:08:33.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:33 smithi012 ceph-mon[152263]: from='osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:08:33.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:33 smithi012 ceph-mon[152263]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:08:33.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 9146/264 objects degraded (3464.394%), 19 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:33.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:33.290 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:33.427 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:33.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:33.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:33.442 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:08:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:08:33.068+0000 7feb893aa700 -1 osd.4 238 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:08:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:33 smithi110 ceph-mon[136032]: pgmap v688: 89 pgs: 1 peering, 1 active+clean+remapped, 24 active+undersized, 7 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 9146/264 objects degraded (3464.394%); 1314/264 objects misplaced (497.727%); 0 B/s, 29 objects/s recovering 2023-10-09T10:08:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:33 smithi110 ceph-mon[136032]: from='osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:08:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:33 smithi110 ceph-mon[136032]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:08:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 9146/264 objects degraded (3464.394%), 19 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:33.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:33.566 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:33.652 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:33.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:33.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:33.786 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:33.827 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:33.828 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:33.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:33.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:33.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:33.960 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T10:08:33.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T10:08:33.976 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:34.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[152263]: failed to encode map e265 with expected crc 2023-10-09T10:08:34.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[152263]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T10:08:34.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[152263]: osdmap e265: 8 total, 7 up, 8 in 2023-10-09T10:08:34.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[152263]: from='osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T10:08:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[152263]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T10:08:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[152263]: pgmap v690: 89 pgs: 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 26 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 7566/72 objects degraded (10508.333%); 1314/72 objects misplaced (1825.000%); 0 B/s, 204 objects/s recovering 2023-10-09T10:08:34.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[147926]: failed to encode map e265 with expected crc 2023-10-09T10:08:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[147926]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T10:08:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[147926]: osdmap e265: 8 total, 7 up, 8 in 2023-10-09T10:08:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[147926]: from='osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T10:08:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[147926]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T10:08:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:34.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:34 smithi012 ceph-mon[147926]: pgmap v690: 89 pgs: 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 26 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 7566/72 objects degraded (10508.333%); 1314/72 objects misplaced (1825.000%); 0 B/s, 204 objects/s recovering 2023-10-09T10:08:34.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:08:34.293 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:08:34.315 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:08:34.338 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:08:34.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:08:34.379 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:34.384 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:08:34.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:34.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:34.406 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:08:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:34 smithi110 ceph-mon[136032]: failed to encode map e265 with expected crc 2023-10-09T10:08:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:34 smithi110 ceph-mon[136032]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T10:08:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:34 smithi110 ceph-mon[136032]: osdmap e265: 8 total, 7 up, 8 in 2023-10-09T10:08:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:34 smithi110 ceph-mon[136032]: from='osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T10:08:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:34 smithi110 ceph-mon[136032]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2023-10-09T10:08:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:34 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:34.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:34 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:34.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:34 smithi110 ceph-mon[136032]: pgmap v690: 89 pgs: 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 26 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 44 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 7566/72 objects degraded (10508.333%); 1314/72 objects misplaced (1825.000%); 0 B/s, 204 objects/s recovering 2023-10-09T10:08:34.444 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:08:34.444 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:08:34.444 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:08:34.444 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:08:34.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:08:34.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:08:34.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:08:34.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:08:34.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:08:34.463 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:34.463 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:34.490 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:34.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:34.550 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:34.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:34.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:34.654 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:34.680 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:34.715 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:34.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:34.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:34.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:34.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:34.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:34.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:35.008 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:35.042 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:35.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:35.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:35.145 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:35.171 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.171 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.171 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:08:35.171 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686] boot 2023-10-09T10:08:35.171 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T10:08:35.171 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2911202236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:35.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:35 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:35.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:35.208 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:35.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:35.311 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:35.337 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:35.370 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:35.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:35.386 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686] boot 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2911202236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: osd.4 [v2:172.21.15.110:6800/881762686,v1:172.21.15.110:6801/881762686] boot 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.388 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:08:35.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: failed to encode map e265 with expected crc 2023-10-09T10:08:35.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: failed to encode map e266 with expected crc 2023-10-09T10:08:35.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2911202236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:35.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:35.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:35 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:35.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:35.504 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:35.590 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:35.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:08:35.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:35.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:35.692 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:35.697 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:35.724 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:35.757 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:35.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:35.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:35.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:35.896 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:35.941 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:35.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:36.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:36.076 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: failed to encode map e239 with expected crc 2023-10-09T10:08:36.076 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: failed to encode map e266 with expected crc 2023-10-09T10:08:36.076 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.076 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T10:08:36.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: pgmap v693: 89 pgs: 3 remapped+peering, 20 peering, 2 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 6054/72 objects degraded (8408.333%); 0 B/s, 261 objects/s recovering 2023-10-09T10:08:36.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:08:36.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:08:36.077 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:36 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:36.104 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:36.153 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:36.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:36.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:36.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:08:36.229 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: failed to encode map e239 with expected crc 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: failed to encode map e266 with expected crc 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: pgmap v693: 89 pgs: 3 remapped+peering, 20 peering, 2 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 6054/72 objects degraded (8408.333%); 0 B/s, 261 objects/s recovering 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:08:36.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: failed to encode map e239 with expected crc 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: failed to encode map e266 with expected crc 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: failed to encode map e267 with expected crc 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: pgmap v693: 89 pgs: 3 remapped+peering, 20 peering, 2 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 6054/72 objects degraded (8408.333%); 0 B/s, 261 objects/s recovering 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:08:36.231 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:36 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:36.263 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:36.308 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:36.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:36.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:36.445 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:36.472 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:36.508 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:36.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:36.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:36.645 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:36.687 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:36.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:36.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:36.742 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:36.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:36.824 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:36.825 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:36.889 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:36.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:36.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:36.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:37.030 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:37.099 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:08:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[147926]: failed to encode map e268 with expected crc 2023-10-09T10:08:37.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[147926]: failed to encode map e268 with expected crc 2023-10-09T10:08:37.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[147926]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T10:08:37.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3691655874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:37.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:37.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[152263]: failed to encode map e268 with expected crc 2023-10-09T10:08:37.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[152263]: failed to encode map e268 with expected crc 2023-10-09T10:08:37.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[152263]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T10:08:37.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3691655874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:37.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:37.358 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:37 smithi110 ceph-mon[136032]: failed to encode map e268 with expected crc 2023-10-09T10:08:37.358 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:37 smithi110 ceph-mon[136032]: failed to encode map e268 with expected crc 2023-10-09T10:08:37.358 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:37 smithi110 ceph-mon[136032]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T10:08:37.358 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3691655874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:37.358 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:08:37.375 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:37.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:37.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:37.481 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:37.513 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:37.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:08:37.918 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_1' removed 2023-10-09T10:08:38.029 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:38.029 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:08:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:08:37] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.33.4" 2023-10-09T10:08:38.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:38.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: pgmap v696: 89 pgs: 4 remapped+peering, 19 peering, 2 active+recovering+undersized+remapped, 7 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 50 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6053/72 objects degraded (8406.944%); 0 B/s, 27 objects/s recovering 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/531889220' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6054/72 objects degraded (8408.333%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/531889220' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[147926]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:08:38.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T10:08:38.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: pgmap v696: 89 pgs: 4 remapped+peering, 19 peering, 2 active+recovering+undersized+remapped, 7 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 50 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6053/72 objects degraded (8406.944%); 0 B/s, 27 objects/s recovering 2023-10-09T10:08:38.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/531889220' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:08:38.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6054/72 objects degraded (8408.333%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:38.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/531889220' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:08:38.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:38 smithi012 ceph-mon[152263]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T10:08:38.336 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:38.336 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: failed to encode map e269 with expected crc 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: pgmap v696: 89 pgs: 4 remapped+peering, 19 peering, 2 active+recovering+undersized+remapped, 7 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 50 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6053/72 objects degraded (8406.944%); 0 B/s, 27 objects/s recovering 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/531889220' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6054/72 objects degraded (8408.333%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/531889220' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:08:38.337 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:38 smithi110 ceph-mon[136032]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T10:08:38.352 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:38.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:38.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:38.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:38.498 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:38.624 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:38.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:38.678 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:08:38.679 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:08:38.679 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T10:08:38.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T10:08:38.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:38.754 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:38.907 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:38.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:38.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:39.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:39.047 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:39.150 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:39.153 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:39.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:39.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:39.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:39.283 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:39.418 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:39.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:39 smithi012 ceph-mon[152263]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:39 smithi012 ceph-mon[152263]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:39 smithi012 ceph-mon[152263]: failed to encode map e269 with expected crc 2023-10-09T10:08:39.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:39 smithi012 ceph-mon[152263]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.421 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:39 smithi012 ceph-mon[147926]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.421 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:39 smithi012 ceph-mon[147926]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.421 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:39 smithi012 ceph-mon[147926]: failed to encode map e269 with expected crc 2023-10-09T10:08:39.421 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:39 smithi012 ceph-mon[147926]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:39.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:39.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:39 smithi110 ceph-mon[136032]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:39 smithi110 ceph-mon[136032]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:39 smithi110 ceph-mon[136032]: failed to encode map e269 with expected crc 2023-10-09T10:08:39.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:39 smithi110 ceph-mon[136032]: failed to encode map e270 with expected crc 2023-10-09T10:08:39.465 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:39.466 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:39.523 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:39.532 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:39.561 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:39.600 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:39.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:39.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:39.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:39.733 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:40.349 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:40 smithi012 ceph-mon[147926]: pgmap v698: 73 pgs: 1 remapped+peering, 19 peering, 53 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:40.349 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:40 smithi012 ceph-mon[152263]: pgmap v698: 73 pgs: 1 remapped+peering, 19 peering, 53 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:40.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:40 smithi110 ceph-mon[136032]: pgmap v698: 73 pgs: 1 remapped+peering, 19 peering, 53 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:40.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:08:40.561 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:08:40.586 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:40.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:40.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:40.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:40.721 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:40.973 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:40.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:41.079 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:41.108 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:41.140 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:41.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:41.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:41.161 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:08:41.169 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:41.182 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:08:41.202 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:08:41.221 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:08:41.238 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:08:41.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:41.256 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:08:41.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:41 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6053/72 objects degraded (8406.944%), 7 pgs degraded, 9 pgs undersized) 2023-10-09T10:08:41.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:41 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6053/72 objects degraded (8406.944%), 7 pgs degraded, 9 pgs undersized) 2023-10-09T10:08:41.277 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:08:41.277 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:41.299 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:08:41.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:08:41.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:08:41.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:08:41.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:08:41.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:08:41.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:08:41.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:08:41.302 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:08:41.320 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:41.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:41.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:41.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:41 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6053/72 objects degraded (8406.944%), 7 pgs degraded, 9 pgs undersized) 2023-10-09T10:08:41.453 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:41.508 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:41.538 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:41.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:41.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:41.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:41.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:41.708 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:41.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:41.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:41.812 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:41.843 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:42.157 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:42.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:42.220 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_2' created 2023-10-09T10:08:42.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2564643227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:42.242 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2564643227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:42.261 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:42.289 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2564643227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:42.765 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:42.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:42.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:42.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:42.901 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:42.921 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T10:08:42.987 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:42.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:43.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:43.122 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:43.161 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:43.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:43.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:43.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:43.294 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:43.335 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:43.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:43.361 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 3, 4] 2023-10-09T10:08:43.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T10:08:43.402 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[147926]: pgmap v699: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:43.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4189510836' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:08:43.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:08:43.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[147926]: failed to encode map e271 with expected crc 2023-10-09T10:08:43.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[147926]: failed to encode map e271 with expected crc 2023-10-09T10:08:43.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T10:08:43.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[147926]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T10:08:43.403 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[152263]: pgmap v699: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:43.403 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4189510836' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:08:43.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:08:43.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[152263]: failed to encode map e271 with expected crc 2023-10-09T10:08:43.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[152263]: failed to encode map e271 with expected crc 2023-10-09T10:08:43.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T10:08:43.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:43 smithi012 ceph-mon[152263]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T10:08:43.439 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:43.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:43 smithi110 ceph-mon[136032]: pgmap v699: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:43.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4189510836' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:08:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:08:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:43 smithi110 ceph-mon[136032]: failed to encode map e271 with expected crc 2023-10-09T10:08:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:43 smithi110 ceph-mon[136032]: failed to encode map e271 with expected crc 2023-10-09T10:08:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T10:08:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:43 smithi110 ceph-mon[136032]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T10:08:43.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:43.553 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:43.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:43.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:43.598 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:43.657 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:43.685 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:43.769 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:43.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:43.848 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:08:43.849 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:08:43.849 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:08:43.849 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:08:43.873 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:43.901 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:44.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:08:44.191 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:44.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:44.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:44.224 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:08:44.290 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:08:44.295 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:44.360 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:44.393 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:44.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T10:08:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: pgmap v702: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2204678419' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: pgmap v702: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:44.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2204678419' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:44.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:44.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:44.497 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.523 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: failed to encode map e272 with expected crc 2023-10-09T10:08:44.523 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: pgmap v702: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:44.523 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2204678419' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:08:44.523 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:08:44.523 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:44.524 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:44.557 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:44.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:44.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:44.689 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:44.822 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:44.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:44.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:44.889 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:44.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:44.954 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:45.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:45 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:08:45.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:45 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:08:45.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:45 smithi110 ceph-mon[136032]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.356 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:45 smithi110 ceph-mon[136032]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.356 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2204678419' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:45.356 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:45 smithi110 ceph-mon[136032]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T10:08:45.356 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:45 smithi110 ceph-mon[136032]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.356 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:45 smithi110 ceph-mon[136032]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:08:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:08:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[147926]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[147926]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2204678419' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[147926]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[147926]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[147926]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[152263]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[152263]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2204678419' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[152263]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T10:08:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[152263]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:45 smithi012 ceph-mon[152263]: failed to encode map e273 with expected crc 2023-10-09T10:08:45.998 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:08:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:08:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:08:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:08:45.999 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:08:46.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:46 smithi110 ceph-mon[136032]: pgmap v704: 89 pgs: 8 creating+peering, 3 unknown, 78 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:46.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:08:46.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:46 smithi012 ceph-mon[147926]: pgmap v704: 89 pgs: 8 creating+peering, 3 unknown, 78 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:46.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:46 smithi012 ceph-mon[152263]: pgmap v704: 89 pgs: 8 creating+peering, 3 unknown, 78 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:47.006 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:47.010 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:47.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:47.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:47.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:47.142 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:47.278 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:47.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:47.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:47.411 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:47.444 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:47.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:47.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:47.547 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:47.576 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:47.590 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:47.591 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:47.687 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:47.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:47.791 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:47.819 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:47.913 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:47.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:48.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:48.059 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:48.091 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:48.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:08:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:08:47] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.33.4" 2023-10-09T10:08:48.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:48.219 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:124332 6.17 6:eb9d4fcd:::benchmark_data_smithi012_182576_object18811:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:59.922816+0000", 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 468.29354037000002, 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1875801510000001, 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.220 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124332 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922816+0000", 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922816+0000", 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922814+0000", 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4519999999999997e-06 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922823+0000", 2023-10-09T10:08:48.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6069000000000001e-05 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922839+0000", 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3810000000000004e-06 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.922845+0000", 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026576590000000001 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.925503+0000", 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3853999999999999e-05 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.925527+0000", 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98917412900000001 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.914701+0000", 2023-10-09T10:08:48.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2989e-05 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.914714+0000", 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.18281733 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.097531+0000", 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1924999999999998e-05 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.097623+0000", 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7617999999999999e-05 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.097721+0000", 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034036079999999998 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.101124+0000", 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9483e-05 2023-10-09T10:08:48.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.101174+0000", 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079896740000000004 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.109163+0000", 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8235999999999998e-05 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.109202+0000", 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011949619999999999 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.110397+0000", 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1875801510000001 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:124344 6.7 6:e214d4f7:::benchmark_data_smithi012_182576_object18823:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:59.948524+0000", 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 468.26783254200001, 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.157156058, 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124344 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948524+0000", 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948524+0000", 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948523+0000", 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3069999999999999e-06 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948529+0000", 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1191e-05 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948540+0000", 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7899999999999999e-06 2023-10-09T10:08:48.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948545+0000", 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4625e-05 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948590+0000", 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6800000000000004e-06 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.948596+0000", 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96706273700000001 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.915659+0000", 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.619e-06 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.915668+0000", 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1799493320000001 2023-10-09T10:08:48.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.095617+0000", 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111927 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.095729+0000", 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000116551 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:08:48.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.095845+0000", 2023-10-09T10:08:48.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027622850000000002 2023-10-09T10:08:48.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.098608+0000", 2023-10-09T10:08:48.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023244139999999999 2023-10-09T10:08:48.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.100932+0000", 2023-10-09T10:08:48.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037536100000000001 2023-10-09T10:08:48.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.104686+0000", 2023-10-09T10:08:48.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011579200000000001 2023-10-09T10:08:48.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.104802+0000", 2023-10-09T10:08:48.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00087872500000000001 2023-10-09T10:08:48.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.105680+0000", 2023-10-09T10:08:48.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.157156058 2023-10-09T10:08:48.243 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.243 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.244 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.244 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:124349 6.17 6:ea801b85:::benchmark_data_smithi012_182576_object18828:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T10:08:48.245 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:00:59.957332+0000", 2023-10-09T10:08:48.245 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 468.25902496600003, 2023-10-09T10:08:48.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1541461819999999, 2023-10-09T10:08:48.245 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.246 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.246 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.247 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:08:48.247 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:08:48.247 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 124349 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957332+0000", 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957332+0000", 2023-10-09T10:08:48.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957330+0000", 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8789999999999998e-06 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957340+0000", 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2724999999999999e-05 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957353+0000", 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2390000000000002e-06 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957358+0000", 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8668999999999997e-05 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957417+0000", 2023-10-09T10:08:48.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3715e-05 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:00:59.957431+0000", 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95730658400000002 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.914737+0000", 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9449999999999999e-06 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:00.914746+0000", 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.183858906 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.098605+0000", 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011053 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.098716+0000", 2023-10-09T10:08:48.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010681800000000001 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.098822+0000", 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023075550000000002 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.101130+0000", 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2167000000000002e-05 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.101202+0000", 2023-10-09T10:08:48.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0092419330000000008 2023-10-09T10:08:48.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.110444+0000", 2023-10-09T10:08:48.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3112000000000003e-05 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.110487+0000", 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099079500000000004 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:01:02.111478+0000", 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1541461819999999 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.255 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.256 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.34797.0:168959 6.a 6:5b4f3c85:::benchmark_data_smithi012_182576_object63438:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T10:08:48.256 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:02:02.187477+0000", 2023-10-09T10:08:48.256 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 406.02887996099997, 2023-10-09T10:08:48.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.83604466, 2023-10-09T10:08:48.256 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.256 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.34797", 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 168959 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187477+0000", 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187477+0000", 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187475+0000", 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9630000000000001e-06 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187482+0000", 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3553000000000001e-05 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187496+0000", 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5649999999999998e-06 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187502+0000", 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5254000000000003e-05 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187558+0000", 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7540000000000001e-06 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.187567+0000", 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.50423419199999997 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.691802+0000", 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.269e-05 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:02.691814+0000", 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.266860579 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.958675+0000", 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3975999999999997e-05 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.958759+0000", 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00059436199999999997 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2,6", 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.959353+0000", 2023-10-09T10:08:48.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028813258000000001 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.988166+0000", 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042397880000000004 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:03.992406+0000", 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012154336 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.004561+0000", 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018792485000000001 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023353+0000", 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5186999999999998e-05 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023418+0000", 2023-10-09T10:08:48.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103275 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:02:04.023522+0000", 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.83604466 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189822 7.e 7:75c53490:::benchmark_data_smithi012_221426_object82044:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e235)", 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.352313+0000", 2023-10-09T10:08:48.265 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.864044233, 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1682787279999998, 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189822 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352313+0000", 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352313+0000", 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352311+0000", 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1600000000000001e-05 2023-10-09T10:08:48.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352322+0000", 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6101000000000001e-05 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352338+0000", 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3510000000000008e-06 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352348+0000", 2023-10-09T10:08:48.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000208784 2023-10-09T10:08:48.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352556+0000", 2023-10-09T10:08:48.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.38e-05 2023-10-09T10:08:48.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-10-09T10:08:48.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352570+0000", 2023-10-09T10:08:48.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028903000000000002 2023-10-09T10:08:48.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.355460+0000", 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4120000000000007e-06 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.355470+0000", 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0820199669999999 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.437490+0000", 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033238400000000002 2023-10-09T10:08:48.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.440814+0000", 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079646506000000006 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.520460+0000", 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013110199999999999 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.520591+0000", 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1682787279999998 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189857 7.e 7:72010e1c:::benchmark_data_smithi012_221426_object82079:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e235)", 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.352444+0000", 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.86391313, 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2263163940000004, 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.273 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189857 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352444+0000", 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352444+0000", 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352442+0000", 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6990000000000003e-06 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352450+0000", 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5135e-05 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352465+0000", 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1009999999999992e-06 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352473+0000", 2023-10-09T10:08:48.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017449699999999999 2023-10-09T10:08:48.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352648+0000", 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0151e-05 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352658+0000", 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028324719999999999 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.355490+0000", 2023-10-09T10:08:48.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5820000000000001e-06 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.355496+0000", 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1057662949999996 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.461262+0000", 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037577500000000001 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.461638+0000", 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010975377 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.472614+0000", 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7334e-05 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.472711+0000", 2023-10-09T10:08:48.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015850099999999999 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.472869+0000", 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054263319999999999 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478296+0000", 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.656e-05 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478362+0000", 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.099068411999999995 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.577431+0000", 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1085000000000003e-05 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.577482+0000", 2023-10-09T10:08:48.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012783359999999999 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578760+0000", 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2263163940000004 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189895 7.e 7:706a3b6c:::benchmark_data_smithi012_221426_object82117:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.283 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.472620+0000", 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.74373674, 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1068740220000004, 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189895 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472620+0000", 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472620+0000", 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472618+0000", 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6149999999999999e-06 2023-10-09T10:08:48.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472626+0000", 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3054000000000001e-05 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472639+0000", 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3929999999999996e-06 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472645+0000", 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0260999999999998e-05 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472706+0000", 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1963999999999999e-05 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472717+0000", 2023-10-09T10:08:48.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9890024180000001 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.461720+0000", 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029004900000000001 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462010+0000", 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016384402999999999 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478394+0000", 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100259 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478495+0000", 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016704099999999999 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478662+0000", 2023-10-09T10:08:48.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096639559999999996 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.488326+0000", 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017766220000000001 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.490102+0000", 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.088771355999999996 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578874+0000", 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8349000000000002e-05 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578912+0000", 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058215199999999995 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579494+0000", 2023-10-09T10:08:48.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1068740220000004 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189935 7.e 7:7b9491a5:::benchmark_data_smithi012_221426_object82157:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.525002+0000", 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.691355233, 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0551137519999996, 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189935 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525002+0000", 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525002+0000", 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525000+0000", 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0190999999999999e-05 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525010+0000", 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1472999999999999e-05 2023-10-09T10:08:48.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525031+0000", 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.191e-06 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525039+0000", 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4003000000000004e-05 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525103+0000", 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2624e-05 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525116+0000", 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9369911579999997 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462107+0000", 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000335633 2023-10-09T10:08:48.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462443+0000", 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021951134000000001 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.484394+0000", 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110223 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.484504+0000", 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000190457 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.484695+0000", 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010366334 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.495061+0000", 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1199999999999996e-06 2023-10-09T10:08:48.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.495068+0000", 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084530545999999998 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579599+0000", 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2638000000000002e-05 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579631+0000", 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048402200000000002 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580115+0000", 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0551137519999996 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189951 7.e 7:72370faa:::benchmark_data_smithi012_221426_object82173:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.300 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.550258+0000", 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.66609853600001, 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0304842059999997, 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189951 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550258+0000", 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550258+0000", 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550256+0000", 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0445000000000001e-05 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550267+0000", 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0763000000000001e-05 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550288+0000", 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0390000000000005e-06 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550296+0000", 2023-10-09T10:08:48.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1124999999999995e-05 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550357+0000", 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7599999999999997e-06 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550366+0000", 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9122201030000001 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462587+0000", 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032054199999999999 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462907+0000", 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027249079999999998 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.490156+0000", 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118588 2023-10-09T10:08:48.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.490275+0000", 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019666 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.490471+0000", 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016806707000000001 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.507278+0000", 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011562615 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.518841+0000", 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.061361996000000002 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580203+0000", 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2552999999999998e-05 2023-10-09T10:08:48.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580235+0000", 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050716400000000005 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580742+0000", 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0304842059999997 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.308 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189992 7.e 7:7e96e1e7:::benchmark_data_smithi012_221426_object82214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.627928+0000", 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.58842832000001, 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9534130049999998, 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.309 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189992 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627928+0000", 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627928+0000", 2023-10-09T10:08:48.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627927+0000", 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1340000000000004e-06 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627935+0000", 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6446000000000001e-05 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627951+0000", 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0209999999999996e-06 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627957+0000", 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.333e-05 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.628001+0000", 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9800000000000004e-06 2023-10-09T10:08:48.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.628010+0000", 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.834941336 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462951+0000", 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023041399999999999 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463181+0000", 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.034042462000000002 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.497224+0000", 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1834000000000002e-05 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.497316+0000", 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017792799999999999 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.497494+0000", 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015801953000000001 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.513296+0000", 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013098742 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526394+0000", 2023-10-09T10:08:48.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054431805999999999 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580826+0000", 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1927000000000001e-05 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580858+0000", 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048334099999999998 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581341+0000", 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9534130049999998 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189997 7.e 7:7601f845:::benchmark_data_smithi012_221426_object82219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.637282+0000", 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.579074847, 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9446827339999997, 2023-10-09T10:08:48.317 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189997 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637282+0000", 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637282+0000", 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637280+0000", 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6240000000000002e-06 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637288+0000", 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4859e-05 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637303+0000", 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4529999999999999e-06 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637309+0000", 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9356e-05 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637359+0000", 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1171e-05 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637370+0000", 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8259225219999999 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463292+0000", 2023-10-09T10:08:48.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023791400000000001 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463530+0000", 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.039758138999999998 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.503288+0000", 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108609 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.503397+0000", 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000200063 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.503597+0000", 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015166198000000001 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.518763+0000", 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076912940000000004 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526454+0000", 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054970941000000002 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581425+0000", 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3358000000000001e-05 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581469+0000", 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00049589800000000004 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581965+0000", 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9446827339999997 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190020 7.e 7:726633f4:::benchmark_data_smithi012_221426_object82242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.665505+0000", 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.55085164099999, 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.917070517, 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190020 2023-10-09T10:08:48.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665505+0000", 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665505+0000", 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665505+0000", 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.492e-06 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665507+0000", 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0849000000000001e-05 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665518+0000", 2023-10-09T10:08:48.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.233e-06 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665522+0000", 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6986e-05 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665569+0000", 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0998999999999999e-05 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665580+0000", 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7979873040000003 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463568+0000", 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026484300000000001 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463832+0000", 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.045393021999999998 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.509225+0000", 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010867800000000001 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.509334+0000", 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018993699999999999 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.509524+0000", 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010873582 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.520398+0000", 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061112220000000004 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526509+0000", 2023-10-09T10:08:48.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055545221999999998 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582054+0000", 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2280000000000003e-05 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582086+0000", 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048930100000000004 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582576+0000", 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.917070517 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190037 7.e 7:7898f990:::benchmark_data_smithi012_221426_object82259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.690696+0000", 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.52566122499999, 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8928224829999998, 2023-10-09T10:08:48.334 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190037 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690696+0000", 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690696+0000", 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690695+0000", 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7029999999999997e-06 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690700+0000", 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0607e-05 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690710+0000", 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5809999999999998e-06 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690715+0000", 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8108000000000001e-05 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690763+0000", 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5320000000000001e-06 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690770+0000", 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7731051459999998 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463876+0000", 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000234665 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464110+0000", 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051168810000000002 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.515279+0000", 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8494999999999998e-05 2023-10-09T10:08:48.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.515378+0000", 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017001200000000001 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.515548+0000", 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061315839999999998 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.521679+0000", 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048848420000000004 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526564+0000", 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056062085999999997 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582626+0000", 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6119999999999998e-05 2023-10-09T10:08:48.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582682+0000", 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00083589099999999996 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.583518+0000", 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8928224829999998 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190038 7.e 7:7015ad23:::benchmark_data_smithi012_221426_object82260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.690725+0000", 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.525631985, 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8935087460000002, 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190038 2023-10-09T10:08:48.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690725+0000", 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690725+0000", 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690724+0000", 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2280000000000001e-06 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690727+0000", 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6109999999999998e-06 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690730+0000", 2023-10-09T10:08:48.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4559999999999999e-06 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690733+0000", 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7816999999999999e-05 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690791+0000", 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1880000000000004e-06 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690795+0000", 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7733637140000003 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464158+0000", 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042128800000000002 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464580+0000", 2023-10-09T10:08:48.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057145987000000002 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.521726+0000", 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7033999999999994e-05 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.521813+0000", 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015166100000000001 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.521964+0000", 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046447090000000003 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526609+0000", 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9210000000000003e-06 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526616+0000", 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056952488000000002 2023-10-09T10:08:48.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.583569+0000", 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4563e-05 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.583613+0000", 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062047100000000004 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.584234+0000", 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8935087460000002 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190119 7.e 7:73998f4a:::benchmark_data_smithi012_221426_object82341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.813673+0000", 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.40268407799999, 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7712925779999997, 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.351 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190119 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813673+0000", 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813673+0000", 2023-10-09T10:08:48.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:48.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813671+0000", 2023-10-09T10:08:48.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4869999999999999e-06 2023-10-09T10:08:48.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813678+0000", 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0417999999999998e-05 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813718+0000", 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7849999999999999e-06 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813723+0000", 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6807999999999997e-05 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813760+0000", 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9610000000000002e-06 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813766+0000", 2023-10-09T10:08:48.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6508705859999999 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464636+0000", 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034053700000000002 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464977+0000", 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.077599679000000005 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.542577+0000", 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8681999999999998e-05 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.542635+0000", 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3194000000000003e-05 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.542719+0000", 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014864018 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.557583+0000", 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021250937000000001 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578834+0000", 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054931679999999997 2023-10-09T10:08:48.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.584327+0000", 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4540999999999998e-05 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.584381+0000", 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058408199999999998 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.584965+0000", 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7712925779999997 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190129 7.e 7:77d17918:::benchmark_data_smithi012_221426_object82351:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.831462+0000", 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.38489515800001, 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7543196769999998, 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.360 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190129 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831462+0000", 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831462+0000", 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831460+0000", 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1391e-05 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831471+0000", 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1126e-05 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831482+0000", 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3470000000000004e-06 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831489+0000", 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7751000000000003e-05 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831547+0000", 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1390000000000001e-06 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831556+0000", 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6334554919999995 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465011+0000", 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036681399999999997 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465378+0000", 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082331029 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.547709+0000", 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4887000000000001e-05 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.547804+0000", 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015043500000000001 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.547954+0000", 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016570558999999999 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.564525+0000", 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015033075999999999 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579558+0000", 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054733459999999996 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585031+0000", 2023-10-09T10:08:48.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1279999999999997e-05 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585083+0000", 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00069872300000000005 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585781+0000", 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7543196769999998 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.368 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190148 7.e 7:73a672b0:::benchmark_data_smithi012_221426_object82370:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.862419+0000", 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.35393785700001, 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7241475929999996, 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.369 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190148 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862419+0000", 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862419+0000", 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862417+0000", 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6969999999999999e-06 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862426+0000", 2023-10-09T10:08:48.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6313000000000001e-05 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862442+0000", 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.319e-06 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862448+0000", 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9616000000000001e-05 2023-10-09T10:08:48.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862498+0000", 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0275000000000001e-05 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862508+0000", 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6029239119999996 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465432+0000", 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039124599999999999 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465823+0000", 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.087867027 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.553690+0000", 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6755999999999994e-05 2023-10-09T10:08:48.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.553777+0000", 2023-10-09T10:08:48.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000137775 2023-10-09T10:08:48.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.553915+0000", 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017295237000000001 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.571210+0000", 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089565500000000006 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580167+0000", 2023-10-09T10:08:48.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056861580000000002 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585853+0000", 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2290000000000002e-05 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585915+0000", 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00065133699999999999 2023-10-09T10:08:48.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.586567+0000", 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7241475929999996 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190161 7.e 7:78ca3d82:::benchmark_data_smithi012_221426_object82383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.885594+0000", 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.33076274699999, 2023-10-09T10:08:48.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7015902470000004, 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190161 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885594+0000", 2023-10-09T10:08:48.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885594+0000", 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885593+0000", 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4420000000000001e-06 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885600+0000", 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.325e-05 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885613+0000", 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2089999999999999e-06 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885621+0000", 2023-10-09T10:08:48.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6032999999999998e-05 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885667+0000", 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0569999999999998e-06 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885673+0000", 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5802007260000002 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465873+0000", 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000347401 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466221+0000", 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.093410134000000006 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.559631+0000", 2023-10-09T10:08:48.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119565 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.559750+0000", 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018952699999999999 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.559940+0000", 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017418316 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.577358+0000", 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003434739 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580793+0000", 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058168040000000001 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.586610+0000", 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3302000000000001e-05 2023-10-09T10:08:48.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.586643+0000", 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00054114400000000002 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587184+0000", 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7015902470000004 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.386 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190171 7.e 7:7bc672db:::benchmark_data_smithi012_221426_object82393:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.906118+0000", 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.310238579, 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6816790670000001, 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.387 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190171 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906118+0000", 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906118+0000", 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906117+0000", 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3710000000000002e-06 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906122+0000", 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0206e-05 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906132+0000", 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6589999999999998e-06 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906137+0000", 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0462e-05 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906177+0000", 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9599999999999997e-06 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906183+0000", 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5600940379999999 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466277+0000", 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000401865 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466679+0000", 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.099618462000000005 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.566297+0000", 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115087 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.566413+0000", 2023-10-09T10:08:48.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020621100000000001 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.566619+0000", 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012227658000000001 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578846+0000", 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025430679999999999 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581390+0000", 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058345560000000003 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587224+0000", 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2561999999999998e-05 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587257+0000", 2023-10-09T10:08:48.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00054065999999999999 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587797+0000", 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6816790670000001 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.395 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190180 7.e 7:709efe90:::benchmark_data_smithi012_221426_object82402:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.945202+0000", 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.271155147, 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6432413459999999, 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190180 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:48.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945202+0000", 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945202+0000", 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:48.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945201+0000", 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4689999999999999e-06 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945205+0000", 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0219999999999993e-06 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945214+0000", 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0990000000000001e-06 2023-10-09T10:08:48.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945218+0000", 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6171999999999999e-05 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945255+0000", 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4559999999999999e-06 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945260+0000", 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5214660279999999 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466726+0000", 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025202900000000002 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466978+0000", 2023-10-09T10:08:48.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10637545399999999 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.573354+0000", 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6915000000000003e-05 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.573451+0000", 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000172957 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.573623+0000", 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059456320000000002 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579569+0000", 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024467289999999999 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582016+0000", 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058284809999999999 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587844+0000", 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4780000000000002e-05 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587889+0000", 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00055391200000000005 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:48.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.588443+0000", 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6432413459999999 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:48.404 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:08:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:48.406 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:48.406 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:48.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:48.507 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:48.540 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:48.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:48.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:48.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:48.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:08:48.732 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:48.757 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:48.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:48.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:48.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:48.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:48.937 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:49.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:08:49.167 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:48 smithi012 ceph-mon[147926]: pgmap v705: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:49.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:48 smithi012 ceph-mon[152263]: pgmap v705: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:49.168 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:49.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:49.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:49.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:48 smithi110 ceph-mon[136032]: pgmap v705: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:49.244 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:49.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:49.302 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:49.441 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:49.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:49.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:49.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:49.579 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:49.651 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:08:49.666 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:49.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:49.670 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:08:49.691 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:08:49.711 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:08:49.730 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:08:49.749 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:08:49.767 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:08:49.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:49.785 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:08:49.787 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:08:49.787 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:08:49.787 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:08:49.788 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:08:49.788 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:08:49.788 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:08:49.788 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:08:49.788 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:08:49.799 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:49.833 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:49.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:49.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:49.930 INFO:teuthology.orchestra.run.smithi012.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T10:08:49.938 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:49.961 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3949605897' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:08:49.961 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:49 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:08:49.961 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2298347853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:49.961 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3949605897' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:08:49.961 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:49 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:08:49.961 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2298347853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:49.962 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3949605897' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:08:49.962 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:49 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:08:49.962 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2298347853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:49.965 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:50.000 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:50.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:50.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:50.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:50.133 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:50.328 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:50.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:50.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:50.425 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:50.426 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:50.432 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:50.460 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:50.494 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:50.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:50.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:50.597 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:50.608 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:50.626 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:50.759 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:50.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:50.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:50.769 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:08:50.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:50.892 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:50.929 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T10:08:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[147926]: pgmap v706: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[147926]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:08:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[147926]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T10:08:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1114849347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:08:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:08:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[147926]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[147926]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[152263]: pgmap v706: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:51.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[152263]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:08:51.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[152263]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T10:08:51.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1114849347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:08:51.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:08:51.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[152263]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:50 smithi012 ceph-mon[152263]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:50 smithi110 ceph-mon[136032]: pgmap v706: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:50 smithi110 ceph-mon[136032]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:50 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:08:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:50 smithi110 ceph-mon[136032]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T10:08:51.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1114849347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:08:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:50 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:08:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:50 smithi110 ceph-mon[136032]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:50 smithi110 ceph-mon[136032]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T10:08:51.354 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:51.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:51.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:51.405 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:51.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:51.488 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:51.561 DEBUG:teuthology.orchestra.run.smithi012:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T10:08:51.563 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:08:51.564 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:08:51.612 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:hints = 1 2023-10-09T10:08:51.612 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:08:51.612 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object prefix: benchmark_data_smithi012_271002 2023-10-09T10:08:51.612 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:08:51.613 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:08:51.693 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:51.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:51.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:51.892 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:51.947 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[147926]: failed to encode map e274 with expected crc 2023-10-09T10:08:51.947 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[147926]: failed to encode map e275 with expected crc 2023-10-09T10:08:51.947 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:08:51.947 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[147926]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T10:08:51.948 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[147926]: failed to encode map e275 with expected crc 2023-10-09T10:08:51.948 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[147926]: failed to encode map e275 with expected crc 2023-10-09T10:08:51.948 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[147926]: failed to encode map e275 with expected crc 2023-10-09T10:08:51.957 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:51.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:51.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:52.064 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:52.112 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:52.215 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:52.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:52.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:52.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:51 smithi110 ceph-mon[136032]: failed to encode map e274 with expected crc 2023-10-09T10:08:52.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:51 smithi110 ceph-mon[136032]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:08:52.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:51 smithi110 ceph-mon[136032]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T10:08:52.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:51 smithi110 ceph-mon[136032]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.220 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:51 smithi110 ceph-mon[136032]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:51 smithi110 ceph-mon[136032]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:51 smithi110 ceph-mon[136032]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.320 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:52.329 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:52.364 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:52.414 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:52.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:52.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[147926]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[152263]: failed to encode map e274 with expected crc 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[152263]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[152263]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[152263]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[152263]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[152263]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:51 smithi012 ceph-mon[152263]: failed to encode map e275 with expected crc 2023-10-09T10:08:52.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:52.560 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:08:52.569 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:52.612 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 1 16 3260 3244 202.739 202.75 0.00683624 0.00490862 2023-10-09T10:08:52.670 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:52.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:52.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:52.766 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:52.766 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:52.775 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:52.813 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:52.985 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:08:53.092 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:52 smithi110 ceph-mon[136032]: pgmap v709: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:53.092 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2195958098' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:53.102 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:53.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:53.161 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:08:53.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:53.221 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:52 smithi012 ceph-mon[147926]: pgmap v709: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:53.221 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2195958098' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:53.221 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:52 smithi012 ceph-mon[152263]: pgmap v709: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:08:53.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2195958098' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:08:53.270 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:53.494 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:53.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:53.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:53.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:53.613 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 2 16 6215 6199 193.705 184.688 0.0100861 0.00514889 2023-10-09T10:08:53.679 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:53.846 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:53.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:53.930 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:53.952 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:54.001 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:54.078 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:54.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:54.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:54.106 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:54.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[152263]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2195958098' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:08:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[152263]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T10:08:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[152263]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[152263]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:54.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[147926]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2195958098' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:08:54.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[147926]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T10:08:54.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[147926]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:53 smithi012 ceph-mon[147926]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:08:54.228 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:53 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:08:54.228 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:53 smithi110 ceph-mon[136032]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.228 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2195958098' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:08:54.228 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:53 smithi110 ceph-mon[136032]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T10:08:54.229 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:53 smithi110 ceph-mon[136032]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.229 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:53 smithi110 ceph-mon[136032]: failed to encode map e276 with expected crc 2023-10-09T10:08:54.242 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:54.305 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:08:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:54.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:08:54.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:54.487 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:54.552 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:54.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:54.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:54.613 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 3 16 8929 8913 185.659 169.625 0.00382633 0.0053785 2023-10-09T10:08:54.657 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:54.718 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:54.772 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:54.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:54.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:54.879 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:54.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:54.991 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T10:08:54.996 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:55.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:55.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:55.106 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:54 smithi012 ceph-mon[152263]: pgmap v711: 89 pgs: 89 active+clean; 110 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 292 op/s 2023-10-09T10:08:55.106 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:54 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3342655381' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:08:55.106 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:54 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:08:55.107 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:54 smithi012 ceph-mon[147926]: pgmap v711: 89 pgs: 89 active+clean; 110 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 292 op/s 2023-10-09T10:08:55.107 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:54 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3342655381' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:08:55.107 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:54 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:08:55.111 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:08:55.111 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:08:55.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:55.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:54 smithi110 ceph-mon[136032]: pgmap v711: 89 pgs: 89 active+clean; 110 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 292 op/s 2023-10-09T10:08:55.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:54 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3342655381' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:08:55.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:54 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:08:55.391 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:55.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:55.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:55.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:08:55.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:55.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T10:08:55.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:08:55.570 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:55.613 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 4 16 11687 11671 182.334 172.375 0.00869431 0.00547048 2023-10-09T10:08:55.644 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:55.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:55.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:55.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:55.882 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:55.956 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:55.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:56.002 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:08:56.042 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:08:56.056 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:56.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:56.083 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:08:56.129 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:08:56.130 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:56.175 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:08:56.206 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:08:56.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:56.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:08:56.216 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:08:56.250 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:08:56.281 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:56 smithi110 ceph-mon[136032]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.281 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:56 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:08:56.282 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:56 smithi110 ceph-mon[136032]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T10:08:56.282 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:56 smithi110 ceph-mon[136032]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.282 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:56 smithi110 ceph-mon[136032]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.282 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:56 smithi110 ceph-mon[136032]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.282 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/126565659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:56.286 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:08:56.296 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:08:56.296 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:08:56.296 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:08:56.296 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:08:56.297 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:08:56.297 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:08:56.297 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:08:56.297 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:08:56.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:56.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[147926]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:08:56.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[147926]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T10:08:56.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[147926]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[147926]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[147926]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/126565659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:56.335 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[152263]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.335 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:08:56.335 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[152263]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T10:08:56.335 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[152263]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.335 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[152263]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.335 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[152263]: failed to encode map e277 with expected crc 2023-10-09T10:08:56.335 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/126565659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:08:56.350 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:56.392 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:56.489 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:08:56.524 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:56.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:56.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:56.614 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 5 15 13683 13668 170.83 124.812 0.0046172 0.00584794 2023-10-09T10:08:56.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:56.673 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:56.828 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:56.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:56.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:56.975 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:57.033 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:57.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:57.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:57.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:08:57.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:57 smithi012 ceph-mon[147926]: pgmap v713: 89 pgs: 89 active+clean; 283 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T10:08:57.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:57 smithi012 ceph-mon[152263]: pgmap v713: 89 pgs: 89 active+clean; 283 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T10:08:57.197 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:57.298 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:15389 8.5 8:ae5fcb18:::benchmark_data_smithi012_271002_object15388:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e277)", 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:08:57.293949+0000", 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0036840330000000002, 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037191540000000001, 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:57.299 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 15389 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.293949+0000", 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:57.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.293949+0000", 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.293947+0000", 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2149000000000002e-05 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.294029+0000", 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1795e-05 2023-10-09T10:08:57.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.294041+0000", 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.84e-06 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.294047+0000", 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4083000000000003e-05 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:57.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.294111+0000", 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121891 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.294233+0000", 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000202499 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.294435+0000", 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024417430000000001 2023-10-09T10:08:57.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.296877+0000", 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3859999999999999e-05 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.296891+0000", 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029422150000000002 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:57.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:15394 8.5 8:a269677a:::benchmark_data_smithi012_271002_object15393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e277)", 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:08:57.295682+0000", 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0019508819999999999, 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020895140000000002, 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 15394 2023-10-09T10:08:57.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.295682+0000", 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.295682+0000", 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:08:57.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.295680+0000", 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2281999999999998e-05 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.295743+0000", 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2228e-05 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.295755+0000", 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4919999999999998e-06 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.295761+0000", 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2627000000000003e-05 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.295814+0000", 2023-10-09T10:08:57.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013537799999999999 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.295949+0000", 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017693599999999999 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.296126+0000", 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044440400000000002 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:57.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:15393 8.c e277/271 8:3ebabba6:::benchmark_data_smithi012_271002_object15392:head v 277'942, mlcod=277'940)", 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:08:57.297054+0000", 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00057929699999999999, 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00080075899999999995, 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:08:57.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.297054+0000", 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.297054+0000", 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.297053+0000", 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2076999999999997e-05 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.297095+0000", 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4061e-05 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.297109+0000", 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.533e-06 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.297115+0000", 2023-10-09T10:08:57.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000113686 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.297229+0000", 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4649000000000001e-05 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:57.297254+0000", 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020015399999999999 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:57.313 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:08:57.314 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:57.314 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:08:57.314 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:08:57.314 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 3 2023-10-09T10:08:57.314 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:08:57.331 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:57 smithi110 ceph-mon[136032]: pgmap v713: 89 pgs: 89 active+clean; 283 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T10:08:57.502 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:08:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:57.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:08:57.605 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:08:57.613 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 6 16 16154 16138 168.085 154.375 0.00285055 0.00594325 2023-10-09T10:08:57.626 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:08:57.669 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:57.737 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:08:57.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:57.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:08:57.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:08:57.905 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:57.915 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:08:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:08:57] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.33.4" 2023-10-09T10:08:57.976 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:08:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:57.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:08:58.081 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:08:58.137 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:58.200 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:08:58.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:58.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:08:58.307 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:08:58.365 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:58.427 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:08:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:58.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:08:58.480 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:08:58.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:08:58.594 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:08:58.613 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 7 16 18712 18696 166.91 159.875 0.00440736 0.0059858 2023-10-09T10:08:59.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:08:59.332 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:59 smithi110 ceph-mon[136032]: pgmap v714: 89 pgs: 89 active+clean; 429 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T10:08:59.332 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:08:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:59.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:59 smithi012 ceph-mon[152263]: pgmap v714: 89 pgs: 89 active+clean; 429 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T10:08:59.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:08:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:59 smithi012 ceph-mon[147926]: pgmap v714: 89 pgs: 89 active+clean; 429 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T10:08:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:08:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:08:59.613 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 8 16 21024 21008 164.107 144.5 0.00578979 0.00608594 2023-10-09T10:08:59.650 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:08:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:08:59.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:08:59.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:08:59.832 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:00.099 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:00.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:00.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:00.209 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:00.224 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:09:00.224 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:09:00.224 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:09:00.224 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:09:00.224 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:09:00.265 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:00.311 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:00.318 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:00.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:00.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:00.425 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:00.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:00.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:00 smithi012 ceph-mon[147926]: pgmap v715: 89 pgs: 89 active+clean; 831 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T10:09:00.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:00 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T10:09:00.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T10:09:00.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:00.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:00 smithi012 ceph-mon[152263]: pgmap v715: 89 pgs: 89 active+clean; 831 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T10:09:00.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:00 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T10:09:00.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T10:09:00.484 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:00.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:00.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:00 smithi110 ceph-mon[136032]: pgmap v715: 89 pgs: 89 active+clean; 831 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T10:09:00.553 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:00 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T10:09:00.554 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T10:09:00.557 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:09:00.611 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:00.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:00.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:00.621 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 9 16 22880 22864 158.629 116 0.0109164 0.00629685 2023-10-09T10:09:00.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:00.786 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:00.855 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:00.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:00.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:01.026 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:01.153 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:01.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:01.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:01.255 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T10:09:01.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:01.260 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:09:01.289 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:01.300 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:09:01.344 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:02.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[152263]: failed to encode map e278 with expected crc 2023-10-09T10:09:02.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[152263]: failed to encode map e278 with expected crc 2023-10-09T10:09:02.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T10:09:02.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[152263]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T10:09:02.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/850232876' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:02.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:02.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3059136575' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:09:02.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:09:02.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[147926]: failed to encode map e278 with expected crc 2023-10-09T10:09:02.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[147926]: failed to encode map e278 with expected crc 2023-10-09T10:09:02.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T10:09:02.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[147926]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T10:09:02.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/850232876' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:02.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:02.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3059136575' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:09:02.187 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:09:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:02.188 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 10 16 24896 24880 155.365 126 0.00699018 0.00643054 2023-10-09T10:09:02.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:02.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:01 smithi110 ceph-mon[136032]: failed to encode map e278 with expected crc 2023-10-09T10:09:02.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:01 smithi110 ceph-mon[136032]: failed to encode map e278 with expected crc 2023-10-09T10:09:02.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T10:09:02.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:01 smithi110 ceph-mon[136032]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T10:09:02.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/850232876' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:02.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:02.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3059136575' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:09:02.189 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:09:02.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T10:09:02.190 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:09:02.191 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:09:02.191 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:09:02.226 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:09:02.263 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:09:02.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:02.301 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:09:02.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:09:02.370 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:09:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:09:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:09:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:09:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:09:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:09:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:09:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:09:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:09:02.389 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:02.461 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:02.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:02.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:02.548 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:02.548 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.548 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.548 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:02.548 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:09:02.548 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T10:09:02.549 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.549 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: pgmap v718: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2023-10-09T10:09:02.549 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1132787778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:02.549 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:02.549 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.549 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.549 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:02.549 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:09:02.550 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T10:09:02.550 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.550 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: pgmap v718: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2023-10-09T10:09:02.550 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1132787778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:02.560 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:09:02.560 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:09:02.578 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:02.588 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:02.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:02.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:09:02.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T10:09:02.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: failed to encode map e279 with expected crc 2023-10-09T10:09:02.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: pgmap v718: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2023-10-09T10:09:02.589 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1132787778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:02.621 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 11 16 26812 26796 152.128 119.75 0.00913157 0.00656718 2023-10-09T10:09:02.645 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:02.784 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:02.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:02.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:02.963 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:02.983 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:09:03.021 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:03.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:03.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:03.195 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:03.233 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:03.376 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:03.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:03.482 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:03.530 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:03.581 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:03.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:03.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:03.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 12 16 28697 28681 149.264 117.812 0.00352725 0.00669465 2023-10-09T10:09:03.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:03.745 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:04.106 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:04.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:04.112 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:04.213 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:04.263 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:04.317 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:04.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:04.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:04.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:09:04.422 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:04.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:04.507 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:04.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:04.611 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:04.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 13 16 30579 30563 146.831 117.625 0.0111715 0.00680505 2023-10-09T10:09:04.654 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:04.705 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:04.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:04.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:04.839 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:04.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:09:04.932 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:04 smithi110 ceph-mon[136032]: pgmap v719: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-10-09T10:09:05.046 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:05.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:05.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:05.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:05.243 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:04 smithi012 ceph-mon[147926]: pgmap v719: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-10-09T10:09:05.243 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:04 smithi012 ceph-mon[152263]: pgmap v719: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-10-09T10:09:05.251 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:05.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 14 16 32259 32243 143.844 105 0.00757354 0.00694657 2023-10-09T10:09:05.957 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T10:09:05.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/429309571' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:09:05.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:05 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:09:05.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/429309571' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:09:05.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:05 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:09:06.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/429309571' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:09:06.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:05 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:09:06.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 15 16 34309 34293 142.796 128.125 0.00957985 0.00699732 2023-10-09T10:09:07.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[147926]: pgmap v720: 89 pgs: 89 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.57k op/s 2023-10-09T10:09:07.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[147926]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[147926]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:09:07.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[147926]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T10:09:07.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[147926]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.390 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[152263]: pgmap v720: 89 pgs: 89 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.57k op/s 2023-10-09T10:09:07.391 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[152263]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.391 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[152263]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.391 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:09:07.391 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[152263]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T10:09:07.391 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:06 smithi012 ceph-mon[152263]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:06 smithi110 ceph-mon[136032]: pgmap v720: 89 pgs: 89 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.57k op/s 2023-10-09T10:09:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:06 smithi110 ceph-mon[136032]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:06 smithi110 ceph-mon[136032]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:06 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:09:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:06 smithi110 ceph-mon[136032]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T10:09:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:06 smithi110 ceph-mon[136032]: failed to encode map e280 with expected crc 2023-10-09T10:09:07.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 16 16 36281 36265 141.573 123.25 0.0031505 0.00705855 2023-10-09T10:09:07.691 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:07.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:07.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:07.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:07.863 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:09:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:09:07] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.33.4" 2023-10-09T10:09:08.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 17 16 38075 38059 139.841 112.125 0.00705791 0.0071453 2023-10-09T10:09:09.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:08 smithi012 ceph-mon[147926]: pgmap v722: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:09:09.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:08 smithi012 ceph-mon[152263]: pgmap v722: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:09:09.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:08 smithi110 ceph-mon[136032]: pgmap v722: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:09:09.618 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:09.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:09.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 18 16 40009 39993 138.787 120.875 0.012528 0.0071997 2023-10-09T10:09:09.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:09.752 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:10.500 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T10:09:10.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 19 16 41550 41534 136.552 96.3125 0.00729522 0.00730598 2023-10-09T10:09:10.864 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:10.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:10.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:10.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:10.979 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:11.013 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:11.073 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:11.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:11.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:11.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:10 smithi012 ceph-mon[147926]: pgmap v723: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T10:09:11.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:10 smithi012 ceph-mon[152263]: pgmap v723: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T10:09:11.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:11.239 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:11.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:10 smithi110 ceph-mon[136032]: pgmap v723: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T10:09:11.452 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:11.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:11.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:11.622 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:09:11.623087+0000 min lat: 0.00120497 max lat: 0.0868591 avg lat: 0.00742229 2023-10-09T10:09:11.623 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:09:11.623 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 20 16 43120 43104 134.631 98.125 0.00791695 0.00742229 2023-10-09T10:09:11.650 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:11.830 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:11.832 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:11.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:11.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:11.862 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:09:11.862 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:09:11.942 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:11.991 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:12.053 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:12.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:12.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:12.161 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:12.201 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:12.258 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:12.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:12.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:12.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:12.433 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:12.623 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 21 16 44673 44657 132.842 97.0625 0.00664887 0.00752188 2023-10-09T10:09:12.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:09:12.803 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:12.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:12.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:12.912 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:12.952 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:12.983 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:12 smithi012 ceph-mon[152263]: pgmap v724: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T10:09:12.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:12 smithi012 ceph-mon[147926]: pgmap v724: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T10:09:12.999 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:13.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:13.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:13.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:13.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:13.396 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:13.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:13.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:12 smithi110 ceph-mon[136032]: pgmap v724: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T10:09:13.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:13.546 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:13.599 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:13.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:13.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:13.624 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 22 16 46086 46070 130.812 88.3125 0.0175551 0.00763706 2023-10-09T10:09:13.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:13.793 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:14.156 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:14.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:14.263 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:14.266 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:13 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:14.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:13 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:14.295 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:13 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:14.298 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:14.348 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:14.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:14.453 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:14.524 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:14.545 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:14.593 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:14.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:14.624 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:09:14.624 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 23 16 47435 47419 128.789 84.3125 0.0126116 0.00775856 2023-10-09T10:09:14.666 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:09:14.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:14.711 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:09:14.744 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:09:14.784 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:14.802 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:09:14.828 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:09:14.865 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:09:14.904 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:09:14.915 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:09:14.916 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:09:14.916 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:09:14.916 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:09:14.916 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:09:14.916 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:09:14.916 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:09:14.916 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:09:14.946 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:14.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:14.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:15.033 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:09:15.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:15 smithi012 ceph-mon[152263]: pgmap v725: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T10:09:15.042 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2756717805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:09:15.042 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3830443836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:15.042 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:15 smithi012 ceph-mon[147926]: pgmap v725: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T10:09:15.043 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2756717805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:09:15.043 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3830443836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:15.059 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:15.093 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:15.135 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:15.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:15.240 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:15.280 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:15 smithi110 ceph-mon[136032]: pgmap v725: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T10:09:15.281 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2756717805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:09:15.281 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3830443836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:15.285 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:15.338 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:15.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:15.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:15.530 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:15.624 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 24 15 48751 48736 126.853 82.3125 0.0116138 0.00787782 2023-10-09T10:09:15.959 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:15.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:16.040 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T10:09:16.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:16.167 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2756717805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3973904568' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:09:16.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2756717805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3973904568' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:09:16.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:16 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:09:16.434 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:16.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:16.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:16.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2756717805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:09:16.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T10:09:16.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: failed to encode map e281 with expected crc 2023-10-09T10:09:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3973904568' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:09:16.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:16 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:09:16.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T10:09:16.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:16.630 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 25 16 50245 50229 125.485 93.3125 0.0180519 0.00796393 2023-10-09T10:09:16.662 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:16.726 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:16.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:16.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:16.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:16.947 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:17.009 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:17.010 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:09:17.010 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:09:17.010 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:09:17.010 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:09:17.011 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:09:17.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:17.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:17.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:17.165 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:17.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: pgmap v727: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T10:09:17.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:09:17.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[152263]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: pgmap v727: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T10:09:17.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.362 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.362 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.362 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.362 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-mon[147926]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.394 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:17.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:17.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:17.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: pgmap v727: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:17 smithi110 ceph-mon[136032]: failed to encode map e282 with expected crc 2023-10-09T10:09:17.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:17.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:09:17.546 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:17.587 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:17.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:17.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:17.630 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 26 16 51586 51570 123.883 83.8125 0.00725023 0.00806598 2023-10-09T10:09:17.684 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:17.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:17.724 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:17.774 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:17.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:17.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:17.878 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:17.916 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:09:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:09:17] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T10:09:17.929 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:17.979 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:17.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:18.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:18.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:09:18.161 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:18.631 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 27 16 52823 52807 122.157 77.3125 0.01599 0.00818028 2023-10-09T10:09:18.650 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:18.659 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:18.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:18.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:18.762 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:18.819 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:18.850 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:18.941 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:18.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:18.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:19.046 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:19.092 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:19.149 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:19.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:19.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:19.248 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:19 smithi012 ceph-mon[152263]: pgmap v729: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T10:09:19.248 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:19 smithi012 ceph-mon[147926]: pgmap v729: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T10:09:19.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:19.291 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:19.408 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:19.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:19.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:19 smithi110 ceph-mon[136032]: pgmap v729: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T10:09:19.525 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:19.545 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:19.561 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:19.610 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:19.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:19.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:19.630 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 28 16 54018 54002 120.462 74.6875 0.0181453 0.00829529 2023-10-09T10:09:19.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:19.868 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:20.634 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 29 16 55231 55215 118.907 75.8125 0.00833866 0.00840519 2023-10-09T10:09:21.074 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T10:09:21.222 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:21.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:21.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:21.333 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:21.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:21 smithi012 ceph-mon[147926]: pgmap v730: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T10:09:21.435 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/136143678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:09:21.436 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:21 smithi012 ceph-mon[152263]: pgmap v730: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T10:09:21.436 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/136143678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:09:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:21 smithi110 ceph-mon[136032]: pgmap v730: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T10:09:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/136143678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:09:21.635 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 30 16 56623 56607 117.843 87 0.00581486 0.00848121 2023-10-09T10:09:22.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/136143678' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:09:22.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T10:09:22.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[147926]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/136143678' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:09:22.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T10:09:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:22 smithi012 ceph-mon[152263]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.401 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:22.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/136143678' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:09:22.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T10:09:22.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:22 smithi110 ceph-mon[136032]: failed to encode map e283 with expected crc 2023-10-09T10:09:22.635 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 31 16 58085 58069 116.99 91.375 0.011267 0.00854266 2023-10-09T10:09:23.401 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:23 smithi012 ceph-mon[147926]: pgmap v732: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T10:09:23.402 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:23 smithi012 ceph-mon[152263]: pgmap v732: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T10:09:23.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:23 smithi110 ceph-mon[136032]: pgmap v732: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T10:09:23.637 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 32 16 59449 59433 115.99 85.25 0.0134576 0.00861655 2023-10-09T10:09:24.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:24 smithi012 ceph-mon[152263]: pgmap v733: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:09:24.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:24 smithi012 ceph-mon[147926]: pgmap v733: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:09:24.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:24 smithi110 ceph-mon[136032]: pgmap v733: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:09:24.500 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:24.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:24.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:24.608 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:24.637 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 33 16 60911 60895 115.245 91.375 0.0098963 0.0086729 2023-10-09T10:09:24.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:24.690 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:24.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:24.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:24.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:25.245 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:25.637 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 34 16 62300 62284 114.409 86.8125 0.00843053 0.008735 2023-10-09T10:09:26.302 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:26.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:26.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:26.500 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:26.640 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 35 16 63673 63657 113.586 85.8125 0.0106181 0.00879901 2023-10-09T10:09:26.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:26 smithi012 ceph-mon[147926]: pgmap v734: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T10:09:26.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:26 smithi012 ceph-mon[152263]: pgmap v734: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T10:09:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:26 smithi110 ceph-mon[136032]: pgmap v734: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T10:09:26.986 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:26.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:26.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:27.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T10:09:27.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:27.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:27.189 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:27.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:27.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:27.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:27.339 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:27.639 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 36 16 64944 64928 112.638 79.4375 0.0113437 0.00887336 2023-10-09T10:09:28.181 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:09:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:09:27] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T10:09:28.324 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:28.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:28.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:28.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:28.508 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:28.718 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 37 16 66240 66224 111.783 81 0.0149588 0.00894081 2023-10-09T10:09:29.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:09:29.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:09:29.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:09:29.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:09:29.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:09:29.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:09:29.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:09:29.244 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:09:29.245 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:09:29.245 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:09:29.245 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:09:29.245 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:09:29.245 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:09:29.245 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:09:29.245 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:09:29.245 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:09:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:28 smithi012 ceph-mon[147926]: pgmap v735: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:09:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3113664401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:28 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:28 smithi012 ceph-mon[152263]: pgmap v735: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:09:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3113664401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:28 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:29.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:28 smithi110 ceph-mon[136032]: pgmap v735: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:09:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3113664401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:28 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:29.568 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:29.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:29.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:29.639 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 38 16 67477 67461 110.876 77.3125 0.00749751 0.00901402 2023-10-09T10:09:29.673 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:30.015 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:30.640 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 39 16 68675 68659 109.953 74.875 0.0153633 0.00908923 2023-10-09T10:09:30.764 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:30.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:30.850 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:09:30.850 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:09:30.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:30.930 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:30.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:30.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:31.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:31.101 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:31.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:31.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:30 smithi012 ceph-mon[147926]: pgmap v736: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:09:31.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:30 smithi012 ceph-mon[152263]: pgmap v736: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:09:31.188 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:30 smithi110 ceph-mon[136032]: pgmap v736: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:09:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:31.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:31.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:31.333 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:31.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:31.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:31.491 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:31.550 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:31.600 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:31.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:31.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:31.640 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:09:31.640276+0000 min lat: 0.00120497 max lat: 0.120692 avg lat: 0.00917047 2023-10-09T10:09:31.640 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:09:31.640 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 40 16 69816 69800 108.988 71.3125 0.0150124 0.00917047 2023-10-09T10:09:31.653 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:09:31.708 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:31.755 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:31.757 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:31.808 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:31.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:31.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:31.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:31.981 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:32.004 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:09:32.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:31 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3709504606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:32.376 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:31 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3709504606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:32.376 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:31 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3709504606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:32.569 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:32.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:32.671 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 41 16 70947 70931 108.051 70.6875 0.00601894 0.00924915 2023-10-09T10:09:32.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:32.815 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:32.886 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:32.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:32.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:32.973 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:32.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:33.065 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:33.141 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:33.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:33.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:33.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:32 smithi110 ceph-mon[136032]: pgmap v737: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:09:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:32 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:09:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:32 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:09:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:32 smithi110 ceph-mon[136032]: failed to encode map e284 with expected crc 2023-10-09T10:09:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3709504606' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:33.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:32 smithi110 ceph-mon[136032]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T10:09:33.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[147926]: pgmap v737: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:09:33.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:09:33.237 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:09:33.237 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[147926]: failed to encode map e284 with expected crc 2023-10-09T10:09:33.237 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3709504606' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:33.237 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[147926]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T10:09:33.238 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[152263]: pgmap v737: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:09:33.239 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:09:33.239 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:09:33.239 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[152263]: failed to encode map e284 with expected crc 2023-10-09T10:09:33.239 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3709504606' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:33.239 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:32 smithi012 ceph-mon[152263]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T10:09:33.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:33.319 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:33.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:09:33.491 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:33.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:33.599 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:33.647 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 42 16 72046 72030 107.114 68.6875 0.0113442 0.0093302 2023-10-09T10:09:33.688 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:33.813 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:33.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:33.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:33.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:33.990 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:34.641 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 43 16 73165 73149 106.25 69.9375 0.00666475 0.00940657 2023-10-09T10:09:34.673 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:09:34.718 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:34.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:34.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:34.824 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:34.860 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:34.920 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:34.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:34.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:34.985 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:34 smithi012 ceph-mon[147926]: pgmap v739: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T10:09:34.986 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:34 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2379227691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:09:35.013 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T10:09:35.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:35.110 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:35.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:34 smithi110 ceph-mon[136032]: pgmap v739: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T10:09:35.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:34 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2379227691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:09:35.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:34 smithi012 ceph-mon[152263]: pgmap v739: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T10:09:35.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:34 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2379227691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:09:35.641 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 44 16 74280 74264 105.419 69.6875 0.0103161 0.00948116 2023-10-09T10:09:35.707 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:35.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:35.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:35.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:35.862 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:35.918 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:35.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:35.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:36.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:36.098 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:36.165 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:36.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:36.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:36.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:36 smithi110 ceph-mon[136032]: failed to encode map e285 with expected crc 2023-10-09T10:09:36.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2379227691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:09:36.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:36 smithi110 ceph-mon[136032]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T10:09:36.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:36 smithi110 ceph-mon[136032]: failed to encode map e285 with expected crc 2023-10-09T10:09:36.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:36 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:09:36.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[147926]: failed to encode map e285 with expected crc 2023-10-09T10:09:36.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2379227691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:09:36.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[147926]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T10:09:36.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[147926]: failed to encode map e285 with expected crc 2023-10-09T10:09:36.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:09:36.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[152263]: failed to encode map e285 with expected crc 2023-10-09T10:09:36.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2379227691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:09:36.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[152263]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T10:09:36.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[152263]: failed to encode map e285 with expected crc 2023-10-09T10:09:36.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:36 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:09:36.291 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:36.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:36.409 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:36.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:36.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:36.520 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:36.553 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:36.603 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:36.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:36.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:36.641 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 45 16 75371 75355 104.592 68.1875 0.0114493 0.0095564 2023-10-09T10:09:36.715 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:09:36.765 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:36.833 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:09:36.833 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:09:37.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:37 smithi012 ceph-mon[152263]: pgmap v741: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T10:09:37.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:37 smithi012 ceph-mon[147926]: pgmap v741: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T10:09:37.324 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:09:37.326 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:37 smithi110 ceph-mon[136032]: pgmap v741: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T10:09:37.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:37.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:09:37.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T10:09:37.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:37.484 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:37.543 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:37.641 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 46 16 76518 76502 103.877 71.6875 0.0114931 0.00962236 2023-10-09T10:09:37.772 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:37.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:37.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:37.881 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:37.983 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:38.180 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:09:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:09:37] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.33.4" 2023-10-09T10:09:38.641 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 47 16 77595 77579 103.099 67.3125 0.0102032 0.009694 2023-10-09T10:09:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:39 smithi012 ceph-mon[152263]: pgmap v742: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T10:09:39.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:39 smithi012 ceph-mon[147926]: pgmap v742: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T10:09:39.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:39 smithi110 ceph-mon[136032]: pgmap v742: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T10:09:39.642 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 48 16 78744 78728 102.448 71.8125 0.00353122 0.00975563 2023-10-09T10:09:39.757 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:39.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:39.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:39.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:39.922 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:40.614 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:09:40.648 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 49 16 79791 79775 101.684 65.4375 0.0245584 0.0098291 2023-10-09T10:09:40.671 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:40.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:40.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:40.775 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:40.842 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:40.868 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:40.942 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:40.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:40.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:41.046 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:41.182 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:41.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:41 smithi110 ceph-mon[136032]: pgmap v743: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:09:41.236 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:41.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:41.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:41.404 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:41.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:41 smithi012 ceph-mon[147926]: pgmap v743: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:09:41.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:41 smithi012 ceph-mon[152263]: pgmap v743: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:09:41.469 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:41.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:41.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:41.581 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:41.643 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:41.646 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 50 16 80799 80783 100.911 63 0.0161138 0.009905 2023-10-09T10:09:41.700 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:41.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:41.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:41.809 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:41.871 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:41.875 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:41.994 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:42.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:42.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:42.173 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:42.646 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 51 16 81765 81749 100.116 60.375 0.0210625 0.00998358 2023-10-09T10:09:42.909 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:42.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:42.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:43.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:43.064 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:43.121 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:43.122 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:42 smithi012 ceph-mon[147926]: pgmap v744: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:09:43.122 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:42 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:43.122 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:42 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:43.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:42 smithi012 ceph-mon[152263]: pgmap v744: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:09:43.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:42 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:43.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:42 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:43.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:43.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:43.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:42 smithi110 ceph-mon[136032]: pgmap v744: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:09:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:42 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:42 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:43.225 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:43.267 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:43.317 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:43.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:43.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:43.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:43.452 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:43.512 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:43.592 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:43.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:43.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:43.646 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 52 16 82800 82784 99.4351 64.6875 0.0139919 0.0100519 2023-10-09T10:09:43.707 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:43.762 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:43.823 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:43.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:43.934 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:43.963 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:43.977 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:09:44.021 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:44.023 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:09:44.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:44.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:44.067 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:09:44.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:09:44.131 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:44.138 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:09:44.181 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:09:44.221 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:44.222 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:09:44.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:09:44.281 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:09:44.281 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:09:44.281 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:09:44.281 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:09:44.281 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:09:44.281 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:09:44.281 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:09:44.281 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:09:44.375 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:44.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:44.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:44.580 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:44.649 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 53 16 83647 83631 98.5536 52.9375 0.0146696 0.0101425 2023-10-09T10:09:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[147926]: pgmap v745: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T10:09:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/817225307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3847805945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:09:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:09:45.017 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:09:45.170 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:45.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:45.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[152263]: pgmap v745: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T10:09:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/817225307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3847805945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:09:45.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:44 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:09:45.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:44 smithi110 ceph-mon[136032]: pgmap v745: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T10:09:45.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/817225307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:45.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:45.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:09:45.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3847805945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:09:45.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:44 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:09:45.280 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:45.323 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:45.379 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:45.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:45.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:45.567 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:45.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:45.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:45.649 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 54 16 84623 84607 97.8584 61 0.0104306 0.0102144 2023-10-09T10:09:45.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:45.811 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:45.876 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:45.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:45.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:46.012 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T10:09:46.079 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:46.133 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:46.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:46.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:46.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[147926]: failed to encode map e286 with expected crc 2023-10-09T10:09:46.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:09:46.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[147926]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T10:09:46.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[147926]: failed to encode map e286 with expected crc 2023-10-09T10:09:46.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2256269989' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:09:46.143 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:09:46.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:46.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[152263]: failed to encode map e286 with expected crc 2023-10-09T10:09:46.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:09:46.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[152263]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T10:09:46.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[152263]: failed to encode map e286 with expected crc 2023-10-09T10:09:46.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2256269989' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:09:46.144 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:45 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:09:46.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:45 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:46.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:45 smithi110 ceph-mon[136032]: failed to encode map e286 with expected crc 2023-10-09T10:09:46.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:45 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:09:46.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:45 smithi110 ceph-mon[136032]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T10:09:46.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:45 smithi110 ceph-mon[136032]: failed to encode map e286 with expected crc 2023-10-09T10:09:46.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2256269989' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:09:46.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:45 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:09:46.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:46.271 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:46.405 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:09:46.405 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:09:46.405 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:09:46.405 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:09:46.405 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:09:46.631 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:46.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:46.649 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 55 16 85075 85059 96.5935 28.25 0.00443722 0.0102423 2023-10-09T10:09:46.660 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T10:09:46.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:09:46.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:46.778 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:46.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:46 smithi012 ceph-mon[147926]: pgmap v747: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:09:47.095 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:09:47.200 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:47.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:47.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:47.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:09:47.297 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:46 smithi012 ceph-mon[147926]: failed to encode map e287 with expected crc 2023-10-09T10:09:47.297 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:46 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:09:47.297 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:46 smithi012 ceph-mon[147926]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T10:09:47.297 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:46 smithi012 ceph-mon[152263]: pgmap v747: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:09:47.297 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:46 smithi012 ceph-mon[152263]: failed to encode map e287 with expected crc 2023-10-09T10:09:47.297 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:46 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:09:47.297 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:46 smithi012 ceph-mon[152263]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T10:09:47.304 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:47.332 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:46 smithi110 ceph-mon[136032]: pgmap v747: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:09:47.333 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:46 smithi110 ceph-mon[136032]: failed to encode map e287 with expected crc 2023-10-09T10:09:47.333 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:46 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:09:47.333 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:46 smithi110 ceph-mon[136032]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T10:09:47.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:47.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:47.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:47.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:09:47.513 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:47.600 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:09:47.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:47.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:09:47.649 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 56 16 85075 85059 94.8696 0 - 0.0102423 2023-10-09T10:09:47.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:09:47.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:47.749 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:47.793 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:47.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:47.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:47.870 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:47.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:47.915 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:09:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:09:47] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-10-09T10:09:47.928 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:48.011 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:48.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:48.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:48.152 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[147926]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[147926]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T10:09:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[147926]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[147926]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[147926]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[152263]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[152263]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T10:09:48.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[152263]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[152263]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:48 smithi012 ceph-mon[152263]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.205 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:48.238 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:48.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:48.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:48.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:48.369 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:48 smithi110 ceph-mon[136032]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.369 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:48 smithi110 ceph-mon[136032]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T10:09:48.370 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:48 smithi110 ceph-mon[136032]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.370 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:48 smithi110 ceph-mon[136032]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.370 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:48 smithi110 ceph-mon[136032]: failed to encode map e288 with expected crc 2023-10-09T10:09:48.373 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:48.408 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:48.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:48.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:48.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:48.555 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:48.600 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:48.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:48.649 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 57 16 85262 85246 93.4111 5.84375 0.0144921 0.0106988 2023-10-09T10:09:48.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:48.795 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:49.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[152263]: pgmap v750: 89 pgs: 3 peering, 86 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T10:09:49.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[152263]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[152263]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[152263]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T10:09:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[152263]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[147926]: pgmap v750: 89 pgs: 3 peering, 86 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T10:09:49.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[147926]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[147926]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[147926]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T10:09:49.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:49 smithi012 ceph-mon[147926]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:49 smithi110 ceph-mon[136032]: pgmap v750: 89 pgs: 3 peering, 86 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-09T10:09:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:49 smithi110 ceph-mon[136032]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:49 smithi110 ceph-mon[136032]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:49 smithi110 ceph-mon[136032]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T10:09:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:49 smithi110 ceph-mon[136032]: failed to encode map e289 with expected crc 2023-10-09T10:09:49.526 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:49.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:49.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:49.546 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:49.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:49.650 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 58 16 86075 86059 92.6769 50.8125 0.0227315 0.0107861 2023-10-09T10:09:49.677 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:49.717 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:49.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:49.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:49.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:49.913 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:50.342 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:50.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:50.453 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:50.527 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:50.651 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 59 16 86857 86841 91.9327 48.875 0.0238673 0.0108727 2023-10-09T10:09:50.700 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:50.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:50.808 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:50.844 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:50.881 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:50.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:50.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:50.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:51.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:51 smithi012 ceph-mon[147926]: pgmap v752: 89 pgs: 1 remapped+peering, 3 peering, 85 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 128 B/s, 0 objects/s recovering 2023-10-09T10:09:51.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:51 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:51.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:51 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:51.083 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:51.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:51 smithi110 ceph-mon[136032]: pgmap v752: 89 pgs: 1 remapped+peering, 3 peering, 85 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 128 B/s, 0 objects/s recovering 2023-10-09T10:09:51.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:51 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:51.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:51 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:51.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:51 smithi012 ceph-mon[152263]: pgmap v752: 89 pgs: 1 remapped+peering, 3 peering, 85 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 128 B/s, 0 objects/s recovering 2023-10-09T10:09:51.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:51 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:51.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:51 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:51.651 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:09:51.651418+0000 min lat: 0.00120497 max lat: 2.42059 avg lat: 0.0109511 2023-10-09T10:09:51.651 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:09:51.651 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 60 16 87672 87656 91.2497 50.9375 0.0113131 0.0109511 2023-10-09T10:09:52.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:52 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 1/257334 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:09:52.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:52 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 1/257334 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:09:52.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:52 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 1/257334 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:09:52.652 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 61 16 88518 88502 90.6191 52.875 0.00748982 0.0110298 2023-10-09T10:09:53.365 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:53 smithi012 ceph-mon[152263]: pgmap v753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 1/257334 objects degraded (0.000%); 5236/257334 objects misplaced (2.035%); 501 KiB/s, 8 objects/s recovering 2023-10-09T10:09:53.365 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:53 smithi012 ceph-mon[147926]: pgmap v753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 1/257334 objects degraded (0.000%); 5236/257334 objects misplaced (2.035%); 501 KiB/s, 8 objects/s recovering 2023-10-09T10:09:53.432 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:53.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:53.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:53 smithi110 ceph-mon[136032]: pgmap v753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 1/257334 objects degraded (0.000%); 5236/257334 objects misplaced (2.035%); 501 KiB/s, 8 objects/s recovering 2023-10-09T10:09:53.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:53.637 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:53.652 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 62 16 89192 89176 89.8373 42.125 0.00591669 0.0111251 2023-10-09T10:09:53.842 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:53.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:53.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:53.899 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:53.965 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:53.997 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:54.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:54.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:54.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:54.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:54.181 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:54.234 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:54.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:54.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:54.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:54.378 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:54 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:09:54.378 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:54 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:09:54.378 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:54 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:54.381 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:54 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:09:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:54 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:09:54.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:54 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:54.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:54 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:09:54.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:54 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:09:54.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:54 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:09:54.440 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:54.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:54.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:54.545 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:09:54.585 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:54.636 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:09:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:54.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:09:54.652 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 63 16 89979 89963 89.1923 49.1875 0.053479 0.0112063 2023-10-09T10:09:54.693 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:09:54.736 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:09:54.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:54.775 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:09:54.846 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:09:54.869 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:54.884 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:09:54.922 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:09:54.964 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:09:54.980 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:09:54.996 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:09:55.002 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:09:55.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:09:55.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:09:55.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:09:55.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:09:55.026 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:09:55.027 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:09:55.027 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:09:55.027 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:09:55.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:55 smithi012 ceph-mon[147926]: pgmap v754: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 5374/262683 objects degraded (2.046%); 5236/262683 objects misplaced (1.993%); 412 KiB/s, 6 objects/s recovering 2023-10-09T10:09:55.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/463997312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:55.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:55 smithi012 ceph-mon[147926]: from='client.45572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:09:55.327 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:55 smithi012 ceph-mon[152263]: pgmap v754: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 5374/262683 objects degraded (2.046%); 5236/262683 objects misplaced (1.993%); 412 KiB/s, 6 objects/s recovering 2023-10-09T10:09:55.328 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/463997312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:55.328 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:55 smithi012 ceph-mon[152263]: from='client.45572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:09:55.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:55 smithi110 ceph-mon[136032]: pgmap v754: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 5374/262683 objects degraded (2.046%); 5236/262683 objects misplaced (1.993%); 412 KiB/s, 6 objects/s recovering 2023-10-09T10:09:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/463997312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:09:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:55 smithi110 ceph-mon[136032]: from='client.45572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:09:55.564 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:55.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:55.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:55.652 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 64 16 90686 90670 88.4894 44.1875 0.02429 0.0112953 2023-10-09T10:09:55.673 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:55.763 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:55.832 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:55.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:55.938 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:55.952 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":754,"stamp":"2023-10-09T10:09:53.301652+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5737420704,"num_objects":87561,"num_object_clones":0,"num_object_copies":262683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":5236,"num_objects_unfound":0,"num_objects_dirty":87561,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":87797,"num_write_kb":5604813,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":4270796,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87731,"ondisk_log_size":87731,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19062976,"kb_used_data":16700672,"kb_used_omap":167,"kb_used_meta":2362072,"kb_avail":730898240,"statfs":{"total":767960285184,"available":748439797760,"internally_reserved":0,"allocated":17101488128,"data_stored":17095261820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2418762329},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,6,6,3,0,0,0,0,0,0,0,8],"upper_bound":8192},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":354484224,"num_objects":5409,"num_object_clones":0,"num_object_copies":16227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":5236,"num_objects_unfound":0,"num_objects_dirty":5409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5409,"num_write_kb":346176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":3080746,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.299911"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":130,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.004092+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:09:47.004092+0000","last_peered":"2023-10-09T10:09:47.004092+0000","last_clean":"2023-10-09T10:09:47.004092+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:09:47.004092+0000","last_undegraded":"2023-10-09T10:09:47.004092+0000","last_fullsized":"2023-10-09T10:09:47.004092+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":245,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996826+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:09:46.996826+0000","last_peered":"2023-10-09T10:09:46.996826+0000","last_clean":"2023-10-09T10:09:46.996826+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:09:46.996826+0000","last_undegraded":"2023-10-09T10:09:46.996826+0000","last_fullsized":"2023-10-09T10:09:46.996826+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":291,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.005753+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:09:47.005753+0000","last_peered":"2023-10-09T10:09:47.005753+0000","last_clean":"2023-10-09T10:09:47.005753+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:09:47.005753+0000","last_undegraded":"2023-10-09T10:09:47.005753+0000","last_fullsized":"2023-10-09T10:09:47.005753+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":154,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.003146+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:09:47.003146+0000","last_peered":"2023-10-09T10:09:47.003146+0000","last_clean":"2023-10-09T10:09:47.003146+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:09:47.003146+0000","last_undegraded":"2023-10-09T10:09:47.003146+0000","last_fullsized":"2023-10-09T10:09:47.003146+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":259,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.995613+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:09:46.995613+0000","last_peered":"2023-10-09T10:09:46.995613+0000","last_clean":"2023-10-09T10:09:46.995613+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:09:46.995613+0000","last_undegraded":"2023-10-09T10:09:46.995613+0000","last_fullsized":"2023-10-09T10:09:46.995613+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001037+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:09:47.001037+0000","last_peered":"2023-10-09T10:09:47.001037+0000","last_clean":"2023-10-09T10:09:47.001037+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:09:47.001037+0000","last_undegraded":"2023-10-09T10:09:47.001037+0000","last_fullsized":"2023-10-09T10:09:47.001037+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":172,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.700906+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:09:47.700906+0000","last_peered":"2023-10-09T10:09:47.700906+0000","last_clean":"2023-10-09T10:09:47.700906+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:09:47.700906+0000","last_undegraded":"2023-10-09T10:09:47.700906+0000","last_fullsized":"2023-10-09T10:09:47.700906+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":240,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.995527+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:09:46.995527+0000","last_peered":"2023-10-09T10:09:46.995527+0000","last_clean":"2023-10-09T10:09:46.995527+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:09:46.995527+0000","last_undegraded":"2023-10-09T10:09:46.995527+0000","last_fullsized":"2023-10-09T10:09:46.995527+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000651+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:09:47.000651+0000","last_peered":"2023-10-09T10:09:47.000651+0000","last_clean":"2023-10-09T10:09:47.000651+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:09:47.000651+0000","last_undegraded":"2023-10-09T10:09:47.000651+0000","last_fullsized":"2023-10-09T10:09:47.000651+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":201,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996392+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:09:46.996392+0000","last_peered":"2023-10-09T10:09:46.996392+0000","last_clean":"2023-10-09T10:09:46.996392+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:09:46.996392+0000","last_undegraded":"2023-10-09T10:09:46.996392+0000","last_fullsized":"2023-10-09T10:09:46.996392+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":327,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222174+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:09:47.222174+0000","last_peered":"2023-10-09T10:09:47.222174+0000","last_clean":"2023-10-09T10:09:47.222174+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:09:47.222174+0000","last_undegraded":"2023-10-09T10:09:47.222174+0000","last_fullsized":"2023-10-09T10:09:47.222174+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":283,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001527+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:09:47.001527+0000","last_peered":"2023-10-09T10:09:47.001527+0000","last_clean":"2023-10-09T10:09:47.001527+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:09:47.001527+0000","last_undegraded":"2023-10-09T10:09:47.001527+0000","last_fullsized":"2023-10-09T10:09:47.001527+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":176,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.221960+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:09:47.221960+0000","last_peered":"2023-10-09T10:09:47.221960+0000","last_clean":"2023-10-09T10:09:47.221960+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:09:47.221960+0000","last_undegraded":"2023-10-09T10:09:47.221960+0000","last_fullsized":"2023-10-09T10:09:47.221960+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":155,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.005359+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:09:47.005359+0000","last_peered":"2023-10-09T10:09:47.005359+0000","last_clean":"2023-10-09T10:09:47.005359+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:09:47.005359+0000","last_undegraded":"2023-10-09T10:09:47.005359+0000","last_fullsized":"2023-10-09T10:09:47.005359+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":360,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222167+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:09:47.222167+0000","last_peered":"2023-10-09T10:09:47.222167+0000","last_clean":"2023-10-09T10:09:47.222167+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:09:47.222167+0000","last_undegraded":"2023-10-09T10:09:47.222167+0000","last_fullsized":"2023-10-09T10:09:47.222167+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":227,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996905+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:09:46.996905+0000","last_peered":"2023-10-09T10:09:46.996905+0000","last_clean":"2023-10-09T10:09:46.996905+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:09:46.996905+0000","last_undegraded":"2023-10-09T10:09:46.996905+0000","last_fullsized":"2023-10-09T10:09:46.996905+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":154,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000106+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:09:47.000106+0000","last_peered":"2023-10-09T10:09:47.000106+0000","last_clean":"2023-10-09T10:09:47.000106+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:09:47.000106+0000","last_undegraded":"2023-10-09T10:09:47.000106+0000","last_fullsized":"2023-10-09T10:09:47.000106+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000651+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:09:47.000651+0000","last_peered":"2023-10-09T10:09:47.000651+0000","last_clean":"2023-10-09T10:09:47.000651+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:09:47.000651+0000","last_undegraded":"2023-10-09T10:09:47.000651+0000","last_fullsized":"2023-10-09T10:09:47.000651+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":157,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.221936+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:09:47.221936+0000","last_peered":"2023-10-09T10:09:47.221936+0000","last_clean":"2023-10-09T10:09:47.221936+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:09:47.221936+0000","last_undegraded":"2023-10-09T10:09:47.221936+0000","last_fullsized":"2023-10-09T10:09:47.221936+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":154,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.700861+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:09:47.700861+0000","last_peered":"2023-10-09T10:09:47.700861+0000","last_clean":"2023-10-09T10:09:47.700861+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:09:47.700861+0000","last_undegraded":"2023-10-09T10:09:47.700861+0000","last_fullsized":"2023-10-09T10:09:47.700861+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":177,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.999692+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:09:46.999692+0000","last_peered":"2023-10-09T10:09:46.999692+0000","last_clean":"2023-10-09T10:09:46.999692+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:09:46.999692+0000","last_undegraded":"2023-10-09T10:09:46.999692+0000","last_fullsized":"2023-10-09T10:09:46.999692+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":240,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996416+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:09:46.996416+0000","last_peered":"2023-10-09T10:09:46.996416+0000","last_clean":"2023-10-09T10:09:46.996416+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:09:46.996416+0000","last_undegraded":"2023-10-09T10:09:46.996416+0000","last_fullsized":"2023-10-09T10:09:46.996416+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"289'5629","reported_seq":5663,"reported_epoch":289,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:09:52.822475+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:09:52.822475+0000","last_peered":"2023-10-09T10:09:52.822475+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:09:52.822475+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5629,"log_dups_size":0,"ondisk_log_size":5629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368902144,"num_objects":5629,"num_object_clones":0,"num_object_copies":16887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5629,"num_write_kb":360256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":256},{"shards":"4,5,6","objects":5373}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":321,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222092+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:09:47.222092+0000","last_peered":"2023-10-09T10:09:47.222092+0000","last_clean":"2023-10-09T10:09:47.222092+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:09:47.222092+0000","last_undegraded":"2023-10-09T10:09:47.222092+0000","last_fullsized":"2023-10-09T10:09:47.222092+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":107,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000089+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:09:47.000089+0000","last_peered":"2023-10-09T10:09:47.000089+0000","last_clean":"2023-10-09T10:09:47.000089+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:09:47.000089+0000","last_undegraded":"2023-10-09T10:09:47.000089+0000","last_fullsized":"2023-10-09T10:09:47.000089+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"289'5442","reported_seq":5468,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:49.972243+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:09:49.972243+0000","last_peered":"2023-10-09T10:09:49.972243+0000","last_clean":"2023-10-09T10:09:49.972243+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:09:49.972243+0000","last_undegraded":"2023-10-09T10:09:49.972243+0000","last_fullsized":"2023-10-09T10:09:49.972243+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356646912,"num_objects":5442,"num_object_clones":0,"num_object_copies":16326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5442,"num_write_kb":348288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":309,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222027+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:09:47.222027+0000","last_peered":"2023-10-09T10:09:47.222027+0000","last_clean":"2023-10-09T10:09:47.222027+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:09:47.222027+0000","last_undegraded":"2023-10-09T10:09:47.222027+0000","last_fullsized":"2023-10-09T10:09:47.222027+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":199,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.005666+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:09:47.005666+0000","last_peered":"2023-10-09T10:09:47.005666+0000","last_clean":"2023-10-09T10:09:47.005666+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:09:47.005666+0000","last_undegraded":"2023-10-09T10:09:47.005666+0000","last_fullsized":"2023-10-09T10:09:47.005666+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"289'5507","reported_seq":5534,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:52.653577+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:09:52.653577+0000","last_peered":"2023-10-09T10:09:52.653577+0000","last_clean":"2023-10-09T10:09:52.653577+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:09:52.653577+0000","last_undegraded":"2023-10-09T10:09:52.653577+0000","last_fullsized":"2023-10-09T10:09:52.653577+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5507,"log_dups_size":0,"ondisk_log_size":5507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360906752,"num_objects":5507,"num_object_clones":0,"num_object_copies":16521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5507,"num_write_kb":352448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":111,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000786+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:09:47.000786+0000","last_peered":"2023-10-09T10:09:47.000786+0000","last_clean":"2023-10-09T10:09:47.000786+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:09:47.000786+0000","last_undegraded":"2023-10-09T10:09:47.000786+0000","last_fullsized":"2023-10-09T10:09:47.000786+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":132,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001525+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:09:47.001525+0000","last_peered":"2023-10-09T10:09:47.001525+0000","last_clean":"2023-10-09T10:09:47.001525+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:09:47.001525+0000","last_undegraded":"2023-10-09T10:09:47.001525+0000","last_fullsized":"2023-10-09T10:09:47.001525+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"289'5322","reported_seq":5349,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:49.478507+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:09:49.478507+0000","last_peered":"2023-10-09T10:09:49.478507+0000","last_clean":"2023-10-09T10:09:49.478507+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:09:49.478507+0000","last_undegraded":"2023-10-09T10:09:49.478507+0000","last_fullsized":"2023-10-09T10:09:49.478507+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5322,"log_dups_size":0,"ondisk_log_size":5322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348782592,"num_objects":5322,"num_object_clones":0,"num_object_copies":15966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5322,"num_write_kb":340608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":122,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001624+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:09:47.001624+0000","last_peered":"2023-10-09T10:09:47.001624+0000","last_clean":"2023-10-09T10:09:47.001624+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:09:47.001624+0000","last_undegraded":"2023-10-09T10:09:47.001624+0000","last_fullsized":"2023-10-09T10:09:47.001624+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":155,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001074+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:09:47.001074+0000","last_peered":"2023-10-09T10:09:47.001074+0000","last_clean":"2023-10-09T10:09:47.001074+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:09:47.001074+0000","last_undegraded":"2023-10-09T10:09:47.001074+0000","last_fullsized":"2023-10-09T10:09:47.001074+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"289'5549","reported_seq":5576,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:51.070488+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:09:51.070488+0000","last_peered":"2023-10-09T10:09:51.070488+0000","last_clean":"2023-10-09T10:09:51.070488+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:09:51.070488+0000","last_undegraded":"2023-10-09T10:09:51.070488+0000","last_fullsized":"2023-10-09T10:09:51.070488+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5549,"log_dups_size":0,"ondisk_log_size":5549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363659264,"num_objects":5549,"num_object_clones":0,"num_object_copies":16647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5549,"num_write_kb":355136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":281,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.997934+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:09:46.997934+0000","last_peered":"2023-10-09T10:09:46.997934+0000","last_clean":"2023-10-09T10:09:46.997934+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:09:46.997934+0000","last_undegraded":"2023-10-09T10:09:46.997934+0000","last_fullsized":"2023-10-09T10:09:46.997934+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996774+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:09:46.996774+0000","last_peered":"2023-10-09T10:09:46.996774+0000","last_clean":"2023-10-09T10:09:46.996774+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:09:46.996774+0000","last_undegraded":"2023-10-09T10:09:46.996774+0000","last_fullsized":"2023-10-09T10:09:46.996774+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"289'5487","reported_seq":5513,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:49.479371+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:09:49.479371+0000","last_peered":"2023-10-09T10:09:49.479371+0000","last_clean":"2023-10-09T10:09:49.479371+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:09:49.479371+0000","last_undegraded":"2023-10-09T10:09:49.479371+0000","last_fullsized":"2023-10-09T10:09:49.479371+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5487,"log_dups_size":0,"ondisk_log_size":5487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359596032,"num_objects":5487,"num_object_clones":0,"num_object_copies":16461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5487,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":213,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001473+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:09:47.001473+0000","last_peered":"2023-10-09T10:09:47.001473+0000","last_clean":"2023-10-09T10:09:47.001473+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:09:47.001473+0000","last_undegraded":"2023-10-09T10:09:47.001473+0000","last_fullsized":"2023-10-09T10:09:47.001473+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":219,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.008124+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:09:47.008124+0000","last_peered":"2023-10-09T10:09:47.008124+0000","last_clean":"2023-10-09T10:09:47.008124+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:09:47.008124+0000","last_undegraded":"2023-10-09T10:09:47.008124+0000","last_fullsized":"2023-10-09T10:09:47.008124+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"289'5426","reported_seq":5453,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:52.680578+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:09:52.680578+0000","last_peered":"2023-10-09T10:09:52.680578+0000","last_clean":"2023-10-09T10:09:52.680578+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:09:52.680578+0000","last_undegraded":"2023-10-09T10:09:52.680578+0000","last_fullsized":"2023-10-09T10:09:52.680578+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5426,"log_dups_size":0,"ondisk_log_size":5426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355598336,"num_objects":5426,"num_object_clones":0,"num_object_copies":16278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5426,"num_write_kb":347264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":183,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222134+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:09:47.222134+0000","last_peered":"2023-10-09T10:09:47.222134+0000","last_clean":"2023-10-09T10:09:47.222134+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:09:47.222134+0000","last_undegraded":"2023-10-09T10:09:47.222134+0000","last_fullsized":"2023-10-09T10:09:47.222134+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":155,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001042+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:09:47.001042+0000","last_peered":"2023-10-09T10:09:47.001042+0000","last_clean":"2023-10-09T10:09:47.001042+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:09:47.001042+0000","last_undegraded":"2023-10-09T10:09:47.001042+0000","last_fullsized":"2023-10-09T10:09:47.001042+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"289'5404","reported_seq":5431,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:49.490602+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:09:49.490602+0000","last_peered":"2023-10-09T10:09:49.490602+0000","last_clean":"2023-10-09T10:09:49.490602+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:09:49.490602+0000","last_undegraded":"2023-10-09T10:09:49.490602+0000","last_fullsized":"2023-10-09T10:09:49.490602+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354156544,"num_objects":5404,"num_object_clones":0,"num_object_copies":16212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5404,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":242,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.003026+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:09:48.003026+0000","last_peered":"2023-10-09T10:09:48.003026+0000","last_clean":"2023-10-09T10:09:48.003026+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:09:48.003026+0000","last_undegraded":"2023-10-09T10:09:48.003026+0000","last_fullsized":"2023-10-09T10:09:48.003026+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":291,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001264+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:09:47.001264+0000","last_peered":"2023-10-09T10:09:47.001264+0000","last_clean":"2023-10-09T10:09:47.001264+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:09:47.001264+0000","last_undegraded":"2023-10-09T10:09:47.001264+0000","last_fullsized":"2023-10-09T10:09:47.001264+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"289'5549","reported_seq":5576,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:52.653693+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:09:52.653693+0000","last_peered":"2023-10-09T10:09:52.653693+0000","last_clean":"2023-10-09T10:09:52.653693+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:09:52.653693+0000","last_undegraded":"2023-10-09T10:09:52.653693+0000","last_fullsized":"2023-10-09T10:09:52.653693+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5549,"log_dups_size":0,"ondisk_log_size":5549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363659264,"num_objects":5549,"num_object_clones":0,"num_object_copies":16647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5549,"num_write_kb":355136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":291,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000785+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:09:47.000785+0000","last_peered":"2023-10-09T10:09:47.000785+0000","last_clean":"2023-10-09T10:09:47.000785+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:09:47.000785+0000","last_undegraded":"2023-10-09T10:09:47.000785+0000","last_fullsized":"2023-10-09T10:09:47.000785+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000632+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:09:47.000632+0000","last_peered":"2023-10-09T10:09:47.000632+0000","last_clean":"2023-10-09T10:09:47.000632+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:09:47.000632+0000","last_undegraded":"2023-10-09T10:09:47.000632+0000","last_fullsized":"2023-10-09T10:09:47.000632+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":247,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.005199+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:09:47.005199+0000","last_peered":"2023-10-09T10:09:47.005199+0000","last_clean":"2023-10-09T10:09:47.005199+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:09:47.005199+0000","last_undegraded":"2023-10-09T10:09:47.005199+0000","last_fullsized":"2023-10-09T10:09:47.005199+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"289'5545","reported_seq":5571,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:52.846846+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:09:52.846846+0000","last_peered":"2023-10-09T10:09:52.846846+0000","last_clean":"2023-10-09T10:09:52.846846+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:09:52.846846+0000","last_undegraded":"2023-10-09T10:09:52.846846+0000","last_fullsized":"2023-10-09T10:09:52.846846+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363397120,"num_objects":5545,"num_object_clones":0,"num_object_copies":16635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5545,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":107,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000339+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:09:47.000339+0000","last_peered":"2023-10-09T10:09:47.000339+0000","last_clean":"2023-10-09T10:09:47.000339+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:09:47.000339+0000","last_undegraded":"2023-10-09T10:09:47.000339+0000","last_fullsized":"2023-10-09T10:09:47.000339+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":104,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000067+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:09:47.000067+0000","last_peered":"2023-10-09T10:09:47.000067+0000","last_clean":"2023-10-09T10:09:47.000067+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:09:47.000067+0000","last_undegraded":"2023-10-09T10:09:47.000067+0000","last_fullsized":"2023-10-09T10:09:47.000067+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":314,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000702+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:09:47.000702+0000","last_peered":"2023-10-09T10:09:47.000702+0000","last_clean":"2023-10-09T10:09:47.000702+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:09:47.000702+0000","last_undegraded":"2023-10-09T10:09:47.000702+0000","last_fullsized":"2023-10-09T10:09:47.000702+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"289'5404","reported_seq":5431,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:51.082697+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:09:51.082697+0000","last_peered":"2023-10-09T10:09:51.082697+0000","last_clean":"2023-10-09T10:09:51.082697+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:09:51.082697+0000","last_undegraded":"2023-10-09T10:09:51.082697+0000","last_fullsized":"2023-10-09T10:09:51.082697+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354156544,"num_objects":5404,"num_object_clones":0,"num_object_copies":16212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5404,"num_write_kb":345856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":208,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.003025+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:09:48.003025+0000","last_peered":"2023-10-09T10:09:48.003025+0000","last_clean":"2023-10-09T10:09:48.003025+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:09:48.003025+0000","last_undegraded":"2023-10-09T10:09:48.003025+0000","last_fullsized":"2023-10-09T10:09:48.003025+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":283,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.002127+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:09:47.002127+0000","last_peered":"2023-10-09T10:09:47.002127+0000","last_clean":"2023-10-09T10:09:47.002127+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:09:47.002127+0000","last_undegraded":"2023-10-09T10:09:47.002127+0000","last_fullsized":"2023-10-09T10:09:47.002127+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":304,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001189+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:09:47.001189+0000","last_peered":"2023-10-09T10:09:47.001189+0000","last_clean":"2023-10-09T10:09:47.001189+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:09:47.001189+0000","last_undegraded":"2023-10-09T10:09:47.001189+0000","last_fullsized":"2023-10-09T10:09:47.001189+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"289'5473","reported_seq":5499,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:52.821632+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:09:52.821632+0000","last_peered":"2023-10-09T10:09:52.821632+0000","last_clean":"2023-10-09T10:09:52.821632+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:09:52.821632+0000","last_undegraded":"2023-10-09T10:09:52.821632+0000","last_fullsized":"2023-10-09T10:09:52.821632+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358678528,"num_objects":5473,"num_object_clones":0,"num_object_copies":16419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5473,"num_write_kb":350272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":116,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000430+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:09:47.000430+0000","last_peered":"2023-10-09T10:09:47.000430+0000","last_clean":"2023-10-09T10:09:47.000430+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:09:47.000430+0000","last_undegraded":"2023-10-09T10:09:47.000430+0000","last_fullsized":"2023-10-09T10:09:47.000430+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":535,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222251+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:09:47.222251+0000","last_peered":"2023-10-09T10:09:47.222251+0000","last_clean":"2023-10-09T10:09:47.222251+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:09:47.222251+0000","last_undegraded":"2023-10-09T10:09:47.222251+0000","last_fullsized":"2023-10-09T10:09:47.222251+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":192,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996686+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:09:46.996686+0000","last_peered":"2023-10-09T10:09:46.996686+0000","last_clean":"2023-10-09T10:09:46.996686+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:09:46.996686+0000","last_undegraded":"2023-10-09T10:09:46.996686+0000","last_fullsized":"2023-10-09T10:09:46.996686+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":204,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.221987+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:09:47.221987+0000","last_peered":"2023-10-09T10:09:47.221987+0000","last_clean":"2023-10-09T10:09:47.221987+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:09:47.221987+0000","last_undegraded":"2023-10-09T10:09:47.221987+0000","last_fullsized":"2023-10-09T10:09:47.221987+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"289'5419","reported_seq":5446,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:49.446272+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:09:49.446272+0000","last_peered":"2023-10-09T10:09:49.446272+0000","last_clean":"2023-10-09T10:09:49.446272+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:09:49.446272+0000","last_undegraded":"2023-10-09T10:09:49.446272+0000","last_fullsized":"2023-10-09T10:09:49.446272+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5419,"log_dups_size":0,"ondisk_log_size":5419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355139584,"num_objects":5419,"num_object_clones":0,"num_object_copies":16257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5419,"num_write_kb":346816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":263,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.002236+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:09:47.002236+0000","last_peered":"2023-10-09T10:09:47.002236+0000","last_clean":"2023-10-09T10:09:47.002236+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:09:47.002236+0000","last_undegraded":"2023-10-09T10:09:47.002236+0000","last_fullsized":"2023-10-09T10:09:47.002236+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":165,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000542+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:09:47.000542+0000","last_peered":"2023-10-09T10:09:47.000542+0000","last_clean":"2023-10-09T10:09:47.000542+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:09:47.000542+0000","last_undegraded":"2023-10-09T10:09:47.000542+0000","last_fullsized":"2023-10-09T10:09:47.000542+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":193,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.009133+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:09:47.009133+0000","last_peered":"2023-10-09T10:09:47.009133+0000","last_clean":"2023-10-09T10:09:47.009133+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:09:47.009133+0000","last_undegraded":"2023-10-09T10:09:47.009133+0000","last_fullsized":"2023-10-09T10:09:47.009133+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"289'5434","reported_seq":5559,"reported_epoch":289,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:09:51.085260+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:09:51.085260+0000","last_peered":"2023-10-09T10:09:51.085260+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:09:51.085260+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356122624,"num_objects":5434,"num_object_clones":0,"num_object_copies":16302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":5236,"num_objects_unfound":0,"num_objects_dirty":5434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5434,"num_write_kb":347776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":3080192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":197},{"shards":"1,3,4","objects":5236}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":308,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222522+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:09:47.222522+0000","last_peered":"2023-10-09T10:09:47.222522+0000","last_clean":"2023-10-09T10:09:47.222522+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:09:47.222522+0000","last_undegraded":"2023-10-09T10:09:47.222522+0000","last_fullsized":"2023-10-09T10:09:47.222522+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":320,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.006422+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:09:47.006422+0000","last_peered":"2023-10-09T10:09:47.006422+0000","last_clean":"2023-10-09T10:09:47.006422+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:09:47.006422+0000","last_undegraded":"2023-10-09T10:09:47.006422+0000","last_fullsized":"2023-10-09T10:09:47.006422+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":157,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000141+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:09:47.000141+0000","last_peered":"2023-10-09T10:09:47.000141+0000","last_clean":"2023-10-09T10:09:47.000141+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:09:47.000141+0000","last_undegraded":"2023-10-09T10:09:47.000141+0000","last_fullsized":"2023-10-09T10:09:47.000141+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"289'5498","reported_seq":5525,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:52.636958+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:09:52.636958+0000","last_peered":"2023-10-09T10:09:52.636958+0000","last_clean":"2023-10-09T10:09:52.636958+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:09:52.636958+0000","last_undegraded":"2023-10-09T10:09:52.636958+0000","last_fullsized":"2023-10-09T10:09:52.636958+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5498,"log_dups_size":0,"ondisk_log_size":5498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360316928,"num_objects":5498,"num_object_clones":0,"num_object_copies":16494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5498,"num_write_kb":351872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.003609+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:09:48.003609+0000","last_peered":"2023-10-09T10:09:48.003609+0000","last_clean":"2023-10-09T10:09:48.003609+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:09:48.003609+0000","last_undegraded":"2023-10-09T10:09:48.003609+0000","last_fullsized":"2023-10-09T10:09:48.003609+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":265,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001124+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:09:47.001124+0000","last_peered":"2023-10-09T10:09:47.001124+0000","last_clean":"2023-10-09T10:09:47.001124+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:09:47.001124+0000","last_undegraded":"2023-10-09T10:09:47.001124+0000","last_fullsized":"2023-10-09T10:09:47.001124+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":240,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.995362+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:09:46.995362+0000","last_peered":"2023-10-09T10:09:46.995362+0000","last_clean":"2023-10-09T10:09:46.995362+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:09:46.995362+0000","last_undegraded":"2023-10-09T10:09:46.995362+0000","last_fullsized":"2023-10-09T10:09:46.995362+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"289'5449","reported_seq":5475,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:52.834425+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:09:52.834425+0000","last_peered":"2023-10-09T10:09:52.834425+0000","last_clean":"2023-10-09T10:09:52.834425+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:09:52.834425+0000","last_undegraded":"2023-10-09T10:09:52.834425+0000","last_fullsized":"2023-10-09T10:09:52.834425+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5449,"log_dups_size":0,"ondisk_log_size":5449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357105664,"num_objects":5449,"num_object_clones":0,"num_object_copies":16347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5449,"num_write_kb":348736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":197,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.002759+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:09:47.002759+0000","last_peered":"2023-10-09T10:09:47.002759+0000","last_clean":"2023-10-09T10:09:47.002759+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:09:47.002759+0000","last_undegraded":"2023-10-09T10:09:47.002759+0000","last_fullsized":"2023-10-09T10:09:47.002759+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":159,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.002662+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:09:47.002662+0000","last_peered":"2023-10-09T10:09:47.002662+0000","last_clean":"2023-10-09T10:09:47.002662+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:09:47.002662+0000","last_undegraded":"2023-10-09T10:09:47.002662+0000","last_fullsized":"2023-10-09T10:09:47.002662+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":142,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000229+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:09:47.000229+0000","last_peered":"2023-10-09T10:09:47.000229+0000","last_clean":"2023-10-09T10:09:47.000229+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:09:47.000229+0000","last_undegraded":"2023-10-09T10:09:47.000229+0000","last_fullsized":"2023-10-09T10:09:47.000229+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":153,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.999972+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:09:46.999972+0000","last_peered":"2023-10-09T10:09:46.999972+0000","last_clean":"2023-10-09T10:09:46.999972+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:09:46.999972+0000","last_undegraded":"2023-10-09T10:09:46.999972+0000","last_fullsized":"2023-10-09T10:09:46.999972+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":226,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.011779+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:09:47.011779+0000","last_peered":"2023-10-09T10:09:47.011779+0000","last_clean":"2023-10-09T10:09:47.011779+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:09:47.011779+0000","last_undegraded":"2023-10-09T10:09:47.011779+0000","last_fullsized":"2023-10-09T10:09:47.011779+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":126,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001508+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:09:47.001508+0000","last_peered":"2023-10-09T10:09:47.001508+0000","last_clean":"2023-10-09T10:09:47.001508+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:09:47.001508+0000","last_undegraded":"2023-10-09T10:09:47.001508+0000","last_fullsized":"2023-10-09T10:09:47.001508+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":166,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.997849+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:09:46.997849+0000","last_peered":"2023-10-09T10:09:46.997849+0000","last_clean":"2023-10-09T10:09:46.997849+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:09:46.997849+0000","last_undegraded":"2023-10-09T10:09:46.997849+0000","last_fullsized":"2023-10-09T10:09:46.997849+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":126,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000561+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:09:47.000561+0000","last_peered":"2023-10-09T10:09:47.000561+0000","last_clean":"2023-10-09T10:09:47.000561+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:09:47.000561+0000","last_undegraded":"2023-10-09T10:09:47.000561+0000","last_fullsized":"2023-10-09T10:09:47.000561+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":175,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996611+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:09:46.996611+0000","last_peered":"2023-10-09T10:09:46.996611+0000","last_clean":"2023-10-09T10:09:46.996611+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:09:46.996611+0000","last_undegraded":"2023-10-09T10:09:46.996611+0000","last_fullsized":"2023-10-09T10:09:46.996611+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":305,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.010746+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:09:47.010746+0000","last_peered":"2023-10-09T10:09:47.010746+0000","last_clean":"2023-10-09T10:09:47.010746+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:09:47.010746+0000","last_undegraded":"2023-10-09T10:09:47.010746+0000","last_fullsized":"2023-10-09T10:09:47.010746+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":325,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996519+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:09:46.996519+0000","last_peered":"2023-10-09T10:09:46.996519+0000","last_clean":"2023-10-09T10:09:46.996519+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:09:46.996519+0000","last_undegraded":"2023-10-09T10:09:46.996519+0000","last_fullsized":"2023-10-09T10:09:46.996519+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":235,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.014298+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:09:47.014298+0000","last_peered":"2023-10-09T10:09:47.014298+0000","last_clean":"2023-10-09T10:09:47.014298+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:09:47.014298+0000","last_undegraded":"2023-10-09T10:09:47.014298+0000","last_fullsized":"2023-10-09T10:09:47.014298+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":155,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001929+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:09:47.001929+0000","last_peered":"2023-10-09T10:09:47.001929+0000","last_clean":"2023-10-09T10:09:47.001929+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:09:47.001929+0000","last_undegraded":"2023-10-09T10:09:47.001929+0000","last_fullsized":"2023-10-09T10:09:47.001929+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5736824832,"num_objects":87537,"num_object_clones":0,"num_object_copies":262611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":5236,"num_objects_unfound":0,"num_objects_dirty":87537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87537,"num_write_kb":5602368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":3080192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17153261568,"data_stored":17153261568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87537,"ondisk_log_size":87537,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430482,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2018228,"kb_used_data":1712812,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91726924,"statfs":{"total":95995035648,"available":93928370176,"internally_reserved":0,"allocated":1753919488,"data_stored":1753387863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5940000000000001,"15min":1.5740000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":5.625,"5min":9.0690000000000008,"15min":50.591999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.548,"5min":1.8,"15min":1.6040000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":6.0670000000000002,"5min":8.2300000000000004,"15min":50.747},"last":0.46100000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:09:08 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":1.9470000000000001,"15min":1.429},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.0670000000000002,"5min":16.352,"15min":25.338000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.885,"15min":1.2929999999999999},"min":{"1min":0.27500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.0970000000000004,"5min":18.27,"15min":18.27},"last":0.73099999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":1.99,"15min":2.3559999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.4669999999999996,"5min":7.952,"15min":7.952},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.919,"15min":2.1269999999999998},"min":{"1min":0.35899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.4779999999999998,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.53600000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:09:47 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.0049999999999999,"15min":1.835},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.058,"5min":29.058,"15min":29.058},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.214,"5min":1.7989999999999999,"15min":1.897},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.577,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":0.56799999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1130000000000004,"5min":4.1130000000000004,"15min":4.1130000000000004},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.70799999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:08:54 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.86199999999999999,"15min":0.83199999999999996},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":2.8029999999999999,"5min":4.5720000000000001,"15min":12.958},"last":0.498},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.94599999999999995,"15min":0.83299999999999996},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.7090000000000001,"5min":5.8470000000000004,"15min":6.6120000000000001},"last":0.60999999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:08:54 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.0640000000000001,"15min":0.84799999999999998},"min":{"1min":0.30199999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.675,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.505,"5min":1.024,"15min":0.81100000000000005},"min":{"1min":0.27300000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.272,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":0.51200000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626710,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2363880,"kb_used_data":2058464,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91381272,"statfs":{"total":95995035648,"available":93574422528,"internally_reserved":0,"allocated":2107867136,"data_stored":2107348753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.9259999999999999,"15min":1.319},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":15.435,"5min":15.435,"15min":15.435},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.853,"5min":1.931,"15min":1.365},"min":{"1min":0.252,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.308999999999999,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.34599999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:08:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":1.887,"15min":1.4399999999999999},"min":{"1min":0.30099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.897,"5min":15.897,"15min":18.853000000000002},"last":35.609999999999999},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.0790000000000002,"15min":1.405},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":5.2720000000000002,"5min":21.561,"15min":21.561},"last":32.396000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.3340000000000001,"15min":1.6919999999999999},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8149999999999999,"5min":7.6760000000000002,"15min":7.6760000000000002},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.234,"5min":2.3239999999999998,"15min":3.0990000000000002},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.5600000000000005,"5min":10.26,"15min":10.26},"last":0.40799999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.321,"15min":1.347},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3879999999999999,"5min":6.0999999999999996,"15min":6.0999999999999996},"last":1.8009999999999999},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.4220000000000002,"15min":2.3940000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":11.211,"5min":11.211,"15min":11.211},"last":4.3890000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.48,"5min":1.48,"15min":1.48},"last":0.36399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.71999999999999997,"15min":0.72199999999999998},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.611,"5min":7.7080000000000002,"15min":14.09},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.87,"15min":0.77300000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":2.464,"5min":7.508,"15min":7.508},"last":0.33600000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:08:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.79600000000000004,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.69699999999999995,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.83999999999999997,"15min":0.80800000000000005},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72599999999999998,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.46899999999999997}]}]},{"osd":1,"up_from":84,"seq":360777253023,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2790076,"kb_used_data":2418676,"kb_used_omap":9,"kb_used_meta":371382,"kb_avail":90955076,"statfs":{"total":95995035648,"available":93137997824,"internally_reserved":0,"allocated":2476724224,"data_stored":2475713783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":380296169},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4139999999999999,"5min":3.2530000000000001,"15min":1.9239999999999999},"min":{"1min":0.19400000000000001,"5min":0.192,"15min":0.185},"max":{"1min":7.7699999999999996,"5min":25.59,"15min":25.59},"last":2.4180000000000001},{"interface":"front","average":{"1min":1.248,"5min":2.754,"15min":1.7549999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.192},"max":{"1min":11.423999999999999,"5min":16.629000000000001,"15min":16.629000000000001},"last":2.3780000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4900000000000002,"5min":3.734,"15min":4.0430000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.487,"5min":17.369,"15min":17.369},"last":5.8029999999999999},{"interface":"front","average":{"1min":2.8220000000000001,"5min":3.4529999999999998,"15min":4.056},"min":{"1min":0.29399999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":10.366,"5min":16.023,"15min":16.023},"last":2.4780000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":3.1619999999999999,"15min":3.4100000000000001},"min":{"1min":0.36399999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.196999999999999,"5min":16.931000000000001,"15min":16.931000000000001},"last":1.484},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.266,"15min":4.6900000000000004},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.8569999999999993,"5min":17.289000000000001,"15min":17.289000000000001},"last":6.2210000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3860000000000001,"5min":5.3860000000000001,"15min":5.3860000000000001},"last":2.5009999999999999},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":10.132,"5min":10.132,"15min":10.132},"last":1.4019999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.891,"15min":1.536},"min":{"1min":0.41499999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7719999999999998,"5min":14.416,"15min":15.531000000000001},"last":1.5149999999999999},{"interface":"front","average":{"1min":1.3380000000000001,"5min":2.613,"15min":1.8939999999999999},"min":{"1min":0.308,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.2569999999999997,"5min":16.277999999999999,"15min":19.706},"last":2.452}]},{"osd":6,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":2.2309999999999999,"15min":1.6299999999999999},"min":{"1min":0.36799999999999999,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.8449999999999998,"5min":14.246,"15min":16.193999999999999},"last":2.5289999999999999},{"interface":"front","average":{"1min":0.86899999999999999,"5min":2.0499999999999998,"15min":1.6140000000000001},"min":{"1min":0.378,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":2.343,"5min":14.471,"15min":15.481999999999999},"last":1.4490000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":2.831,"15min":1.901},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.2149999999999999,"5min":16.331,"15min":16.331},"last":2.5099999999999998},{"interface":"front","average":{"1min":1.585,"5min":2.3090000000000002,"15min":1.6930000000000001},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.8879999999999999,"5min":14.281000000000001,"15min":16.100999999999999},"last":2.5710000000000002}]}]},{"osd":0,"up_from":79,"seq":339302416550,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2459312,"kb_used_data":2106472,"kb_used_omap":28,"kb_used_meta":352803,"kb_avail":91285840,"statfs":{"total":95995035648,"available":93476700160,"internally_reserved":0,"allocated":2157027328,"data_stored":2156027053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":361271120},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.8580000000000001,"15min":1.3819999999999999},"min":{"1min":0.224,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":6.0510000000000002,"5min":13.084,"15min":13.084},"last":1.1639999999999999},{"interface":"front","average":{"1min":2.2109999999999999,"5min":3.4489999999999998,"15min":1.994},"min":{"1min":0.27200000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.516,"5min":22.178999999999998,"15min":22.178999999999998},"last":2.0569999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.7210000000000001,"15min":2.121},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.5579999999999998,"5min":10.034000000000001,"15min":10.034000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.353,"5min":2.2879999999999998,"15min":3.0910000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.4480000000000004,"5min":14.693,"15min":14.693},"last":5.3979999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4990000000000001,"5min":4.391,"15min":4.7069999999999999},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.867000000000001,"5min":25.734999999999999,"15min":25.734999999999999},"last":5.6749999999999998},{"interface":"front","average":{"1min":2.4809999999999999,"5min":4.0250000000000004,"15min":4.282},"min":{"1min":0.44,"5min":0.254,"15min":0.254},"max":{"1min":13.975,"5min":21.923999999999999,"15min":21.923999999999999},"last":4.4509999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.3760000000000003,"5min":4.3760000000000003,"15min":4.3760000000000003},"last":1.8640000000000001},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":13.789999999999999,"5min":13.789999999999999,"15min":13.789999999999999},"last":4.3550000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":2.8149999999999999,"15min":1.847},"min":{"1min":0.38900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.736000000000001,"5min":17.096,"15min":17.096},"last":4.399},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.419,"15min":1.2470000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.4870000000000001,"5min":12.478,"15min":12.478},"last":0.505}]},{"osd":6,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.494,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5289999999999999,"5min":9.7260000000000009,"15min":11.699},"last":1.204},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.6549999999999998,"15min":1.954},"min":{"1min":0.45100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.4130000000000003,"5min":14.384,"15min":15.108000000000001},"last":1.8320000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.9929999999999999,"15min":1.986},"min":{"1min":0.51400000000000001,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":13.829000000000001,"5min":14.631,"15min":26.978000000000002},"last":4.4249999999999998},{"interface":"front","average":{"1min":1.085,"5min":1.6819999999999999,"15min":1.544},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.5209999999999999,"5min":12.428000000000001,"15min":26.768000000000001},"last":1.2330000000000001}]}]},{"osd":2,"up_from":217,"seq":932007903283,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1569508,"kb_used_data":1380508,"kb_used_omap":83,"kb_used_meta":188908,"kb_avail":92175644,"statfs":{"total":95995035648,"available":94387859456,"internally_reserved":0,"allocated":1413640192,"data_stored":1412652201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":193442287},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.28,"15min":1.6060000000000001},"min":{"1min":0.247,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.5139999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.8849999999999998,"15min":2.923},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.891999999999999,"5min":16.309999999999999,"15min":16.309999999999999},"last":12.891999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.0209999999999999,"15min":2.4649999999999999},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.971,"5min":11.971,"15min":11.971},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.8849999999999998,"5min":2.492,"15min":2.6280000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.009,"5min":15.569000000000001,"15min":15.569000000000001},"last":0.27800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.798,"15min":1.948},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.8859999999999999,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.1779999999999999,"5min":3.367,"15min":3.7320000000000002},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.8440000000000003,"5min":15.811,"15min":15.811},"last":0.60299999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.633,"5min":2.633,"15min":2.633},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.8939999999999999,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.5580000000000001,"15min":1.502},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.0870000000000002,"5min":12.423999999999999,"15min":12.423999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.21,"5min":1.2789999999999999,"15min":1.6000000000000001},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.7709999999999999,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":0.52200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2629999999999999,"15min":1.0489999999999999},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.6040000000000001,"5min":15.292999999999999,"15min":15.292999999999999},"last":0.379},{"interface":"front","average":{"1min":1.032,"5min":1.6499999999999999,"15min":1.712},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.984,"5min":12.394,"15min":12.394},"last":0.56599999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.2809999999999999,"15min":1.0760000000000001},"min":{"1min":0.48499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1800000000000002,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.7170000000000001,"15min":1.613},"min":{"1min":0.51200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.5010000000000003,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.59099999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824027,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2299084,"kb_used_data":2089028,"kb_used_omap":10,"kb_used_meta":210037,"kb_avail":91446068,"statfs":{"total":95995035648,"available":93640773632,"internally_reserved":0,"allocated":2139164672,"data_stored":2138023045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":215077955},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,8],"upper_bound":8192},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":4.2489999999999997,"15min":4.5519999999999996},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.465,"5min":23.381,"15min":23.381},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.853,"5min":3.7130000000000001,"15min":4.0229999999999997},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.5190000000000001,"5min":14.289999999999999,"15min":14.289999999999999},"last":0.30499999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5329999999999999,"15min":3.8130000000000002},"min":{"1min":0.28799999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.415,"5min":20.407,"15min":20.407},"last":0.28799999999999998},{"interface":"front","average":{"1min":2.702,"5min":3.2850000000000001,"15min":3.3820000000000001},"min":{"1min":0.40300000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.494,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":3.3410000000000002,"15min":3.6070000000000002},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.1020000000000003,"5min":20.356000000000002,"15min":20.356000000000002},"last":0.371},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.7290000000000001,"15min":2.819},"min":{"1min":0.34799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":9.3379999999999992,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.57499999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":3.7570000000000001},"last":0.44},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.5739999999999998,"5min":8.5739999999999998,"15min":8.5739999999999998},"last":0.42399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.4319999999999999,"15min":2.5529999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.6199999999999992,"5min":11.836,"15min":11.836},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.274,"15min":2.327},"min":{"1min":0.55600000000000005,"5min":0.309,"15min":0.309},"max":{"1min":6.9249999999999998,"5min":14.164999999999999,"15min":14.164999999999999},"last":0.55600000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.5379999999999998,"15min":2.6720000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.532,"5min":11.887,"15min":11.887},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.496,"5min":2.3820000000000001,"15min":2.5299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.8710000000000004,"5min":14.125,"15min":14.125},"last":0.48099999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.4039999999999999,"15min":1.3939999999999999},"min":{"1min":0.45400000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0609999999999999,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":0.496},{"interface":"front","average":{"1min":1.8089999999999999,"5min":2.5009999999999999,"15min":2.617},"min":{"1min":0.42199999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":8.6549999999999994,"5min":11.801,"15min":11.801},"last":0.61199999999999999}]}]},{"osd":4,"up_from":266,"seq":1142461300753,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2435244,"kb_used_data":2104292,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":91309908,"statfs":{"total":95995035648,"available":93501345792,"internally_reserved":0,"allocated":2154795008,"data_stored":2154270961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.3799999999999999,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":2.4489999999999998},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3959999999999999,"5min":2.3959999999999999,"15min":2.3959999999999999},"last":0.48599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":2.7160000000000002,"5min":2.7160000000000002,"15min":2.7160000000000002},"last":4.3129999999999997},{"interface":"front","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":4.7619999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.661,"5min":3.661,"15min":3.661},"last":2.335},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6449999999999996,"5min":4.6449999999999996,"15min":4.6449999999999996},"last":0.35599999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.262,"15min":1.262},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.1259999999999999,"5min":3.1259999999999999,"15min":3.1259999999999999},"last":1.9119999999999999},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.4470000000000001,"5min":4.4470000000000001,"15min":4.4470000000000001},"last":0.39600000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.6970000000000001,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.5609999999999999,"5min":2.5609999999999999,"15min":2.5609999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.917,"5min":1.917,"15min":1.917},"last":0.44}]},{"osd":7,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.5630000000000002,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.63400000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822938,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3127644,"kb_used_data":2830420,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":90617508,"statfs":{"total":95995035648,"available":92792328192,"internally_reserved":0,"allocated":2898350080,"data_stored":2897838161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":1.5860000000000001,"15min":1.3460000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.3090000000000002,"5min":10.92,"15min":15.773999999999999},"last":1.357},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.673,"15min":1.4790000000000001},"min":{"1min":0.25600000000000001,"5min":0.23899999999999999,"15min":0.23400000000000001},"max":{"1min":6.3090000000000002,"5min":11.747,"15min":15.699},"last":1.4870000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.3680000000000001,"15min":1.3300000000000001},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.2670000000000003,"5min":11.335000000000001,"15min":15.689},"last":0.316},{"interface":"front","average":{"1min":1.845,"5min":1.895,"15min":1.5269999999999999},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":19.402999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":1.087}]},{"osd":2,"last update":"Mon Oct 9 10:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.8480000000000001,"15min":2.4369999999999998},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.3819999999999997,"5min":15.640000000000001,"15min":15.640000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.302,"5min":1.8220000000000001,"15min":2.2949999999999999},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.3659999999999997,"5min":10.656000000000001,"15min":10.656000000000001},"last":1.0649999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.734,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5860000000000003,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.2,"5min":2.6549999999999998,"15min":2.8969999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":2.9710000000000001,"5min":17.34,"15min":17.34},"last":3.4239999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.0819999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":1.2290000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":1.0109999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.995,"15min":0.90000000000000002},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.4159999999999999,"5min":13.483000000000001,"15min":27.716000000000001},"last":1.198},{"interface":"front","average":{"1min":0.754,"5min":0.98999999999999999,"15min":0.81100000000000005},"min":{"1min":0.317,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.3490000000000002,"5min":18.600999999999999,"15min":18.600999999999999},"last":1.2909999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:08:53 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.92000000000000004,"15min":0.83999999999999997},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.1739999999999999,"5min":7.4290000000000003,"15min":10.334},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.78300000000000003,"15min":0.879},"min":{"1min":0.317,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.849,"5min":8.0210000000000008,"15min":12.356},"last":1.0369999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2169307136,"data_stored":2169307136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2476081152,"data_stored":2476081152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1425801216,"data_stored":1425801216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2140274688,"data_stored":2140274688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2162819072,"data_stored":2162819072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2903048192,"data_stored":2903048192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2112487424,"data_stored":2112487424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1763442688,"data_stored":1763442688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:09:55.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:09:56.014 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:56 smithi012 ceph-mon[147926]: pgmap v755: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 5374/266292 objects degraded (2.018%); 5236/266292 objects misplaced (1.966%); 362 KiB/s, 5 objects/s recovering 2023-10-09T10:09:56.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:56 smithi012 ceph-mon[152263]: pgmap v755: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 5374/266292 objects degraded (2.018%); 5236/266292 objects misplaced (1.966%); 362 KiB/s, 5 objects/s recovering 2023-10-09T10:09:56.357 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:56.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:56.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:56.408 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:09:56.408 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:09:56.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:56 smithi110 ceph-mon[136032]: pgmap v755: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 5374/266292 objects degraded (2.018%); 5236/266292 objects misplaced (1.966%); 362 KiB/s, 5 objects/s recovering 2023-10-09T10:09:56.470 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:56.558 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:56.655 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 65 16 91363 91347 87.7777 42.3125 0.0476836 0.0113884 2023-10-09T10:09:56.740 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:56.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:56.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:56.933 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:57.004 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:57.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:57.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:09:57.108 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:57.176 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:57.226 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:57.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:57.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:57.315 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:57.326 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:09:57.333 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:57.369 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2802712754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:57.369 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:57 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:57.370 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2802712754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:57.370 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:57 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:57.408 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:57.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2802712754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:57.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:57 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:09:57.655 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 66 16 92071 92055 87.1185 44.25 0.0292674 0.0114722 2023-10-09T10:09:57.819 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:09:58.179 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:09:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:09:57] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-10-09T10:09:58.212 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:09:58.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:58.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:09:58.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:09:58.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:09:58.474 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:58.580 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:09:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:58.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:58.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:58.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T10:09:58.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 5374/268416 objects degraded (2.002%); 5137/268416 objects misplaced (1.914%); 1006 KiB/s, 15 objects/s recovering 2023-10-09T10:09:58.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.585 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.585 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/266292 objects degraded (2.018%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:09:58.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 5374/268416 objects degraded (2.002%); 5137/268416 objects misplaced (1.914%); 1006 KiB/s, 15 objects/s recovering 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/266292 objects degraded (2.018%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:09:58.654 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 67 16 92853 92837 86.5479 48.875 0.0173257 0.0115488 2023-10-09T10:09:58.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:09:58.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:09:58.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:09:58.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T10:09:58.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 5374/268416 objects degraded (2.002%); 5137/268416 objects misplaced (1.914%); 1006 KiB/s, 15 objects/s recovering 2023-10-09T10:09:58.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: failed to encode map e290 with expected crc 2023-10-09T10:09:58.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/266292 objects degraded (2.018%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:09:58.726 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:58.775 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:09:58.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:58.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:09:58.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:09:58.993 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:59.056 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:09:59.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:59.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:09:59.100 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:09:59.101 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:09:59.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:59.214 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:09:59.215 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:59.316 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:59.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:59.400 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:09:59.421 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:09:59.422 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:09:59.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:59.518 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:09:59.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:09:59.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:09:59.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:09:59.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:09:59.660 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 68 16 93555 93539 85.9174 43.875 0.0446088 0.0116339 2023-10-09T10:09:59.734 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:09:59.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:10:00.006 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:10:00.006 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:00.006 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:09:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:10:00.007 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:10:00.007 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:00.007 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:09:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:10:00.034 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":758,"stamp":"2023-10-09T10:09:59.303872+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5980034976,"num_objects":91263,"num_object_clones":0,"num_object_copies":273789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":5137,"num_objects_unfound":0,"num_objects_dirty":91263,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":91499,"num_write_kb":5841741,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":215,"num_bytes_recovered":10758860,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91433,"ondisk_log_size":91433,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19783504,"kb_used_data":17429392,"kb_used_omap":167,"kb_used_meta":2353880,"kb_avail":730177712,"statfs":{"total":767960285184,"available":747701977088,"internally_reserved":0,"allocated":17847697408,"data_stored":17841470624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2410373721},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,4,5,2,1,0,8],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":434831360,"num_objects":6635,"num_object_clones":0,"num_object_copies":19905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":5137,"num_objects_unfound":0,"num_objects_dirty":6635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6635,"num_write_kb":424640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":146,"num_bytes_recovered":9568256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006678"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":130,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.004092+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:09:47.004092+0000","last_peered":"2023-10-09T10:09:47.004092+0000","last_clean":"2023-10-09T10:09:47.004092+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:09:47.004092+0000","last_undegraded":"2023-10-09T10:09:47.004092+0000","last_fullsized":"2023-10-09T10:09:47.004092+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":245,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996826+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:09:46.996826+0000","last_peered":"2023-10-09T10:09:46.996826+0000","last_clean":"2023-10-09T10:09:46.996826+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:09:46.996826+0000","last_undegraded":"2023-10-09T10:09:46.996826+0000","last_fullsized":"2023-10-09T10:09:46.996826+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":291,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.005753+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:09:47.005753+0000","last_peered":"2023-10-09T10:09:47.005753+0000","last_clean":"2023-10-09T10:09:47.005753+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:09:47.005753+0000","last_undegraded":"2023-10-09T10:09:47.005753+0000","last_fullsized":"2023-10-09T10:09:47.005753+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":155,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.267085+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:09:48.267085+0000","last_peered":"2023-10-09T10:09:48.267085+0000","last_clean":"2023-10-09T10:09:48.267085+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:09:48.267085+0000","last_undegraded":"2023-10-09T10:09:48.267085+0000","last_fullsized":"2023-10-09T10:09:48.267085+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":259,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.995613+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:09:46.995613+0000","last_peered":"2023-10-09T10:09:46.995613+0000","last_clean":"2023-10-09T10:09:46.995613+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:09:46.995613+0000","last_undegraded":"2023-10-09T10:09:46.995613+0000","last_fullsized":"2023-10-09T10:09:46.995613+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":108,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.011243+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:09:48.011243+0000","last_peered":"2023-10-09T10:09:48.011243+0000","last_clean":"2023-10-09T10:09:48.011243+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:09:48.011243+0000","last_undegraded":"2023-10-09T10:09:48.011243+0000","last_fullsized":"2023-10-09T10:09:48.011243+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":173,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.266498+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:09:48.266498+0000","last_peered":"2023-10-09T10:09:48.266498+0000","last_clean":"2023-10-09T10:09:48.266498+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:09:48.266498+0000","last_undegraded":"2023-10-09T10:09:48.266498+0000","last_fullsized":"2023-10-09T10:09:48.266498+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":240,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.995527+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:09:46.995527+0000","last_peered":"2023-10-09T10:09:46.995527+0000","last_clean":"2023-10-09T10:09:46.995527+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:09:46.995527+0000","last_undegraded":"2023-10-09T10:09:46.995527+0000","last_fullsized":"2023-10-09T10:09:46.995527+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":107,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.011153+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:09:48.011153+0000","last_peered":"2023-10-09T10:09:48.011153+0000","last_clean":"2023-10-09T10:09:48.011153+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:09:48.011153+0000","last_undegraded":"2023-10-09T10:09:48.011153+0000","last_fullsized":"2023-10-09T10:09:48.011153+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":201,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996392+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:09:46.996392+0000","last_peered":"2023-10-09T10:09:46.996392+0000","last_clean":"2023-10-09T10:09:46.996392+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:09:46.996392+0000","last_undegraded":"2023-10-09T10:09:46.996392+0000","last_fullsized":"2023-10-09T10:09:46.996392+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":327,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222174+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:09:47.222174+0000","last_peered":"2023-10-09T10:09:47.222174+0000","last_clean":"2023-10-09T10:09:47.222174+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:09:47.222174+0000","last_undegraded":"2023-10-09T10:09:47.222174+0000","last_fullsized":"2023-10-09T10:09:47.222174+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":283,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001527+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:09:47.001527+0000","last_peered":"2023-10-09T10:09:47.001527+0000","last_clean":"2023-10-09T10:09:47.001527+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:09:47.001527+0000","last_undegraded":"2023-10-09T10:09:47.001527+0000","last_fullsized":"2023-10-09T10:09:47.001527+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":176,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.221960+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:09:47.221960+0000","last_peered":"2023-10-09T10:09:47.221960+0000","last_clean":"2023-10-09T10:09:47.221960+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:09:47.221960+0000","last_undegraded":"2023-10-09T10:09:47.221960+0000","last_fullsized":"2023-10-09T10:09:47.221960+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":155,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.005359+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:09:47.005359+0000","last_peered":"2023-10-09T10:09:47.005359+0000","last_clean":"2023-10-09T10:09:47.005359+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:09:47.005359+0000","last_undegraded":"2023-10-09T10:09:47.005359+0000","last_fullsized":"2023-10-09T10:09:47.005359+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":360,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222167+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:09:47.222167+0000","last_peered":"2023-10-09T10:09:47.222167+0000","last_clean":"2023-10-09T10:09:47.222167+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:09:47.222167+0000","last_undegraded":"2023-10-09T10:09:47.222167+0000","last_fullsized":"2023-10-09T10:09:47.222167+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":227,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996905+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:09:46.996905+0000","last_peered":"2023-10-09T10:09:46.996905+0000","last_clean":"2023-10-09T10:09:46.996905+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:09:46.996905+0000","last_undegraded":"2023-10-09T10:09:46.996905+0000","last_fullsized":"2023-10-09T10:09:46.996905+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":155,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.265962+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:09:48.265962+0000","last_peered":"2023-10-09T10:09:48.265962+0000","last_clean":"2023-10-09T10:09:48.265962+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:09:48.265962+0000","last_undegraded":"2023-10-09T10:09:48.265962+0000","last_fullsized":"2023-10-09T10:09:48.265962+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.011253+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:09:48.011253+0000","last_peered":"2023-10-09T10:09:48.011253+0000","last_clean":"2023-10-09T10:09:48.011253+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:09:48.011253+0000","last_undegraded":"2023-10-09T10:09:48.011253+0000","last_fullsized":"2023-10-09T10:09:48.011253+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":157,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.221936+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:09:47.221936+0000","last_peered":"2023-10-09T10:09:47.221936+0000","last_clean":"2023-10-09T10:09:47.221936+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:09:47.221936+0000","last_undegraded":"2023-10-09T10:09:47.221936+0000","last_fullsized":"2023-10-09T10:09:47.221936+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":155,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.267159+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:09:48.267159+0000","last_peered":"2023-10-09T10:09:48.267159+0000","last_clean":"2023-10-09T10:09:48.267159+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:09:48.267159+0000","last_undegraded":"2023-10-09T10:09:48.267159+0000","last_fullsized":"2023-10-09T10:09:48.267159+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":178,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.265899+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:09:48.265899+0000","last_peered":"2023-10-09T10:09:48.265899+0000","last_clean":"2023-10-09T10:09:48.265899+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:09:48.265899+0000","last_undegraded":"2023-10-09T10:09:48.265899+0000","last_fullsized":"2023-10-09T10:09:48.265899+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":240,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996416+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:09:46.996416+0000","last_peered":"2023-10-09T10:09:46.996416+0000","last_clean":"2023-10-09T10:09:46.996416+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:09:46.996416+0000","last_undegraded":"2023-10-09T10:09:46.996416+0000","last_fullsized":"2023-10-09T10:09:46.996416+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"290'5847","reported_seq":5882,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:09:57.826464+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:09:57.826464+0000","last_peered":"2023-10-09T10:09:57.826464+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:09:57.826464+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5847,"log_dups_size":0,"ondisk_log_size":5847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383188992,"num_objects":5847,"num_object_clones":0,"num_object_copies":17541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5847,"num_write_kb":374208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":474},{"shards":"4,5,6","objects":5373}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":321,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222092+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:09:47.222092+0000","last_peered":"2023-10-09T10:09:47.222092+0000","last_clean":"2023-10-09T10:09:47.222092+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:09:47.222092+0000","last_undegraded":"2023-10-09T10:09:47.222092+0000","last_fullsized":"2023-10-09T10:09:47.222092+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":108,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.010634+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:09:48.010634+0000","last_peered":"2023-10-09T10:09:48.010634+0000","last_clean":"2023-10-09T10:09:48.010634+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:09:48.010634+0000","last_undegraded":"2023-10-09T10:09:48.010634+0000","last_fullsized":"2023-10-09T10:09:48.010634+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"289'5672","reported_seq":5698,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:54.996727+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:09:54.996727+0000","last_peered":"2023-10-09T10:09:54.996727+0000","last_clean":"2023-10-09T10:09:54.996727+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:09:54.996727+0000","last_undegraded":"2023-10-09T10:09:54.996727+0000","last_fullsized":"2023-10-09T10:09:54.996727+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5672,"log_dups_size":0,"ondisk_log_size":5672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371720192,"num_objects":5672,"num_object_clones":0,"num_object_copies":17016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5672,"num_write_kb":363008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":309,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222027+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:09:47.222027+0000","last_peered":"2023-10-09T10:09:47.222027+0000","last_clean":"2023-10-09T10:09:47.222027+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:09:47.222027+0000","last_undegraded":"2023-10-09T10:09:47.222027+0000","last_fullsized":"2023-10-09T10:09:47.222027+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":199,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.005666+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:09:47.005666+0000","last_peered":"2023-10-09T10:09:47.005666+0000","last_clean":"2023-10-09T10:09:47.005666+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:09:47.005666+0000","last_undegraded":"2023-10-09T10:09:47.005666+0000","last_fullsized":"2023-10-09T10:09:47.005666+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"290'5745","reported_seq":5773,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.651774+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:09:57.651774+0000","last_peered":"2023-10-09T10:09:57.651774+0000","last_clean":"2023-10-09T10:09:57.651774+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:09:57.651774+0000","last_undegraded":"2023-10-09T10:09:57.651774+0000","last_fullsized":"2023-10-09T10:09:57.651774+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376504320,"num_objects":5745,"num_object_clones":0,"num_object_copies":17235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5745,"num_write_kb":367680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":112,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.011271+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:09:48.011271+0000","last_peered":"2023-10-09T10:09:48.011271+0000","last_clean":"2023-10-09T10:09:48.011271+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:09:48.011271+0000","last_undegraded":"2023-10-09T10:09:48.011271+0000","last_fullsized":"2023-10-09T10:09:48.011271+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":132,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001525+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:09:47.001525+0000","last_peered":"2023-10-09T10:09:47.001525+0000","last_clean":"2023-10-09T10:09:47.001525+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:09:47.001525+0000","last_undegraded":"2023-10-09T10:09:47.001525+0000","last_fullsized":"2023-10-09T10:09:47.001525+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"289'5579","reported_seq":5605,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:54.509723+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:09:54.509723+0000","last_peered":"2023-10-09T10:09:54.509723+0000","last_clean":"2023-10-09T10:09:54.509723+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:09:54.509723+0000","last_undegraded":"2023-10-09T10:09:54.509723+0000","last_fullsized":"2023-10-09T10:09:54.509723+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5579,"log_dups_size":0,"ondisk_log_size":5579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365625344,"num_objects":5579,"num_object_clones":0,"num_object_copies":16737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5579,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.039230+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:09:48.039230+0000","last_peered":"2023-10-09T10:09:48.039230+0000","last_clean":"2023-10-09T10:09:48.039230+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:09:48.039230+0000","last_undegraded":"2023-10-09T10:09:48.039230+0000","last_fullsized":"2023-10-09T10:09:48.039230+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":156,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.266226+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:09:48.266226+0000","last_peered":"2023-10-09T10:09:48.266226+0000","last_clean":"2023-10-09T10:09:48.266226+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:09:48.266226+0000","last_undegraded":"2023-10-09T10:09:48.266226+0000","last_fullsized":"2023-10-09T10:09:48.266226+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"289'5774","reported_seq":5801,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:56.045856+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:09:56.045856+0000","last_peered":"2023-10-09T10:09:56.045856+0000","last_clean":"2023-10-09T10:09:56.045856+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:09:56.045856+0000","last_undegraded":"2023-10-09T10:09:56.045856+0000","last_fullsized":"2023-10-09T10:09:56.045856+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378404864,"num_objects":5774,"num_object_clones":0,"num_object_copies":17322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5774,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":282,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.003250+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:09:48.003250+0000","last_peered":"2023-10-09T10:09:48.003250+0000","last_clean":"2023-10-09T10:09:48.003250+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:09:48.003250+0000","last_undegraded":"2023-10-09T10:09:48.003250+0000","last_fullsized":"2023-10-09T10:09:48.003250+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":152,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.003376+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:09:48.003376+0000","last_peered":"2023-10-09T10:09:48.003376+0000","last_clean":"2023-10-09T10:09:48.003376+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:09:48.003376+0000","last_undegraded":"2023-10-09T10:09:48.003376+0000","last_fullsized":"2023-10-09T10:09:48.003376+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"289'5730","reported_seq":5757,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:54.484017+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:09:54.484017+0000","last_peered":"2023-10-09T10:09:54.484017+0000","last_clean":"2023-10-09T10:09:54.484017+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:09:54.484017+0000","last_undegraded":"2023-10-09T10:09:54.484017+0000","last_fullsized":"2023-10-09T10:09:54.484017+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375521280,"num_objects":5730,"num_object_clones":0,"num_object_copies":17190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5730,"num_write_kb":366720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":213,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001473+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:09:47.001473+0000","last_peered":"2023-10-09T10:09:47.001473+0000","last_clean":"2023-10-09T10:09:47.001473+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:09:47.001473+0000","last_undegraded":"2023-10-09T10:09:47.001473+0000","last_fullsized":"2023-10-09T10:09:47.001473+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":219,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.008124+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:09:47.008124+0000","last_peered":"2023-10-09T10:09:47.008124+0000","last_clean":"2023-10-09T10:09:47.008124+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:09:47.008124+0000","last_undegraded":"2023-10-09T10:09:47.008124+0000","last_fullsized":"2023-10-09T10:09:47.008124+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"290'5654","reported_seq":5682,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.670982+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:09:57.670982+0000","last_peered":"2023-10-09T10:09:57.670982+0000","last_clean":"2023-10-09T10:09:57.670982+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:09:57.670982+0000","last_undegraded":"2023-10-09T10:09:57.670982+0000","last_fullsized":"2023-10-09T10:09:57.670982+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5654,"log_dups_size":0,"ondisk_log_size":5654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370540544,"num_objects":5654,"num_object_clones":0,"num_object_copies":16962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":183,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222134+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:09:47.222134+0000","last_peered":"2023-10-09T10:09:47.222134+0000","last_clean":"2023-10-09T10:09:47.222134+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:09:47.222134+0000","last_undegraded":"2023-10-09T10:09:47.222134+0000","last_fullsized":"2023-10-09T10:09:47.222134+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":155,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001042+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:09:47.001042+0000","last_peered":"2023-10-09T10:09:47.001042+0000","last_clean":"2023-10-09T10:09:47.001042+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:09:47.001042+0000","last_undegraded":"2023-10-09T10:09:47.001042+0000","last_fullsized":"2023-10-09T10:09:47.001042+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"289'5642","reported_seq":5667,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:54.486707+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:09:54.486707+0000","last_peered":"2023-10-09T10:09:54.486707+0000","last_clean":"2023-10-09T10:09:54.486707+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:09:54.486707+0000","last_undegraded":"2023-10-09T10:09:54.486707+0000","last_fullsized":"2023-10-09T10:09:54.486707+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369754112,"num_objects":5642,"num_object_clones":0,"num_object_copies":16926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5642,"num_write_kb":361088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":242,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.003026+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:09:48.003026+0000","last_peered":"2023-10-09T10:09:48.003026+0000","last_clean":"2023-10-09T10:09:48.003026+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:09:48.003026+0000","last_undegraded":"2023-10-09T10:09:48.003026+0000","last_fullsized":"2023-10-09T10:09:48.003026+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":291,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001264+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:09:47.001264+0000","last_peered":"2023-10-09T10:09:47.001264+0000","last_clean":"2023-10-09T10:09:47.001264+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:09:47.001264+0000","last_undegraded":"2023-10-09T10:09:47.001264+0000","last_fullsized":"2023-10-09T10:09:47.001264+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"290'5756","reported_seq":5784,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.569254+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:09:57.569254+0000","last_peered":"2023-10-09T10:09:57.569254+0000","last_clean":"2023-10-09T10:09:57.569254+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:09:57.569254+0000","last_undegraded":"2023-10-09T10:09:57.569254+0000","last_fullsized":"2023-10-09T10:09:57.569254+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377225216,"num_objects":5756,"num_object_clones":0,"num_object_copies":17268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5756,"num_write_kb":368384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":291,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000785+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:09:47.000785+0000","last_peered":"2023-10-09T10:09:47.000785+0000","last_clean":"2023-10-09T10:09:47.000785+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:09:47.000785+0000","last_undegraded":"2023-10-09T10:09:47.000785+0000","last_fullsized":"2023-10-09T10:09:47.000785+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":132,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.011141+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:09:48.011141+0000","last_peered":"2023-10-09T10:09:48.011141+0000","last_clean":"2023-10-09T10:09:48.011141+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:09:48.011141+0000","last_undegraded":"2023-10-09T10:09:48.011141+0000","last_fullsized":"2023-10-09T10:09:48.011141+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":247,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.005199+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:09:47.005199+0000","last_peered":"2023-10-09T10:09:47.005199+0000","last_clean":"2023-10-09T10:09:47.005199+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:09:47.005199+0000","last_undegraded":"2023-10-09T10:09:47.005199+0000","last_fullsized":"2023-10-09T10:09:47.005199+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"290'5741","reported_seq":5768,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.820010+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:09:57.820010+0000","last_peered":"2023-10-09T10:09:57.820010+0000","last_clean":"2023-10-09T10:09:57.820010+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:09:57.820010+0000","last_undegraded":"2023-10-09T10:09:57.820010+0000","last_fullsized":"2023-10-09T10:09:57.820010+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5741,"log_dups_size":0,"ondisk_log_size":5741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376242176,"num_objects":5741,"num_object_clones":0,"num_object_copies":17223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5741,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":108,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.010839+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:09:48.010839+0000","last_peered":"2023-10-09T10:09:48.010839+0000","last_clean":"2023-10-09T10:09:48.010839+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:09:48.010839+0000","last_undegraded":"2023-10-09T10:09:48.010839+0000","last_fullsized":"2023-10-09T10:09:48.010839+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":105,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.265896+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:09:48.265896+0000","last_peered":"2023-10-09T10:09:48.265896+0000","last_clean":"2023-10-09T10:09:48.265896+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:09:48.265896+0000","last_undegraded":"2023-10-09T10:09:48.265896+0000","last_fullsized":"2023-10-09T10:09:48.265896+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":314,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.000702+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:09:47.000702+0000","last_peered":"2023-10-09T10:09:47.000702+0000","last_clean":"2023-10-09T10:09:47.000702+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:09:47.000702+0000","last_undegraded":"2023-10-09T10:09:47.000702+0000","last_fullsized":"2023-10-09T10:09:47.000702+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"289'5620","reported_seq":5647,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:56.038574+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:09:56.038574+0000","last_peered":"2023-10-09T10:09:56.038574+0000","last_clean":"2023-10-09T10:09:56.038574+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:09:56.038574+0000","last_undegraded":"2023-10-09T10:09:56.038574+0000","last_fullsized":"2023-10-09T10:09:56.038574+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5620,"log_dups_size":0,"ondisk_log_size":5620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368312320,"num_objects":5620,"num_object_clones":0,"num_object_copies":16860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5620,"num_write_kb":359680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":208,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.003025+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:09:48.003025+0000","last_peered":"2023-10-09T10:09:48.003025+0000","last_clean":"2023-10-09T10:09:48.003025+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:09:48.003025+0000","last_undegraded":"2023-10-09T10:09:48.003025+0000","last_fullsized":"2023-10-09T10:09:48.003025+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":283,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.002127+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:09:47.002127+0000","last_peered":"2023-10-09T10:09:47.002127+0000","last_clean":"2023-10-09T10:09:47.002127+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:09:47.002127+0000","last_undegraded":"2023-10-09T10:09:47.002127+0000","last_fullsized":"2023-10-09T10:09:47.002127+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":304,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001189+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:09:47.001189+0000","last_peered":"2023-10-09T10:09:47.001189+0000","last_clean":"2023-10-09T10:09:47.001189+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:09:47.001189+0000","last_undegraded":"2023-10-09T10:09:47.001189+0000","last_fullsized":"2023-10-09T10:09:47.001189+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"290'5704","reported_seq":5731,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.788847+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:09:57.788847+0000","last_peered":"2023-10-09T10:09:57.788847+0000","last_clean":"2023-10-09T10:09:57.788847+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:09:57.788847+0000","last_undegraded":"2023-10-09T10:09:57.788847+0000","last_fullsized":"2023-10-09T10:09:57.788847+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373817344,"num_objects":5704,"num_object_clones":0,"num_object_copies":17112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5704,"num_write_kb":365056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":117,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.010973+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:09:48.010973+0000","last_peered":"2023-10-09T10:09:48.010973+0000","last_clean":"2023-10-09T10:09:48.010973+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:09:48.010973+0000","last_undegraded":"2023-10-09T10:09:48.010973+0000","last_fullsized":"2023-10-09T10:09:48.010973+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":535,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222251+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:09:47.222251+0000","last_peered":"2023-10-09T10:09:47.222251+0000","last_clean":"2023-10-09T10:09:47.222251+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:09:47.222251+0000","last_undegraded":"2023-10-09T10:09:47.222251+0000","last_fullsized":"2023-10-09T10:09:47.222251+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":192,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996686+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:09:46.996686+0000","last_peered":"2023-10-09T10:09:46.996686+0000","last_clean":"2023-10-09T10:09:46.996686+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:09:46.996686+0000","last_undegraded":"2023-10-09T10:09:46.996686+0000","last_fullsized":"2023-10-09T10:09:46.996686+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":204,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.221987+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:09:47.221987+0000","last_peered":"2023-10-09T10:09:47.221987+0000","last_clean":"2023-10-09T10:09:47.221987+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:09:47.221987+0000","last_undegraded":"2023-10-09T10:09:47.221987+0000","last_fullsized":"2023-10-09T10:09:47.221987+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"289'5654","reported_seq":5681,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:54.510143+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:09:54.510143+0000","last_peered":"2023-10-09T10:09:54.510143+0000","last_clean":"2023-10-09T10:09:54.510143+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:09:54.510143+0000","last_undegraded":"2023-10-09T10:09:54.510143+0000","last_fullsized":"2023-10-09T10:09:54.510143+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5654,"log_dups_size":0,"ondisk_log_size":5654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370540544,"num_objects":5654,"num_object_clones":0,"num_object_copies":16962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":263,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.002236+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:09:47.002236+0000","last_peered":"2023-10-09T10:09:47.002236+0000","last_clean":"2023-10-09T10:09:47.002236+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:09:47.002236+0000","last_undegraded":"2023-10-09T10:09:47.002236+0000","last_fullsized":"2023-10-09T10:09:47.002236+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":166,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.266311+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:09:48.266311+0000","last_peered":"2023-10-09T10:09:48.266311+0000","last_clean":"2023-10-09T10:09:48.266311+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:09:48.266311+0000","last_undegraded":"2023-10-09T10:09:48.266311+0000","last_fullsized":"2023-10-09T10:09:48.266311+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":194,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.267011+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:09:48.267011+0000","last_peered":"2023-10-09T10:09:48.267011+0000","last_clean":"2023-10-09T10:09:48.267011+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:09:48.267011+0000","last_undegraded":"2023-10-09T10:09:48.267011+0000","last_fullsized":"2023-10-09T10:09:48.267011+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"289'5701","reported_seq":6026,"reported_epoch":289,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:09:56.080448+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:09:56.080448+0000","last_peered":"2023-10-09T10:09:56.080448+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:09:56.080448+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373620736,"num_objects":5701,"num_object_clones":0,"num_object_copies":17103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":5137,"num_objects_unfound":0,"num_objects_dirty":5701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5701,"num_write_kb":364864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":146,"num_bytes_recovered":9568256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":563},{"shards":"1,3,4","objects":5137}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":308,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.222522+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:09:47.222522+0000","last_peered":"2023-10-09T10:09:47.222522+0000","last_clean":"2023-10-09T10:09:47.222522+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:09:47.222522+0000","last_undegraded":"2023-10-09T10:09:47.222522+0000","last_fullsized":"2023-10-09T10:09:47.222522+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":320,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.006422+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:09:47.006422+0000","last_peered":"2023-10-09T10:09:47.006422+0000","last_clean":"2023-10-09T10:09:47.006422+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:09:47.006422+0000","last_undegraded":"2023-10-09T10:09:47.006422+0000","last_fullsized":"2023-10-09T10:09:47.006422+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":158,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.266084+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:09:48.266084+0000","last_peered":"2023-10-09T10:09:48.266084+0000","last_clean":"2023-10-09T10:09:48.266084+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:09:48.266084+0000","last_undegraded":"2023-10-09T10:09:48.266084+0000","last_fullsized":"2023-10-09T10:09:48.266084+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"290'5739","reported_seq":5767,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.705375+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:09:57.705375+0000","last_peered":"2023-10-09T10:09:57.705375+0000","last_clean":"2023-10-09T10:09:57.705375+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:09:57.705375+0000","last_undegraded":"2023-10-09T10:09:57.705375+0000","last_fullsized":"2023-10-09T10:09:57.705375+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376111104,"num_objects":5739,"num_object_clones":0,"num_object_copies":17217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5739,"num_write_kb":367296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.003609+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:09:48.003609+0000","last_peered":"2023-10-09T10:09:48.003609+0000","last_clean":"2023-10-09T10:09:48.003609+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:09:48.003609+0000","last_undegraded":"2023-10-09T10:09:48.003609+0000","last_fullsized":"2023-10-09T10:09:48.003609+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":265,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001124+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:09:47.001124+0000","last_peered":"2023-10-09T10:09:47.001124+0000","last_clean":"2023-10-09T10:09:47.001124+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:09:47.001124+0000","last_undegraded":"2023-10-09T10:09:47.001124+0000","last_fullsized":"2023-10-09T10:09:47.001124+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":240,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.995362+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:09:46.995362+0000","last_peered":"2023-10-09T10:09:46.995362+0000","last_clean":"2023-10-09T10:09:46.995362+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:09:46.995362+0000","last_undegraded":"2023-10-09T10:09:46.995362+0000","last_fullsized":"2023-10-09T10:09:46.995362+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"290'5681","reported_seq":5707,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.859113+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:09:57.859113+0000","last_peered":"2023-10-09T10:09:57.859113+0000","last_clean":"2023-10-09T10:09:57.859113+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:09:57.859113+0000","last_undegraded":"2023-10-09T10:09:57.859113+0000","last_fullsized":"2023-10-09T10:09:57.859113+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":5681,"log_dups_size":0,"ondisk_log_size":5681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372310016,"num_objects":5681,"num_object_clones":0,"num_object_copies":17043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5681,"num_write_kb":363584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":197,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.002759+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:09:47.002759+0000","last_peered":"2023-10-09T10:09:47.002759+0000","last_clean":"2023-10-09T10:09:47.002759+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:09:47.002759+0000","last_undegraded":"2023-10-09T10:09:47.002759+0000","last_fullsized":"2023-10-09T10:09:47.002759+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":160,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.266935+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:09:48.266935+0000","last_peered":"2023-10-09T10:09:48.266935+0000","last_clean":"2023-10-09T10:09:48.266935+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:09:48.266935+0000","last_undegraded":"2023-10-09T10:09:48.266935+0000","last_fullsized":"2023-10-09T10:09:48.266935+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":143,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.010686+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:09:48.010686+0000","last_peered":"2023-10-09T10:09:48.010686+0000","last_clean":"2023-10-09T10:09:48.010686+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:09:48.010686+0000","last_undegraded":"2023-10-09T10:09:48.010686+0000","last_fullsized":"2023-10-09T10:09:48.010686+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":154,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.265808+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:09:48.265808+0000","last_peered":"2023-10-09T10:09:48.265808+0000","last_clean":"2023-10-09T10:09:48.265808+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:09:48.265808+0000","last_undegraded":"2023-10-09T10:09:48.265808+0000","last_fullsized":"2023-10-09T10:09:48.265808+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":226,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.011779+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:09:47.011779+0000","last_peered":"2023-10-09T10:09:47.011779+0000","last_clean":"2023-10-09T10:09:47.011779+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:09:47.011779+0000","last_undegraded":"2023-10-09T10:09:47.011779+0000","last_fullsized":"2023-10-09T10:09:47.011779+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":127,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.039086+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:09:48.039086+0000","last_peered":"2023-10-09T10:09:48.039086+0000","last_clean":"2023-10-09T10:09:48.039086+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:09:48.039086+0000","last_undegraded":"2023-10-09T10:09:48.039086+0000","last_fullsized":"2023-10-09T10:09:48.039086+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":166,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.997849+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:09:46.997849+0000","last_peered":"2023-10-09T10:09:46.997849+0000","last_clean":"2023-10-09T10:09:46.997849+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:09:46.997849+0000","last_undegraded":"2023-10-09T10:09:46.997849+0000","last_fullsized":"2023-10-09T10:09:46.997849+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":127,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.011055+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:09:48.011055+0000","last_peered":"2023-10-09T10:09:48.011055+0000","last_clean":"2023-10-09T10:09:48.011055+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:09:48.011055+0000","last_undegraded":"2023-10-09T10:09:48.011055+0000","last_fullsized":"2023-10-09T10:09:48.011055+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":175,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996611+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:09:46.996611+0000","last_peered":"2023-10-09T10:09:46.996611+0000","last_clean":"2023-10-09T10:09:46.996611+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:09:46.996611+0000","last_undegraded":"2023-10-09T10:09:46.996611+0000","last_fullsized":"2023-10-09T10:09:46.996611+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":305,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.010746+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:09:47.010746+0000","last_peered":"2023-10-09T10:09:47.010746+0000","last_clean":"2023-10-09T10:09:47.010746+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:09:47.010746+0000","last_undegraded":"2023-10-09T10:09:47.010746+0000","last_fullsized":"2023-10-09T10:09:47.010746+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":325,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:46.996519+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:09:46.996519+0000","last_peered":"2023-10-09T10:09:46.996519+0000","last_clean":"2023-10-09T10:09:46.996519+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:09:46.996519+0000","last_undegraded":"2023-10-09T10:09:46.996519+0000","last_fullsized":"2023-10-09T10:09:46.996519+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":236,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T10:09:48.266370+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:09:48.266370+0000","last_peered":"2023-10-09T10:09:48.266370+0000","last_clean":"2023-10-09T10:09:48.266370+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:09:48.266370+0000","last_undegraded":"2023-10-09T10:09:48.266370+0000","last_fullsized":"2023-10-09T10:09:48.266370+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":155,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T10:09:47.001929+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:09:47.001929+0000","last_peered":"2023-10-09T10:09:47.001929+0000","last_clean":"2023-10-09T10:09:47.001929+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:09:47.001929+0000","last_undegraded":"2023-10-09T10:09:47.001929+0000","last_fullsized":"2023-10-09T10:09:47.001929+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5979439104,"num_objects":91239,"num_object_clones":0,"num_object_copies":273717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":5137,"num_objects_unfound":0,"num_objects_dirty":91239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":91239,"num_write_kb":5839296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":146,"num_bytes_recovered":9568256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17901027328,"data_stored":17901027328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91239,"ondisk_log_size":91239,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430483,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2095732,"kb_used_data":1786220,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91649420,"statfs":{"total":95995035648,"available":93849006080,"internally_reserved":0,"allocated":1829089280,"data_stored":1828557655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5940000000000001,"15min":1.5740000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":5.625,"5min":9.0690000000000008,"15min":50.591999999999999},"last":2.7389999999999999},{"interface":"front","average":{"1min":1.548,"5min":1.8,"15min":1.6040000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":6.0670000000000002,"5min":8.2300000000000004,"15min":50.747},"last":2.3820000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:09:08 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":1.9470000000000001,"15min":1.429},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.0670000000000002,"5min":16.352,"15min":25.338000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.885,"15min":1.2929999999999999},"min":{"1min":0.27500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.0970000000000004,"5min":18.27,"15min":18.27},"last":0.47699999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.7190000000000001,"15min":2.266},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":3.8370000000000002,"5min":7.952,"15min":7.952},"last":1.538},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.7310000000000001,"15min":2.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.2039999999999997,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":4.1989999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:09:47 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.0049999999999999,"15min":1.835},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.058,"5min":29.058,"15min":29.058},"last":2.5739999999999998},{"interface":"front","average":{"1min":1.214,"5min":1.7989999999999999,"15min":1.897},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.577,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":0.41099999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1130000000000004,"5min":4.1130000000000004,"15min":4.1130000000000004},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":2.7919999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.149,"15min":0.93899999999999995},"min":{"1min":0.31900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":25.277999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":2.4529999999999998,"5min":1.2589999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.754000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":2.02}]},{"osd":6,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.1299999999999999,"15min":0.88600000000000001},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5369999999999999,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.98599999999999999,"15min":0.82499999999999996},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.998,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":1.998}]}]},{"osd":6,"up_from":42,"seq":180388626711,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2447784,"kb_used_data":2138272,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91297368,"statfs":{"total":95995035648,"available":93488504832,"internally_reserved":0,"allocated":2189590528,"data_stored":2189072145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.9259999999999999,"15min":1.319},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":15.435,"5min":15.435,"15min":15.435},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.853,"5min":1.931,"15min":1.365},"min":{"1min":0.252,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.308999999999999,"5min":15.308999999999999,"15min":15.308999999999999},"last":1.204}]},{"osd":1,"last update":"Mon Oct 9 10:08:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":1.887,"15min":1.4399999999999999},"min":{"1min":0.30099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.897,"5min":15.897,"15min":18.853000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.0790000000000002,"15min":1.405},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":5.2720000000000002,"5min":21.561,"15min":21.561},"last":1.177}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.3340000000000001,"15min":1.6919999999999999},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8149999999999999,"5min":7.6760000000000002,"15min":7.6760000000000002},"last":0.376},{"interface":"front","average":{"1min":1.234,"5min":2.3239999999999998,"15min":3.0990000000000002},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.5600000000000005,"5min":10.26,"15min":10.26},"last":0.442}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.321,"15min":1.347},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3879999999999999,"5min":6.0999999999999996,"15min":6.0999999999999996},"last":2.1019999999999999},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.4220000000000002,"15min":2.3940000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":11.211,"5min":11.211,"15min":11.211},"last":0.56999999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.48,"5min":1.48,"15min":1.48},"last":0.48399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.71999999999999997,"15min":0.72199999999999998},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.611,"5min":7.7080000000000002,"15min":14.09},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.87,"15min":0.77300000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":2.464,"5min":7.508,"15min":7.508},"last":0.46500000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:08:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.79600000000000004,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.69699999999999995,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.502},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.83999999999999997,"15min":0.80800000000000005},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72599999999999998,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.53300000000000003}]}]},{"osd":1,"up_from":84,"seq":360777253024,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2907004,"kb_used_data":2527412,"kb_used_omap":9,"kb_used_meta":379574,"kb_avail":90838148,"statfs":{"total":95995035648,"available":93018263552,"internally_reserved":0,"allocated":2588069888,"data_stored":2587059447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":388684777},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,8],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4139999999999999,"5min":3.2530000000000001,"15min":1.9239999999999999},"min":{"1min":0.19400000000000001,"5min":0.192,"15min":0.185},"max":{"1min":7.7699999999999996,"5min":25.59,"15min":25.59},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.248,"5min":2.754,"15min":1.7549999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.192},"max":{"1min":11.423999999999999,"5min":16.629000000000001,"15min":16.629000000000001},"last":0.67500000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":3.2160000000000002,"15min":3.8700000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.1539999999999999,"5min":17.369,"15min":17.369},"last":1.3200000000000001},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.907,"15min":3.8740000000000001},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5960000000000001,"5min":16.023,"15min":16.023},"last":3.4199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":3.1619999999999999,"15min":3.4100000000000001},"min":{"1min":0.36399999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.196999999999999,"5min":16.931000000000001,"15min":16.931000000000001},"last":1.032},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.266,"15min":4.6900000000000004},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.8569999999999993,"5min":17.289000000000001,"15min":17.289000000000001},"last":1.214}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3860000000000001,"5min":5.3860000000000001,"15min":5.3860000000000001},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":10.132,"5min":10.132,"15min":10.132},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.891,"15min":1.536},"min":{"1min":0.41499999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7719999999999998,"5min":14.416,"15min":15.531000000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.3380000000000001,"5min":2.613,"15min":1.8939999999999999},"min":{"1min":0.308,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.2569999999999997,"5min":16.277999999999999,"15min":19.706},"last":0.55700000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":2.2309999999999999,"15min":1.6299999999999999},"min":{"1min":0.36799999999999999,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.8449999999999998,"5min":14.246,"15min":16.193999999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.86899999999999999,"5min":2.0499999999999998,"15min":1.6140000000000001},"min":{"1min":0.378,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":2.343,"5min":14.471,"15min":15.481999999999999},"last":0.65000000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":2.831,"15min":1.901},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.2149999999999999,"5min":16.331,"15min":16.331},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.585,"5min":2.3090000000000002,"15min":1.6930000000000001},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.8879999999999999,"5min":14.281000000000001,"15min":16.100999999999999},"last":1.085}]}]},{"osd":0,"up_from":79,"seq":339302416551,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2536496,"kb_used_data":2195944,"kb_used_omap":28,"kb_used_meta":340515,"kb_avail":91208656,"statfs":{"total":95995035648,"available":93397663744,"internally_reserved":0,"allocated":2248646656,"data_stored":2247646381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":348688208},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.8580000000000001,"15min":1.3819999999999999},"min":{"1min":0.224,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":6.0510000000000002,"5min":13.084,"15min":13.084},"last":4.0780000000000003},{"interface":"front","average":{"1min":2.2109999999999999,"5min":3.4489999999999998,"15min":1.994},"min":{"1min":0.27200000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.516,"5min":22.178999999999998,"15min":22.178999999999998},"last":21.949000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.7210000000000001,"15min":2.121},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.5579999999999998,"5min":10.034000000000001,"15min":10.034000000000001},"last":1.923},{"interface":"front","average":{"1min":1.353,"5min":2.2879999999999998,"15min":3.0910000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.4480000000000004,"5min":14.693,"15min":14.693},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4990000000000001,"5min":4.391,"15min":4.7069999999999999},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.867000000000001,"5min":25.734999999999999,"15min":25.734999999999999},"last":9.1159999999999997},{"interface":"front","average":{"1min":2.4809999999999999,"5min":4.0250000000000004,"15min":4.282},"min":{"1min":0.44,"5min":0.254,"15min":0.254},"max":{"1min":13.975,"5min":21.923999999999999,"15min":21.923999999999999},"last":7.9829999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.3760000000000003,"5min":4.3760000000000003,"15min":4.3760000000000003},"last":9.0180000000000007},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":13.789999999999999,"5min":13.789999999999999,"15min":13.789999999999999},"last":5.0460000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":2.8149999999999999,"15min":1.847},"min":{"1min":0.38900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.736000000000001,"5min":17.096,"15min":17.096},"last":5.0759999999999996},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.419,"15min":1.2470000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.4870000000000001,"5min":12.478,"15min":12.478},"last":0.45700000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.494,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5289999999999999,"5min":9.7260000000000009,"15min":11.699},"last":0.502},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.6549999999999998,"15min":1.954},"min":{"1min":0.45100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.4130000000000003,"5min":14.384,"15min":15.108000000000001},"last":9.0760000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.9929999999999999,"15min":1.986},"min":{"1min":0.51400000000000001,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":13.829000000000001,"5min":14.631,"15min":26.978000000000002},"last":5.0030000000000001},{"interface":"front","average":{"1min":1.085,"5min":1.6819999999999999,"15min":1.544},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.5209999999999999,"5min":12.428000000000001,"15min":26.768000000000001},"last":0.47999999999999998}]}]},{"osd":2,"up_from":217,"seq":932007903284,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1666084,"kb_used_data":1485276,"kb_used_omap":83,"kb_used_meta":180716,"kb_avail":92079068,"statfs":{"total":95995035648,"available":94288965632,"internally_reserved":0,"allocated":1520922624,"data_stored":1519934633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":185053679},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.28,"15min":1.6060000000000001},"min":{"1min":0.247,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.5139999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":3.6880000000000002},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.8849999999999998,"15min":2.923},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.891999999999999,"5min":16.309999999999999,"15min":16.309999999999999},"last":1.351}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.0209999999999999,"15min":2.4649999999999999},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.971,"5min":11.971,"15min":11.971},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.8849999999999998,"5min":2.492,"15min":2.6280000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.009,"5min":15.569000000000001,"15min":15.569000000000001},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.798,"15min":1.948},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.8859999999999999,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.1779999999999999,"5min":3.367,"15min":3.7320000000000002},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.8440000000000003,"5min":15.811,"15min":15.811},"last":0.42399999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.633,"5min":2.633,"15min":2.633},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.8939999999999999,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":0.55900000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.5580000000000001,"15min":1.502},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.0870000000000002,"5min":12.423999999999999,"15min":12.423999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.21,"5min":1.2789999999999999,"15min":1.6000000000000001},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.7709999999999999,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":3.6469999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2629999999999999,"15min":1.0489999999999999},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.6040000000000001,"5min":15.292999999999999,"15min":15.292999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.6499999999999999,"15min":1.712},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.984,"5min":12.394,"15min":12.394},"last":0.50900000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.2809999999999999,"15min":1.0760000000000001},"min":{"1min":0.48499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1800000000000002,"5min":15.343999999999999,"15min":15.343999999999999},"last":1.3129999999999999},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.7170000000000001,"15min":1.613},"min":{"1min":0.51200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.5010000000000003,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.53600000000000003}]}]},{"osd":3,"up_from":250,"seq":1073741824028,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2392396,"kb_used_data":2178244,"kb_used_omap":10,"kb_used_meta":214133,"kb_avail":91352756,"statfs":{"total":95995035648,"available":93545222144,"internally_reserved":0,"allocated":2230521856,"data_stored":2229380229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":219272259},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":4.2489999999999997,"15min":4.5519999999999996},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.465,"5min":23.381,"15min":23.381},"last":1.296},{"interface":"front","average":{"1min":1.853,"5min":3.7130000000000001,"15min":4.0229999999999997},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.5190000000000001,"5min":14.289999999999999,"15min":14.289999999999999},"last":1.0700000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5329999999999999,"15min":3.8130000000000002},"min":{"1min":0.28799999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.415,"5min":20.407,"15min":20.407},"last":0.84899999999999998},{"interface":"front","average":{"1min":2.702,"5min":3.2850000000000001,"15min":3.3820000000000001},"min":{"1min":0.40300000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.494,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.877}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":3.3410000000000002,"15min":3.6070000000000002},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.1020000000000003,"5min":20.356000000000002,"15min":20.356000000000002},"last":0.89900000000000002},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.7290000000000001,"15min":2.819},"min":{"1min":0.34799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":9.3379999999999992,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.38200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":3.7570000000000001},"last":0.435},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.5739999999999998,"5min":8.5739999999999998,"15min":8.5739999999999998},"last":0.29499999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.4319999999999999,"15min":2.5529999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.6199999999999992,"5min":11.836,"15min":11.836},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.274,"15min":2.327},"min":{"1min":0.55600000000000005,"5min":0.309,"15min":0.309},"max":{"1min":6.9249999999999998,"5min":14.164999999999999,"15min":14.164999999999999},"last":0.96699999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.5379999999999998,"15min":2.6720000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.532,"5min":11.887,"15min":11.887},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.496,"5min":2.3820000000000001,"15min":2.5299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.8710000000000004,"5min":14.125,"15min":14.125},"last":0.93600000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.4039999999999999,"15min":1.3939999999999999},"min":{"1min":0.45400000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0609999999999999,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.8089999999999999,"5min":2.5009999999999999,"15min":2.617},"min":{"1min":0.42199999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":8.6549999999999994,"5min":11.801,"15min":11.801},"last":0.40799999999999997}]}]},{"osd":4,"up_from":266,"seq":1142461300754,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2494004,"kb_used_data":2175340,"kb_used_omap":35,"kb_used_meta":318620,"kb_avail":91251148,"statfs":{"total":95995035648,"available":93441175552,"internally_reserved":0,"allocated":2227548160,"data_stored":2227023875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":326267118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.3799999999999999,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3959999999999999,"5min":2.3959999999999999,"15min":2.3959999999999999},"last":0.72699999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":2.7160000000000002,"5min":2.7160000000000002,"15min":2.7160000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":4.6829999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.661,"5min":3.661,"15min":3.661},"last":1.3120000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6449999999999996,"5min":4.6449999999999996,"15min":4.6449999999999996},"last":0.27700000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.262,"15min":1.262},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.1259999999999999,"5min":3.1259999999999999,"15min":3.1259999999999999},"last":2.8660000000000001},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.4470000000000001,"5min":4.4470000000000001,"15min":4.4470000000000001},"last":0.66700000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":1.335},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.6970000000000001,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.30199999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.5609999999999999,"5min":2.5609999999999999,"15min":2.5609999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.917,"5min":1.917,"15min":1.917},"last":0.60199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.5630000000000002,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":1.282}]}]},{"osd":5,"up_from":36,"seq":154618822939,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3244004,"kb_used_data":2942684,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":90501148,"statfs":{"total":95995035648,"available":92673175552,"internally_reserved":0,"allocated":3013308416,"data_stored":3012796259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":1.5860000000000001,"15min":1.3460000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.3090000000000002,"5min":10.92,"15min":15.773999999999999},"last":0.503},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.673,"15min":1.4790000000000001},"min":{"1min":0.25600000000000001,"5min":0.23899999999999999,"15min":0.23400000000000001},"max":{"1min":6.3090000000000002,"5min":11.747,"15min":15.699},"last":0.43099999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.3680000000000001,"15min":1.3300000000000001},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.2670000000000003,"5min":11.335000000000001,"15min":15.689},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.845,"5min":1.895,"15min":1.5269999999999999},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":19.402999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.38400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.8480000000000001,"15min":2.4369999999999998},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.3819999999999997,"5min":15.640000000000001,"15min":15.640000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.302,"5min":1.8220000000000001,"15min":2.2949999999999999},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.3659999999999997,"5min":10.656000000000001,"15min":10.656000000000001},"last":0.47299999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.734,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5860000000000003,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":4.3810000000000002},{"interface":"front","average":{"1min":1.2,"5min":2.6549999999999998,"15min":2.8969999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":2.9710000000000001,"5min":17.34,"15min":17.34},"last":3.9870000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.0819999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":1.8220000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.41099999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.995,"15min":0.90000000000000002},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.4159999999999999,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.754,"5min":0.98999999999999999,"15min":0.81100000000000005},"min":{"1min":0.317,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.3490000000000002,"5min":18.600999999999999,"15min":18.600999999999999},"last":1.8580000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0589999999999999,"15min":0.88},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.282,"5min":7.4290000000000003,"15min":10.334},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.749,"15min":0.88600000000000001},"min":{"1min":0.35099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.3500000000000001,"5min":8.0210000000000008,"15min":12.356},"last":0.44900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2258108416,"data_stored":2258108416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2588475392,"data_stored":2588475392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1530789888,"data_stored":1530789888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2234056704,"data_stored":2234056704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2236481536,"data_stored":2236481536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3020292096,"data_stored":3020292096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2193620992,"data_stored":2193620992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1839202304,"data_stored":1839202304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:10:00.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:10:00.061 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:00.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:00.169 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:10:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:00.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:09:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:10:00.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:00.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:00.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:00.301 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:00.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:00.514 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:00.601 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:00.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:00.657 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 69 16 94247 94231 85.2993 43.25 0.0140529 0.0117175 2023-10-09T10:10:00.713 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:00.753 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:00.813 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:00.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:00.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:00.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:01.020 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:01.071 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:00 smithi110 ceph-mon[136032]: pgmap v758: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 663 op/s; 5374/273789 objects degraded (1.963%); 5137/273789 objects misplaced (1.876%); 934 KiB/s, 14 objects/s recovering 2023-10-09T10:10:01.071 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:00 smithi110 ceph-mon[136032]: from='client.35628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:01.071 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:00 smithi110 ceph-mon[136032]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 5374/268416 objects degraded (2.002%), 2 pgs degraded 2023-10-09T10:10:01.071 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:00 smithi110 ceph-mon[136032]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T10:10:01.071 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:00 smithi110 ceph-mon[136032]: [WRN] PG_DEGRADED: Degraded data redundancy: 5374/268416 objects degraded (2.002%), 2 pgs degraded 2023-10-09T10:10:01.071 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:00 smithi110 ceph-mon[136032]: pg 8.4 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-10-09T10:10:01.071 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:00 smithi110 ceph-mon[136032]: pg 8.f is active+recovering+undersized+degraded+remapped, acting [3,1] 2023-10-09T10:10:01.102 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:01.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[152263]: pgmap v758: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 663 op/s; 5374/273789 objects degraded (1.963%); 5137/273789 objects misplaced (1.876%); 934 KiB/s, 14 objects/s recovering 2023-10-09T10:10:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[152263]: from='client.35628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[152263]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 5374/268416 objects degraded (2.002%), 2 pgs degraded 2023-10-09T10:10:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[152263]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T10:10:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[152263]: [WRN] PG_DEGRADED: Degraded data redundancy: 5374/268416 objects degraded (2.002%), 2 pgs degraded 2023-10-09T10:10:01.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[152263]: pg 8.4 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-10-09T10:10:01.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[152263]: pg 8.f is active+recovering+undersized+degraded+remapped, acting [3,1] 2023-10-09T10:10:01.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[147926]: pgmap v758: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 663 op/s; 5374/273789 objects degraded (1.963%); 5137/273789 objects misplaced (1.876%); 934 KiB/s, 14 objects/s recovering 2023-10-09T10:10:01.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[147926]: from='client.35628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:01.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[147926]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 5374/268416 objects degraded (2.002%), 2 pgs degraded 2023-10-09T10:10:01.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[147926]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T10:10:01.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[147926]: [WRN] PG_DEGRADED: Degraded data redundancy: 5374/268416 objects degraded (2.002%), 2 pgs degraded 2023-10-09T10:10:01.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[147926]: pg 8.4 is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2023-10-09T10:10:01.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:00 smithi012 ceph-mon[147926]: pg 8.f is active+recovering+undersized+degraded+remapped, acting [3,1] 2023-10-09T10:10:01.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:01.273 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:01.435 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:10:01.435 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:10:01.435 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:10:01.445 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:10:01.466 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:01.466 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:01.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:01.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:01.489 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:10:01.504 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:01.537 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:10:01.551 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:10:01.552 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:10:01.552 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:10:01.552 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:10:01.552 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:10:01.572 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:10:01.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:01.631 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:01.657 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 70 16 94981 94965 84.7364 45.875 0.0076175 0.011795 2023-10-09T10:10:01.680 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:10:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:10:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:10:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:10:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:10:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:10:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:10:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:10:01.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:10:01.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:10:02.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3815454100' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:02.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3815454100' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:02.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3815454100' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:02.623 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:02.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:02.657 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 71 16 95781 95765 84.2473 50 0.010208 0.0118633 2023-10-09T10:10:02.698 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:02.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:02.799 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:03.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:02 smithi110 ceph-mon[136032]: pgmap v759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 5374/279276 objects degraded (1.924%); 5040/279276 objects misplaced (1.805%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:03.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:02 smithi012 ceph-mon[152263]: pgmap v759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 5374/279276 objects degraded (1.924%); 5040/279276 objects misplaced (1.805%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:03.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:02 smithi012 ceph-mon[147926]: pgmap v759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 5374/279276 objects degraded (1.924%); 5040/279276 objects misplaced (1.805%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:03.657 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 72 16 96555 96539 83.7493 48.375 0.00964621 0.0119349 2023-10-09T10:10:03.858 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:03.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:03.945 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:10:03.945 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:10:03.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:10:03.946 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:10:03.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.946 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:18874 8.b e277/271 8:d3307fe1:::benchmark_data_smithi012_271002_object18873:head v 277'1224, mlcod=277'1220)", 2023-10-09T10:10:03.946 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:08:58.682346+0000", 2023-10-09T10:10:03.946 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 65.256156961000002, 2023-10-09T10:10:03.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017869765999999999, 2023-10-09T10:10:03.947 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:03.947 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:03.947 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:03.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:03.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682346+0000", 2023-10-09T10:10:03.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:03.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:03.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682346+0000", 2023-10-09T10:10:03.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:03.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:03.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682346+0000", 2023-10-09T10:10:03.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0373e-05 2023-10-09T10:10:03.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:03.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682396+0000", 2023-10-09T10:10:03.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0125e-05 2023-10-09T10:10:03.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:03.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682406+0000", 2023-10-09T10:10:03.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016295928000000001 2023-10-09T10:10:03.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:03.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.698702+0000", 2023-10-09T10:10:03.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2420000000000002e-05 2023-10-09T10:10:03.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:03.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.698765+0000", 2023-10-09T10:10:03.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3952999999999999e-05 2023-10-09T10:10:03.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:03.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.698789+0000", 2023-10-09T10:10:03.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013978129999999999 2023-10-09T10:10:03.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:03.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.700186+0000", 2023-10-09T10:10:03.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9802e-05 2023-10-09T10:10:03.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.700216+0000", 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017869765999999999 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.954 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:18876 8.2 e277/271 8:4bafa7ad:::benchmark_data_smithi012_271002_object18875:head v 277'1135, mlcod=277'1132)", 2023-10-09T10:10:03.954 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:08:58.682537+0000", 2023-10-09T10:10:03.954 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 65.255966454000003, 2023-10-09T10:10:03.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017532289, 2023-10-09T10:10:03.954 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:03.954 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:03.954 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682537+0000", 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682537+0000", 2023-10-09T10:10:03.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:03.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:03.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682536+0000", 2023-10-09T10:10:03.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5120000000000003e-05 2023-10-09T10:10:03.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.682571+0000", 2023-10-09T10:10:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016132951999999999 2023-10-09T10:10:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.698704+0000", 2023-10-09T10:10:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1714e-05 2023-10-09T10:10:03.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.698716+0000", 2023-10-09T10:10:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9614000000000003e-05 2023-10-09T10:10:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:03.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.698766+0000", 2023-10-09T10:10:03.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2779999999999999e-05 2023-10-09T10:10:03.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:03.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.698788+0000", 2023-10-09T10:10:03.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001245183 2023-10-09T10:10:03.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.700034+0000", 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5642000000000002e-05 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:08:58.700069+0000", 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017532289 2023-10-09T10:10:03.960 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.961 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:03.961 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.961 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:53367 8.5 e282/271 8:aa687311:::benchmark_data_smithi012_271002_object53366:head v 282'3372, mlcod=282'3369)", 2023-10-09T10:10:03.961 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:19.090533+0000", 2023-10-09T10:10:03.961 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 44.847970801000002, 2023-10-09T10:10:03.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012652305000000001, 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.090533+0000", 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:03.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.090533+0000", 2023-10-09T10:10:03.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:03.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:03.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.090532+0000", 2023-10-09T10:10:03.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4211000000000002e-05 2023-10-09T10:10:03.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:03.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.090566+0000", 2023-10-09T10:10:03.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1078e-05 2023-10-09T10:10:03.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:03.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.090577+0000", 2023-10-09T10:10:03.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.392e-06 2023-10-09T10:10:03.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:03.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.090583+0000", 2023-10-09T10:10:03.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024268419999999998 2023-10-09T10:10:03.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:03.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.093010+0000", 2023-10-09T10:10:03.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2563999999999999e-05 2023-10-09T10:10:03.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:03.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.093033+0000", 2023-10-09T10:10:03.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010113139 2023-10-09T10:10:03.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:03.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.103146+0000", 2023-10-09T10:10:03.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8889e-05 2023-10-09T10:10:03.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:03.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:19.103185+0000", 2023-10-09T10:10:03.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012652305000000001 2023-10-09T10:10:03.969 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.969 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:03.969 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.969 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:62588 8.b e283/271 8:d7f00efb:::benchmark_data_smithi012_271002_object62587:head v 283'3927, mlcod=283'3924)", 2023-10-09T10:10:03.969 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:25.839868+0000", 2023-10-09T10:10:03.970 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 38.098635362000003, 2023-10-09T10:10:03.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012951519999999999, 2023-10-09T10:10:03.970 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:03.970 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:03.970 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:03.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:03.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.839868+0000", 2023-10-09T10:10:03.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:03.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:03.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.839868+0000", 2023-10-09T10:10:03.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:03.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:03.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.839867+0000", 2023-10-09T10:10:03.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7419e-05 2023-10-09T10:10:03.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:03.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.839925+0000", 2023-10-09T10:10:03.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0693000000000001e-05 2023-10-09T10:10:03.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:03.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.839935+0000", 2023-10-09T10:10:03.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.741e-06 2023-10-09T10:10:03.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:03.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.839940+0000", 2023-10-09T10:10:03.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016388520000000001 2023-10-09T10:10:03.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:03.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.841579+0000", 2023-10-09T10:10:03.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8538000000000001e-05 2023-10-09T10:10:03.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:03.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.841597+0000", 2023-10-09T10:10:03.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011175078999999999 2023-10-09T10:10:03.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:03.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.852773+0000", 2023-10-09T10:10:03.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7036999999999997e-05 2023-10-09T10:10:03.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:03.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.852820+0000", 2023-10-09T10:10:03.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012951519999999999 2023-10-09T10:10:03.978 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.978 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:03.978 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.978 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:62771 8.b e283/271 8:da088a62:::benchmark_data_smithi012_271002_object62770:head v 283'3945, mlcod=283'3943)", 2023-10-09T10:10:03.978 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:25.967763+0000", 2023-10-09T10:10:03.979 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 37.970740087000003, 2023-10-09T10:10:03.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012341069, 2023-10-09T10:10:03.979 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:03.979 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:03.979 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:03.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:03.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.967763+0000", 2023-10-09T10:10:03.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:03.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:03.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.967763+0000", 2023-10-09T10:10:03.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:03.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:03.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.967762+0000", 2023-10-09T10:10:03.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1529e-05 2023-10-09T10:10:03.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:03.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.967824+0000", 2023-10-09T10:10:03.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0682999999999999e-05 2023-10-09T10:10:03.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:03.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.967835+0000", 2023-10-09T10:10:03.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9329999999999998e-06 2023-10-09T10:10:03.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:03.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.967841+0000", 2023-10-09T10:10:03.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8763000000000001e-05 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.967879+0000", 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9089e-05 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.967898+0000", 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012164483 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.980063+0000", 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1501999999999997e-05 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:25.980104+0000", 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012341069 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:64616 8.4 e283/271 8:2466e3da:::benchmark_data_smithi012_271002_object64615:head v 283'4126, mlcod=283'4124)", 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:27.400209+0000", 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.538294026999999, 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011914944, 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.400209+0000", 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:03.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.400209+0000", 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.400208+0000", 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1342e-05 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.400230+0000", 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1052e-05 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.400241+0000", 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.178e-06 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:03.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.400247+0000", 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4799000000000001e-05 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.400312+0000", 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.206e-05 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.400354+0000", 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011735496999999999 2023-10-09T10:10:03.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.412089+0000", 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4928999999999997e-05 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:27.412124+0000", 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011914944 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:03.993 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.994 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:83379 8.5 e285/271 8:a71bf671:::benchmark_data_smithi012_271002_object83378:head v 285'5265, mlcod=285'5263)", 2023-10-09T10:10:03.994 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:44.324592+0000", 2023-10-09T10:10:03.994 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.613911740999999, 2023-10-09T10:10:03.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01186915, 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.324592+0000", 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.324592+0000", 2023-10-09T10:10:03.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.324591+0000", 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1897999999999999e-05 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.324613+0000", 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4989e-05 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.324628+0000", 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4030000000000004e-06 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.324634+0000", 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2811000000000006e-05 2023-10-09T10:10:03.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.324697+0000", 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9216e-05 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.324716+0000", 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011702819999999999 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.336419+0000", 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1743000000000003e-05 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:44.336461+0000", 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01186915 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:03.999 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:84230 8.5 e286/271 8:ab09a08a:::benchmark_data_smithi012_271002_object84229:head v 286'5319, mlcod=286'5316)", 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:45.294913+0000", 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.643590238000002, 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012042412000000001, 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.294913+0000", 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.294913+0000", 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.294912+0000", 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4790000000000002e-05 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.294937+0000", 2023-10-09T10:10:04.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1284000000000001e-05 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.294948+0000", 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.37e-06 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.294955+0000", 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039497550000000001 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.298904+0000", 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0438999999999999e-05 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.298925+0000", 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080005470000000002 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.306925+0000", 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0120999999999998e-05 2023-10-09T10:10:04.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.306956+0000", 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012042412000000001 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:84267 8.e e286/271 8:75ba2ae3:::benchmark_data_smithi012_271002_object84266:head v 286'5244, mlcod=286'5241)", 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:45.324614+0000", 2023-10-09T10:10:04.004 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.613888974999998, 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020611930000000001, 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.324614+0000", 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.324614+0000", 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.324614+0000", 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031674260000000001 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.327781+0000", 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5937999999999999e-05 2023-10-09T10:10:04.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.327797+0000", 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2869999999999999e-06 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.327804+0000", 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001578072 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.329382+0000", 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3731999999999999e-05 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.329406+0000", 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015773947999999999 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345180+0000", 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6295999999999999e-05 2023-10-09T10:10:04.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345226+0000", 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020611930000000001 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:84271 8.f e286/271 8:f5c6da19:::benchmark_data_smithi012_271002_object84270:head v 286'5247, mlcod=286'5245)", 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:45.331095+0000", 2023-10-09T10:10:04.009 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.607408079999999, 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014262436, 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.331095+0000", 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.331095+0000", 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.331094+0000", 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9060000000000001e-05 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.331143+0000", 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9592000000000001e-05 2023-10-09T10:10:04.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.331163+0000", 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8660000000000006e-06 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.331171+0000", 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5289e-05 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.331216+0000", 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5922e-05 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.331242+0000", 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014080617 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345323+0000", 2023-10-09T10:10:04.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5271000000000002e-05 2023-10-09T10:10:04.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345358+0000", 2023-10-09T10:10:04.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014262436 2023-10-09T10:10:04.014 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.014 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.014 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:84272 8.e e286/271 8:706b6520:::benchmark_data_smithi012_271002_object84271:head v 286'5245, mlcod=286'5241)", 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:45.332561+0000", 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.605942492, 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012686998999999999, 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.332561+0000", 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.332561+0000", 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.332560+0000", 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7248999999999999e-05 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.332587+0000", 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1827e-05 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.332599+0000", 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0409999999999997e-06 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.332604+0000", 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4771999999999998e-05 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.332649+0000", 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.529e-05 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.332674+0000", 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01255583 2023-10-09T10:10:04.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345230+0000", 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7867999999999999e-05 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345248+0000", 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012686998999999999 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.020 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:84275 8.5 e286/271 8:ab81792a:::benchmark_data_smithi012_271002_object84274:head v 286'5322, mlcod=286'5320)", 2023-10-09T10:10:04.020 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:45.333854+0000", 2023-10-09T10:10:04.020 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.604649698999999, 2023-10-09T10:10:04.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011418736000000001, 2023-10-09T10:10:04.020 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.333854+0000", 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.333854+0000", 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.333853+0000", 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4037999999999999e-05 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.333877+0000", 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4521999999999999e-05 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.333892+0000", 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.601e-06 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.333898+0000", 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0176e-05 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.333968+0000", 2023-10-09T10:10:04.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9755e-05 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.333988+0000", 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011252975 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345241+0000", 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1420000000000001e-05 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345272+0000", 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011418736000000001 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:84278 8.2 e286/271 8:4fffb096:::benchmark_data_smithi012_271002_object84277:head v 286'5143, mlcod=286'5141)", 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:45.334584+0000", 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.603919768000001, 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013316982999999999, 2023-10-09T10:10:04.025 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.334584+0000", 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.334584+0000", 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:04.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.334583+0000", 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8448e-05 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.334641+0000", 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2945000000000001e-05 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.334654+0000", 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.609e-06 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.334660+0000", 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002234205 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.336894+0000", 2023-10-09T10:10:04.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9809e-05 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.336914+0000", 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010940521999999999 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.347854+0000", 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6140000000000002e-05 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.347901+0000", 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013316982999999999 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:84689 8.b e286/271 8:ddac58c4:::benchmark_data_smithi012_271002_object84688:head v 286'5264, mlcod=286'5261)", 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:45.713841+0000", 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.224662608999999, 2023-10-09T10:10:04.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011502745999999999, 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.713841+0000", 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.713841+0000", 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.713840+0000", 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0948999999999997e-05 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.713881+0000", 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5167999999999999e-05 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.713896+0000", 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.319e-06 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.713902+0000", 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028401590000000001 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.716742+0000", 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0302e-05 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.716763+0000", 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085444040000000002 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.725307+0000", 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6600999999999997e-05 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.725344+0000", 2023-10-09T10:10:04.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011502745999999999 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:86702 8.2 e289/271 8:4ae248e0:::benchmark_data_smithi012_271002_object86701:head v 289'5312, mlcod=289'5310)", 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:50.453281+0000", 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 13.485222134000001, 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011301153, 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.453281+0000", 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.453281+0000", 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.453280+0000", 2023-10-09T10:10:04.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9228999999999998e-05 2023-10-09T10:10:04.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.453340+0000", 2023-10-09T10:10:04.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4895999999999999e-05 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.453354+0000", 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8329999999999999e-06 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.453360+0000", 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024662740000000001 2023-10-09T10:10:04.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.455827+0000", 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0451000000000001e-05 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.455847+0000", 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086994629999999993 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.464546+0000", 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6019000000000002e-05 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:50.464582+0000", 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011301153 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.040 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:89800 8.b e289/271 8:dffceed6:::benchmark_data_smithi012_271002_object89799:head v 289'5559, mlcod=289'5557)", 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:54.409229+0000", 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 9.5292743600000005, 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011749071999999999, 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.409229+0000", 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.409229+0000", 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.409228+0000", 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7546999999999997e-05 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.409276+0000", 2023-10-09T10:10:04.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3662999999999999e-05 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.409290+0000", 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8069999999999996e-06 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.409295+0000", 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3441e-05 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.409339+0000", 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0313e-05 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.409359+0000", 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01157642 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.420935+0000", 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2648000000000002e-05 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:54.420978+0000", 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011749071999999999 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:95001 8.2 e290/271 8:4f18b7d6:::benchmark_data_smithi012_271002_object95000:head v 290'5825, mlcod=290'5822)", 2023-10-09T10:10:04.045 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:01.706794+0000", 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.2317093529999998, 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011653165, 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.706794+0000", 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.706794+0000", 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.706793+0000", 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8452000000000003e-05 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.706841+0000", 2023-10-09T10:10:04.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8321e-05 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.706860+0000", 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4180000000000002e-06 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.706866+0000", 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7628e-05 2023-10-09T10:10:04.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.706904+0000", 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4142999999999999e-05 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.706928+0000", 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011493951000000001 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.718422+0000", 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5361e-05 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:01.718447+0000", 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011653165 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:95487 8.4 e290/288 8:21ebf5dc:::benchmark_data_smithi012_271002_object95486:head v 290'6043, mlcod=0'0)", 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:02.265631+0000", 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.67287224, 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011506071, 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.265631+0000", 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.265631+0000", 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.265630+0000", 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4253999999999999e-05 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.265655+0000", 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5917e-05 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.265670+0000", 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0886999999999999e-05 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.265681+0000", 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6199999999999998e-05 2023-10-09T10:10:04.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.265728+0000", 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3787000000000001e-05 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.265751+0000", 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011332729 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.277084+0000", 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3216e-05 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:02.277137+0000", 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011506071 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:96714 8.e e290/271 8:76dd1331:::benchmark_data_smithi012_271002_object96713:head v 290'6027, mlcod=290'6025)", 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:03.861861+0000", 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.076642163999999999, 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014219647, 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861861+0000", 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861861+0000", 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861860+0000", 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4375000000000001e-05 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861904+0000", 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6534000000000001e-05 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861921+0000", 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3369e-05 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861934+0000", 2023-10-09T10:10:04.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7049000000000002e-05 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861991+0000", 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1744999999999999e-05 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.862013+0000", 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014015282 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.876028+0000", 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2476999999999999e-05 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.876081+0000", 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014219647 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:96716 8.e e290/271 8:716a2fcb:::benchmark_data_smithi012_271002_object96715:head v 290'6028, mlcod=290'6025)", 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:03.861950+0000", 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.076553414, 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014162054, 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861950+0000", 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861950+0000", 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861950+0000", 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9584999999999999e-05 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:04.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861979+0000", 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5969999999999992e-06 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861988+0000", 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4530000000000004e-06 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.861992+0000", 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021567650000000002 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.864149+0000", 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6798e-05 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.864176+0000", 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011910604 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:04.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.876086+0000", 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5723000000000001e-05 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:03.876112+0000", 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014162054 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:04.065 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:10:04.066 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:04.112 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:04.138 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/279276 objects degraded (1.924%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:04.141 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/279276 objects degraded (1.924%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:04.268 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:04.271 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/279276 objects degraded (1.924%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:04.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:04.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:10:04.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:04.419 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:04.458 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:04.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:04.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:04.562 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:04.600 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:04.653 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:04.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:04.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:04.658 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 73 16 97348 97332 83.2812 49.5625 0.0119914 0.0120024 2023-10-09T10:10:04.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:04.888 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:04.961 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:04.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:04.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:05.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:05.156 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:05.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:04 smithi012 ceph-mon[147926]: pgmap v760: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 5374/284802 objects degraded (1.887%); 5040/284802 objects misplaced (1.770%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:05.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:04 smithi012 ceph-mon[152263]: pgmap v760: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 5374/284802 objects degraded (1.887%); 5040/284802 objects misplaced (1.770%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:05.223 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:05.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:05.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:05.336 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:05.427 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:05.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:04 smithi110 ceph-mon[136032]: pgmap v760: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 5374/284802 objects degraded (1.887%); 5040/284802 objects misplaced (1.770%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:05.615 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:05.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:05.657 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 74 16 98065 98049 82.7615 44.8125 0.0212274 0.0120773 2023-10-09T10:10:05.733 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:05.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:05.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:05.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:05.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:05.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:06.017 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:06.109 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:06.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:06.155 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:06.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:06.328 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:06.530 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:06.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:06.638 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:06.661 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 75 16 98717 98701 82.1975 40.75 0.0169116 0.0121605 2023-10-09T10:10:06.673 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:06.709 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:06.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:06.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:06.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:06.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:10:06.889 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:07.035 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:07.046 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:07.046 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5374/288309 objects degraded (1.864%), 2 pgs degraded","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":924,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":291,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":96103,"data_bytes":6297229216,"bytes_used":21349543936,"bytes_avail":746610741248,"bytes_total":767960285184,"degraded_objects":5374,"degraded_total":288309,"degraded_ratio":0.018639723352375404,"misplaced_objects":5040,"misplaced_total":288309,"misplaced_ratio":0.017481244081870493,"recovering_objects_per_sec":19,"recovering_bytes_per_sec":1283886,"recovering_keys_per_sec":0,"num_objects_recovered":196,"num_bytes_recovered":12845056,"num_keys_recovered":0,"write_bytes_sec":48073687,"read_op_per_sec":0,"write_op_per_sec":733},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T10:09:33.293227+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T10:10:07.047 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:10:07.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:07.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:07.159 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:07.184 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:07 smithi012 ceph-mon[147926]: pgmap v761: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 5374/288309 objects degraded (1.864%); 5040/288309 objects misplaced (1.748%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:07.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1520624100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:07.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:07 smithi012 ceph-mon[152263]: pgmap v761: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 5374/288309 objects degraded (1.864%); 5040/288309 objects misplaced (1.748%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:07.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1520624100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:07.197 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:07.236 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:07.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:07.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:07.388 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:07 smithi110 ceph-mon[136032]: pgmap v761: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 5374/288309 objects degraded (1.864%); 5040/288309 objects misplaced (1.748%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:07.388 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1520624100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:07.391 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:07.448 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:07.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:07.552 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:07.588 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:07.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:07.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:07.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:07.661 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 76 16 99417 99401 81.6918 43.75 0.00851429 0.0122358 2023-10-09T10:10:07.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:07.835 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:07.916 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:07.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:07.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:07.953 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:10:07.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:10:08.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:08.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:08.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[147926]: failed to encode map e291 with expected crc 2023-10-09T10:10:08.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1520624100' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:10:08.048 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[147926]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T10:10:08.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[147926]: failed to encode map e291 with expected crc 2023-10-09T10:10:08.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4172996357' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:08.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/288309 objects degraded (1.864%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:08.049 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:10:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:10:07] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.33.4" 2023-10-09T10:10:08.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:08.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[152263]: failed to encode map e291 with expected crc 2023-10-09T10:10:08.050 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1520624100' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:10:08.050 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[152263]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T10:10:08.050 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[152263]: failed to encode map e291 with expected crc 2023-10-09T10:10:08.050 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4172996357' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:08.050 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/288309 objects degraded (1.864%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:08.127 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:08.326 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:08.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:08.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:08.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:08.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:08 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:08 smithi110 ceph-mon[136032]: failed to encode map e291 with expected crc 2023-10-09T10:10:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1520624100' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:10:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:08 smithi110 ceph-mon[136032]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T10:10:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:08 smithi110 ceph-mon[136032]: failed to encode map e291 with expected crc 2023-10-09T10:10:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4172996357' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:08.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/288309 objects degraded (1.864%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:08.473 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:08.520 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:08.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:08.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:08.596 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:08.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:08.650 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:10:08.650 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:10:08.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.650 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:100083 8.e 8:75426c90:::benchmark_data_smithi012_271002_object100082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-09T10:10:08.650 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:08.630605+0000", 2023-10-09T10:10:08.650 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014413275, 2023-10-09T10:10:08.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014449257, 2023-10-09T10:10:08.650 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 100083 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630605+0000", 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630605+0000", 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630604+0000", 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630634+0000", 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9408e-05 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630641+0000", 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0709999999999997e-06 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630647+0000", 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.088e-06 2023-10-09T10:10:08.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634909+0000", 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042621899999999999 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.635031+0000", 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121997 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.635161+0000", 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012976700000000001 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.643893+0000", 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087321410000000006 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:08.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:100082 8.9 e291/271 8:90bb4f2d:::benchmark_data_smithi012_271002_object100081:head v 291'6202, mlcod=291'6200)", 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:08.634496+0000", 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.010521882999999999, 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010652330999999999, 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:08.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634496+0000", 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634496+0000", 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634495+0000", 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:10:08.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634549+0000", 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3868e-05 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634565+0000", 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6198e-05 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634575+0000", 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0268999999999999e-05 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634777+0000", 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020190799999999999 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634800+0000", 2023-10-09T10:10:08.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3212000000000001e-05 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:100074 8.e 8:79d30fa2:::benchmark_data_smithi012_271002_object100073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:08.630363+0000", 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014654825999999999, 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014867905000000001, 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:10:08.660 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 100074 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630363+0000", 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630363+0000", 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630361+0000", 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630394+0000", 2023-10-09T10:10:08.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3176999999999998e-05 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630420+0000", 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5316e-05 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630433+0000", 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2959000000000001e-05 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630693+0000", 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026053399999999998 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630890+0000", 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019704799999999999 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:10:08.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.631026+0000", 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013550300000000001 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.638611+0000", 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0075856500000000002 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.640361+0000", 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017497400000000001 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.665 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:100078 8.c 8:391094b5:::benchmark_data_smithi012_271002_object100077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:08.630457+0000", 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014561368999999999, 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014878529999999999, 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:08.666 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 100078 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630457+0000", 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:08.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630457+0000", 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630456+0000", 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630479+0000", 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3249999999999999e-05 2023-10-09T10:10:08.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630487+0000", 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1790000000000008e-06 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630493+0000", 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.037e-06 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:08.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634654+0000", 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041606179999999996 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634784+0000", 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012968699999999999 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634935+0000", 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015081400000000001 2023-10-09T10:10:08.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:100081 8.6 8:65b6f1fc:::benchmark_data_smithi012_271002_object100080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:08.630561+0000", 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014456482999999999, 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014866442000000001, 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:08.671 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 100081 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630561+0000", 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630561+0000", 2023-10-09T10:10:08.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630561+0000", 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630585+0000", 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4539e-05 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630592+0000", 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9450000000000004e-06 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.630598+0000", 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2890000000000003e-06 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634575+0000", 2023-10-09T10:10:08.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039768529999999998 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634757+0000", 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018124600000000001 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:08.634935+0000", 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017831800000000001 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:08.675 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.676 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:08.676 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:10:08.676 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 5 2023-10-09T10:10:08.676 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:10:08.676 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 77 16 100090 100074 81.1774 42.0625 0.0269367 0.0123109 2023-10-09T10:10:08.707 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:09.074 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:09.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:09.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:09.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:09 smithi012 ceph-mon[147926]: pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 5374/290469 objects degraded (1.850%); 4939/290469 objects misplaced (1.700%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:09.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:09 smithi012 ceph-mon[152263]: pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 5374/290469 objects degraded (1.850%); 4939/290469 objects misplaced (1.700%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:09.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:10:09.308 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:09.405 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:09.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:09.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:09.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:09 smithi110 ceph-mon[136032]: pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 5374/290469 objects degraded (1.850%); 4939/290469 objects misplaced (1.700%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:09.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:09.567 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:09.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:09.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:09.661 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 78 16 100808 100792 80.7121 44.875 0.0122422 0.0123855 2023-10-09T10:10:09.685 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:09.719 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:09.759 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:09.808 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:09.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:09.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:09.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:10:09.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:10.049 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:10.152 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:10.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:10.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:10.263 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:10.290 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:10.333 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:10.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:10.438 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:10.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:10.507 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:10.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:10.557 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:10:10.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:10.627 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:10:10.661 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 79 16 101433 101417 80.1852 39.0625 0.028049 0.0124656 2023-10-09T10:10:10.680 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:10:10.703 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:10.722 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:10:10.768 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:10:10.815 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:10:10.854 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:10:10.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:10:10.899 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:10:10.899 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:10:10.899 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:10:10.899 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:10:10.899 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:10:10.899 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:10:10.899 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:10:10.900 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:10:10.939 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:10.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:10.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:11.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:11.129 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:11.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:11 smithi012 ceph-mon[152263]: pgmap v764: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 5374/296034 objects degraded (1.815%); 4939/296034 objects misplaced (1.668%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:11.371 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1349289615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:11.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:11 smithi012 ceph-mon[147926]: pgmap v764: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 5374/296034 objects degraded (1.815%); 4939/296034 objects misplaced (1.668%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:11.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1349289615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:11.372 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:11.387 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:11.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:11.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:11 smithi110 ceph-mon[136032]: pgmap v764: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 5374/296034 objects degraded (1.815%); 4939/296034 objects misplaced (1.668%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T10:10:11.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1349289615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:11.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:11.570 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:11.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:11.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:11.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:11.661 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:10:11.662034+0000 min lat: 0.00120497 max lat: 2.42059 avg lat: 0.012538 2023-10-09T10:10:11.662 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:10:11.662 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 80 16 102129 102113 79.7268 43.5 0.0122484 0.012538 2023-10-09T10:10:11.743 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:11.799 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:11.857 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:11.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:11.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:11.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:12.006 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:12.038 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:12.054 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:12.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:12.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:12.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:12.250 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:12.513 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:12.514 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:12.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:12.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:12.542 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:10:12.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:12.662 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 81 16 102845 102829 79.2951 44.75 0.00494851 0.0126066 2023-10-09T10:10:12.720 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:12.808 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:12.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:12.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:12.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:12.995 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:13.060 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:13.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:13.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:13.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:13.272 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:13.319 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:13 smithi012 ceph-mon[147926]: pgmap v765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 5374/301104 objects degraded (1.785%); 4835/301104 objects misplaced (1.606%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:13.319 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/301104 objects degraded (1.785%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:13.320 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:13 smithi012 ceph-mon[152263]: pgmap v765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 5374/301104 objects degraded (1.785%); 4835/301104 objects misplaced (1.606%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:13.320 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/301104 objects degraded (1.785%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:13.321 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":765,"stamp":"2023-10-09T10:10:11.314139+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6576740256,"num_objects":100368,"num_object_clones":0,"num_object_copies":301104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":4835,"num_objects_unfound":0,"num_objects_dirty":100368,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":100604,"num_write_kb":6424461,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":517,"num_bytes_recovered":30550732,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100538,"ondisk_log_size":100538,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21630344,"kb_used_data":19262280,"kb_used_omap":167,"kb_used_meta":2367832,"kb_avail":728330872,"statfs":{"total":767960285184,"available":745810812928,"internally_reserved":0,"allocated":19724574720,"data_stored":19718329782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2424660569},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,7,4,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":476839936,"num_objects":7276,"num_object_clones":0,"num_object_copies":21828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-205,"num_objects_unfound":0,"num_objects_dirty":7276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7276,"num_write_kb":465664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":205,"num_bytes_recovered":13434880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009650"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":132,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311647+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:09:57.311647+0000","last_peered":"2023-10-09T10:09:57.311647+0000","last_clean":"2023-10-09T10:09:57.311647+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:09:57.311647+0000","last_undegraded":"2023-10-09T10:09:57.311647+0000","last_fullsized":"2023-10-09T10:09:57.311647+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":248,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.024367+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:10:07.024367+0000","last_peered":"2023-10-09T10:10:07.024367+0000","last_clean":"2023-10-09T10:10:07.024367+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:10:07.024367+0000","last_undegraded":"2023-10-09T10:10:07.024367+0000","last_fullsized":"2023-10-09T10:10:07.024367+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":293,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311259+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:09:57.311259+0000","last_peered":"2023-10-09T10:09:57.311259+0000","last_clean":"2023-10-09T10:09:57.311259+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:09:57.311259+0000","last_undegraded":"2023-10-09T10:09:57.311259+0000","last_fullsized":"2023-10-09T10:09:57.311259+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":156,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.316965+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:09:57.316965+0000","last_peered":"2023-10-09T10:09:57.316965+0000","last_clean":"2023-10-09T10:09:57.316965+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:09:57.316965+0000","last_undegraded":"2023-10-09T10:09:57.316965+0000","last_fullsized":"2023-10-09T10:09:57.316965+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":262,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:10:06.140551+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:10:06.140551+0000","last_peered":"2023-10-09T10:10:06.140551+0000","last_clean":"2023-10-09T10:10:06.140551+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:10:06.140551+0000","last_undegraded":"2023-10-09T10:10:06.140551+0000","last_fullsized":"2023-10-09T10:10:06.140551+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.319827+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:09:57.319827+0000","last_peered":"2023-10-09T10:09:57.319827+0000","last_clean":"2023-10-09T10:09:57.319827+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:09:57.319827+0000","last_undegraded":"2023-10-09T10:09:57.319827+0000","last_fullsized":"2023-10-09T10:09:57.319827+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":174,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.317210+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:09:57.317210+0000","last_peered":"2023-10-09T10:09:57.317210+0000","last_clean":"2023-10-09T10:09:57.317210+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:09:57.317210+0000","last_undegraded":"2023-10-09T10:09:57.317210+0000","last_fullsized":"2023-10-09T10:09:57.317210+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":243,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:10:01.124633+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:10:01.124633+0000","last_peered":"2023-10-09T10:10:01.124633+0000","last_clean":"2023-10-09T10:10:01.124633+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:10:01.124633+0000","last_undegraded":"2023-10-09T10:10:01.124633+0000","last_fullsized":"2023-10-09T10:10:01.124633+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":108,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.319693+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:09:57.319693+0000","last_peered":"2023-10-09T10:09:57.319693+0000","last_clean":"2023-10-09T10:09:57.319693+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:09:57.319693+0000","last_undegraded":"2023-10-09T10:09:57.319693+0000","last_fullsized":"2023-10-09T10:09:57.319693+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":204,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:10:01.124671+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:10:01.124671+0000","last_peered":"2023-10-09T10:10:01.124671+0000","last_clean":"2023-10-09T10:10:01.124671+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:10:01.124671+0000","last_undegraded":"2023-10-09T10:10:01.124671+0000","last_fullsized":"2023-10-09T10:10:01.124671+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":329,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322981+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:09:57.322981+0000","last_peered":"2023-10-09T10:09:57.322981+0000","last_clean":"2023-10-09T10:09:57.322981+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:09:57.322981+0000","last_undegraded":"2023-10-09T10:09:57.322981+0000","last_fullsized":"2023-10-09T10:09:57.322981+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":285,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311199+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:09:57.311199+0000","last_peered":"2023-10-09T10:09:57.311199+0000","last_clean":"2023-10-09T10:09:57.311199+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:09:57.311199+0000","last_undegraded":"2023-10-09T10:09:57.311199+0000","last_fullsized":"2023-10-09T10:09:57.311199+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":178,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.323257+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:09:57.323257+0000","last_peered":"2023-10-09T10:09:57.323257+0000","last_clean":"2023-10-09T10:09:57.323257+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:09:57.323257+0000","last_undegraded":"2023-10-09T10:09:57.323257+0000","last_fullsized":"2023-10-09T10:09:57.323257+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":157,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311540+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:09:57.311540+0000","last_peered":"2023-10-09T10:09:57.311540+0000","last_clean":"2023-10-09T10:09:57.311540+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:09:57.311540+0000","last_undegraded":"2023-10-09T10:09:57.311540+0000","last_fullsized":"2023-10-09T10:09:57.311540+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":362,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322126+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:09:57.322126+0000","last_peered":"2023-10-09T10:09:57.322126+0000","last_clean":"2023-10-09T10:09:57.322126+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:09:57.322126+0000","last_undegraded":"2023-10-09T10:09:57.322126+0000","last_fullsized":"2023-10-09T10:09:57.322126+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":230,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:10:01.124753+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:10:01.124753+0000","last_peered":"2023-10-09T10:10:01.124753+0000","last_clean":"2023-10-09T10:10:01.124753+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:10:01.124753+0000","last_undegraded":"2023-10-09T10:10:01.124753+0000","last_fullsized":"2023-10-09T10:10:01.124753+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":156,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.317393+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:09:57.317393+0000","last_peered":"2023-10-09T10:09:57.317393+0000","last_clean":"2023-10-09T10:09:57.317393+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:09:57.317393+0000","last_undegraded":"2023-10-09T10:09:57.317393+0000","last_fullsized":"2023-10-09T10:09:57.317393+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":124,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.320012+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:09:57.320012+0000","last_peered":"2023-10-09T10:09:57.320012+0000","last_clean":"2023-10-09T10:09:57.320012+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:09:57.320012+0000","last_undegraded":"2023-10-09T10:09:57.320012+0000","last_fullsized":"2023-10-09T10:09:57.320012+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":159,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322008+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:09:57.322008+0000","last_peered":"2023-10-09T10:09:57.322008+0000","last_clean":"2023-10-09T10:09:57.322008+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:09:57.322008+0000","last_undegraded":"2023-10-09T10:09:57.322008+0000","last_fullsized":"2023-10-09T10:09:57.322008+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":156,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.317038+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:09:57.317038+0000","last_peered":"2023-10-09T10:09:57.317038+0000","last_clean":"2023-10-09T10:09:57.317038+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:09:57.317038+0000","last_undegraded":"2023-10-09T10:09:57.317038+0000","last_fullsized":"2023-10-09T10:09:57.317038+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":179,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.313352+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:09:57.313352+0000","last_peered":"2023-10-09T10:09:57.313352+0000","last_clean":"2023-10-09T10:09:57.313352+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:09:57.313352+0000","last_undegraded":"2023-10-09T10:09:57.313352+0000","last_fullsized":"2023-10-09T10:09:57.313352+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":243,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:10:06.140807+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:10:06.140807+0000","last_peered":"2023-10-09T10:10:06.140807+0000","last_clean":"2023-10-09T10:10:06.140807+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:10:06.140807+0000","last_undegraded":"2023-10-09T10:10:06.140807+0000","last_fullsized":"2023-10-09T10:10:06.140807+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"291'6293","reported_seq":6329,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:07.813223+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:10:07.813223+0000","last_peered":"2023-10-09T10:10:07.813223+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:10:07.813223+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6293,"log_dups_size":0,"ondisk_log_size":6293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412418048,"num_objects":6293,"num_object_clones":0,"num_object_copies":18879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6293,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":920},{"shards":"4,5,6","objects":5373}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":323,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322885+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:09:57.322885+0000","last_peered":"2023-10-09T10:09:57.322885+0000","last_clean":"2023-10-09T10:09:57.322885+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:09:57.322885+0000","last_undegraded":"2023-10-09T10:09:57.322885+0000","last_fullsized":"2023-10-09T10:09:57.322885+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":109,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.321633+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:09:57.321633+0000","last_peered":"2023-10-09T10:09:57.321633+0000","last_clean":"2023-10-09T10:09:57.321633+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:09:57.321633+0000","last_undegraded":"2023-10-09T10:09:57.321633+0000","last_fullsized":"2023-10-09T10:09:57.321633+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"291'6408","reported_seq":6436,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:09.991043+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:10:09.991043+0000","last_peered":"2023-10-09T10:10:09.991043+0000","last_clean":"2023-10-09T10:10:09.991043+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:10:09.991043+0000","last_undegraded":"2023-10-09T10:10:09.991043+0000","last_fullsized":"2023-10-09T10:10:09.991043+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419954688,"num_objects":6408,"num_object_clones":0,"num_object_copies":19224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6408,"num_write_kb":410112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":311,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322786+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:09:57.322786+0000","last_peered":"2023-10-09T10:09:57.322786+0000","last_clean":"2023-10-09T10:09:57.322786+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:09:57.322786+0000","last_undegraded":"2023-10-09T10:09:57.322786+0000","last_fullsized":"2023-10-09T10:09:57.322786+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":201,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311189+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:09:57.311189+0000","last_peered":"2023-10-09T10:09:57.311189+0000","last_clean":"2023-10-09T10:09:57.311189+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:09:57.311189+0000","last_undegraded":"2023-10-09T10:09:57.311189+0000","last_fullsized":"2023-10-09T10:09:57.311189+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"291'6232","reported_seq":6261,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.640952+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:10:07.640952+0000","last_peered":"2023-10-09T10:10:07.640952+0000","last_clean":"2023-10-09T10:10:07.640952+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:10:07.640952+0000","last_undegraded":"2023-10-09T10:10:07.640952+0000","last_fullsized":"2023-10-09T10:10:07.640952+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6232,"log_dups_size":0,"ondisk_log_size":6232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408420352,"num_objects":6232,"num_object_clones":0,"num_object_copies":18696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6232,"num_write_kb":398848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":113,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.319974+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:09:57.319974+0000","last_peered":"2023-10-09T10:09:57.319974+0000","last_clean":"2023-10-09T10:09:57.319974+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:09:57.319974+0000","last_undegraded":"2023-10-09T10:09:57.319974+0000","last_fullsized":"2023-10-09T10:09:57.319974+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.324020+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:09:57.324020+0000","last_peered":"2023-10-09T10:09:57.324020+0000","last_clean":"2023-10-09T10:09:57.324020+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:09:57.324020+0000","last_undegraded":"2023-10-09T10:09:57.324020+0000","last_fullsized":"2023-10-09T10:09:57.324020+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"291'6229","reported_seq":6258,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:09.467087+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:10:09.467087+0000","last_peered":"2023-10-09T10:10:09.467087+0000","last_clean":"2023-10-09T10:10:09.467087+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:10:09.467087+0000","last_undegraded":"2023-10-09T10:10:09.467087+0000","last_fullsized":"2023-10-09T10:10:09.467087+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6229,"log_dups_size":0,"ondisk_log_size":6229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408223744,"num_objects":6229,"num_object_clones":0,"num_object_copies":18687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6229,"num_write_kb":398656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":124,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322285+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:09:57.322285+0000","last_peered":"2023-10-09T10:09:57.322285+0000","last_clean":"2023-10-09T10:09:57.322285+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:09:57.322285+0000","last_undegraded":"2023-10-09T10:09:57.322285+0000","last_fullsized":"2023-10-09T10:09:57.322285+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":157,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.316950+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:09:57.316950+0000","last_peered":"2023-10-09T10:09:57.316950+0000","last_clean":"2023-10-09T10:09:57.316950+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:09:57.316950+0000","last_undegraded":"2023-10-09T10:09:57.316950+0000","last_fullsized":"2023-10-09T10:09:57.316950+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"291'6482","reported_seq":6511,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:11.104769+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:10:11.104769+0000","last_peered":"2023-10-09T10:10:11.104769+0000","last_clean":"2023-10-09T10:10:11.104769+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:10:11.104769+0000","last_undegraded":"2023-10-09T10:10:11.104769+0000","last_fullsized":"2023-10-09T10:10:11.104769+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6482,"log_dups_size":0,"ondisk_log_size":6482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424804352,"num_objects":6482,"num_object_clones":0,"num_object_copies":19446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6482,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":284,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.309982+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:09:57.309982+0000","last_peered":"2023-10-09T10:09:57.309982+0000","last_clean":"2023-10-09T10:09:57.309982+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:09:57.309982+0000","last_undegraded":"2023-10-09T10:09:57.309982+0000","last_fullsized":"2023-10-09T10:09:57.309982+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":154,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.309356+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:09:57.309356+0000","last_peered":"2023-10-09T10:09:57.309356+0000","last_clean":"2023-10-09T10:09:57.309356+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:09:57.309356+0000","last_undegraded":"2023-10-09T10:09:57.309356+0000","last_fullsized":"2023-10-09T10:09:57.309356+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"291'6367","reported_seq":6396,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:09.490503+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:10:09.490503+0000","last_peered":"2023-10-09T10:10:09.490503+0000","last_clean":"2023-10-09T10:10:09.490503+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:10:09.490503+0000","last_undegraded":"2023-10-09T10:10:09.490503+0000","last_fullsized":"2023-10-09T10:10:09.490503+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6367,"log_dups_size":0,"ondisk_log_size":6367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417267712,"num_objects":6367,"num_object_clones":0,"num_object_copies":19101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6367,"num_write_kb":407488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":215,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311110+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:09:57.311110+0000","last_peered":"2023-10-09T10:09:57.311110+0000","last_clean":"2023-10-09T10:09:57.311110+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:09:57.311110+0000","last_undegraded":"2023-10-09T10:09:57.311110+0000","last_fullsized":"2023-10-09T10:09:57.311110+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":221,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311329+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:09:57.311329+0000","last_peered":"2023-10-09T10:09:57.311329+0000","last_clean":"2023-10-09T10:09:57.311329+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:09:57.311329+0000","last_undegraded":"2023-10-09T10:09:57.311329+0000","last_fullsized":"2023-10-09T10:09:57.311329+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"291'6121","reported_seq":6150,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.640937+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:10:07.640937+0000","last_peered":"2023-10-09T10:10:07.640937+0000","last_clean":"2023-10-09T10:10:07.640937+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:10:07.640937+0000","last_undegraded":"2023-10-09T10:10:07.640937+0000","last_fullsized":"2023-10-09T10:10:07.640937+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6121,"log_dups_size":0,"ondisk_log_size":6121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401145856,"num_objects":6121,"num_object_clones":0,"num_object_copies":18363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6121,"num_write_kb":391744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":185,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.321954+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:09:57.321954+0000","last_peered":"2023-10-09T10:09:57.321954+0000","last_clean":"2023-10-09T10:09:57.321954+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:09:57.321954+0000","last_undegraded":"2023-10-09T10:09:57.321954+0000","last_fullsized":"2023-10-09T10:09:57.321954+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":157,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311495+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:09:57.311495+0000","last_peered":"2023-10-09T10:09:57.311495+0000","last_clean":"2023-10-09T10:09:57.311495+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:09:57.311495+0000","last_undegraded":"2023-10-09T10:09:57.311495+0000","last_fullsized":"2023-10-09T10:09:57.311495+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"291'6289","reported_seq":6318,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:09.490128+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:10:09.490128+0000","last_peered":"2023-10-09T10:10:09.490128+0000","last_clean":"2023-10-09T10:10:09.490128+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:10:09.490128+0000","last_undegraded":"2023-10-09T10:10:09.490128+0000","last_fullsized":"2023-10-09T10:10:09.490128+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6289,"log_dups_size":0,"ondisk_log_size":6289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412155904,"num_objects":6289,"num_object_clones":0,"num_object_copies":18867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6289,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":244,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.309333+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:09:57.309333+0000","last_peered":"2023-10-09T10:09:57.309333+0000","last_clean":"2023-10-09T10:09:57.309333+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:09:57.309333+0000","last_undegraded":"2023-10-09T10:09:57.309333+0000","last_fullsized":"2023-10-09T10:09:57.309333+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":293,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.328141+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:09:57.328141+0000","last_peered":"2023-10-09T10:09:57.328141+0000","last_clean":"2023-10-09T10:09:57.328141+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:09:57.328141+0000","last_undegraded":"2023-10-09T10:09:57.328141+0000","last_fullsized":"2023-10-09T10:09:57.328141+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"291'6200","reported_seq":6229,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.654907+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:10:07.654907+0000","last_peered":"2023-10-09T10:10:07.654907+0000","last_clean":"2023-10-09T10:10:07.654907+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:10:07.654907+0000","last_undegraded":"2023-10-09T10:10:07.654907+0000","last_fullsized":"2023-10-09T10:10:07.654907+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406323200,"num_objects":6200,"num_object_clones":0,"num_object_copies":18600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6200,"num_write_kb":396800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":293,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311843+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:09:57.311843+0000","last_peered":"2023-10-09T10:09:57.311843+0000","last_clean":"2023-10-09T10:09:57.311843+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:09:57.311843+0000","last_undegraded":"2023-10-09T10:09:57.311843+0000","last_fullsized":"2023-10-09T10:09:57.311843+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":133,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.319884+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:09:57.319884+0000","last_peered":"2023-10-09T10:09:57.319884+0000","last_clean":"2023-10-09T10:09:57.319884+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:09:57.319884+0000","last_undegraded":"2023-10-09T10:09:57.319884+0000","last_fullsized":"2023-10-09T10:09:57.319884+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":249,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311728+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:09:57.311728+0000","last_peered":"2023-10-09T10:09:57.311728+0000","last_clean":"2023-10-09T10:09:57.311728+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:09:57.311728+0000","last_undegraded":"2023-10-09T10:09:57.311728+0000","last_fullsized":"2023-10-09T10:09:57.311728+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"291'6210","reported_seq":6238,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.840913+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:10:07.840913+0000","last_peered":"2023-10-09T10:10:07.840913+0000","last_clean":"2023-10-09T10:10:07.840913+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:10:07.840913+0000","last_undegraded":"2023-10-09T10:10:07.840913+0000","last_fullsized":"2023-10-09T10:10:07.840913+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6210,"log_dups_size":0,"ondisk_log_size":6210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406978560,"num_objects":6210,"num_object_clones":0,"num_object_copies":18630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6210,"num_write_kb":397440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":109,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.319622+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:09:57.319622+0000","last_peered":"2023-10-09T10:09:57.319622+0000","last_clean":"2023-10-09T10:09:57.319622+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:09:57.319622+0000","last_undegraded":"2023-10-09T10:09:57.319622+0000","last_fullsized":"2023-10-09T10:09:57.319622+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":106,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.316725+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:09:57.316725+0000","last_peered":"2023-10-09T10:09:57.316725+0000","last_clean":"2023-10-09T10:09:57.316725+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:09:57.316725+0000","last_undegraded":"2023-10-09T10:09:57.316725+0000","last_fullsized":"2023-10-09T10:09:57.316725+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":316,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311093+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:09:57.311093+0000","last_peered":"2023-10-09T10:09:57.311093+0000","last_clean":"2023-10-09T10:09:57.311093+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:09:57.311093+0000","last_undegraded":"2023-10-09T10:09:57.311093+0000","last_fullsized":"2023-10-09T10:09:57.311093+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"291'6307","reported_seq":6336,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:11.114648+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:10:11.114648+0000","last_peered":"2023-10-09T10:10:11.114648+0000","last_clean":"2023-10-09T10:10:11.114648+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:10:11.114648+0000","last_undegraded":"2023-10-09T10:10:11.114648+0000","last_fullsized":"2023-10-09T10:10:11.114648+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413335552,"num_objects":6307,"num_object_clones":0,"num_object_copies":18921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6307,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":210,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311343+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:09:57.311343+0000","last_peered":"2023-10-09T10:09:57.311343+0000","last_clean":"2023-10-09T10:09:57.311343+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:09:57.311343+0000","last_undegraded":"2023-10-09T10:09:57.311343+0000","last_fullsized":"2023-10-09T10:09:57.311343+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":285,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311305+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:09:57.311305+0000","last_peered":"2023-10-09T10:09:57.311305+0000","last_clean":"2023-10-09T10:09:57.311305+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:09:57.311305+0000","last_undegraded":"2023-10-09T10:09:57.311305+0000","last_fullsized":"2023-10-09T10:09:57.311305+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":306,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.312186+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:09:57.312186+0000","last_peered":"2023-10-09T10:09:57.312186+0000","last_clean":"2023-10-09T10:09:57.312186+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:09:57.312186+0000","last_undegraded":"2023-10-09T10:09:57.312186+0000","last_fullsized":"2023-10-09T10:09:57.312186+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"291'6165","reported_seq":6193,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.814538+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:10:07.814538+0000","last_peered":"2023-10-09T10:10:07.814538+0000","last_clean":"2023-10-09T10:10:07.814538+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:10:07.814538+0000","last_undegraded":"2023-10-09T10:10:07.814538+0000","last_fullsized":"2023-10-09T10:10:07.814538+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6165,"log_dups_size":0,"ondisk_log_size":6165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404029440,"num_objects":6165,"num_object_clones":0,"num_object_copies":18495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6165,"num_write_kb":394560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":118,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.320055+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:09:57.320055+0000","last_peered":"2023-10-09T10:09:57.320055+0000","last_clean":"2023-10-09T10:09:57.320055+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:09:57.320055+0000","last_undegraded":"2023-10-09T10:09:57.320055+0000","last_fullsized":"2023-10-09T10:09:57.320055+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":537,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322009+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:09:57.322009+0000","last_peered":"2023-10-09T10:09:57.322009+0000","last_clean":"2023-10-09T10:09:57.322009+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:09:57.322009+0000","last_undegraded":"2023-10-09T10:09:57.322009+0000","last_fullsized":"2023-10-09T10:09:57.322009+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":194,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.309688+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:09:57.309688+0000","last_peered":"2023-10-09T10:09:57.309688+0000","last_clean":"2023-10-09T10:09:57.309688+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:09:57.309688+0000","last_undegraded":"2023-10-09T10:09:57.309688+0000","last_fullsized":"2023-10-09T10:09:57.309688+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":206,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.323059+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:09:57.323059+0000","last_peered":"2023-10-09T10:09:57.323059+0000","last_clean":"2023-10-09T10:09:57.323059+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:09:57.323059+0000","last_undegraded":"2023-10-09T10:09:57.323059+0000","last_fullsized":"2023-10-09T10:09:57.323059+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"291'6312","reported_seq":6341,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:09.493918+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:10:09.493918+0000","last_peered":"2023-10-09T10:10:09.493918+0000","last_clean":"2023-10-09T10:10:09.493918+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:10:09.493918+0000","last_undegraded":"2023-10-09T10:10:09.493918+0000","last_fullsized":"2023-10-09T10:10:09.493918+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6312,"log_dups_size":0,"ondisk_log_size":6312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413663232,"num_objects":6312,"num_object_clones":0,"num_object_copies":18936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6312,"num_write_kb":403968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":265,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311464+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:09:57.311464+0000","last_peered":"2023-10-09T10:09:57.311464+0000","last_clean":"2023-10-09T10:09:57.311464+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:09:57.311464+0000","last_undegraded":"2023-10-09T10:09:57.311464+0000","last_fullsized":"2023-10-09T10:09:57.311464+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":167,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.316487+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:09:57.316487+0000","last_peered":"2023-10-09T10:09:57.316487+0000","last_clean":"2023-10-09T10:09:57.316487+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:09:57.316487+0000","last_undegraded":"2023-10-09T10:09:57.316487+0000","last_fullsized":"2023-10-09T10:09:57.316487+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":195,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.316892+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:09:57.316892+0000","last_peered":"2023-10-09T10:09:57.316892+0000","last_clean":"2023-10-09T10:09:57.316892+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:09:57.316892+0000","last_undegraded":"2023-10-09T10:09:57.316892+0000","last_fullsized":"2023-10-09T10:09:57.316892+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"291'6386","reported_seq":7319,"reported_epoch":291,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:11.081331+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:10:11.081331+0000","last_peered":"2023-10-09T10:10:11.081331+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:10:11.081331+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6386,"log_dups_size":0,"ondisk_log_size":6386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418512896,"num_objects":6386,"num_object_clones":0,"num_object_copies":19158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":4835,"num_objects_unfound":0,"num_objects_dirty":6386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6386,"num_write_kb":408704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":448,"num_bytes_recovered":29360128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1550},{"shards":"1,3,4","objects":4835}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":310,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322828+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:09:57.322828+0000","last_peered":"2023-10-09T10:09:57.322828+0000","last_clean":"2023-10-09T10:09:57.322828+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:09:57.322828+0000","last_undegraded":"2023-10-09T10:09:57.322828+0000","last_fullsized":"2023-10-09T10:09:57.322828+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":322,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311918+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:09:57.311918+0000","last_peered":"2023-10-09T10:09:57.311918+0000","last_clean":"2023-10-09T10:09:57.311918+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:09:57.311918+0000","last_undegraded":"2023-10-09T10:09:57.311918+0000","last_fullsized":"2023-10-09T10:09:57.311918+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":159,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.316809+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:09:57.316809+0000","last_peered":"2023-10-09T10:09:57.316809+0000","last_clean":"2023-10-09T10:09:57.316809+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:09:57.316809+0000","last_undegraded":"2023-10-09T10:09:57.316809+0000","last_fullsized":"2023-10-09T10:09:57.316809+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"291'6210","reported_seq":6239,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.688600+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:10:07.688600+0000","last_peered":"2023-10-09T10:10:07.688600+0000","last_clean":"2023-10-09T10:10:07.688600+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:10:07.688600+0000","last_undegraded":"2023-10-09T10:10:07.688600+0000","last_fullsized":"2023-10-09T10:10:07.688600+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6210,"log_dups_size":0,"ondisk_log_size":6210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406978560,"num_objects":6210,"num_object_clones":0,"num_object_copies":18630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6210,"num_write_kb":397440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":253,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.318899+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:09:57.318899+0000","last_peered":"2023-10-09T10:09:57.318899+0000","last_clean":"2023-10-09T10:09:57.318899+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:09:57.318899+0000","last_undegraded":"2023-10-09T10:09:57.318899+0000","last_fullsized":"2023-10-09T10:09:57.318899+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":267,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311572+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:09:57.311572+0000","last_peered":"2023-10-09T10:09:57.311572+0000","last_clean":"2023-10-09T10:09:57.311572+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:09:57.311572+0000","last_undegraded":"2023-10-09T10:09:57.311572+0000","last_fullsized":"2023-10-09T10:09:57.311572+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":242,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.309214+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:09:57.309214+0000","last_peered":"2023-10-09T10:09:57.309214+0000","last_clean":"2023-10-09T10:09:57.309214+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:09:57.309214+0000","last_undegraded":"2023-10-09T10:09:57.309214+0000","last_fullsized":"2023-10-09T10:09:57.309214+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"291'6133","reported_seq":6161,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.830453+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:10:07.830453+0000","last_peered":"2023-10-09T10:10:07.830453+0000","last_clean":"2023-10-09T10:10:07.830453+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:10:07.830453+0000","last_undegraded":"2023-10-09T10:10:07.830453+0000","last_fullsized":"2023-10-09T10:10:07.830453+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6133,"log_dups_size":0,"ondisk_log_size":6133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401932288,"num_objects":6133,"num_object_clones":0,"num_object_copies":18399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6133,"num_write_kb":392512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":199,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.326632+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:09:57.326632+0000","last_peered":"2023-10-09T10:09:57.326632+0000","last_clean":"2023-10-09T10:09:57.326632+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:09:57.326632+0000","last_undegraded":"2023-10-09T10:09:57.326632+0000","last_fullsized":"2023-10-09T10:09:57.326632+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":161,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.316816+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:09:57.316816+0000","last_peered":"2023-10-09T10:09:57.316816+0000","last_clean":"2023-10-09T10:09:57.316816+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:09:57.316816+0000","last_undegraded":"2023-10-09T10:09:57.316816+0000","last_fullsized":"2023-10-09T10:09:57.316816+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":144,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.319481+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:09:57.319481+0000","last_peered":"2023-10-09T10:09:57.319481+0000","last_clean":"2023-10-09T10:09:57.319481+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:09:57.319481+0000","last_undegraded":"2023-10-09T10:09:57.319481+0000","last_fullsized":"2023-10-09T10:09:57.319481+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":156,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:10:02.713895+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:10:02.713895+0000","last_peered":"2023-10-09T10:10:02.713895+0000","last_clean":"2023-10-09T10:10:02.713895+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:10:02.713895+0000","last_undegraded":"2023-10-09T10:10:02.713895+0000","last_fullsized":"2023-10-09T10:10:02.713895+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":228,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311988+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:09:57.311988+0000","last_peered":"2023-10-09T10:09:57.311988+0000","last_clean":"2023-10-09T10:09:57.311988+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:09:57.311988+0000","last_undegraded":"2023-10-09T10:09:57.311988+0000","last_fullsized":"2023-10-09T10:09:57.311988+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":128,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322209+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:09:57.322209+0000","last_peered":"2023-10-09T10:09:57.322209+0000","last_clean":"2023-10-09T10:09:57.322209+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:09:57.322209+0000","last_undegraded":"2023-10-09T10:09:57.322209+0000","last_fullsized":"2023-10-09T10:09:57.322209+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":169,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020775+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:10:07.020775+0000","last_peered":"2023-10-09T10:10:07.020775+0000","last_clean":"2023-10-09T10:10:07.020775+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:10:07.020775+0000","last_undegraded":"2023-10-09T10:10:07.020775+0000","last_fullsized":"2023-10-09T10:10:07.020775+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":128,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.319871+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:09:57.319871+0000","last_peered":"2023-10-09T10:09:57.319871+0000","last_clean":"2023-10-09T10:09:57.319871+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:09:57.319871+0000","last_undegraded":"2023-10-09T10:09:57.319871+0000","last_fullsized":"2023-10-09T10:09:57.319871+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":178,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021641+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:10:07.021641+0000","last_peered":"2023-10-09T10:10:07.021641+0000","last_clean":"2023-10-09T10:10:07.021641+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:10:07.021641+0000","last_undegraded":"2023-10-09T10:10:07.021641+0000","last_fullsized":"2023-10-09T10:10:07.021641+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":307,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.309333+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:09:57.309333+0000","last_peered":"2023-10-09T10:09:57.309333+0000","last_clean":"2023-10-09T10:09:57.309333+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:09:57.309333+0000","last_undegraded":"2023-10-09T10:09:57.309333+0000","last_fullsized":"2023-10-09T10:09:57.309333+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":328,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:10:06.140717+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:10:06.140717+0000","last_peered":"2023-10-09T10:10:06.140717+0000","last_clean":"2023-10-09T10:10:06.140717+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:10:06.140717+0000","last_undegraded":"2023-10-09T10:10:06.140717+0000","last_fullsized":"2023-10-09T10:10:06.140717+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":237,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.317101+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:09:57.317101+0000","last_peered":"2023-10-09T10:09:57.317101+0000","last_clean":"2023-10-09T10:09:57.317101+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:09:57.317101+0000","last_undegraded":"2023-10-09T10:09:57.317101+0000","last_fullsized":"2023-10-09T10:09:57.317101+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.310984+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:09:57.310984+0000","last_peered":"2023-10-09T10:09:57.310984+0000","last_clean":"2023-10-09T10:09:57.310984+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:09:57.310984+0000","last_undegraded":"2023-10-09T10:09:57.310984+0000","last_fullsized":"2023-10-09T10:09:57.310984+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6576144384,"num_objects":100344,"num_object_clones":0,"num_object_copies":301032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":4835,"num_objects_unfound":0,"num_objects_dirty":100344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":100344,"num_write_kb":6422016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":448,"num_bytes_recovered":29360128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19776536576,"data_stored":19776536576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100344,"ondisk_log_size":100344,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430486,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2315140,"kb_used_data":2009724,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91430012,"statfs":{"total":95995035648,"available":93624332288,"internally_reserved":0,"allocated":2057957376,"data_stored":2057425275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5940000000000001,"15min":1.5740000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":5.625,"5min":9.0690000000000008,"15min":50.591999999999999},"last":7.8419999999999996},{"interface":"front","average":{"1min":1.548,"5min":1.8,"15min":1.6040000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":6.0670000000000002,"5min":8.2300000000000004,"15min":50.747},"last":0.28799999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:10:08 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":1.8440000000000001,"15min":1.5169999999999999},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2669999999999995,"5min":16.352,"15min":25.338000000000001},"last":3.056},{"interface":"front","average":{"1min":1.53,"5min":1.8839999999999999,"15min":1.3560000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.9269999999999996,"5min":18.27,"15min":18.27},"last":4.9269999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.7190000000000001,"15min":2.266},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":3.8370000000000002,"5min":7.952,"15min":7.952},"last":3.8929999999999998},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.7310000000000001,"15min":2.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.2039999999999997,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.93799999999999994}]},{"osd":3,"last update":"Mon Oct 9 10:09:47 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.0049999999999999,"15min":1.835},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.058,"5min":29.058,"15min":29.058},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.214,"5min":1.7989999999999999,"15min":1.897},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.577,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":3.8220000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1130000000000004,"5min":4.1130000000000004,"15min":4.1130000000000004},"last":2.4889999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":2.2890000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.149,"15min":0.93899999999999995},"min":{"1min":0.31900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":25.277999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":2.5219999999999998},{"interface":"front","average":{"1min":2.4529999999999998,"5min":1.2589999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.754000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":2.1899999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.1299999999999999,"15min":0.88600000000000001},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5369999999999999,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.371},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.98599999999999999,"15min":0.82499999999999996},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.998,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":2.548}]}]},{"osd":6,"up_from":42,"seq":180388626714,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2664632,"kb_used_data":2359216,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91080520,"statfs":{"total":95995035648,"available":93266452480,"internally_reserved":0,"allocated":2415837184,"data_stored":2415318325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.9259999999999999,"15min":1.319},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":15.435,"5min":15.435,"15min":15.435},"last":9.2870000000000008},{"interface":"front","average":{"1min":1.853,"5min":1.931,"15min":1.365},"min":{"1min":0.252,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.308999999999999,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.442}]},{"osd":1,"last update":"Mon Oct 9 10:10:02 2023","interfaces":[{"interface":"back","average":{"1min":4.641,"5min":2.4300000000000002,"15min":1.7130000000000001},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":35.609999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.3540000000000001,"15min":1.6319999999999999},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":32.396000000000001,"5min":32.396000000000001,"15min":32.396000000000001},"last":3.7480000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.3340000000000001,"15min":1.6919999999999999},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8149999999999999,"5min":7.6760000000000002,"15min":7.6760000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.234,"5min":2.3239999999999998,"15min":3.0990000000000002},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.5600000000000005,"5min":10.26,"15min":10.26},"last":6.5640000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.321,"15min":1.347},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3879999999999999,"5min":6.0999999999999996,"15min":6.0999999999999996},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.4220000000000002,"15min":2.3940000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":11.211,"5min":11.211,"15min":11.211},"last":0.64400000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.48,"5min":1.48,"15min":1.48},"last":0.73299999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.71999999999999997,"15min":0.72199999999999998},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.611,"5min":7.7080000000000002,"15min":14.09},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.87,"15min":0.77300000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":2.464,"5min":7.508,"15min":7.508},"last":0.66800000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.77000000000000002,"15min":0.755},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.5800000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.81499999999999995,"15min":0.80500000000000005},"min":{"1min":0.39000000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.6080000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.75800000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253027,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3201552,"kb_used_data":2826056,"kb_used_omap":9,"kb_used_meta":375478,"kb_avail":90543600,"statfs":{"total":95995035648,"available":92716646400,"internally_reserved":0,"allocated":2893881344,"data_stored":2892866662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":384490473},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4139999999999999,"5min":3.2530000000000001,"15min":1.9239999999999999},"min":{"1min":0.19400000000000001,"5min":0.192,"15min":0.185},"max":{"1min":7.7699999999999996,"5min":25.59,"15min":25.59},"last":3.2930000000000001},{"interface":"front","average":{"1min":1.248,"5min":2.754,"15min":1.7549999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.192},"max":{"1min":11.423999999999999,"5min":16.629000000000001,"15min":16.629000000000001},"last":2.9239999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":3.2160000000000002,"15min":3.8700000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.1539999999999999,"5min":17.369,"15min":17.369},"last":4.0359999999999996},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.907,"15min":3.8740000000000001},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5960000000000001,"5min":16.023,"15min":16.023},"last":3.028}]},{"osd":3,"last update":"Mon Oct 9 10:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":3.1619999999999999,"15min":3.4100000000000001},"min":{"1min":0.36399999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.196999999999999,"5min":16.931000000000001,"15min":16.931000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.266,"15min":4.6900000000000004},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.8569999999999993,"5min":17.289000000000001,"15min":17.289000000000001},"last":3.1970000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3860000000000001,"5min":5.3860000000000001,"15min":5.3860000000000001},"last":3.0720000000000001},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":10.132,"5min":10.132,"15min":10.132},"last":0.45500000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.891,"15min":1.536},"min":{"1min":0.41499999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7719999999999998,"5min":14.416,"15min":15.531000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.3380000000000001,"5min":2.613,"15min":1.8939999999999999},"min":{"1min":0.308,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.2569999999999997,"5min":16.277999999999999,"15min":19.706},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":2.2309999999999999,"15min":1.6299999999999999},"min":{"1min":0.36799999999999999,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.8449999999999998,"5min":14.246,"15min":16.193999999999999},"last":3.1070000000000002},{"interface":"front","average":{"1min":0.86899999999999999,"5min":2.0499999999999998,"15min":1.6140000000000001},"min":{"1min":0.378,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":2.343,"5min":14.471,"15min":15.481999999999999},"last":0.48499999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":2.831,"15min":1.901},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.2149999999999999,"5min":16.331,"15min":16.331},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.585,"5min":2.3090000000000002,"15min":1.6930000000000001},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.8879999999999999,"5min":14.281000000000001,"15min":16.100999999999999},"last":3.1320000000000001}]}]},{"osd":0,"up_from":79,"seq":339302416553,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2719932,"kb_used_data":2371188,"kb_used_omap":28,"kb_used_meta":348707,"kb_avail":91025220,"statfs":{"total":95995035648,"available":93209825280,"internally_reserved":0,"allocated":2428096512,"data_stored":2427092234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":357076816},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.8580000000000001,"15min":1.3819999999999999},"min":{"1min":0.224,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":6.0510000000000002,"5min":13.084,"15min":13.084},"last":1.8129999999999999},{"interface":"front","average":{"1min":2.2109999999999999,"5min":3.4489999999999998,"15min":1.994},"min":{"1min":0.27200000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.516,"5min":22.178999999999998,"15min":22.178999999999998},"last":1.909}]},{"osd":2,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.7210000000000001,"15min":2.121},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.5579999999999998,"5min":10.034000000000001,"15min":10.034000000000001},"last":2.2650000000000001},{"interface":"front","average":{"1min":1.353,"5min":2.2879999999999998,"15min":3.0910000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.4480000000000004,"5min":14.693,"15min":14.693},"last":0.27600000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4990000000000001,"5min":4.391,"15min":4.7069999999999999},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.867000000000001,"5min":25.734999999999999,"15min":25.734999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.4809999999999999,"5min":4.0250000000000004,"15min":4.282},"min":{"1min":0.44,"5min":0.254,"15min":0.254},"max":{"1min":13.975,"5min":21.923999999999999,"15min":21.923999999999999},"last":0.88200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.3760000000000003,"5min":4.3760000000000003,"15min":4.3760000000000003},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":13.789999999999999,"5min":13.789999999999999,"15min":13.789999999999999},"last":0.626}]},{"osd":5,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":2.8149999999999999,"15min":1.847},"min":{"1min":0.38900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.736000000000001,"5min":17.096,"15min":17.096},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.419,"15min":1.2470000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.4870000000000001,"5min":12.478,"15min":12.478},"last":0.59599999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.494,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5289999999999999,"5min":9.7260000000000009,"15min":11.699},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.6549999999999998,"15min":1.954},"min":{"1min":0.45100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.4130000000000003,"5min":14.384,"15min":15.108000000000001},"last":0.38800000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.9929999999999999,"15min":1.986},"min":{"1min":0.51400000000000001,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":13.829000000000001,"5min":14.631,"15min":26.978000000000002},"last":0.69899999999999995},{"interface":"front","average":{"1min":1.085,"5min":1.6819999999999999,"15min":1.544},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.5209999999999999,"5min":12.428000000000001,"15min":26.768000000000001},"last":0.52800000000000002}]}]},{"osd":2,"up_from":217,"seq":932007903287,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1931188,"kb_used_data":1754476,"kb_used_omap":83,"kb_used_meta":176620,"kb_avail":91813964,"statfs":{"total":95995035648,"available":94017499136,"internally_reserved":0,"allocated":1796583424,"data_stored":1795594957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":180859375},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.28,"15min":1.6060000000000001},"min":{"1min":0.247,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.5139999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":19.372},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.8849999999999998,"15min":2.923},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.891999999999999,"5min":16.309999999999999,"15min":16.309999999999999},"last":6.1550000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.0209999999999999,"15min":2.4649999999999999},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.971,"5min":11.971,"15min":11.971},"last":6.819},{"interface":"front","average":{"1min":2.8849999999999998,"5min":2.492,"15min":2.6280000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.009,"5min":15.569000000000001,"15min":15.569000000000001},"last":6.4119999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.798,"15min":1.948},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.8859999999999999,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":6.7939999999999996},{"interface":"front","average":{"1min":1.1779999999999999,"5min":3.367,"15min":3.7320000000000002},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.8440000000000003,"5min":15.811,"15min":15.811},"last":6.1230000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.633,"5min":2.633,"15min":2.633},"last":6.29},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.8939999999999999,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":3.0089999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.5580000000000001,"15min":1.502},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.0870000000000002,"5min":12.423999999999999,"15min":12.423999999999999},"last":2.9500000000000002},{"interface":"front","average":{"1min":1.21,"5min":1.2789999999999999,"15min":1.6000000000000001},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.7709999999999999,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":6.7549999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2629999999999999,"15min":1.0489999999999999},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.6040000000000001,"5min":15.292999999999999,"15min":15.292999999999999},"last":6.2599999999999998},{"interface":"front","average":{"1min":1.032,"5min":1.6499999999999999,"15min":1.712},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.984,"5min":12.394,"15min":12.394},"last":2.9780000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.2809999999999999,"15min":1.0760000000000001},"min":{"1min":0.48499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1800000000000002,"5min":15.343999999999999,"15min":15.343999999999999},"last":6.2190000000000003},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.7170000000000001,"15min":1.613},"min":{"1min":0.51200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.5010000000000003,"5min":12.356999999999999,"15min":12.356999999999999},"last":2.9940000000000002}]}]},{"osd":3,"up_from":250,"seq":1073741824031,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2667044,"kb_used_data":2438940,"kb_used_omap":10,"kb_used_meta":228085,"kb_avail":91078108,"statfs":{"total":95995035648,"available":93263982592,"internally_reserved":0,"allocated":2497474560,"data_stored":2496324927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":233559107},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":4.2489999999999997,"15min":4.5519999999999996},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.465,"5min":23.381,"15min":23.381},"last":4.1500000000000004},{"interface":"front","average":{"1min":1.853,"5min":3.7130000000000001,"15min":4.0229999999999997},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.5190000000000001,"5min":14.289999999999999,"15min":14.289999999999999},"last":20.239999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5329999999999999,"15min":3.8130000000000002},"min":{"1min":0.28799999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.415,"5min":20.407,"15min":20.407},"last":20.422999999999998},{"interface":"front","average":{"1min":2.702,"5min":3.2850000000000001,"15min":3.3820000000000001},"min":{"1min":0.40300000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.494,"5min":23.423999999999999,"15min":23.423999999999999},"last":1.554}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":3.3410000000000002,"15min":3.6070000000000002},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.1020000000000003,"5min":20.356000000000002,"15min":20.356000000000002},"last":20.594999999999999},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.7290000000000001,"15min":2.819},"min":{"1min":0.34799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":9.3379999999999992,"5min":23.452000000000002,"15min":23.452000000000002},"last":2.7879999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":3.7570000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.5739999999999998,"5min":8.5739999999999998,"15min":8.5739999999999998},"last":1.1379999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.4319999999999999,"15min":2.5529999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.6199999999999992,"5min":11.836,"15min":11.836},"last":1.2270000000000001},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.274,"15min":2.327},"min":{"1min":0.55600000000000005,"5min":0.309,"15min":0.309},"max":{"1min":6.9249999999999998,"5min":14.164999999999999,"15min":14.164999999999999},"last":11.446}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.5379999999999998,"15min":2.6720000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.532,"5min":11.887,"15min":11.887},"last":1.202},{"interface":"front","average":{"1min":1.496,"5min":2.3820000000000001,"15min":2.5299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.8710000000000004,"5min":14.125,"15min":14.125},"last":11.500999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.4039999999999999,"15min":1.3939999999999999},"min":{"1min":0.45400000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0609999999999999,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.8089999999999999,"5min":2.5009999999999999,"15min":2.617},"min":{"1min":0.42199999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":8.6549999999999994,"5min":11.801,"15min":11.801},"last":1.175}]}]},{"osd":4,"up_from":266,"seq":1142461300756,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2652412,"kb_used_data":2325556,"kb_used_omap":35,"kb_used_meta":326812,"kb_avail":91092740,"statfs":{"total":95995035648,"available":93278965760,"internally_reserved":0,"allocated":2381369344,"data_stored":2380844821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":334655726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.3799999999999999,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":2.2130000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3959999999999999,"5min":2.3959999999999999,"15min":2.3959999999999999},"last":2.5899999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":2.7160000000000002,"5min":2.7160000000000002,"15min":2.7160000000000002},"last":2.2320000000000002},{"interface":"front","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":2.5129999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.661,"5min":3.661,"15min":3.661},"last":1.4890000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6449999999999996,"5min":4.6449999999999996,"15min":4.6449999999999996},"last":0.33200000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.262,"15min":1.262},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.1259999999999999,"5min":3.1259999999999999,"15min":3.1259999999999999},"last":2.157},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.4470000000000001,"5min":4.4470000000000001,"15min":4.4470000000000001},"last":2.9060000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":1.4530000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.6970000000000001,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.40699999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.5609999999999999,"5min":2.5609999999999999,"15min":2.5609999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.917,"5min":1.917,"15min":1.917},"last":1.284}]},{"osd":7,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.439},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.5630000000000002,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":1.5169999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822941,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3478444,"kb_used_data":3177124,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":90266708,"statfs":{"total":95995035648,"available":92433108992,"internally_reserved":0,"allocated":3253374976,"data_stored":3252862581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":1.5860000000000001,"15min":1.3460000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.3090000000000002,"5min":10.92,"15min":15.773999999999999},"last":5.6980000000000004},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.673,"15min":1.4790000000000001},"min":{"1min":0.25600000000000001,"5min":0.23899999999999999,"15min":0.23400000000000001},"max":{"1min":6.3090000000000002,"5min":11.747,"15min":15.699},"last":0.36099999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.3680000000000001,"15min":1.3300000000000001},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.2670000000000003,"5min":11.335000000000001,"15min":15.689},"last":3.335},{"interface":"front","average":{"1min":1.845,"5min":1.895,"15min":1.5269999999999999},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":19.402999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":3.1480000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.8480000000000001,"15min":2.4369999999999998},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.3819999999999997,"5min":15.640000000000001,"15min":15.640000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.302,"5min":1.8220000000000001,"15min":2.2949999999999999},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.3659999999999997,"5min":10.656000000000001,"15min":10.656000000000001},"last":1.0309999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.734,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5860000000000003,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":2.9820000000000002},{"interface":"front","average":{"1min":1.2,"5min":2.6549999999999998,"15min":2.8969999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":2.9710000000000001,"5min":17.34,"15min":17.34},"last":1.0649999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.0819999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":1.0900000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.995,"15min":0.90000000000000002},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.4159999999999999,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.752},{"interface":"front","average":{"1min":0.754,"5min":0.98999999999999999,"15min":0.81100000000000005},"min":{"1min":0.317,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.3490000000000002,"5min":18.600999999999999,"15min":18.600999999999999},"last":2.6920000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0589999999999999,"15min":0.88},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.282,"5min":7.4290000000000003,"15min":10.334},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.749,"15min":0.88600000000000001},"min":{"1min":0.35099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.3500000000000001,"5min":8.0210000000000008,"15min":12.356},"last":1.1639999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2439053312,"data_stored":2439053312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2894004224,"data_stored":2894004224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1808596992,"data_stored":1808596992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2501246976,"data_stored":2501246976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2387017728,"data_stored":2387017728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3258187776,"data_stored":3258187776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2418999296,"data_stored":2418999296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2069430272,"data_stored":2069430272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:10:13.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:10:13.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:13 smithi110 ceph-mon[136032]: pgmap v765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 5374/301104 objects degraded (1.785%); 4835/301104 objects misplaced (1.606%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:13.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/301104 objects degraded (1.785%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:13.585 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:13.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:13.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:13.662 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 82 16 103532 103516 78.8519 42.9375 0.0186083 0.0126762 2023-10-09T10:10:13.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:13.724 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:13.766 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:13.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:13.870 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:13.899 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:13.941 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:13.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:13.988 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:10:13.989 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:10:14.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:14.149 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:14.226 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:14.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:14.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:14.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:14 smithi012 ceph-mon[147926]: from='client.55669 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:14.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:14.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:14 smithi012 ceph-mon[152263]: from='client.55669 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:14.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:14.443 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:14 smithi110 ceph-mon[136032]: from='client.55669 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:14.614 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:14.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:14.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:14.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:10:14.666 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 83 16 104134 104118 78.3517 37.625 0.0549844 0.012757 2023-10-09T10:10:14.673 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:14.717 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:14.834 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:14.968 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:14.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:14.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:15.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:15.178 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:10:15.186 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:15.290 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:15 smithi012 ceph-mon[147926]: pgmap v766: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 5374/306243 objects degraded (1.755%); 4835/306243 objects misplaced (1.579%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:15.290 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1790375418' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:15.290 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:15.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:15 smithi012 ceph-mon[152263]: pgmap v766: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 5374/306243 objects degraded (1.755%); 4835/306243 objects misplaced (1.579%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:15.291 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1790375418' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:15.291 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:15.391 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:15.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:15.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:15 smithi110 ceph-mon[136032]: pgmap v766: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 5374/306243 objects degraded (1.755%); 4835/306243 objects misplaced (1.579%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1790375418' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:15.495 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:15.540 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:15.583 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:15.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:15.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 84 16 104813 104797 77.9243 42.4375 0.0160411 0.0128279 2023-10-09T10:10:15.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:15.730 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:15.804 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:15.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:15.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:15.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:15.991 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:16.142 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:16.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:16.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:16.151 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:16.173 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:16 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:10:16.173 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:16 smithi110 ceph-mon[136032]: failed to encode map e292 with expected crc 2023-10-09T10:10:16.173 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:16 smithi110 ceph-mon[136032]: failed to encode map e292 with expected crc 2023-10-09T10:10:16.174 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:16 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:16.174 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:16 smithi110 ceph-mon[136032]: osdmap e292: 8 total, 8 up, 8 in 2023-10-09T10:10:16.174 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:16 smithi110 ceph-mon[136032]: pgmap v768: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 5374/309279 objects degraded (1.738%); 4835/309279 objects misplaced (1.563%); 801 KiB/s, 12 objects/s recovering 2023-10-09T10:10:16.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:16.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[147926]: failed to encode map e292 with expected crc 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[147926]: failed to encode map e292 with expected crc 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[147926]: osdmap e292: 8 total, 8 up, 8 in 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[147926]: pgmap v768: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 5374/309279 objects degraded (1.738%); 4835/309279 objects misplaced (1.563%); 801 KiB/s, 12 objects/s recovering 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[152263]: failed to encode map e292 with expected crc 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[152263]: failed to encode map e292 with expected crc 2023-10-09T10:10:16.312 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:16.313 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[152263]: osdmap e292: 8 total, 8 up, 8 in 2023-10-09T10:10:16.313 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:16 smithi012 ceph-mon[152263]: pgmap v768: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 5374/309279 objects degraded (1.738%); 4835/309279 objects misplaced (1.563%); 801 KiB/s, 12 objects/s recovering 2023-10-09T10:10:16.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:10:16.340 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:16.414 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:16.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:16.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:16.550 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:16.593 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:16.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:16.669 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 85 16 105483 105467 77.4976 41.875 0.0217293 0.0128976 2023-10-09T10:10:16.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:16.798 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:16.821 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:10:16.868 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:10:16.890 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:16.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:16.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:16.914 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:10:16.960 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:10:16.992 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:10:17.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:17.030 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:10:17.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:10:17.112 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:10:17.114 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:10:17.115 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:10:17.115 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:10:17.115 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:10:17.115 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:10:17.115 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:10:17.115 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:10:17.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:10:17.149 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:17.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:17 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1619207329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:17.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:17 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1619207329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:17.442 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:17.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:17 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1619207329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:17.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:17.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:17.552 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:17.586 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:17.639 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:17.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:17.669 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 86 16 106133 106117 77.0691 40.625 0.0216115 0.0129703 2023-10-09T10:10:17.745 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:17.762 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:17.784 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:17.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:17.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:17.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:17.934 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:17.969 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:10:17.970 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:18.020 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:18.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:18.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:10:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:10:17] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T10:10:18.241 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:18.337 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:18.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:18.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:18.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:18.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:18 smithi110 ceph-mon[136032]: pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 5374/311250 objects degraded (1.727%); 4729/311250 objects misplaced (1.519%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:18.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/309279 objects degraded (1.738%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:18.474 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:18.477 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:10:18.477 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:10:18.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:10:18.477 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:10:18.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.477 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191677 6.19 6:9839b5ab:::benchmark_data_smithi012_182576_object86156:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e165)", 2023-10-09T10:10:18.477 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:10:18.477 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.30240256100001, 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5138246070000001, 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191677 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167168+0000", 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167175+0000", 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4989999999999999e-06 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167184+0000", 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7329999999999998e-06 2023-10-09T10:10:18.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167193+0000", 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1599999999999998e-06 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174376+0000", 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071834530000000002 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174387+0000", 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1457999999999999e-05 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.658706+0000", 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4843186290000001 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.658834+0000", 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012832699999999999 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680928+0000", 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022093278000000001 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680994+0000", 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6519000000000005e-05 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191684 6.19 6:9f873d67:::benchmark_data_smithi012_182576_object86163:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e165)", 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.302373488, 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5032064950000001, 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.484 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191684 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167198+0000", 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167201+0000", 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3e-06 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167206+0000", 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8479999999999997e-06 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167212+0000", 2023-10-09T10:10:18.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1420000000000001e-06 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174497+0000", 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072854699999999996 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174507+0000", 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0071000000000001e-05 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659145+0000", 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4846380749999999 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659343+0000", 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019807000000000001 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659860+0000", 2023-10-09T10:10:18.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051712700000000004 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670116+0000", 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010255966 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670223+0000", 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010616 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670251+0000", 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8269999999999999e-05 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670302+0000", 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1674e-05 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670405+0000", 2023-10-09T10:10:18.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102691 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191706 6.19 6:9a3f0857:::benchmark_data_smithi012_182576_object86185:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.28220238400002, 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.483309049, 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.492 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191706 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187368+0000", 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187373+0000", 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.087e-06 2023-10-09T10:10:18.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187381+0000", 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6139999999999992e-06 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187388+0000", 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9029999999999998e-06 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191191+0000", 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038029449999999998 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191201+0000", 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0463e-05 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.662490+0000", 2023-10-09T10:10:18.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.471288079 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.662641+0000", 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015126899999999999 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.663089+0000", 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00044850599999999999 2023-10-09T10:10:18.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670456+0000", 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007366956 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670469+0000", 2023-10-09T10:10:18.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2931e-05 2023-10-09T10:10:18.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670520+0000", 2023-10-09T10:10:18.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1273000000000002e-05 2023-10-09T10:10:18.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670579+0000", 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8789999999999998e-05 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670679+0000", 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9545999999999999e-05 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.503 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191710 6.19 6:9b680818:::benchmark_data_smithi012_182576_object86189:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.28217891999998, 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4835924069999999, 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191710 2023-10-09T10:10:18.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187395+0000", 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.238e-06 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187400+0000", 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.074e-06 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187405+0000", 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5569999999999997e-06 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191222+0000", 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038174189999999998 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191229+0000", 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.665559+0000", 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4743292960000001 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.665709+0000", 2023-10-09T10:10:18.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015078499999999999 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.666141+0000", 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043125999999999999 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670478+0000", 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043378319999999998 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670789+0000", 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031025900000000001 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670831+0000", 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2349999999999999e-05 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670880+0000", 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9397999999999997e-05 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670986+0000", 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105147 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191721 6.19 6:98cdd829:::benchmark_data_smithi012_182576_object86200:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:10:18.511 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.264147106, 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4866997529999999, 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191721 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205423+0000", 2023-10-09T10:10:18.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205432+0000", 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1829999999999994e-06 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205449+0000", 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7813e-05 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205460+0000", 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0966e-05 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205526+0000", 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5363000000000004e-05 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205537+0000", 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.148e-05 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671019+0000", 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.465481982 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671151+0000", 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013229600000000001 2023-10-09T10:10:18.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671734+0000", 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058208800000000003 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680757+0000", 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090230250000000005 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681028+0000", 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027170299999999998 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.691971+0000", 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010942708000000001 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692026+0000", 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5075999999999999e-05 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692125+0000", 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8728999999999996e-05 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191734 6.19 6:9ad5bbea:::benchmark_data_smithi012_182576_object86213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.253959227, 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.476698235, 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.520 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191734 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215612+0000", 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215617+0000", 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1109999999999999e-06 2023-10-09T10:10:18.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215630+0000", 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.347e-05 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215639+0000", 2023-10-09T10:10:18.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2190000000000004e-06 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215696+0000", 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7519000000000003e-05 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215707+0000", 2023-10-09T10:10:18.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1391e-05 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678298+0000", 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.462590391 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678424+0000", 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012647799999999999 2023-10-09T10:10:18.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678879+0000", 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045511999999999999 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689119+0000", 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010239244999999999 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692138+0000", 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030192180000000002 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692181+0000", 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2972999999999997e-05 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692224+0000", 2023-10-09T10:10:18.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3328999999999998e-05 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692311+0000", 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6976000000000002e-05 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191762 6.19 6:98c0cf1b:::benchmark_data_smithi012_182576_object86241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.221823282, 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.470776208, 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191762 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.529 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:10:18.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247747+0000", 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247756+0000", 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.003e-06 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247771+0000", 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4816e-05 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247779+0000", 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.551e-06 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247839+0000", 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9710999999999998e-05 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247851+0000", 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.221e-05 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681049+0000", 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4331976559999999 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681140+0000", 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1250999999999998e-05 2023-10-09T10:10:18.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681494+0000", 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035398299999999999 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689149+0000", 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076551509999999998 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692143+0000", 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029938090000000001 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718366+0000", 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026223026999999999 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718421+0000", 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4262999999999999e-05 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718525+0000", 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104549 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.537 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191773 6.19 6:9b9ae800:::benchmark_data_smithi012_182576_object86252:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.21022276899998, 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.45934027, 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.538 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191773 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:10:18.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259348+0000", 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259355+0000", 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8310000000000003e-06 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259368+0000", 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3388e-05 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259378+0000", 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4809999999999995e-06 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.262058+0000", 2023-10-09T10:10:18.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026803130000000001 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.262070+0000", 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1612e-05 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.683593+0000", 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.421522669 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.683700+0000", 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107054 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.684074+0000", 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037416800000000003 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689177+0000", 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051031940000000001 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692342+0000", 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003165026 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718562+0000", 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026219606999999999 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718600+0000", 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8834999999999997e-05 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718690+0000", 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9263000000000002e-05 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191842 6.19 6:9f5df5a7:::benchmark_data_smithi012_182576_object86321:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.12462831300002, 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.373904045, 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191842 2023-10-09T10:10:18.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344942+0000", 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344950+0000", 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2900000000000002e-06 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344964+0000", 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3373000000000001e-05 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344973+0000", 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8200000000000003e-06 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.345026+0000", 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3804999999999997e-05 2023-10-09T10:10:18.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.345038+0000", 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1443999999999999e-05 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686361+0000", 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3413228020000001 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686486+0000", 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012562000000000001 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686879+0000", 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039255099999999999 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689204+0000", 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002325503 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692347+0000", 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031431340000000001 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718723+0000", 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026375604 2023-10-09T10:10:18.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718760+0000", 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7398999999999999e-05 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718848+0000", 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7406999999999998e-05 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191919 6.19 6:9f034d99:::benchmark_data_smithi012_182576_object86398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 462.02006746299998, 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2695003030000001, 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191919 2023-10-09T10:10:18.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449503+0000", 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449510+0000", 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.08e-06 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449527+0000", 2023-10-09T10:10:18.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6714999999999998e-05 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449537+0000", 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5950000000000005e-06 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449603+0000", 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5975000000000004e-05 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449617+0000", 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4163e-05 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689226+0000", 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.239609637 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689349+0000", 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012278500000000001 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689741+0000", 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039135500000000002 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.697237+0000", 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074966369999999996 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.697277+0000", 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9959000000000001e-05 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718881+0000", 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021603832999999999 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718918+0000", 2023-10-09T10:10:18.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7262999999999999e-05 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.719005+0000", 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6716000000000001e-05 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195708 6.a 6:5cc4cade:::benchmark_data_smithi012_182576_object90187:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:10:18.561 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 456.19817741000003, 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.548119284, 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195708 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271393+0000", 2023-10-09T10:10:18.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271398+0000", 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9450000000000001e-06 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271408+0000", 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0183000000000001e-05 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271416+0000", 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7559999999999999e-06 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271479+0000", 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2669000000000006e-05 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271486+0000", 2023-10-09T10:10:18.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5900000000000002e-06 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717349+0000", 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.44586224400000002 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717362+0000", 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3382e-05 2023-10-09T10:10:18.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.803595+0000", 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0862334440000001 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.806135+0000", 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025396630000000002 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.806223+0000", 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7700000000000004e-05 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.813674+0000", 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074509019999999997 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819055+0000", 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053811429999999997 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819227+0000", 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017252200000000001 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819303+0000", 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5587000000000007e-05 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819381+0000", 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7582000000000004e-05 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819514+0000", 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013344400000000001 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195730 6.a 6:51d763b1:::benchmark_data_smithi012_182576_object90209:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 456.140208105, 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4903754600000001, 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195730 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329363+0000", 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329371+0000", 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1990000000000006e-06 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329394+0000", 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2909999999999999e-05 2023-10-09T10:10:18.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329404+0000", 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9809999999999998e-06 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329466+0000", 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2092000000000001e-05 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329479+0000", 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.359e-05 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717394+0000", 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38791446600000001 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717405+0000", 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1208e-05 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809765+0000", 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0923597 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809844+0000", 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8974999999999995e-05 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809918+0000", 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4696000000000004e-05 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819132+0000", 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092136140000000002 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819148+0000", 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6092000000000001e-05 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819267+0000", 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119096 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819562+0000", 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029480100000000003 2023-10-09T10:10:18.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819615+0000", 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2502e-05 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819740+0000", 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012496799999999999 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195764 6.a 6:515d6e24:::benchmark_data_smithi012_182576_object90243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 456.06981941599997, 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.426156247, 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:18.580 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195764 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399751+0000", 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399758+0000", 2023-10-09T10:10:18.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3080000000000002e-06 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399771+0000", 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2680000000000001e-05 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399779+0000", 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1049999999999995e-06 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399843+0000", 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3441999999999993e-05 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399856+0000", 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3008e-05 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717436+0000", 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.317580268 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717445+0000", 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7660000000000004e-06 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814690+0000", 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0972454300000001 2023-10-09T10:10:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814827+0000", 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013737299999999999 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814985+0000", 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015749700000000001 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819155+0000", 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041702220000000003 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.823975+0000", 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048196610000000003 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825698+0000", 2023-10-09T10:10:18.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017233140000000001 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825737+0000", 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8433000000000003e-05 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825794+0000", 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7871000000000002e-05 2023-10-09T10:10:18.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825909+0000", 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114583 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154067 7.e 7:729cdabe:::benchmark_data_smithi012_221426_object46289:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:10:18.589 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 255.63642641000001, 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.711856684, 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154067 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833144+0000", 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:18.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833150+0000", 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0079999999999999e-06 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833167+0000", 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6198999999999999e-05 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833176+0000", 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4490000000000005e-06 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833926+0000", 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00074946799999999999 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:10:18.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833934+0000", 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1589999999999993e-06 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841253+0000", 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073188610000000003 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841264+0000", 2023-10-09T10:10:18.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1069e-05 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497295+0000", 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6560310870000001 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497435+0000", 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013998400000000001 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.544884+0000", 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.047449005000000002 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.545002+0000", 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118678 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154074 7.e 7:7a8c6f84:::benchmark_data_smithi012_221426_object46296:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 255.63569273499999, 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7164543729999999, 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154074 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.597 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:10:18.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:10:18.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833884+0000", 2023-10-09T10:10:18.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1059999999999999e-06 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833895+0000", 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1639e-05 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833906+0000", 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1104000000000001e-05 2023-10-09T10:10:18.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833999+0000", 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2387999999999997e-05 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841286+0000", 2023-10-09T10:10:18.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072812739999999999 2023-10-09T10:10:18.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841293+0000", 2023-10-09T10:10:18.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7290000000000003e-06 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500331+0000", 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.659038185 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500464+0000", 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013265799999999999 2023-10-09T10:10:18.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550212+0000", 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049747711999999999 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550334+0000", 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012227200000000001 2023-10-09T10:10:18.604 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154080 7.5 7:a9f132ab:::benchmark_data_smithi012_221426_object46302:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 255.635656275, 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5768529529999999, 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.605 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154080 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833915+0000", 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833918+0000", 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6740000000000001e-06 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833924+0000", 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2179999999999996e-06 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833929+0000", 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2009999999999998e-06 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834017+0000", 2023-10-09T10:10:18.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7529999999999997e-05 2023-10-09T10:10:18.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:10:18.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834024+0000", 2023-10-09T10:10:18.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2350000000000002e-06 2023-10-09T10:10:18.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837176+0000", 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031517139999999999 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837194+0000", 2023-10-09T10:10:18.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7634000000000001e-05 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395551+0000", 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5583572540000001 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395681+0000", 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130059 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:10:18.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395763+0000", 2023-10-09T10:10:18.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1928999999999994e-05 2023-10-09T10:10:18.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400135+0000", 2023-10-09T10:10:18.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043726360000000001 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409675+0000", 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095396509999999997 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409717+0000", 2023-10-09T10:10:18.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2036000000000002e-05 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410592+0000", 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00087471800000000002 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410648+0000", 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6589000000000003e-05 2023-10-09T10:10:18.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410769+0000", 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120383 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154083 7.e 7:757b33cc:::benchmark_data_smithi012_221426_object46305:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:10:18.617 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 255.635628336, 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7033852390000002, 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154083 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833943+0000", 2023-10-09T10:10:18.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833946+0000", 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0000000000000001e-06 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833952+0000", 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9000000000000003e-06 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833957+0000", 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9629999999999997e-06 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834051+0000", 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4141999999999994e-05 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:18.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834060+0000", 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8079999999999994e-06 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841319+0000", 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072592009999999998 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841327+0000", 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.571e-06 2023-10-09T10:10:18.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500500+0000", 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6591724889999999 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500595+0000", 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5142000000000005e-05 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.501295+0000", 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00069987000000000001 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510695+0000", 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094008289999999994 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520701+0000", 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010005623 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534431+0000", 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013730086000000001 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534503+0000", 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2304000000000004e-05 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537329+0000", 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028256510000000002 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.626 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154085 7.5 7:a7e74dfa:::benchmark_data_smithi012_221426_object46307:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 255.635610434, 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5985369870000001, 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154085 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833961+0000", 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833964+0000", 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.294e-06 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833970+0000", 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8540000000000002e-06 2023-10-09T10:10:18.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833975+0000", 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9710000000000003e-06 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834061+0000", 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6417999999999998e-05 2023-10-09T10:10:18.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834067+0000", 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.421e-06 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837215+0000", 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031475790000000002 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837222+0000", 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4370000000000001e-06 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400164+0000", 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5629411879999999 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400262+0000", 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8307000000000006e-05 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400392+0000", 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013009100000000001 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409725+0000", 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093329940000000007 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409759+0000", 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4034e-05 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.425006+0000", 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015246994 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432361+0000", 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073547029999999998 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.635 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432413+0000", 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2305999999999997e-05 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432499+0000", 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5711000000000007e-05 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154090 7.5 7:a5e0254e:::benchmark_data_smithi012_221426_object46312:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 255.63559117899999, 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.598732759, 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154090 2023-10-09T10:10:18.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833983+0000", 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.26e-06 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833988+0000", 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4190000000000001e-06 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833993+0000", 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5419999999999999e-06 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.640 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834104+0000", 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011161499999999999 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834111+0000", 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0650000000000004e-06 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837242+0000", 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031316959999999998 2023-10-09T10:10:18.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837250+0000", 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1799999999999996e-06 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.404888+0000", 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5676373880000001 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.405039+0000", 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151543 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.405181+0000", 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014114600000000001 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409999+0000", 2023-10-09T10:10:18.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048189210000000003 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410086+0000", 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6949999999999999e-05 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.425038+0000", 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014951341999999999 2023-10-09T10:10:18.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432561+0000", 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0075235340000000001 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432611+0000", 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9985000000000003e-05 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432714+0000", 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102497 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154097 7.e 7:7fa3f1e3:::benchmark_data_smithi012_221426_object46319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:10:18.646 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 255.63557409500001, 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.706036967, 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154097 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:18.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:10:18.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834000+0000", 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.384e-06 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2499999999999997e-06 2023-10-09T10:10:18.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834010+0000", 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8359999999999999e-06 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834090+0000", 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9957999999999996e-05 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834096+0000", 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3350000000000003e-06 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841347+0000", 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072508479999999998 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841354+0000", 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7159999999999997e-06 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510724+0000", 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.669369753 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:18.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510903+0000", 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017934200000000001 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.511077+0000", 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017371899999999999 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520762+0000", 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0096847530000000008 2023-10-09T10:10:18.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.525146+0000", 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043842450000000002 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537388+0000", 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012242494 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537446+0000", 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7565999999999997e-05 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.540035+0000", 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025890919999999999 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:18.656 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:18.656 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:10:18.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:18.656 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:18 smithi012 ceph-mon[152263]: pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 5374/311250 objects degraded (1.727%); 4729/311250 objects misplaced (1.519%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:18.656 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/309279 objects degraded (1.738%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:18.657 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:18 smithi012 ceph-mon[147926]: pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 5374/311250 objects degraded (1.727%); 4729/311250 objects misplaced (1.519%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:18.657 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/309279 objects degraded (1.738%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:18.657 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:18.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:18.669 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 87 16 106754 106738 76.6295 38.8125 0.00887159 0.0130431 2023-10-09T10:10:18.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:18.854 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:19.193 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:19.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:19.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:19.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:10:19.312 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:19.446 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:19.497 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:19.534 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:19.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:19.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:19.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:19.669 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 88 16 107448 107432 76.2518 43.375 0.0201106 0.0131097 2023-10-09T10:10:19.684 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:19.726 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:19.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:19.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:19.830 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:19.871 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:19.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:10:19.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:19.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:20.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:20.158 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:20.331 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:20.335 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:20.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:20.351 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:10:20.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:20 smithi012 ceph-mon[152263]: pgmap v770: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 5374/316158 objects degraded (1.700%); 4729/316158 objects misplaced (1.496%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:20.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:20 smithi012 ceph-mon[147926]: pgmap v770: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 5374/316158 objects degraded (1.700%); 4729/316158 objects misplaced (1.496%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:20.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:20.581 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:20.669 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 89 16 108047 108031 75.8159 37.4375 0.0121001 0.013185 2023-10-09T10:10:20.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:20 smithi110 ceph-mon[136032]: pgmap v770: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 5374/316158 objects degraded (1.700%); 4729/316158 objects misplaced (1.496%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:21.040 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:21.042 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:21.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:21.114 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":770,"stamp":"2023-10-09T10:10:19.318254+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6905599904,"num_objects":105386,"num_object_clones":0,"num_object_copies":316158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":4729,"num_objects_unfound":0,"num_objects_dirty":105386,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":105622,"num_write_kb":6745613,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":623,"num_bytes_recovered":37497548,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105556,"ondisk_log_size":105556,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22575476,"kb_used_data":20168884,"kb_used_omap":167,"kb_used_meta":2406360,"kb_avail":727385740,"statfs":{"total":767960285184,"available":744842997760,"internally_reserved":0,"allocated":20652937216,"data_stored":20646687323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2464113241},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,0,2,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":439615488,"num_objects":6708,"num_object_clones":0,"num_object_copies":20124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-210,"num_objects_unfound":0,"num_objects_dirty":6708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6708,"num_write_kb":429312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":210,"num_bytes_recovered":13762560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004710"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":132,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311647+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:09:57.311647+0000","last_peered":"2023-10-09T10:09:57.311647+0000","last_clean":"2023-10-09T10:09:57.311647+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:09:57.311647+0000","last_undegraded":"2023-10-09T10:09:57.311647+0000","last_fullsized":"2023-10-09T10:09:57.311647+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":249,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:11.147275+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:10:11.147275+0000","last_peered":"2023-10-09T10:10:11.147275+0000","last_clean":"2023-10-09T10:10:11.147275+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:10:11.147275+0000","last_undegraded":"2023-10-09T10:10:11.147275+0000","last_fullsized":"2023-10-09T10:10:11.147275+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":293,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311259+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:09:57.311259+0000","last_peered":"2023-10-09T10:09:57.311259+0000","last_clean":"2023-10-09T10:09:57.311259+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:09:57.311259+0000","last_undegraded":"2023-10-09T10:09:57.311259+0000","last_fullsized":"2023-10-09T10:09:57.311259+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":157,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020195+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:10:07.020195+0000","last_peered":"2023-10-09T10:10:07.020195+0000","last_clean":"2023-10-09T10:10:07.020195+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:10:07.020195+0000","last_undegraded":"2023-10-09T10:10:07.020195+0000","last_fullsized":"2023-10-09T10:10:07.020195+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":263,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.024720+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:10:07.024720+0000","last_peered":"2023-10-09T10:10:07.024720+0000","last_clean":"2023-10-09T10:10:07.024720+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:10:07.024720+0000","last_undegraded":"2023-10-09T10:10:07.024720+0000","last_fullsized":"2023-10-09T10:10:07.024720+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021575+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:10:07.021575+0000","last_peered":"2023-10-09T10:10:07.021575+0000","last_clean":"2023-10-09T10:10:07.021575+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:10:07.021575+0000","last_undegraded":"2023-10-09T10:10:07.021575+0000","last_fullsized":"2023-10-09T10:10:07.021575+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":175,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.019599+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:10:07.019599+0000","last_peered":"2023-10-09T10:10:07.019599+0000","last_clean":"2023-10-09T10:10:07.019599+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:10:07.019599+0000","last_undegraded":"2023-10-09T10:10:07.019599+0000","last_fullsized":"2023-10-09T10:10:07.019599+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":244,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.024667+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:10:07.024667+0000","last_peered":"2023-10-09T10:10:07.024667+0000","last_clean":"2023-10-09T10:10:07.024667+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:10:07.024667+0000","last_undegraded":"2023-10-09T10:10:07.024667+0000","last_fullsized":"2023-10-09T10:10:07.024667+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":109,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021321+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:10:07.021321+0000","last_peered":"2023-10-09T10:10:07.021321+0000","last_clean":"2023-10-09T10:10:07.021321+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:10:07.021321+0000","last_undegraded":"2023-10-09T10:10:07.021321+0000","last_fullsized":"2023-10-09T10:10:07.021321+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":205,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020902+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:10:07.020902+0000","last_peered":"2023-10-09T10:10:07.020902+0000","last_clean":"2023-10-09T10:10:07.020902+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:10:07.020902+0000","last_undegraded":"2023-10-09T10:10:07.020902+0000","last_fullsized":"2023-10-09T10:10:07.020902+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":329,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322981+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:09:57.322981+0000","last_peered":"2023-10-09T10:09:57.322981+0000","last_clean":"2023-10-09T10:09:57.322981+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:09:57.322981+0000","last_undegraded":"2023-10-09T10:09:57.322981+0000","last_fullsized":"2023-10-09T10:09:57.322981+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":285,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311199+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:09:57.311199+0000","last_peered":"2023-10-09T10:09:57.311199+0000","last_clean":"2023-10-09T10:09:57.311199+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:09:57.311199+0000","last_undegraded":"2023-10-09T10:09:57.311199+0000","last_fullsized":"2023-10-09T10:09:57.311199+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":178,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.323257+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:09:57.323257+0000","last_peered":"2023-10-09T10:09:57.323257+0000","last_clean":"2023-10-09T10:09:57.323257+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:09:57.323257+0000","last_undegraded":"2023-10-09T10:09:57.323257+0000","last_fullsized":"2023-10-09T10:09:57.323257+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":157,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311540+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:09:57.311540+0000","last_peered":"2023-10-09T10:09:57.311540+0000","last_clean":"2023-10-09T10:09:57.311540+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:09:57.311540+0000","last_undegraded":"2023-10-09T10:09:57.311540+0000","last_fullsized":"2023-10-09T10:09:57.311540+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":362,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322126+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:09:57.322126+0000","last_peered":"2023-10-09T10:09:57.322126+0000","last_clean":"2023-10-09T10:09:57.322126+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:09:57.322126+0000","last_undegraded":"2023-10-09T10:09:57.322126+0000","last_fullsized":"2023-10-09T10:09:57.322126+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":231,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.024231+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:10:07.024231+0000","last_peered":"2023-10-09T10:10:07.024231+0000","last_clean":"2023-10-09T10:10:07.024231+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:10:07.024231+0000","last_undegraded":"2023-10-09T10:10:07.024231+0000","last_fullsized":"2023-10-09T10:10:07.024231+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":157,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.019765+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:10:07.019765+0000","last_peered":"2023-10-09T10:10:07.019765+0000","last_clean":"2023-10-09T10:10:07.019765+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:10:07.019765+0000","last_undegraded":"2023-10-09T10:10:07.019765+0000","last_fullsized":"2023-10-09T10:10:07.019765+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":125,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021447+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:10:07.021447+0000","last_peered":"2023-10-09T10:10:07.021447+0000","last_clean":"2023-10-09T10:10:07.021447+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:10:07.021447+0000","last_undegraded":"2023-10-09T10:10:07.021447+0000","last_fullsized":"2023-10-09T10:10:07.021447+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":159,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322008+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:09:57.322008+0000","last_peered":"2023-10-09T10:09:57.322008+0000","last_clean":"2023-10-09T10:09:57.322008+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:09:57.322008+0000","last_undegraded":"2023-10-09T10:09:57.322008+0000","last_fullsized":"2023-10-09T10:09:57.322008+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":157,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020282+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:10:07.020282+0000","last_peered":"2023-10-09T10:10:07.020282+0000","last_clean":"2023-10-09T10:10:07.020282+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:10:07.020282+0000","last_undegraded":"2023-10-09T10:10:07.020282+0000","last_fullsized":"2023-10-09T10:10:07.020282+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":180,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.018960+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:10:07.018960+0000","last_peered":"2023-10-09T10:10:07.018960+0000","last_clean":"2023-10-09T10:10:07.018960+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:10:07.018960+0000","last_undegraded":"2023-10-09T10:10:07.018960+0000","last_fullsized":"2023-10-09T10:10:07.018960+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":244,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.026966+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:10:07.026966+0000","last_peered":"2023-10-09T10:10:07.026966+0000","last_clean":"2023-10-09T10:10:07.026966+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:10:07.026966+0000","last_undegraded":"2023-10-09T10:10:07.026966+0000","last_fullsized":"2023-10-09T10:10:07.026966+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"292'6691","reported_seq":6728,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:17.835886+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:10:17.835886+0000","last_peered":"2023-10-09T10:10:17.835886+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:10:17.835886+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438501376,"num_objects":6691,"num_object_clones":0,"num_object_copies":20073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6691,"num_write_kb":428224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":1318},{"shards":"4,5,6","objects":5373}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":323,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322885+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:09:57.322885+0000","last_peered":"2023-10-09T10:09:57.322885+0000","last_clean":"2023-10-09T10:09:57.322885+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:09:57.322885+0000","last_undegraded":"2023-10-09T10:09:57.322885+0000","last_fullsized":"2023-10-09T10:09:57.322885+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":110,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020962+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:10:07.020962+0000","last_peered":"2023-10-09T10:10:07.020962+0000","last_clean":"2023-10-09T10:10:07.020962+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:10:07.020962+0000","last_undegraded":"2023-10-09T10:10:07.020962+0000","last_fullsized":"2023-10-09T10:10:07.020962+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"291'6607","reported_seq":6635,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:14.995794+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:10:14.995794+0000","last_peered":"2023-10-09T10:10:14.995794+0000","last_clean":"2023-10-09T10:10:14.995794+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:10:14.995794+0000","last_undegraded":"2023-10-09T10:10:14.995794+0000","last_fullsized":"2023-10-09T10:10:14.995794+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432996352,"num_objects":6607,"num_object_clones":0,"num_object_copies":19821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6607,"num_write_kb":422848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":311,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322786+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:09:57.322786+0000","last_peered":"2023-10-09T10:09:57.322786+0000","last_clean":"2023-10-09T10:09:57.322786+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:09:57.322786+0000","last_undegraded":"2023-10-09T10:09:57.322786+0000","last_fullsized":"2023-10-09T10:09:57.322786+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":201,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311189+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:09:57.311189+0000","last_peered":"2023-10-09T10:09:57.311189+0000","last_clean":"2023-10-09T10:09:57.311189+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:09:57.311189+0000","last_undegraded":"2023-10-09T10:09:57.311189+0000","last_fullsized":"2023-10-09T10:09:57.311189+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"292'6662","reported_seq":6692,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:17.702481+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:10:17.702481+0000","last_peered":"2023-10-09T10:10:17.702481+0000","last_clean":"2023-10-09T10:10:17.702481+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:10:17.702481+0000","last_undegraded":"2023-10-09T10:10:17.702481+0000","last_fullsized":"2023-10-09T10:10:17.702481+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6662,"log_dups_size":0,"ondisk_log_size":6662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436600832,"num_objects":6662,"num_object_clones":0,"num_object_copies":19986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6662,"num_write_kb":426368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":114,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021530+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:10:07.021530+0000","last_peered":"2023-10-09T10:10:07.021530+0000","last_clean":"2023-10-09T10:10:07.021530+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:10:07.021530+0000","last_undegraded":"2023-10-09T10:10:07.021530+0000","last_fullsized":"2023-10-09T10:10:07.021530+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.324020+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:09:57.324020+0000","last_peered":"2023-10-09T10:09:57.324020+0000","last_clean":"2023-10-09T10:09:57.324020+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:09:57.324020+0000","last_undegraded":"2023-10-09T10:09:57.324020+0000","last_fullsized":"2023-10-09T10:09:57.324020+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"291'6419","reported_seq":6447,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:14.434083+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:10:14.434083+0000","last_peered":"2023-10-09T10:10:14.434083+0000","last_clean":"2023-10-09T10:10:14.434083+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:10:14.434083+0000","last_undegraded":"2023-10-09T10:10:14.434083+0000","last_fullsized":"2023-10-09T10:10:14.434083+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420675584,"num_objects":6419,"num_object_clones":0,"num_object_copies":19257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6419,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":125,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.032573+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:10:07.032573+0000","last_peered":"2023-10-09T10:10:07.032573+0000","last_clean":"2023-10-09T10:10:07.032573+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:10:07.032573+0000","last_undegraded":"2023-10-09T10:10:07.032573+0000","last_fullsized":"2023-10-09T10:10:07.032573+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":158,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.019394+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:10:07.019394+0000","last_peered":"2023-10-09T10:10:07.019394+0000","last_clean":"2023-10-09T10:10:07.019394+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:10:07.019394+0000","last_undegraded":"2023-10-09T10:10:07.019394+0000","last_fullsized":"2023-10-09T10:10:07.019394+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"292'6700","reported_seq":6730,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:16.149603+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:10:16.149603+0000","last_peered":"2023-10-09T10:10:16.149603+0000","last_clean":"2023-10-09T10:10:16.149603+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:10:16.149603+0000","last_undegraded":"2023-10-09T10:10:16.149603+0000","last_fullsized":"2023-10-09T10:10:16.149603+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439091200,"num_objects":6700,"num_object_clones":0,"num_object_copies":20100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6700,"num_write_kb":428800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":285,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020860+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:10:07.020860+0000","last_peered":"2023-10-09T10:10:07.020860+0000","last_clean":"2023-10-09T10:10:07.020860+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:10:07.020860+0000","last_undegraded":"2023-10-09T10:10:07.020860+0000","last_fullsized":"2023-10-09T10:10:07.020860+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.024106+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:10:07.024106+0000","last_peered":"2023-10-09T10:10:07.024106+0000","last_clean":"2023-10-09T10:10:07.024106+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:10:07.024106+0000","last_undegraded":"2023-10-09T10:10:07.024106+0000","last_fullsized":"2023-10-09T10:10:07.024106+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"291'6572","reported_seq":6601,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:14.482389+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:10:14.482389+0000","last_peered":"2023-10-09T10:10:14.482389+0000","last_clean":"2023-10-09T10:10:14.482389+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:10:14.482389+0000","last_undegraded":"2023-10-09T10:10:14.482389+0000","last_fullsized":"2023-10-09T10:10:14.482389+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6572,"log_dups_size":0,"ondisk_log_size":6572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430702592,"num_objects":6572,"num_object_clones":0,"num_object_copies":19716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6572,"num_write_kb":420608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":215,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311110+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:09:57.311110+0000","last_peered":"2023-10-09T10:09:57.311110+0000","last_clean":"2023-10-09T10:09:57.311110+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:09:57.311110+0000","last_undegraded":"2023-10-09T10:09:57.311110+0000","last_fullsized":"2023-10-09T10:09:57.311110+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":221,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311329+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:09:57.311329+0000","last_peered":"2023-10-09T10:09:57.311329+0000","last_clean":"2023-10-09T10:09:57.311329+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:09:57.311329+0000","last_undegraded":"2023-10-09T10:09:57.311329+0000","last_fullsized":"2023-10-09T10:09:57.311329+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"292'6539","reported_seq":6568,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:17.719387+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:10:17.719387+0000","last_peered":"2023-10-09T10:10:17.719387+0000","last_clean":"2023-10-09T10:10:17.719387+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:10:17.719387+0000","last_undegraded":"2023-10-09T10:10:17.719387+0000","last_fullsized":"2023-10-09T10:10:17.719387+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6539,"log_dups_size":0,"ondisk_log_size":6539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428539904,"num_objects":6539,"num_object_clones":0,"num_object_copies":19617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6539,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":185,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.321954+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:09:57.321954+0000","last_peered":"2023-10-09T10:09:57.321954+0000","last_clean":"2023-10-09T10:09:57.321954+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:09:57.321954+0000","last_undegraded":"2023-10-09T10:09:57.321954+0000","last_fullsized":"2023-10-09T10:09:57.321954+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":157,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311495+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:09:57.311495+0000","last_peered":"2023-10-09T10:09:57.311495+0000","last_clean":"2023-10-09T10:09:57.311495+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:09:57.311495+0000","last_undegraded":"2023-10-09T10:09:57.311495+0000","last_fullsized":"2023-10-09T10:09:57.311495+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"291'6502","reported_seq":6531,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:14.499269+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:10:14.499269+0000","last_peered":"2023-10-09T10:10:14.499269+0000","last_clean":"2023-10-09T10:10:14.499269+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:10:14.499269+0000","last_undegraded":"2023-10-09T10:10:14.499269+0000","last_fullsized":"2023-10-09T10:10:14.499269+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6502,"log_dups_size":0,"ondisk_log_size":6502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426115072,"num_objects":6502,"num_object_clones":0,"num_object_copies":19506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":416128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":245,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.024640+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:10:07.024640+0000","last_peered":"2023-10-09T10:10:07.024640+0000","last_clean":"2023-10-09T10:10:07.024640+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:10:07.024640+0000","last_undegraded":"2023-10-09T10:10:07.024640+0000","last_fullsized":"2023-10-09T10:10:07.024640+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":294,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.028451+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:10:07.028451+0000","last_peered":"2023-10-09T10:10:07.028451+0000","last_clean":"2023-10-09T10:10:07.028451+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:10:07.028451+0000","last_undegraded":"2023-10-09T10:10:07.028451+0000","last_fullsized":"2023-10-09T10:10:07.028451+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"292'6613","reported_seq":6643,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:17.694995+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:10:17.694995+0000","last_peered":"2023-10-09T10:10:17.694995+0000","last_clean":"2023-10-09T10:10:17.694995+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:10:17.694995+0000","last_undegraded":"2023-10-09T10:10:17.694995+0000","last_fullsized":"2023-10-09T10:10:17.694995+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433389568,"num_objects":6613,"num_object_clones":0,"num_object_copies":19839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6613,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":293,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311843+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:09:57.311843+0000","last_peered":"2023-10-09T10:09:57.311843+0000","last_clean":"2023-10-09T10:09:57.311843+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:09:57.311843+0000","last_undegraded":"2023-10-09T10:09:57.311843+0000","last_fullsized":"2023-10-09T10:09:57.311843+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021377+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:10:07.021377+0000","last_peered":"2023-10-09T10:10:07.021377+0000","last_clean":"2023-10-09T10:10:07.021377+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:10:07.021377+0000","last_undegraded":"2023-10-09T10:10:07.021377+0000","last_fullsized":"2023-10-09T10:10:07.021377+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":249,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311728+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:09:57.311728+0000","last_peered":"2023-10-09T10:09:57.311728+0000","last_clean":"2023-10-09T10:09:57.311728+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:09:57.311728+0000","last_undegraded":"2023-10-09T10:09:57.311728+0000","last_fullsized":"2023-10-09T10:09:57.311728+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"292'6638","reported_seq":6667,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:17.825960+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:10:17.825960+0000","last_peered":"2023-10-09T10:10:17.825960+0000","last_clean":"2023-10-09T10:10:17.825960+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:10:17.825960+0000","last_undegraded":"2023-10-09T10:10:17.825960+0000","last_fullsized":"2023-10-09T10:10:17.825960+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6638,"log_dups_size":0,"ondisk_log_size":6638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435027968,"num_objects":6638,"num_object_clones":0,"num_object_copies":19914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6638,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":110,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021429+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:10:07.021429+0000","last_peered":"2023-10-09T10:10:07.021429+0000","last_clean":"2023-10-09T10:10:07.021429+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:10:07.021429+0000","last_undegraded":"2023-10-09T10:10:07.021429+0000","last_fullsized":"2023-10-09T10:10:07.021429+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":107,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.019174+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:10:07.019174+0000","last_peered":"2023-10-09T10:10:07.019174+0000","last_clean":"2023-10-09T10:10:07.019174+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:10:07.019174+0000","last_undegraded":"2023-10-09T10:10:07.019174+0000","last_fullsized":"2023-10-09T10:10:07.019174+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":316,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311093+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:09:57.311093+0000","last_peered":"2023-10-09T10:09:57.311093+0000","last_clean":"2023-10-09T10:09:57.311093+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:09:57.311093+0000","last_undegraded":"2023-10-09T10:09:57.311093+0000","last_fullsized":"2023-10-09T10:09:57.311093+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"292'6532","reported_seq":6561,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:16.148991+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:10:16.148991+0000","last_peered":"2023-10-09T10:10:16.148991+0000","last_clean":"2023-10-09T10:10:16.148991+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:10:16.148991+0000","last_undegraded":"2023-10-09T10:10:16.148991+0000","last_fullsized":"2023-10-09T10:10:16.148991+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428081152,"num_objects":6532,"num_object_clones":0,"num_object_copies":19596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6532,"num_write_kb":418048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":211,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020778+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:10:07.020778+0000","last_peered":"2023-10-09T10:10:07.020778+0000","last_clean":"2023-10-09T10:10:07.020778+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:10:07.020778+0000","last_undegraded":"2023-10-09T10:10:07.020778+0000","last_fullsized":"2023-10-09T10:10:07.020778+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":285,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311305+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:09:57.311305+0000","last_peered":"2023-10-09T10:09:57.311305+0000","last_clean":"2023-10-09T10:09:57.311305+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:09:57.311305+0000","last_undegraded":"2023-10-09T10:09:57.311305+0000","last_fullsized":"2023-10-09T10:09:57.311305+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":306,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.312186+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:09:57.312186+0000","last_peered":"2023-10-09T10:09:57.312186+0000","last_clean":"2023-10-09T10:09:57.312186+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:09:57.312186+0000","last_undegraded":"2023-10-09T10:09:57.312186+0000","last_fullsized":"2023-10-09T10:09:57.312186+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"292'6573","reported_seq":6602,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:17.823548+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:10:17.823548+0000","last_peered":"2023-10-09T10:10:17.823548+0000","last_clean":"2023-10-09T10:10:17.823548+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:10:17.823548+0000","last_undegraded":"2023-10-09T10:10:17.823548+0000","last_fullsized":"2023-10-09T10:10:17.823548+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430768128,"num_objects":6573,"num_object_clones":0,"num_object_copies":19719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6573,"num_write_kb":420672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":119,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021080+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:10:07.021080+0000","last_peered":"2023-10-09T10:10:07.021080+0000","last_clean":"2023-10-09T10:10:07.021080+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:10:07.021080+0000","last_undegraded":"2023-10-09T10:10:07.021080+0000","last_fullsized":"2023-10-09T10:10:07.021080+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":537,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322009+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:09:57.322009+0000","last_peered":"2023-10-09T10:09:57.322009+0000","last_clean":"2023-10-09T10:09:57.322009+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:09:57.322009+0000","last_undegraded":"2023-10-09T10:09:57.322009+0000","last_fullsized":"2023-10-09T10:09:57.322009+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":195,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021722+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:10:07.021722+0000","last_peered":"2023-10-09T10:10:07.021722+0000","last_clean":"2023-10-09T10:10:07.021722+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:10:07.021722+0000","last_undegraded":"2023-10-09T10:10:07.021722+0000","last_fullsized":"2023-10-09T10:10:07.021722+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":206,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.323059+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:09:57.323059+0000","last_peered":"2023-10-09T10:09:57.323059+0000","last_clean":"2023-10-09T10:09:57.323059+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:09:57.323059+0000","last_undegraded":"2023-10-09T10:09:57.323059+0000","last_fullsized":"2023-10-09T10:09:57.323059+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"291'6517","reported_seq":6546,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:14.435906+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:10:14.435906+0000","last_peered":"2023-10-09T10:10:14.435906+0000","last_clean":"2023-10-09T10:10:14.435906+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:10:14.435906+0000","last_undegraded":"2023-10-09T10:10:14.435906+0000","last_fullsized":"2023-10-09T10:10:14.435906+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6517,"log_dups_size":0,"ondisk_log_size":6517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427098112,"num_objects":6517,"num_object_clones":0,"num_object_copies":19551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6517,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":265,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311464+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:09:57.311464+0000","last_peered":"2023-10-09T10:09:57.311464+0000","last_clean":"2023-10-09T10:09:57.311464+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:09:57.311464+0000","last_undegraded":"2023-10-09T10:09:57.311464+0000","last_fullsized":"2023-10-09T10:09:57.311464+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":168,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.019867+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:10:07.019867+0000","last_peered":"2023-10-09T10:10:07.019867+0000","last_clean":"2023-10-09T10:10:07.019867+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:10:07.019867+0000","last_undegraded":"2023-10-09T10:10:07.019867+0000","last_fullsized":"2023-10-09T10:10:07.019867+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":196,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020120+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:10:07.020120+0000","last_peered":"2023-10-09T10:10:07.020120+0000","last_clean":"2023-10-09T10:10:07.020120+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:10:07.020120+0000","last_undegraded":"2023-10-09T10:10:07.020120+0000","last_fullsized":"2023-10-09T10:10:07.020120+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"292'6600","reported_seq":7742,"reported_epoch":292,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:16.136144+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:10:16.136144+0000","last_peered":"2023-10-09T10:10:16.136144+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:10:16.136144+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6600,"log_dups_size":0,"ondisk_log_size":6600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432537600,"num_objects":6600,"num_object_clones":0,"num_object_copies":19800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":4729,"num_objects_unfound":0,"num_objects_dirty":6600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6600,"num_write_kb":422400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":554,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1870},{"shards":"1,3,4","objects":4729}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":310,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.322828+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:09:57.322828+0000","last_peered":"2023-10-09T10:09:57.322828+0000","last_clean":"2023-10-09T10:09:57.322828+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:09:57.322828+0000","last_undegraded":"2023-10-09T10:09:57.322828+0000","last_fullsized":"2023-10-09T10:09:57.322828+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":322,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311918+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:09:57.311918+0000","last_peered":"2023-10-09T10:09:57.311918+0000","last_clean":"2023-10-09T10:09:57.311918+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:09:57.311918+0000","last_undegraded":"2023-10-09T10:09:57.311918+0000","last_fullsized":"2023-10-09T10:09:57.311918+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":160,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.019246+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:10:07.019246+0000","last_peered":"2023-10-09T10:10:07.019246+0000","last_clean":"2023-10-09T10:10:07.019246+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:10:07.019246+0000","last_undegraded":"2023-10-09T10:10:07.019246+0000","last_fullsized":"2023-10-09T10:10:07.019246+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"292'6618","reported_seq":6647,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:17.702487+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:10:17.702487+0000","last_peered":"2023-10-09T10:10:17.702487+0000","last_clean":"2023-10-09T10:10:17.702487+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:10:17.702487+0000","last_undegraded":"2023-10-09T10:10:17.702487+0000","last_fullsized":"2023-10-09T10:10:17.702487+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6618,"log_dups_size":0,"ondisk_log_size":6618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433717248,"num_objects":6618,"num_object_clones":0,"num_object_copies":19854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6618,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":254,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.027045+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:10:07.027045+0000","last_peered":"2023-10-09T10:10:07.027045+0000","last_clean":"2023-10-09T10:10:07.027045+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:10:07.027045+0000","last_undegraded":"2023-10-09T10:10:07.027045+0000","last_fullsized":"2023-10-09T10:10:07.027045+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":267,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311572+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:09:57.311572+0000","last_peered":"2023-10-09T10:09:57.311572+0000","last_clean":"2023-10-09T10:09:57.311572+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:09:57.311572+0000","last_undegraded":"2023-10-09T10:09:57.311572+0000","last_fullsized":"2023-10-09T10:09:57.311572+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":243,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.024561+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:10:07.024561+0000","last_peered":"2023-10-09T10:10:07.024561+0000","last_clean":"2023-10-09T10:10:07.024561+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:10:07.024561+0000","last_undegraded":"2023-10-09T10:10:07.024561+0000","last_fullsized":"2023-10-09T10:10:07.024561+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"292'6579","reported_seq":6608,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:17.845124+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:10:17.845124+0000","last_peered":"2023-10-09T10:10:17.845124+0000","last_clean":"2023-10-09T10:10:17.845124+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:10:17.845124+0000","last_undegraded":"2023-10-09T10:10:17.845124+0000","last_fullsized":"2023-10-09T10:10:17.845124+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431161344,"num_objects":6579,"num_object_clones":0,"num_object_copies":19737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6579,"num_write_kb":421056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":199,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.326632+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:09:57.326632+0000","last_peered":"2023-10-09T10:09:57.326632+0000","last_clean":"2023-10-09T10:09:57.326632+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:09:57.326632+0000","last_undegraded":"2023-10-09T10:09:57.326632+0000","last_fullsized":"2023-10-09T10:09:57.326632+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":162,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.020039+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:10:07.020039+0000","last_peered":"2023-10-09T10:10:07.020039+0000","last_clean":"2023-10-09T10:10:07.020039+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:10:07.020039+0000","last_undegraded":"2023-10-09T10:10:07.020039+0000","last_fullsized":"2023-10-09T10:10:07.020039+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":145,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.024207+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:10:07.024207+0000","last_peered":"2023-10-09T10:10:07.024207+0000","last_clean":"2023-10-09T10:10:07.024207+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:10:07.024207+0000","last_undegraded":"2023-10-09T10:10:07.024207+0000","last_fullsized":"2023-10-09T10:10:07.024207+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":157,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.019092+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:10:07.019092+0000","last_peered":"2023-10-09T10:10:07.019092+0000","last_clean":"2023-10-09T10:10:07.019092+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:10:07.019092+0000","last_undegraded":"2023-10-09T10:10:07.019092+0000","last_fullsized":"2023-10-09T10:10:07.019092+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":228,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.311988+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:09:57.311988+0000","last_peered":"2023-10-09T10:09:57.311988+0000","last_clean":"2023-10-09T10:09:57.311988+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:09:57.311988+0000","last_undegraded":"2023-10-09T10:09:57.311988+0000","last_fullsized":"2023-10-09T10:09:57.311988+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":129,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.032483+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:10:07.032483+0000","last_peered":"2023-10-09T10:10:07.032483+0000","last_clean":"2023-10-09T10:10:07.032483+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:10:07.032483+0000","last_undegraded":"2023-10-09T10:10:07.032483+0000","last_fullsized":"2023-10-09T10:10:07.032483+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":170,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:11.147232+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:10:11.147232+0000","last_peered":"2023-10-09T10:10:11.147232+0000","last_clean":"2023-10-09T10:10:11.147232+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:10:11.147232+0000","last_undegraded":"2023-10-09T10:10:11.147232+0000","last_fullsized":"2023-10-09T10:10:11.147232+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":129,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021303+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:10:07.021303+0000","last_peered":"2023-10-09T10:10:07.021303+0000","last_clean":"2023-10-09T10:10:07.021303+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:10:07.021303+0000","last_undegraded":"2023-10-09T10:10:07.021303+0000","last_fullsized":"2023-10-09T10:10:07.021303+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":179,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:11.147193+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:10:11.147193+0000","last_peered":"2023-10-09T10:10:11.147193+0000","last_clean":"2023-10-09T10:10:11.147193+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:10:11.147193+0000","last_undegraded":"2023-10-09T10:10:11.147193+0000","last_fullsized":"2023-10-09T10:10:11.147193+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":308,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.023585+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:10:07.023585+0000","last_peered":"2023-10-09T10:10:07.023585+0000","last_clean":"2023-10-09T10:10:07.023585+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:10:07.023585+0000","last_undegraded":"2023-10-09T10:10:07.023585+0000","last_fullsized":"2023-10-09T10:10:07.023585+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":329,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.021041+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:10:07.021041+0000","last_peered":"2023-10-09T10:10:07.021041+0000","last_clean":"2023-10-09T10:10:07.021041+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:10:07.021041+0000","last_undegraded":"2023-10-09T10:10:07.021041+0000","last_fullsized":"2023-10-09T10:10:07.021041+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":238,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T10:10:07.019092+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:10:07.019092+0000","last_peered":"2023-10-09T10:10:07.019092+0000","last_clean":"2023-10-09T10:10:07.019092+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:10:07.019092+0000","last_undegraded":"2023-10-09T10:10:07.019092+0000","last_fullsized":"2023-10-09T10:10:07.019092+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T10:09:57.310984+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:09:57.310984+0000","last_peered":"2023-10-09T10:09:57.310984+0000","last_clean":"2023-10-09T10:09:57.310984+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:09:57.310984+0000","last_undegraded":"2023-10-09T10:09:57.310984+0000","last_fullsized":"2023-10-09T10:09:57.310984+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6905004032,"num_objects":105362,"num_object_clones":0,"num_object_copies":316086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":4729,"num_objects_unfound":0,"num_objects_dirty":105362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":105362,"num_write_kb":6743168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":554,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20703346688,"data_stored":20703346688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105362,"ondisk_log_size":105362,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430487,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2393732,"kb_used_data":2080124,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":91351420,"statfs":{"total":95995035648,"available":93543854080,"internally_reserved":0,"allocated":2130046976,"data_stored":2129514875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5940000000000001,"15min":1.5740000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":5.625,"5min":9.0690000000000008,"15min":50.591999999999999},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.548,"5min":1.8,"15min":1.6040000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":6.0670000000000002,"5min":8.2300000000000004,"15min":50.747},"last":1.0189999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:10:08 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":1.8440000000000001,"15min":1.5169999999999999},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2669999999999995,"5min":16.352,"15min":25.338000000000001},"last":4.1870000000000003},{"interface":"front","average":{"1min":1.53,"5min":1.8839999999999999,"15min":1.3560000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.9269999999999996,"5min":18.27,"15min":18.27},"last":0.30599999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.7190000000000001,"15min":2.266},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":3.8370000000000002,"5min":7.952,"15min":7.952},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.7310000000000001,"15min":2.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.2039999999999997,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.46300000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:47 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.0049999999999999,"15min":1.835},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.058,"5min":29.058,"15min":29.058},"last":4.0380000000000003},{"interface":"front","average":{"1min":1.214,"5min":1.7989999999999999,"15min":1.897},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.577,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":3.508}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1130000000000004,"5min":4.1130000000000004,"15min":4.1130000000000004},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.40100000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.149,"15min":0.93899999999999995},"min":{"1min":0.31900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":25.277999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.4529999999999998,"5min":1.2589999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.754000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.54600000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.1299999999999999,"15min":0.88600000000000001},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5369999999999999,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.98599999999999999,"15min":0.82499999999999996},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.998,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":0.432}]}]},{"osd":6,"up_from":42,"seq":180388626715,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2736248,"kb_used_data":2426736,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91008904,"statfs":{"total":95995035648,"available":93193117696,"internally_reserved":0,"allocated":2484977664,"data_stored":2484458805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.9259999999999999,"15min":1.319},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":15.435,"5min":15.435,"15min":15.435},"last":0.31},{"interface":"front","average":{"1min":1.853,"5min":1.931,"15min":1.365},"min":{"1min":0.252,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.308999999999999,"5min":15.308999999999999,"15min":15.308999999999999},"last":3.4700000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:10:02 2023","interfaces":[{"interface":"back","average":{"1min":4.641,"5min":2.4300000000000002,"15min":1.7130000000000001},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":35.609999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":2.4340000000000002},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.3540000000000001,"15min":1.6319999999999999},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":32.396000000000001,"5min":32.396000000000001,"15min":32.396000000000001},"last":5.9980000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.3340000000000001,"15min":1.6919999999999999},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8149999999999999,"5min":7.6760000000000002,"15min":7.6760000000000002},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.234,"5min":2.3239999999999998,"15min":3.0990000000000002},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.5600000000000005,"5min":10.26,"15min":10.26},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.321,"15min":1.347},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3879999999999999,"5min":6.0999999999999996,"15min":6.0999999999999996},"last":5.157},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.4220000000000002,"15min":2.3940000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":11.211,"5min":11.211,"15min":11.211},"last":2.8140000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.48,"5min":1.48,"15min":1.48},"last":0.33900000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.71999999999999997,"15min":0.72199999999999998},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.611,"5min":7.7080000000000002,"15min":14.09},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.87,"15min":0.77300000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":2.464,"5min":7.508,"15min":7.508},"last":0.373}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.77000000000000002,"15min":0.755},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.5800000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.81499999999999995,"15min":0.80500000000000005},"min":{"1min":0.39000000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.6080000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.436}]}]},{"osd":1,"up_from":84,"seq":360777253028,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3300368,"kb_used_data":2920776,"kb_used_omap":9,"kb_used_meta":379574,"kb_avail":90444784,"statfs":{"total":95995035648,"available":92615458816,"internally_reserved":0,"allocated":2990874624,"data_stored":2989859942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":388684777},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":2.6549999999999998,"15min":2.0299999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.185},"max":{"1min":6.3479999999999999,"5min":25.59,"15min":25.59},"last":1.1739999999999999},{"interface":"front","average":{"1min":3.7109999999999999,"5min":2.77,"15min":1.976},"min":{"1min":0.26900000000000002,"5min":0.218,"15min":0.192},"max":{"1min":11.682,"5min":14.769,"15min":16.629000000000001},"last":9.8710000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":3.2160000000000002,"15min":3.8700000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.1539999999999999,"5min":17.369,"15min":17.369},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.907,"15min":3.8740000000000001},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5960000000000001,"5min":16.023,"15min":16.023},"last":1.1379999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":3.1619999999999999,"15min":3.4100000000000001},"min":{"1min":0.36399999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.196999999999999,"5min":16.931000000000001,"15min":16.931000000000001},"last":9.8989999999999991},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.266,"15min":4.6900000000000004},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.8569999999999993,"5min":17.289000000000001,"15min":17.289000000000001},"last":4.4139999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3860000000000001,"5min":5.3860000000000001,"15min":5.3860000000000001},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":10.132,"5min":10.132,"15min":10.132},"last":9.6940000000000008}]},{"osd":5,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":2.0419999999999998,"15min":1.72},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.473000000000001,"5min":14.416,"15min":15.531000000000001},"last":9.6449999999999996},{"interface":"front","average":{"1min":1.782,"5min":2.0979999999999999,"15min":1.9690000000000001},"min":{"1min":0.34699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.6890000000000001,"5min":16.277999999999999,"15min":19.706},"last":0.55400000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0339999999999998,"15min":1.7230000000000001},"min":{"1min":0.50800000000000001,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.3710000000000004,"5min":14.215,"15min":16.193999999999999},"last":1.2529999999999999},{"interface":"front","average":{"1min":2.75,"5min":1.9930000000000001,"15min":1.7529999999999999},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":10.161,"5min":14.471,"15min":15.481999999999999},"last":9.7249999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":2.153,"15min":1.972},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.7350000000000003,"5min":16.331,"15min":16.331},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1320000000000001,"15min":1.788},"min":{"1min":0.46000000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3319999999999999,"5min":14.242000000000001,"15min":16.100999999999999},"last":1.218}]}]},{"osd":0,"up_from":79,"seq":339302416555,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2870156,"kb_used_data":2529604,"kb_used_omap":28,"kb_used_meta":340515,"kb_avail":90874996,"statfs":{"total":95995035648,"available":93055995904,"internally_reserved":0,"allocated":2590314496,"data_stored":2589309742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":348688208},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,0,2,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.847,"15min":1.4970000000000001},"min":{"1min":0.24099999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.2990000000000004,"5min":12.84,"15min":13.084},"last":3.5569999999999999},{"interface":"front","average":{"1min":3.9020000000000001,"5min":3.379,"15min":2.2269999999999999},"min":{"1min":0.44900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.949000000000002,"5min":22.178999999999998,"15min":22.178999999999998},"last":8.9719999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.7210000000000001,"15min":2.121},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.5579999999999998,"5min":10.034000000000001,"15min":10.034000000000001},"last":2.226},{"interface":"front","average":{"1min":1.353,"5min":2.2879999999999998,"15min":3.0910000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.4480000000000004,"5min":14.693,"15min":14.693},"last":2.1899999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4990000000000001,"5min":4.391,"15min":4.7069999999999999},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.867000000000001,"5min":25.734999999999999,"15min":25.734999999999999},"last":8.9380000000000006},{"interface":"front","average":{"1min":2.4809999999999999,"5min":4.0250000000000004,"15min":4.282},"min":{"1min":0.44,"5min":0.254,"15min":0.254},"max":{"1min":13.975,"5min":21.923999999999999,"15min":21.923999999999999},"last":0.61699999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.3760000000000003,"5min":4.3760000000000003,"15min":4.3760000000000003},"last":1.9419999999999999},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":13.789999999999999,"5min":13.789999999999999,"15min":13.789999999999999},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":2.8149999999999999,"15min":1.847},"min":{"1min":0.38900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.736000000000001,"5min":17.096,"15min":17.096},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.419,"15min":1.2470000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.4870000000000001,"5min":12.478,"15min":12.478},"last":2.4670000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.494,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5289999999999999,"5min":9.7260000000000009,"15min":11.699},"last":2.504},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.6549999999999998,"15min":1.954},"min":{"1min":0.45100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.4130000000000003,"5min":14.384,"15min":15.108000000000001},"last":8.8879999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.9929999999999999,"15min":1.986},"min":{"1min":0.51400000000000001,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":13.829000000000001,"5min":14.631,"15min":26.978000000000002},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.085,"5min":1.6819999999999999,"15min":1.544},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.5209999999999999,"5min":12.428000000000001,"15min":26.768000000000001},"last":2.4300000000000002}]}]},{"osd":2,"up_from":217,"seq":932007903288,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2041460,"kb_used_data":1842604,"kb_used_omap":83,"kb_used_meta":198764,"kb_avail":91703692,"statfs":{"total":95995035648,"available":93904580608,"internally_reserved":0,"allocated":1886826496,"data_stored":1885838029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":203534831},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.28,"15min":1.6060000000000001},"min":{"1min":0.247,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.5139999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":2.0699999999999998},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.8849999999999998,"15min":2.923},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.891999999999999,"5min":16.309999999999999,"15min":16.309999999999999},"last":4.8399999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.0209999999999999,"15min":2.4649999999999999},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.971,"5min":11.971,"15min":11.971},"last":6.4630000000000001},{"interface":"front","average":{"1min":2.8849999999999998,"5min":2.492,"15min":2.6280000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.009,"5min":15.569000000000001,"15min":15.569000000000001},"last":6.1289999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.798,"15min":1.948},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.8859999999999999,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":2.1190000000000002},{"interface":"front","average":{"1min":1.1779999999999999,"5min":3.367,"15min":3.7320000000000002},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.8440000000000003,"5min":15.811,"15min":15.811},"last":4.8780000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.633,"5min":2.633,"15min":2.633},"last":4.9059999999999997},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.8939999999999999,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":5.2800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.5580000000000001,"15min":1.502},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.0870000000000002,"5min":12.423999999999999,"15min":12.423999999999999},"last":2.23},{"interface":"front","average":{"1min":1.21,"5min":1.2789999999999999,"15min":1.6000000000000001},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.7709999999999999,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":2.1469999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2629999999999999,"15min":1.0489999999999999},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.6040000000000001,"5min":15.292999999999999,"15min":15.292999999999999},"last":4.9329999999999998},{"interface":"front","average":{"1min":1.032,"5min":1.6499999999999999,"15min":1.712},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.984,"5min":12.394,"15min":12.394},"last":2.383}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.2809999999999999,"15min":1.0760000000000001},"min":{"1min":0.48499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1800000000000002,"5min":15.343999999999999,"15min":15.343999999999999},"last":4.9589999999999996},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.7170000000000001,"15min":1.613},"min":{"1min":0.51200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.5010000000000003,"5min":12.356999999999999,"15min":12.356999999999999},"last":1.373}]}]},{"osd":3,"up_from":250,"seq":1073741824032,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2750256,"kb_used_data":2518056,"kb_used_omap":10,"kb_used_meta":232181,"kb_avail":90994896,"statfs":{"total":95995035648,"available":93178773504,"internally_reserved":0,"allocated":2578489344,"data_stored":2577335708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":237753411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":4.2489999999999997,"15min":4.5519999999999996},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.465,"5min":23.381,"15min":23.381},"last":4.2400000000000002},{"interface":"front","average":{"1min":1.853,"5min":3.7130000000000001,"15min":4.0229999999999997},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.5190000000000001,"5min":14.289999999999999,"15min":14.289999999999999},"last":11.029999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5329999999999999,"15min":3.8130000000000002},"min":{"1min":0.28799999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.415,"5min":20.407,"15min":20.407},"last":1.2410000000000001},{"interface":"front","average":{"1min":2.702,"5min":3.2850000000000001,"15min":3.3820000000000001},"min":{"1min":0.40300000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.494,"5min":23.423999999999999,"15min":23.423999999999999},"last":4.1109999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":3.3410000000000002,"15min":3.6070000000000002},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.1020000000000003,"5min":20.356000000000002,"15min":20.356000000000002},"last":1.208},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.7290000000000001,"15min":2.819},"min":{"1min":0.34799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":9.3379999999999992,"5min":23.452000000000002,"15min":23.452000000000002},"last":4.0839999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":3.7570000000000001},"last":4.0019999999999998},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.5739999999999998,"5min":8.5739999999999998,"15min":8.5739999999999998},"last":0.98699999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.4319999999999999,"15min":2.5529999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.6199999999999992,"5min":11.836,"15min":11.836},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.274,"15min":2.327},"min":{"1min":0.55600000000000005,"5min":0.309,"15min":0.309},"max":{"1min":6.9249999999999998,"5min":14.164999999999999,"15min":14.164999999999999},"last":0.84699999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.5379999999999998,"15min":2.6720000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.532,"5min":11.887,"15min":11.887},"last":1.036},{"interface":"front","average":{"1min":1.496,"5min":2.3820000000000001,"15min":2.5299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.8710000000000004,"5min":14.125,"15min":14.125},"last":0.81799999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.4039999999999999,"15min":1.3939999999999999},"min":{"1min":0.45400000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0609999999999999,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":4.0529999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":2.5009999999999999,"15min":2.617},"min":{"1min":0.42199999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":8.6549999999999994,"5min":11.801,"15min":11.801},"last":1.0089999999999999}]}]},{"osd":4,"up_from":266,"seq":1142461300758,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2777924,"kb_used_data":2459260,"kb_used_omap":35,"kb_used_meta":318620,"kb_avail":90967228,"statfs":{"total":95995035648,"available":93150441472,"internally_reserved":0,"allocated":2518282240,"data_stored":2517757479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":326267118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.3799999999999999,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3959999999999999,"5min":2.3959999999999999,"15min":2.3959999999999999},"last":0.34599999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":2.7160000000000002,"5min":2.7160000000000002,"15min":2.7160000000000002},"last":2.3570000000000002},{"interface":"front","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":2.5350000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.661,"5min":3.661,"15min":3.661},"last":3.1509999999999998},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6449999999999996,"5min":4.6449999999999996,"15min":4.6449999999999996},"last":3.0550000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.262,"15min":1.262},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.1259999999999999,"5min":3.1259999999999999,"15min":3.1259999999999999},"last":3.5249999999999999},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.4470000000000001,"5min":4.4470000000000001,"15min":4.4470000000000001},"last":3.367}]},{"osd":5,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.6970000000000001,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.5609999999999999,"5min":2.5609999999999999,"15min":2.5609999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.917,"5min":1.917,"15min":1.917},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.375},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.5630000000000002,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.35599999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822943,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3705332,"kb_used_data":3391724,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":90039820,"statfs":{"total":95995035648,"available":92200775680,"internally_reserved":0,"allocated":3473125376,"data_stored":3472612743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":1.5860000000000001,"15min":1.3460000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.3090000000000002,"5min":10.92,"15min":15.773999999999999},"last":4.2450000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.673,"15min":1.4790000000000001},"min":{"1min":0.25600000000000001,"5min":0.23899999999999999,"15min":0.23400000000000001},"max":{"1min":6.3090000000000002,"5min":11.747,"15min":15.699},"last":0.32000000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:12 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.5740000000000001,"15min":1.4430000000000001},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.7720000000000002,"5min":11.335000000000001,"15min":15.689},"last":2.1179999999999999},{"interface":"front","average":{"1min":1.913,"5min":1.861,"15min":1.6180000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.8119999999999994,"5min":19.402999999999999,"15min":19.402999999999999},"last":2.056}]},{"osd":2,"last update":"Mon Oct 9 10:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.8480000000000001,"15min":2.4369999999999998},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.3819999999999997,"5min":15.640000000000001,"15min":15.640000000000001},"last":3.8029999999999999},{"interface":"front","average":{"1min":1.302,"5min":1.8220000000000001,"15min":2.2949999999999999},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.3659999999999997,"5min":10.656000000000001,"15min":10.656000000000001},"last":5.2789999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.734,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5860000000000003,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.2,"5min":2.6549999999999998,"15min":2.8969999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":2.9710000000000001,"5min":17.34,"15min":17.34},"last":5.085}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.0819999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.27900000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.995,"15min":0.90000000000000002},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.4159999999999999,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.754,"5min":0.98999999999999999,"15min":0.81100000000000005},"min":{"1min":0.317,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.3490000000000002,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.26300000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0589999999999999,"15min":0.88},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.282,"5min":7.4290000000000003,"15min":10.334},"last":0.312},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.749,"15min":0.88600000000000001},"min":{"1min":0.35099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.3500000000000001,"5min":8.0210000000000008,"15min":12.356},"last":0.34599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2601648128,"data_stored":2601648128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2990538752,"data_stored":2990538752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1898512384,"data_stored":1898512384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2584281088,"data_stored":2584281088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2524774400,"data_stored":2524774400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3480420352,"data_stored":3480420352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2484666368,"data_stored":2484666368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2138505216,"data_stored":2138505216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:10:21.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:10:21.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:21.282 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:21.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:21 smithi012 ceph-mon[147926]: from='client.45614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:21.455 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:21.608 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:21.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total time run: 90.0229 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total writes made: 108680 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Write size: 65536 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object size: 65536 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Bandwidth (MB/sec): 75.453 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Bandwidth: 39.1217 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max bandwidth (MB/sec): 202.75 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min bandwidth (MB/sec): 0 2023-10-09T10:10:21.670 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average IOPS: 1207 2023-10-09T10:10:21.671 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev IOPS: 625.958 2023-10-09T10:10:21.671 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max IOPS: 3244 2023-10-09T10:10:21.671 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min IOPS: 0 2023-10-09T10:10:21.671 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average Latency(s): 0.0132488 2023-10-09T10:10:21.671 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Latency(s): 0.0303292 2023-10-09T10:10:21.671 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max latency(s): 2.42059 2023-10-09T10:10:21.671 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min latency(s): 0.00120497 2023-10-09T10:10:21.673 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:10:21.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:21 smithi012 ceph-mon[152263]: from='client.45614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:21.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:21 smithi110 ceph-mon[136032]: from='client.45614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:21.720 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:21.751 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:21.794 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:21.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:21.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:21.898 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:21.927 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:21.966 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:21.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:22.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:22.188 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:22.442 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:10:22.662 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:22.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:22.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:22.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:22 smithi110 ceph-mon[136032]: pgmap v771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 5374/321252 objects degraded (1.673%); 4628/321252 objects misplaced (1.441%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:22.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/777483740' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:22.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:22.756 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:22 smithi012 ceph-mon[152263]: pgmap v771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 5374/321252 objects degraded (1.673%); 4628/321252 objects misplaced (1.441%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:22.756 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/777483740' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:22.756 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:22.756 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:22 smithi012 ceph-mon[147926]: pgmap v771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 5374/321252 objects degraded (1.673%); 4628/321252 objects misplaced (1.441%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:22.757 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/777483740' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:22.757 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:22.769 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:22.803 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:22.850 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:22.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:22.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:22.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:23.059 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:23.269 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:23.280 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:23.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:23.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:23.424 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:23.470 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:23.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:23.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:23.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:23.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:10:23.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: osdmap e293: 8 total, 8 up, 8 in 2023-10-09T10:10:23.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.649 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.649 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.649 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/321252 objects degraded (1.673%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:23.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:23.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:10:23.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: osdmap e293: 8 total, 8 up, 8 in 2023-10-09T10:10:23.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/321252 objects degraded (1.673%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:23.665 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:23.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:10:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: osdmap e293: 8 total, 8 up, 8 in 2023-10-09T10:10:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: failed to encode map e293 with expected crc 2023-10-09T10:10:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/321252 objects degraded (1.673%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:23.769 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:23.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:23.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:23.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:23.906 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:23.940 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:23.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:24.044 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:24.051 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:10:24.109 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:10:24.127 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:24.157 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:10:24.204 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:10:24.210 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:10:24.210 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:10:24.240 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:10:24.275 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:10:24.280 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:24.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:24.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:24.328 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:10:24.367 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:10:24.371 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:10:24.371 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:10:24.371 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:10:24.371 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:10:24.371 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:10:24.372 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:10:24.372 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:10:24.372 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:10:24.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:24.470 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:25.072 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:24 smithi012 ceph-mon[147926]: pgmap v773: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 40 MiB/s wr, 673 op/s; 5374/323796 objects degraded (1.660%); 4628/323796 objects misplaced (1.429%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:10:25.072 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/443957090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:25.072 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:24 smithi012 ceph-mon[152263]: pgmap v773: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 40 MiB/s wr, 673 op/s; 5374/323796 objects degraded (1.660%); 4628/323796 objects misplaced (1.429%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:10:25.073 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/443957090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:25.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:10:25.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:24 smithi110 ceph-mon[136032]: pgmap v773: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 125 B/s rd, 40 MiB/s wr, 673 op/s; 5374/323796 objects degraded (1.660%); 4628/323796 objects misplaced (1.429%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:10:25.474 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/443957090' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:25.581 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:25.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:25.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:25.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:25.892 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:26.603 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:26.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:26.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:10:26.709 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:26.744 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:26.783 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:26.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:26.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:26.916 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:26.967 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:26.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:27.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:27.074 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:10:27.167 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:27.396 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:27.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:27.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:27 smithi012 ceph-mon[147926]: pgmap v774: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 651 op/s; 5374/323046 objects degraded (1.664%); 4628/323046 objects misplaced (1.433%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:27.405 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:27 smithi012 ceph-mon[152263]: pgmap v774: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 651 op/s; 5374/323046 objects degraded (1.664%); 4628/323046 objects misplaced (1.433%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:27.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:27 smithi110 ceph-mon[136032]: pgmap v774: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 651 op/s; 5374/323046 objects degraded (1.664%); 4628/323046 objects misplaced (1.433%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:10:27.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:27.613 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:27.636 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:27.798 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:27.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:27.813 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:27.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:28.001 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:28.080 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:28.080 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5374/321336 objects degraded (1.672%), 2 pgs degraded","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":945,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":293,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":107112,"data_bytes":7018649528,"bytes_used":23629365248,"bytes_avail":744330919936,"bytes_total":767960285184,"degraded_objects":5374,"degraded_total":321336,"degraded_ratio":0.016723927602260564,"misplaced_objects":4518,"misplaced_total":321336,"misplaced_ratio":0.014060049294196731,"recovering_objects_per_sec":21,"recovering_bytes_per_sec":1382374,"recovering_keys_per_sec":0,"num_objects_recovered":211,"num_bytes_recovered":13828096,"num_keys_recovered":0,"read_bytes_sec":102,"write_bytes_sec":32456431,"read_op_per_sec":0,"write_op_per_sec":654},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T10:09:33.293227+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T10:10:28.084 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/323046 objects degraded (1.664%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:28.084 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:10:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:10:27] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T10:10:28.085 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/323046 objects degraded (1.664%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:28.211 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:28.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:28.327 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:28.352 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/323046 objects degraded (1.664%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:28.354 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:28.400 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:28.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:28.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:28.505 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:28.594 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:28.983 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:29.005 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:10:29.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:10:29.156 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:29 smithi012 ceph-mon[147926]: pgmap v775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 654 op/s; 5374/321336 objects degraded (1.672%); 4518/321336 objects misplaced (1.406%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:10:29.157 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4079376491' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:29.157 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:29.159 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:29 smithi012 ceph-mon[152263]: pgmap v775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 654 op/s; 5374/321336 objects degraded (1.672%); 4518/321336 objects misplaced (1.406%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:10:29.159 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4079376491' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:29.159 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:29.362 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:29.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:29.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:29 smithi110 ceph-mon[136032]: pgmap v775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 654 op/s; 5374/321336 objects degraded (1.672%); 4518/321336 objects misplaced (1.406%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:10:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4079376491' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:29.478 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:29.513 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:29.548 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:29.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:29.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:29.653 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:29.724 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:30.053 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:30.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:30.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:30.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:30.240 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:30.319 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:10:30.319 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:10:30.322 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:30.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:30.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:30.536 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:30.678 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:30.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:30.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:30.843 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:30.887 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:30.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:30.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:31.092 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:31.165 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:31 smithi012 ceph-mon[147926]: pgmap v776: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 21 MiB/s wr, 668 op/s; 5374/316008 objects degraded (1.701%); 4518/316008 objects misplaced (1.430%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:10:31.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:31 smithi012 ceph-mon[152263]: pgmap v776: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 21 MiB/s wr, 668 op/s; 5374/316008 objects degraded (1.701%); 4518/316008 objects misplaced (1.430%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:10:31.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:10:31.180 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:31.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:31 smithi110 ceph-mon[136032]: pgmap v776: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 21 MiB/s wr, 668 op/s; 5374/316008 objects degraded (1.701%); 4518/316008 objects misplaced (1.430%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:10:31.194 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:31.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:31.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:31.327 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:31.370 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:31.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:31.474 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:31.514 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:31.568 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:31.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:31.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:31.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:31.814 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:31.924 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:31.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:31.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:32.036 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:32.060 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3119633426' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:32.060 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:32 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:32.063 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3119633426' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:32.063 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:32 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:32.079 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:10:32.082 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:32.126 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:32.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:32.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:32.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:32.324 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:32.325 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:10:32.325 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:10:32.325 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:10:32.325 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:10:32.355 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:10:32.396 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:10:32.408 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:32.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:32.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:32.428 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:10:32.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3119633426' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:32.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:32 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:32.464 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:10:32.467 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:10:32.467 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:10:32.467 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:10:32.467 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:10:32.467 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:10:32.467 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:10:32.467 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:10:32.468 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:10:32.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:32.554 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:32.596 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:32.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:32.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:32.763 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:33.338 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: pgmap v777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 663 op/s; 5374/311070 objects degraded (1.728%); 4408/311070 objects misplaced (1.417%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2317684220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/311070 objects degraded (1.728%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:33.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: pgmap v777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 663 op/s; 5374/311070 objects degraded (1.728%); 4408/311070 objects misplaced (1.417%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:33.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:33.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:33.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T10:10:33.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2317684220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:33.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/311070 objects degraded (1.728%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:33.386 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:33.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:33.395 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: pgmap v777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 663 op/s; 5374/311070 objects degraded (1.728%); 4408/311070 objects misplaced (1.417%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T10:10:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: failed to encode map e294 with expected crc 2023-10-09T10:10:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2317684220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:33.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/311070 objects degraded (1.728%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:33.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:33.598 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:33.746 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:33.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:33.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:33.807 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:33.839 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:10:33.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:33.886 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:33.935 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:33.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:33.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:34.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:34.124 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:34.635 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:34.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:34.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:34.744 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:34.746 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":779,"stamp":"2023-10-09T10:10:33.324339+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6684809144,"num_objects":102018,"num_object_clones":0,"num_object_copies":306054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":4408,"num_objects_unfound":0,"num_objects_dirty":102018,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":115628,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":944,"num_bytes_recovered":58534604,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115562,"ondisk_log_size":115562,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22234168,"kb_used_data":19789560,"kb_used_omap":167,"kb_used_meta":2444056,"kb_avail":727727048,"statfs":{"total":767960285184,"available":745192497152,"internally_reserved":0,"allocated":20264509440,"data_stored":20258236469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2502713945},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-387579904,"num_objects":-5914,"num_object_clones":0,"num_object_copies":-17742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-220,"num_objects_unfound":0,"num_objects_dirty":-5914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6784,"num_write_kb":27840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":220,"num_bytes_recovered":14417920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004697"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":134,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171758+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:10:15.171758+0000","last_peered":"2023-10-09T10:10:15.171758+0000","last_clean":"2023-10-09T10:10:15.171758+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:10:15.171758+0000","last_undegraded":"2023-10-09T10:10:15.171758+0000","last_fullsized":"2023-10-09T10:10:15.171758+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":250,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.140832+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:10:15.140832+0000","last_peered":"2023-10-09T10:10:15.140832+0000","last_clean":"2023-10-09T10:10:15.140832+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:10:15.140832+0000","last_undegraded":"2023-10-09T10:10:15.140832+0000","last_fullsized":"2023-10-09T10:10:15.140832+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":295,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170594+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:10:15.170594+0000","last_peered":"2023-10-09T10:10:15.170594+0000","last_clean":"2023-10-09T10:10:15.170594+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:10:15.170594+0000","last_undegraded":"2023-10-09T10:10:15.170594+0000","last_fullsized":"2023-10-09T10:10:15.170594+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":160,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092979+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:10:32.092979+0000","last_peered":"2023-10-09T10:10:32.092979+0000","last_clean":"2023-10-09T10:10:32.092979+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:10:32.092979+0000","last_undegraded":"2023-10-09T10:10:32.092979+0000","last_fullsized":"2023-10-09T10:10:32.092979+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":264,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.142921+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:10:15.142921+0000","last_peered":"2023-10-09T10:10:15.142921+0000","last_clean":"2023-10-09T10:10:15.142921+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:10:15.142921+0000","last_undegraded":"2023-10-09T10:10:15.142921+0000","last_fullsized":"2023-10-09T10:10:15.142921+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":112,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445337+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:10:22.445337+0000","last_peered":"2023-10-09T10:10:22.445337+0000","last_clean":"2023-10-09T10:10:22.445337+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:10:22.445337+0000","last_undegraded":"2023-10-09T10:10:22.445337+0000","last_fullsized":"2023-10-09T10:10:22.445337+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":177,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445862+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:10:22.445862+0000","last_peered":"2023-10-09T10:10:22.445862+0000","last_clean":"2023-10-09T10:10:22.445862+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:10:22.445862+0000","last_undegraded":"2023-10-09T10:10:22.445862+0000","last_fullsized":"2023-10-09T10:10:22.445862+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":245,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.142807+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:10:15.142807+0000","last_peered":"2023-10-09T10:10:15.142807+0000","last_clean":"2023-10-09T10:10:15.142807+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:10:15.142807+0000","last_undegraded":"2023-10-09T10:10:15.142807+0000","last_fullsized":"2023-10-09T10:10:15.142807+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":111,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445088+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:10:22.445088+0000","last_peered":"2023-10-09T10:10:22.445088+0000","last_clean":"2023-10-09T10:10:22.445088+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:10:22.445088+0000","last_undegraded":"2023-10-09T10:10:22.445088+0000","last_fullsized":"2023-10-09T10:10:22.445088+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":206,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.143597+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:10:15.143597+0000","last_peered":"2023-10-09T10:10:15.143597+0000","last_clean":"2023-10-09T10:10:15.143597+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:10:15.143597+0000","last_undegraded":"2023-10-09T10:10:15.143597+0000","last_fullsized":"2023-10-09T10:10:15.143597+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":331,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.165284+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:10:15.165284+0000","last_peered":"2023-10-09T10:10:15.165284+0000","last_clean":"2023-10-09T10:10:15.165284+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:10:15.165284+0000","last_undegraded":"2023-10-09T10:10:15.165284+0000","last_fullsized":"2023-10-09T10:10:15.165284+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":287,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171621+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:10:15.171621+0000","last_peered":"2023-10-09T10:10:15.171621+0000","last_clean":"2023-10-09T10:10:15.171621+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:10:15.171621+0000","last_undegraded":"2023-10-09T10:10:15.171621+0000","last_fullsized":"2023-10-09T10:10:15.171621+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":180,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.165534+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:10:15.165534+0000","last_peered":"2023-10-09T10:10:15.165534+0000","last_clean":"2023-10-09T10:10:15.165534+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:10:15.165534+0000","last_undegraded":"2023-10-09T10:10:15.165534+0000","last_fullsized":"2023-10-09T10:10:15.165534+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":159,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171742+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:10:15.171742+0000","last_peered":"2023-10-09T10:10:15.171742+0000","last_clean":"2023-10-09T10:10:15.171742+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:10:15.171742+0000","last_undegraded":"2023-10-09T10:10:15.171742+0000","last_fullsized":"2023-10-09T10:10:15.171742+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":364,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.165741+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:10:15.165741+0000","last_peered":"2023-10-09T10:10:15.165741+0000","last_clean":"2023-10-09T10:10:15.165741+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:10:15.165741+0000","last_undegraded":"2023-10-09T10:10:15.165741+0000","last_fullsized":"2023-10-09T10:10:15.165741+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":232,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.141100+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:10:15.141100+0000","last_peered":"2023-10-09T10:10:15.141100+0000","last_clean":"2023-10-09T10:10:15.141100+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:10:15.141100+0000","last_undegraded":"2023-10-09T10:10:15.141100+0000","last_fullsized":"2023-10-09T10:10:15.141100+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":159,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445746+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:10:22.445746+0000","last_peered":"2023-10-09T10:10:22.445746+0000","last_clean":"2023-10-09T10:10:22.445746+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:10:22.445746+0000","last_undegraded":"2023-10-09T10:10:22.445746+0000","last_fullsized":"2023-10-09T10:10:22.445746+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":127,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445044+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:10:22.445044+0000","last_peered":"2023-10-09T10:10:22.445044+0000","last_clean":"2023-10-09T10:10:22.445044+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:10:22.445044+0000","last_undegraded":"2023-10-09T10:10:22.445044+0000","last_fullsized":"2023-10-09T10:10:22.445044+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":161,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.164755+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:10:15.164755+0000","last_peered":"2023-10-09T10:10:15.164755+0000","last_clean":"2023-10-09T10:10:15.164755+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:10:15.164755+0000","last_undegraded":"2023-10-09T10:10:15.164755+0000","last_fullsized":"2023-10-09T10:10:15.164755+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":159,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.446079+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:10:22.446079+0000","last_peered":"2023-10-09T10:10:22.446079+0000","last_clean":"2023-10-09T10:10:22.446079+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:10:22.446079+0000","last_undegraded":"2023-10-09T10:10:22.446079+0000","last_fullsized":"2023-10-09T10:10:22.446079+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":182,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445543+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:10:22.445543+0000","last_peered":"2023-10-09T10:10:22.445543+0000","last_clean":"2023-10-09T10:10:22.445543+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:10:22.445543+0000","last_undegraded":"2023-10-09T10:10:22.445543+0000","last_fullsized":"2023-10-09T10:10:22.445543+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":245,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.152500+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:10:15.152500+0000","last_peered":"2023-10-09T10:10:15.152500+0000","last_clean":"2023-10-09T10:10:15.152500+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:10:15.152500+0000","last_undegraded":"2023-10-09T10:10:15.152500+0000","last_fullsized":"2023-10-09T10:10:15.152500+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"294'7309","reported_seq":7348,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:32.824830+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:10:32.824830+0000","last_peered":"2023-10-09T10:10:32.824830+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:10:32.824830+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7309,"log_dups_size":0,"ondisk_log_size":7309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416612352,"num_objects":6357,"num_object_clones":0,"num_object_copies":19071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7309,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":984},{"shards":"4,5,6","objects":4897}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":325,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.165088+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:10:15.165088+0000","last_peered":"2023-10-09T10:10:15.165088+0000","last_clean":"2023-10-09T10:10:15.165088+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:10:15.165088+0000","last_undegraded":"2023-10-09T10:10:15.165088+0000","last_fullsized":"2023-10-09T10:10:15.165088+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":112,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444715+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:10:22.444715+0000","last_peered":"2023-10-09T10:10:22.444715+0000","last_clean":"2023-10-09T10:10:22.444715+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:10:22.444715+0000","last_undegraded":"2023-10-09T10:10:22.444715+0000","last_fullsized":"2023-10-09T10:10:22.444715+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"293'7213","reported_seq":7243,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:29.932285+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:10:29.932285+0000","last_peered":"2023-10-09T10:10:29.932285+0000","last_clean":"2023-10-09T10:10:29.932285+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:10:29.932285+0000","last_undegraded":"2023-10-09T10:10:29.932285+0000","last_fullsized":"2023-10-09T10:10:29.932285+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7213,"log_dups_size":0,"ondisk_log_size":7213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431030272,"num_objects":6577,"num_object_clones":0,"num_object_copies":19731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7213,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":313,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.165434+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:10:15.165434+0000","last_peered":"2023-10-09T10:10:15.165434+0000","last_clean":"2023-10-09T10:10:15.165434+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:10:15.165434+0000","last_undegraded":"2023-10-09T10:10:15.165434+0000","last_fullsized":"2023-10-09T10:10:15.165434+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":203,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170537+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:10:15.170537+0000","last_peered":"2023-10-09T10:10:15.170537+0000","last_clean":"2023-10-09T10:10:15.170537+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:10:15.170537+0000","last_undegraded":"2023-10-09T10:10:15.170537+0000","last_fullsized":"2023-10-09T10:10:15.170537+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"294'7257","reported_seq":7289,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.694663+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:10:32.694663+0000","last_peered":"2023-10-09T10:10:32.694663+0000","last_clean":"2023-10-09T10:10:32.694663+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:10:32.694663+0000","last_undegraded":"2023-10-09T10:10:32.694663+0000","last_fullsized":"2023-10-09T10:10:32.694663+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7257,"log_dups_size":0,"ondisk_log_size":7257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413466624,"num_objects":6309,"num_object_clones":0,"num_object_copies":18927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7257,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":116,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445127+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:10:22.445127+0000","last_peered":"2023-10-09T10:10:22.445127+0000","last_clean":"2023-10-09T10:10:22.445127+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:10:22.445127+0000","last_undegraded":"2023-10-09T10:10:22.445127+0000","last_fullsized":"2023-10-09T10:10:22.445127+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":136,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.146249+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:10:15.146249+0000","last_peered":"2023-10-09T10:10:15.146249+0000","last_clean":"2023-10-09T10:10:15.146249+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:10:15.146249+0000","last_undegraded":"2023-10-09T10:10:15.146249+0000","last_fullsized":"2023-10-09T10:10:15.146249+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"293'7049","reported_seq":7080,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:29.504932+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:10:29.504932+0000","last_peered":"2023-10-09T10:10:29.504932+0000","last_clean":"2023-10-09T10:10:29.504932+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:10:29.504932+0000","last_undegraded":"2023-10-09T10:10:29.504932+0000","last_fullsized":"2023-10-09T10:10:29.504932+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7049,"log_dups_size":0,"ondisk_log_size":7049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416874496,"num_objects":6361,"num_object_clones":0,"num_object_copies":19083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7049,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":127,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445117+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:10:22.445117+0000","last_peered":"2023-10-09T10:10:22.445117+0000","last_clean":"2023-10-09T10:10:22.445117+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:10:22.445117+0000","last_undegraded":"2023-10-09T10:10:22.445117+0000","last_fullsized":"2023-10-09T10:10:22.445117+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":160,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445981+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:10:22.445981+0000","last_peered":"2023-10-09T10:10:22.445981+0000","last_clean":"2023-10-09T10:10:22.445981+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:10:22.445981+0000","last_undegraded":"2023-10-09T10:10:22.445981+0000","last_fullsized":"2023-10-09T10:10:22.445981+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"293'7305","reported_seq":7336,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:31.170097+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:10:31.170097+0000","last_peered":"2023-10-09T10:10:31.170097+0000","last_clean":"2023-10-09T10:10:31.170097+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:10:31.170097+0000","last_undegraded":"2023-10-09T10:10:31.170097+0000","last_fullsized":"2023-10-09T10:10:31.170097+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7305,"log_dups_size":0,"ondisk_log_size":7305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428277760,"num_objects":6535,"num_object_clones":0,"num_object_copies":19605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7305,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":286,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.143130+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:10:15.143130+0000","last_peered":"2023-10-09T10:10:15.143130+0000","last_clean":"2023-10-09T10:10:15.143130+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:10:15.143130+0000","last_undegraded":"2023-10-09T10:10:15.143130+0000","last_fullsized":"2023-10-09T10:10:15.143130+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.140938+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:10:15.140938+0000","last_peered":"2023-10-09T10:10:15.140938+0000","last_clean":"2023-10-09T10:10:15.140938+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:10:15.140938+0000","last_undegraded":"2023-10-09T10:10:15.140938+0000","last_fullsized":"2023-10-09T10:10:15.140938+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"293'7182","reported_seq":7213,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:29.495825+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:10:29.495825+0000","last_peered":"2023-10-09T10:10:29.495825+0000","last_clean":"2023-10-09T10:10:29.495825+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:10:29.495825+0000","last_undegraded":"2023-10-09T10:10:29.495825+0000","last_fullsized":"2023-10-09T10:10:29.495825+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7182,"log_dups_size":0,"ondisk_log_size":7182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427425792,"num_objects":6522,"num_object_clones":0,"num_object_copies":19566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7182,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":217,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171540+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:10:15.171540+0000","last_peered":"2023-10-09T10:10:15.171540+0000","last_clean":"2023-10-09T10:10:15.171540+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:10:15.171540+0000","last_undegraded":"2023-10-09T10:10:15.171540+0000","last_fullsized":"2023-10-09T10:10:15.171540+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":223,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170984+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:10:15.170984+0000","last_peered":"2023-10-09T10:10:15.170984+0000","last_clean":"2023-10-09T10:10:15.170984+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:10:15.170984+0000","last_undegraded":"2023-10-09T10:10:15.170984+0000","last_fullsized":"2023-10-09T10:10:15.170984+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"294'7184","reported_seq":7216,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.713874+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:10:32.713874+0000","last_peered":"2023-10-09T10:10:32.713874+0000","last_clean":"2023-10-09T10:10:32.713874+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:10:32.713874+0000","last_undegraded":"2023-10-09T10:10:32.713874+0000","last_fullsized":"2023-10-09T10:10:32.713874+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7184,"log_dups_size":0,"ondisk_log_size":7184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409468928,"num_objects":6248,"num_object_clones":0,"num_object_copies":18744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7184,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":187,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.164572+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:10:15.164572+0000","last_peered":"2023-10-09T10:10:15.164572+0000","last_clean":"2023-10-09T10:10:15.164572+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:10:15.164572+0000","last_undegraded":"2023-10-09T10:10:15.164572+0000","last_fullsized":"2023-10-09T10:10:15.164572+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170755+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:10:15.170755+0000","last_peered":"2023-10-09T10:10:15.170755+0000","last_clean":"2023-10-09T10:10:15.170755+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:10:15.170755+0000","last_undegraded":"2023-10-09T10:10:15.170755+0000","last_fullsized":"2023-10-09T10:10:15.170755+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"293'7125","reported_seq":7156,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:29.511511+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:10:29.511511+0000","last_peered":"2023-10-09T10:10:29.511511+0000","last_clean":"2023-10-09T10:10:29.511511+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:10:29.511511+0000","last_undegraded":"2023-10-09T10:10:29.511511+0000","last_fullsized":"2023-10-09T10:10:29.511511+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7125,"log_dups_size":0,"ondisk_log_size":7125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424214528,"num_objects":6473,"num_object_clones":0,"num_object_copies":19419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7125,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":246,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.142708+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:10:15.142708+0000","last_peered":"2023-10-09T10:10:15.142708+0000","last_clean":"2023-10-09T10:10:15.142708+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:10:15.142708+0000","last_undegraded":"2023-10-09T10:10:15.142708+0000","last_fullsized":"2023-10-09T10:10:15.142708+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":296,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171396+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:10:15.171396+0000","last_peered":"2023-10-09T10:10:15.171396+0000","last_clean":"2023-10-09T10:10:15.171396+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:10:15.171396+0000","last_undegraded":"2023-10-09T10:10:15.171396+0000","last_fullsized":"2023-10-09T10:10:15.171396+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"294'7218","reported_seq":7250,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.697971+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:10:32.697971+0000","last_peered":"2023-10-09T10:10:32.697971+0000","last_clean":"2023-10-09T10:10:32.697971+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:10:32.697971+0000","last_undegraded":"2023-10-09T10:10:32.697971+0000","last_fullsized":"2023-10-09T10:10:32.697971+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7218,"log_dups_size":0,"ondisk_log_size":7218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409862144,"num_objects":6254,"num_object_clones":0,"num_object_copies":18762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7218,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":295,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170656+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:10:15.170656+0000","last_peered":"2023-10-09T10:10:15.170656+0000","last_clean":"2023-10-09T10:10:15.170656+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:10:15.170656+0000","last_undegraded":"2023-10-09T10:10:15.170656+0000","last_fullsized":"2023-10-09T10:10:15.170656+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445036+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:10:22.445036+0000","last_peered":"2023-10-09T10:10:22.445036+0000","last_clean":"2023-10-09T10:10:22.445036+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:10:22.445036+0000","last_undegraded":"2023-10-09T10:10:22.445036+0000","last_fullsized":"2023-10-09T10:10:22.445036+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":251,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171833+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:10:15.171833+0000","last_peered":"2023-10-09T10:10:15.171833+0000","last_clean":"2023-10-09T10:10:15.171833+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:10:15.171833+0000","last_undegraded":"2023-10-09T10:10:15.171833+0000","last_fullsized":"2023-10-09T10:10:15.171833+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"294'7284","reported_seq":7315,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.810759+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:10:32.810759+0000","last_peered":"2023-10-09T10:10:32.810759+0000","last_clean":"2023-10-09T10:10:32.810759+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:10:32.810759+0000","last_undegraded":"2023-10-09T10:10:32.810759+0000","last_fullsized":"2023-10-09T10:10:32.810759+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7284,"log_dups_size":0,"ondisk_log_size":7284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412221440,"num_objects":6290,"num_object_clones":0,"num_object_copies":18870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7284,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":112,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444749+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:10:22.444749+0000","last_peered":"2023-10-09T10:10:22.444749+0000","last_clean":"2023-10-09T10:10:22.444749+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:10:22.444749+0000","last_undegraded":"2023-10-09T10:10:22.444749+0000","last_fullsized":"2023-10-09T10:10:22.444749+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":110,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:27.734802+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:10:27.734802+0000","last_peered":"2023-10-09T10:10:27.734802+0000","last_clean":"2023-10-09T10:10:27.734802+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:10:27.734802+0000","last_undegraded":"2023-10-09T10:10:27.734802+0000","last_fullsized":"2023-10-09T10:10:27.734802+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":318,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171510+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:10:15.171510+0000","last_peered":"2023-10-09T10:10:15.171510+0000","last_clean":"2023-10-09T10:10:15.171510+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:10:15.171510+0000","last_undegraded":"2023-10-09T10:10:15.171510+0000","last_fullsized":"2023-10-09T10:10:15.171510+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"293'7194","reported_seq":7225,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:31.051409+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:10:31.051409+0000","last_peered":"2023-10-09T10:10:31.051409+0000","last_clean":"2023-10-09T10:10:31.051409+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:10:31.051409+0000","last_undegraded":"2023-10-09T10:10:31.051409+0000","last_fullsized":"2023-10-09T10:10:31.051409+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7194,"log_dups_size":0,"ondisk_log_size":7194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416022528,"num_objects":6348,"num_object_clones":0,"num_object_copies":19044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7194,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":212,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.143484+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:10:15.143484+0000","last_peered":"2023-10-09T10:10:15.143484+0000","last_clean":"2023-10-09T10:10:15.143484+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:10:15.143484+0000","last_undegraded":"2023-10-09T10:10:15.143484+0000","last_fullsized":"2023-10-09T10:10:15.143484+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":287,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171069+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:10:15.171069+0000","last_peered":"2023-10-09T10:10:15.171069+0000","last_clean":"2023-10-09T10:10:15.171069+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:10:15.171069+0000","last_undegraded":"2023-10-09T10:10:15.171069+0000","last_fullsized":"2023-10-09T10:10:15.171069+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":308,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.171298+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:10:15.171298+0000","last_peered":"2023-10-09T10:10:15.171298+0000","last_clean":"2023-10-09T10:10:15.171298+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:10:15.171298+0000","last_undegraded":"2023-10-09T10:10:15.171298+0000","last_fullsized":"2023-10-09T10:10:15.171298+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"294'7183","reported_seq":7214,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.800009+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:10:32.800009+0000","last_peered":"2023-10-09T10:10:32.800009+0000","last_clean":"2023-10-09T10:10:32.800009+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:10:32.800009+0000","last_undegraded":"2023-10-09T10:10:32.800009+0000","last_fullsized":"2023-10-09T10:10:32.800009+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7183,"log_dups_size":0,"ondisk_log_size":7183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411369472,"num_objects":6277,"num_object_clones":0,"num_object_copies":18831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7183,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":121,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444837+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:10:22.444837+0000","last_peered":"2023-10-09T10:10:22.444837+0000","last_clean":"2023-10-09T10:10:22.444837+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:10:22.444837+0000","last_undegraded":"2023-10-09T10:10:22.444837+0000","last_fullsized":"2023-10-09T10:10:22.444837+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":539,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.165841+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:10:15.165841+0000","last_peered":"2023-10-09T10:10:15.165841+0000","last_clean":"2023-10-09T10:10:15.165841+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:10:15.165841+0000","last_undegraded":"2023-10-09T10:10:15.165841+0000","last_fullsized":"2023-10-09T10:10:15.165841+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":196,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.143385+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:10:15.143385+0000","last_peered":"2023-10-09T10:10:15.143385+0000","last_clean":"2023-10-09T10:10:15.143385+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:10:15.143385+0000","last_undegraded":"2023-10-09T10:10:15.143385+0000","last_fullsized":"2023-10-09T10:10:15.143385+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":208,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.165348+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:10:15.165348+0000","last_peered":"2023-10-09T10:10:15.165348+0000","last_clean":"2023-10-09T10:10:15.165348+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:10:15.165348+0000","last_undegraded":"2023-10-09T10:10:15.165348+0000","last_fullsized":"2023-10-09T10:10:15.165348+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"293'7183","reported_seq":7214,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:29.538864+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:10:29.538864+0000","last_peered":"2023-10-09T10:10:29.538864+0000","last_clean":"2023-10-09T10:10:29.538864+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:10:29.538864+0000","last_undegraded":"2023-10-09T10:10:29.538864+0000","last_fullsized":"2023-10-09T10:10:29.538864+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7183,"log_dups_size":0,"ondisk_log_size":7183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421986304,"num_objects":6439,"num_object_clones":0,"num_object_copies":19317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7183,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":268,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434362+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:10:22.434362+0000","last_peered":"2023-10-09T10:10:22.434362+0000","last_clean":"2023-10-09T10:10:22.434362+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:10:22.434362+0000","last_undegraded":"2023-10-09T10:10:22.434362+0000","last_fullsized":"2023-10-09T10:10:22.434362+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":170,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445584+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:10:22.445584+0000","last_peered":"2023-10-09T10:10:22.445584+0000","last_clean":"2023-10-09T10:10:22.445584+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:10:22.445584+0000","last_undegraded":"2023-10-09T10:10:22.445584+0000","last_fullsized":"2023-10-09T10:10:22.445584+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":198,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445990+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:10:22.445990+0000","last_peered":"2023-10-09T10:10:22.445990+0000","last_clean":"2023-10-09T10:10:22.445990+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:10:22.445990+0000","last_undegraded":"2023-10-09T10:10:22.445990+0000","last_fullsized":"2023-10-09T10:10:22.445990+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"293'7232","reported_seq":9020,"reported_epoch":293,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:31.165820+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:10:31.165820+0000","last_peered":"2023-10-09T10:10:31.165820+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:10:31.165820+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7232,"log_dups_size":0,"ondisk_log_size":7232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421134336,"num_objects":6426,"num_object_clones":0,"num_object_copies":19278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":4408,"num_objects_unfound":0,"num_objects_dirty":6426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7232,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":875,"num_bytes_recovered":57344000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":2017},{"shards":"1,3,4","objects":4408}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":312,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.165099+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:10:15.165099+0000","last_peered":"2023-10-09T10:10:15.165099+0000","last_clean":"2023-10-09T10:10:15.165099+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:10:15.165099+0000","last_undegraded":"2023-10-09T10:10:15.165099+0000","last_fullsized":"2023-10-09T10:10:15.165099+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":324,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170744+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:10:15.170744+0000","last_peered":"2023-10-09T10:10:15.170744+0000","last_clean":"2023-10-09T10:10:15.170744+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:10:15.170744+0000","last_undegraded":"2023-10-09T10:10:15.170744+0000","last_fullsized":"2023-10-09T10:10:15.170744+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445861+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:10:22.445861+0000","last_peered":"2023-10-09T10:10:22.445861+0000","last_clean":"2023-10-09T10:10:22.445861+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:10:22.445861+0000","last_undegraded":"2023-10-09T10:10:22.445861+0000","last_fullsized":"2023-10-09T10:10:22.445861+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"294'7246","reported_seq":7278,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.689467+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:10:32.689467+0000","last_peered":"2023-10-09T10:10:32.689467+0000","last_clean":"2023-10-09T10:10:32.689467+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:10:32.689467+0000","last_undegraded":"2023-10-09T10:10:32.689467+0000","last_fullsized":"2023-10-09T10:10:32.689467+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7246,"log_dups_size":0,"ondisk_log_size":7246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413663232,"num_objects":6312,"num_object_clones":0,"num_object_copies":18936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7246,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":255,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.152630+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:10:15.152630+0000","last_peered":"2023-10-09T10:10:15.152630+0000","last_clean":"2023-10-09T10:10:15.152630+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:10:15.152630+0000","last_undegraded":"2023-10-09T10:10:15.152630+0000","last_fullsized":"2023-10-09T10:10:15.152630+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":269,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170825+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:10:15.170825+0000","last_peered":"2023-10-09T10:10:15.170825+0000","last_clean":"2023-10-09T10:10:15.170825+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:10:15.170825+0000","last_undegraded":"2023-10-09T10:10:15.170825+0000","last_fullsized":"2023-10-09T10:10:15.170825+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":244,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.142592+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:10:15.142592+0000","last_peered":"2023-10-09T10:10:15.142592+0000","last_clean":"2023-10-09T10:10:15.142592+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:10:15.142592+0000","last_undegraded":"2023-10-09T10:10:15.142592+0000","last_fullsized":"2023-10-09T10:10:15.142592+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"294'7204","reported_seq":7236,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.860895+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:10:32.860895+0000","last_peered":"2023-10-09T10:10:32.860895+0000","last_clean":"2023-10-09T10:10:32.860895+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:10:32.860895+0000","last_undegraded":"2023-10-09T10:10:32.860895+0000","last_fullsized":"2023-10-09T10:10:32.860895+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7204,"log_dups_size":0,"ondisk_log_size":7204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410583064,"num_objects":6266,"num_object_clones":0,"num_object_copies":18798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6266,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7204,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":201,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.146676+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:10:15.146676+0000","last_peered":"2023-10-09T10:10:15.146676+0000","last_clean":"2023-10-09T10:10:15.146676+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:10:15.146676+0000","last_undegraded":"2023-10-09T10:10:15.146676+0000","last_fullsized":"2023-10-09T10:10:15.146676+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":164,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445916+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:10:22.445916+0000","last_peered":"2023-10-09T10:10:22.445916+0000","last_clean":"2023-10-09T10:10:22.445916+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:10:22.445916+0000","last_undegraded":"2023-10-09T10:10:22.445916+0000","last_fullsized":"2023-10-09T10:10:22.445916+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":147,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444656+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:10:22.444656+0000","last_peered":"2023-10-09T10:10:22.444656+0000","last_clean":"2023-10-09T10:10:22.444656+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:10:22.444656+0000","last_undegraded":"2023-10-09T10:10:22.444656+0000","last_fullsized":"2023-10-09T10:10:22.444656+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":159,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445692+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:10:22.445692+0000","last_peered":"2023-10-09T10:10:22.445692+0000","last_clean":"2023-10-09T10:10:22.445692+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:10:22.445692+0000","last_undegraded":"2023-10-09T10:10:22.445692+0000","last_fullsized":"2023-10-09T10:10:22.445692+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":230,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170824+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:10:15.170824+0000","last_peered":"2023-10-09T10:10:15.170824+0000","last_clean":"2023-10-09T10:10:15.170824+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:10:15.170824+0000","last_undegraded":"2023-10-09T10:10:15.170824+0000","last_fullsized":"2023-10-09T10:10:15.170824+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":131,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445064+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:10:22.445064+0000","last_peered":"2023-10-09T10:10:22.445064+0000","last_clean":"2023-10-09T10:10:22.445064+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:10:22.445064+0000","last_undegraded":"2023-10-09T10:10:22.445064+0000","last_fullsized":"2023-10-09T10:10:22.445064+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":171,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.143027+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:10:15.143027+0000","last_peered":"2023-10-09T10:10:15.143027+0000","last_clean":"2023-10-09T10:10:15.143027+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:10:15.143027+0000","last_undegraded":"2023-10-09T10:10:15.143027+0000","last_fullsized":"2023-10-09T10:10:15.143027+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":131,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444954+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:10:22.444954+0000","last_peered":"2023-10-09T10:10:22.444954+0000","last_clean":"2023-10-09T10:10:22.444954+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:10:22.444954+0000","last_undegraded":"2023-10-09T10:10:22.444954+0000","last_fullsized":"2023-10-09T10:10:22.444954+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.143289+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:10:15.143289+0000","last_peered":"2023-10-09T10:10:15.143289+0000","last_clean":"2023-10-09T10:10:15.143289+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:10:15.143289+0000","last_undegraded":"2023-10-09T10:10:15.143289+0000","last_fullsized":"2023-10-09T10:10:15.143289+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":309,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.141373+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:10:15.141373+0000","last_peered":"2023-10-09T10:10:15.141373+0000","last_clean":"2023-10-09T10:10:15.141373+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:10:15.141373+0000","last_undegraded":"2023-10-09T10:10:15.141373+0000","last_fullsized":"2023-10-09T10:10:15.141373+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":330,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.143751+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:10:15.143751+0000","last_peered":"2023-10-09T10:10:15.143751+0000","last_clean":"2023-10-09T10:10:15.143751+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:10:15.143751+0000","last_undegraded":"2023-10-09T10:10:15.143751+0000","last_fullsized":"2023-10-09T10:10:15.143751+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":240,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445738+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:10:22.445738+0000","last_peered":"2023-10-09T10:10:22.445738+0000","last_clean":"2023-10-09T10:10:22.445738+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:10:22.445738+0000","last_undegraded":"2023-10-09T10:10:22.445738+0000","last_fullsized":"2023-10-09T10:10:22.445738+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":159,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T10:10:15.170374+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:10:15.170374+0000","last_peered":"2023-10-09T10:10:15.170374+0000","last_clean":"2023-10-09T10:10:15.170374+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:10:15.170374+0000","last_undegraded":"2023-10-09T10:10:15.170374+0000","last_fullsized":"2023-10-09T10:10:15.170374+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6684213272,"num_objects":101994,"num_object_clones":0,"num_object_copies":305982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":4408,"num_objects_unfound":0,"num_objects_dirty":101994,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115368,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":875,"num_bytes_recovered":57344000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20177367040,"data_stored":20177354824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115368,"ondisk_log_size":115368,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430490,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2381976,"kb_used_data":2076560,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91363176,"statfs":{"total":95995035648,"available":93555892224,"internally_reserved":0,"allocated":2126397440,"data_stored":2125860791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5940000000000001,"15min":1.5740000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":5.625,"5min":9.0690000000000008,"15min":50.591999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.548,"5min":1.8,"15min":1.6040000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":6.0670000000000002,"5min":8.2300000000000004,"15min":50.747},"last":0.38300000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:08 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":1.8440000000000001,"15min":1.5169999999999999},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2669999999999995,"5min":16.352,"15min":25.338000000000001},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.53,"5min":1.8839999999999999,"15min":1.3560000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.9269999999999996,"5min":18.27,"15min":18.27},"last":0.58999999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.7190000000000001,"15min":2.266},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":3.8370000000000002,"5min":7.952,"15min":7.952},"last":0.496},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.7310000000000001,"15min":2.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.2039999999999997,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.77600000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:47 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.0049999999999999,"15min":1.835},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.058,"5min":29.058,"15min":29.058},"last":2.6240000000000001},{"interface":"front","average":{"1min":1.214,"5min":1.7989999999999999,"15min":1.897},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.577,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":0.52400000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1130000000000004,"5min":4.1130000000000004,"15min":4.1130000000000004},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.51000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.149,"15min":0.93899999999999995},"min":{"1min":0.31900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":25.277999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.4529999999999998,"5min":1.2589999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.754000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.432}]},{"osd":6,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.1299999999999999,"15min":0.88600000000000001},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5369999999999999,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.98599999999999999,"15min":0.82499999999999996},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.998,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":0.48299999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626718,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2720584,"kb_used_data":2415168,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91024568,"statfs":{"total":95995035648,"available":93209157632,"internally_reserved":0,"allocated":2473132032,"data_stored":2472612697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.9259999999999999,"15min":1.319},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":15.435,"5min":15.435,"15min":15.435},"last":3.4220000000000002},{"interface":"front","average":{"1min":1.853,"5min":1.931,"15min":1.365},"min":{"1min":0.252,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.308999999999999,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:10:02 2023","interfaces":[{"interface":"back","average":{"1min":4.641,"5min":2.4300000000000002,"15min":1.7130000000000001},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":35.609999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.3540000000000001,"15min":1.6319999999999999},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":32.396000000000001,"5min":32.396000000000001,"15min":32.396000000000001},"last":5.8979999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.3340000000000001,"15min":1.6919999999999999},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8149999999999999,"5min":7.6760000000000002,"15min":7.6760000000000002},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.234,"5min":2.3239999999999998,"15min":3.0990000000000002},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.5600000000000005,"5min":10.26,"15min":10.26},"last":0.47799999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.321,"15min":1.347},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3879999999999999,"5min":6.0999999999999996,"15min":6.0999999999999996},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.4220000000000002,"15min":2.3940000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":11.211,"5min":11.211,"15min":11.211},"last":0.41899999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.48,"5min":1.48,"15min":1.48},"last":0.33200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.71999999999999997,"15min":0.72199999999999998},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.611,"5min":7.7080000000000002,"15min":14.09},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.87,"15min":0.77300000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":2.464,"5min":7.508,"15min":7.508},"last":0.39000000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.77000000000000002,"15min":0.755},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.5800000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.81499999999999995,"15min":0.80500000000000005},"min":{"1min":0.39000000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.6080000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.51400000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253031,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3284772,"kb_used_data":2905116,"kb_used_omap":9,"kb_used_meta":379574,"kb_avail":90460380,"statfs":{"total":95995035648,"available":92631429120,"internally_reserved":0,"allocated":2974838784,"data_stored":2973819861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":388684777},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":2.6549999999999998,"15min":2.0299999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.185},"max":{"1min":6.3479999999999999,"5min":25.59,"15min":25.59},"last":4.3730000000000002},{"interface":"front","average":{"1min":3.7109999999999999,"5min":2.77,"15min":1.976},"min":{"1min":0.26900000000000002,"5min":0.218,"15min":0.192},"max":{"1min":11.682,"5min":14.769,"15min":16.629000000000001},"last":0.314}]},{"osd":2,"last update":"Mon Oct 9 10:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":3.2160000000000002,"15min":3.8700000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.1539999999999999,"5min":17.369,"15min":17.369},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.907,"15min":3.8740000000000001},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5960000000000001,"5min":16.023,"15min":16.023},"last":8.7940000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":3.1619999999999999,"15min":3.4100000000000001},"min":{"1min":0.36399999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.196999999999999,"5min":16.931000000000001,"15min":16.931000000000001},"last":7.3109999999999999},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.266,"15min":4.6900000000000004},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.8569999999999993,"5min":17.289000000000001,"15min":17.289000000000001},"last":8.3290000000000006}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3860000000000001,"5min":5.3860000000000001,"15min":5.3860000000000001},"last":6.9370000000000003},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":10.132,"5min":10.132,"15min":10.132},"last":0.35399999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":2.0419999999999998,"15min":1.72},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.473000000000001,"5min":14.416,"15min":15.531000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.782,"5min":2.0979999999999999,"15min":1.9690000000000001},"min":{"1min":0.34699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.6890000000000001,"5min":16.277999999999999,"15min":19.706},"last":0.443}]},{"osd":6,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0339999999999998,"15min":1.7230000000000001},"min":{"1min":0.50800000000000001,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.3710000000000004,"5min":14.215,"15min":16.193999999999999},"last":7.0030000000000001},{"interface":"front","average":{"1min":2.75,"5min":1.9930000000000001,"15min":1.7529999999999999},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":10.161,"5min":14.471,"15min":15.481999999999999},"last":0.95199999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":2.153,"15min":1.972},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.7350000000000003,"5min":16.331,"15min":16.331},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1320000000000001,"15min":1.788},"min":{"1min":0.46000000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3319999999999999,"5min":14.242000000000001,"15min":16.100999999999999},"last":7.0369999999999999}]}]},{"osd":0,"up_from":79,"seq":339302416558,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2789332,"kb_used_data":2436300,"kb_used_omap":28,"kb_used_meta":352803,"kb_avail":90955820,"statfs":{"total":95995035648,"available":93138759680,"internally_reserved":0,"allocated":2494771200,"data_stored":2493766208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":361271120},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.847,"15min":1.4970000000000001},"min":{"1min":0.24099999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.2990000000000004,"5min":12.84,"15min":13.084},"last":0.23100000000000001},{"interface":"front","average":{"1min":3.9020000000000001,"5min":3.379,"15min":2.2269999999999999},"min":{"1min":0.44900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.949000000000002,"5min":22.178999999999998,"15min":22.178999999999998},"last":1.6639999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.7210000000000001,"15min":2.121},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.5579999999999998,"5min":10.034000000000001,"15min":10.034000000000001},"last":1.2749999999999999},{"interface":"front","average":{"1min":1.353,"5min":2.2879999999999998,"15min":3.0910000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.4480000000000004,"5min":14.693,"15min":14.693},"last":0.56799999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4990000000000001,"5min":4.391,"15min":4.7069999999999999},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.867000000000001,"5min":25.734999999999999,"15min":25.734999999999999},"last":2.1099999999999999},{"interface":"front","average":{"1min":2.4809999999999999,"5min":4.0250000000000004,"15min":4.282},"min":{"1min":0.44,"5min":0.254,"15min":0.254},"max":{"1min":13.975,"5min":21.923999999999999,"15min":21.923999999999999},"last":0.98999999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.3760000000000003,"5min":4.3760000000000003,"15min":4.3760000000000003},"last":1.6240000000000001},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":13.789999999999999,"5min":13.789999999999999,"15min":13.789999999999999},"last":0.52600000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":2.8149999999999999,"15min":1.847},"min":{"1min":0.38900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.736000000000001,"5min":17.096,"15min":17.096},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.419,"15min":1.2470000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.4870000000000001,"5min":12.478,"15min":12.478},"last":0.501}]},{"osd":6,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.494,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5289999999999999,"5min":9.7260000000000009,"15min":11.699},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.6549999999999998,"15min":1.954},"min":{"1min":0.45100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.4130000000000003,"5min":14.384,"15min":15.108000000000001},"last":1.71}]},{"osd":7,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.9929999999999999,"15min":1.986},"min":{"1min":0.51400000000000001,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":13.829000000000001,"5min":14.631,"15min":26.978000000000002},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.085,"5min":1.6819999999999999,"15min":1.544},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.5209999999999999,"5min":12.428000000000001,"15min":26.768000000000001},"last":0.54100000000000004}]}]},{"osd":2,"up_from":217,"seq":932007903291,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2112328,"kb_used_data":1885632,"kb_used_omap":83,"kb_used_meta":226604,"kb_avail":91632824,"statfs":{"total":95995035648,"available":93832011776,"internally_reserved":0,"allocated":1930887168,"data_stored":1929894153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":232042991},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.28,"15min":1.6060000000000001},"min":{"1min":0.247,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.5139999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":4.3079999999999998},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.8849999999999998,"15min":2.923},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.891999999999999,"5min":16.309999999999999,"15min":16.309999999999999},"last":6.165}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.0209999999999999,"15min":2.4649999999999999},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.971,"5min":11.971,"15min":11.971},"last":4.476},{"interface":"front","average":{"1min":2.8849999999999998,"5min":2.492,"15min":2.6280000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.009,"5min":15.569000000000001,"15min":15.569000000000001},"last":4.5549999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.798,"15min":1.948},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.8859999999999999,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":9.6419999999999995},{"interface":"front","average":{"1min":1.1779999999999999,"5min":3.367,"15min":3.7320000000000002},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.8440000000000003,"5min":15.811,"15min":15.811},"last":9.7240000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.633,"5min":2.633,"15min":2.633},"last":1.343},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.8939999999999999,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":0.46800000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.5580000000000001,"15min":1.502},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.0870000000000002,"5min":12.423999999999999,"15min":12.423999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.21,"5min":1.2789999999999999,"15min":1.6000000000000001},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.7709999999999999,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":0.435}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2629999999999999,"15min":1.0489999999999999},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.6040000000000001,"5min":15.292999999999999,"15min":15.292999999999999},"last":1.2749999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.6499999999999999,"15min":1.712},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.984,"5min":12.394,"15min":12.394},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.2809999999999999,"15min":1.0760000000000001},"min":{"1min":0.48499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1800000000000002,"5min":15.343999999999999,"15min":15.343999999999999},"last":1.3129999999999999},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.7170000000000001,"15min":1.613},"min":{"1min":0.51200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.5010000000000003,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.52300000000000002}]}]},{"osd":3,"up_from":250,"seq":1073741824035,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2708348,"kb_used_data":2462196,"kb_used_omap":10,"kb_used_meta":246133,"kb_avail":91036804,"statfs":{"total":95995035648,"available":93221687296,"internally_reserved":0,"allocated":2521288704,"data_stored":2520131065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":252040259},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":4.2489999999999997,"15min":4.5519999999999996},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.465,"5min":23.381,"15min":23.381},"last":3.048},{"interface":"front","average":{"1min":1.853,"5min":3.7130000000000001,"15min":4.0229999999999997},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.5190000000000001,"5min":14.289999999999999,"15min":14.289999999999999},"last":0.56799999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5329999999999999,"15min":3.8130000000000002},"min":{"1min":0.28799999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.415,"5min":20.407,"15min":20.407},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.702,"5min":3.2850000000000001,"15min":3.3820000000000001},"min":{"1min":0.40300000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.494,"5min":23.423999999999999,"15min":23.423999999999999},"last":3.0979999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":3.3410000000000002,"15min":3.6070000000000002},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.1020000000000003,"5min":20.356000000000002,"15min":20.356000000000002},"last":3.714},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.7290000000000001,"15min":2.819},"min":{"1min":0.34799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":9.3379999999999992,"5min":23.452000000000002,"15min":23.452000000000002},"last":2.9750000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":3.7570000000000001},"last":3.016},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.5739999999999998,"5min":8.5739999999999998,"15min":8.5739999999999998},"last":0.60199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.4319999999999999,"15min":2.5529999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.6199999999999992,"5min":11.836,"15min":11.836},"last":0.626},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.274,"15min":2.327},"min":{"1min":0.55600000000000005,"5min":0.309,"15min":0.309},"max":{"1min":6.9249999999999998,"5min":14.164999999999999,"15min":14.164999999999999},"last":0.64500000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.5379999999999998,"15min":2.6720000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.532,"5min":11.887,"15min":11.887},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.496,"5min":2.3820000000000001,"15min":2.5299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.8710000000000004,"5min":14.125,"15min":14.125},"last":0.61799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.4039999999999999,"15min":1.3939999999999999},"min":{"1min":0.45400000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0609999999999999,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":3.0720000000000001},{"interface":"front","average":{"1min":1.8089999999999999,"5min":2.5009999999999999,"15min":2.617},"min":{"1min":0.42199999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":8.6549999999999994,"5min":11.801,"15min":11.801},"last":0.69299999999999995}]}]},{"osd":4,"up_from":266,"seq":1142461300761,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2699028,"kb_used_data":2368012,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":91046124,"statfs":{"total":95995035648,"available":93231230976,"internally_reserved":0,"allocated":2424844288,"data_stored":2424319051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.3799999999999999,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":3.0950000000000002},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3959999999999999,"5min":2.3959999999999999,"15min":2.3959999999999999},"last":0.82899999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":2.7160000000000002,"5min":2.7160000000000002,"15min":2.7160000000000002},"last":3.335},{"interface":"front","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":3.4289999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.661,"5min":3.661,"15min":3.661},"last":5.7640000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6449999999999996,"5min":4.6449999999999996,"15min":4.6449999999999996},"last":3.129}]},{"osd":3,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.262,"15min":1.262},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.1259999999999999,"5min":3.1259999999999999,"15min":3.1259999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.4470000000000001,"5min":4.4470000000000001,"15min":4.4470000000000001},"last":3.2370000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.6970000000000001,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.33600000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.5609999999999999,"5min":2.5609999999999999,"15min":2.5609999999999999},"last":0.372},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.917,"5min":1.917,"15min":1.917},"last":1.429}]},{"osd":7,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.5630000000000002,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.40000000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822946,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3537800,"kb_used_data":3240576,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":90207352,"statfs":{"total":95995035648,"available":92372328448,"internally_reserved":0,"allocated":3318349824,"data_stored":3317832643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":1.55,"15min":1.488},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.8540000000000001,"5min":7.8540000000000001,"15min":15.773999999999999},"last":3.722},{"interface":"front","average":{"1min":1.339,"5min":1.597,"15min":1.532},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.0629999999999997,"5min":11.747,"15min":15.699},"last":1.1830000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:12 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.5740000000000001,"15min":1.4430000000000001},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.7720000000000002,"5min":11.335000000000001,"15min":15.689},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.913,"5min":1.861,"15min":1.6180000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.8119999999999994,"5min":19.402999999999999,"15min":19.402999999999999},"last":4.5979999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.8480000000000001,"15min":2.4369999999999998},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.3819999999999997,"5min":15.640000000000001,"15min":15.640000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.302,"5min":1.8220000000000001,"15min":2.2949999999999999},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.3659999999999997,"5min":10.656000000000001,"15min":10.656000000000001},"last":4.2610000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.734,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5860000000000003,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":4.5030000000000001},{"interface":"front","average":{"1min":1.2,"5min":2.6549999999999998,"15min":2.8969999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":2.9710000000000001,"5min":17.34,"15min":17.34},"last":4.3369999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.0819999999999999,"5min":2.0819999999999999,"15min":2.0819999999999999},"last":1.9630000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":1.9319999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.014,"15min":0.94499999999999995},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.25},"max":{"1min":5.4930000000000003,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.0189999999999999,"15min":0.88700000000000001},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.8099999999999996,"5min":18.600999999999999,"15min":18.600999999999999},"last":1.4890000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0589999999999999,"15min":0.88},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.282,"5min":7.4290000000000003,"15min":10.334},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.749,"15min":0.88600000000000001},"min":{"1min":0.35099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.3500000000000001,"5min":8.0210000000000008,"15min":12.356},"last":1.4510000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2477785088,"data_stored":2477785088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2967470080,"data_stored":2967470080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1916669952,"data_stored":1916665880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2512191488,"data_stored":2512191488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2412707840,"data_stored":2412707840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3311865856,"data_stored":3311861784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2465333248,"data_stored":2465333248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2113343488,"data_stored":2113339416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:10:34.818 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:34.852 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:34.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:34.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:34.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:35.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:10:35.052 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:35.093 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:35 smithi110 ceph-mon[136032]: pgmap v779: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.7 MiB/s wr, 678 op/s; 5374/306054 objects degraded (1.756%); 4408/306054 objects misplaced (1.440%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:35.128 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:35.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:35.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:35.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:10:35.353 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:10:35.353 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:10:35.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:124353 6.e 6:79973428:::benchmark_data_smithi012_182576_object18832:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:00:59.962929+0000", 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 575.38747396400004, 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1067407619999998, 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.354 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124353 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962929+0000", 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962929+0000", 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962928+0000", 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:35.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962932+0000", 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9859999999999996e-06 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962938+0000", 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1580000000000002e-06 2023-10-09T10:10:35.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962942+0000", 2023-10-09T10:10:35.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8020000000000002e-06 2023-10-09T10:10:35.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962991+0000", 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9018999999999999e-05 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:00:59.962998+0000", 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0489999999999998e-06 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:00.921490+0000", 2023-10-09T10:10:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.95849217399999997 2023-10-09T10:10:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:00.921503+0000", 2023-10-09T10:10:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2435e-05 2023-10-09T10:10:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.044012+0000", 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1225092349999999 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.044162+0000", 2023-10-09T10:10:35.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014947499999999999 2023-10-09T10:10:35.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:10:35.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.044707+0000", 2023-10-09T10:10:35.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00054556799999999996 2023-10-09T10:10:35.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.066186+0000", 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021478902000000001 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.067240+0000", 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001054148 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.068394+0000", 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011534270000000001 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.069529+0000", 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001135547 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.069584+0000", 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4979999999999999e-05 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:01:02.069670+0000", 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5443000000000002e-05 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:178976 6.e 6:79c6aa12:::benchmark_data_smithi012_182576_object73455:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e159)", 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.711636+0000", 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 497.638767083, 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.151123753, 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178976 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.366 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711636+0000", 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711636+0000", 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711634+0000", 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711640+0000", 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5849999999999999e-06 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711651+0000", 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1365e-05 2023-10-09T10:10:35.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.711662+0000", 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0293e-05 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713443+0000", 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001780986 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713457+0000", 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4611e-05 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.849523+0000", 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.13606574 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.849757+0000", 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023429500000000001 2023-10-09T10:10:35.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862706+0000", 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012948382 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862759+0000", 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3903999999999998e-05 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.371 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.373 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179004 6.e 6:76574bf5:::benchmark_data_smithi012_182576_object73483:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e159)", 2023-10-09T10:10:35.373 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.713468+0000", 2023-10-09T10:10:35.373 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 497.63693483499998, 2023-10-09T10:10:35.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.180355152, 2023-10-09T10:10:35.373 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179004 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713468+0000", 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713468+0000", 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713467+0000", 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713473+0000", 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4030000000000004e-06 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713483+0000", 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.73e-06 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713492+0000", 2023-10-09T10:10:35.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3340000000000008e-06 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713588+0000", 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6204000000000003e-05 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713598+0000", 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1910000000000006e-06 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.850004+0000", 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1364062589999999 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.850114+0000", 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109714 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.850241+0000", 2023-10-09T10:10:35.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012723899999999999 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.859149+0000", 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089080449999999999 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862489+0000", 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033404979999999999 2023-10-09T10:10:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862510+0000", 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0551e-05 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893616+0000", 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.031106249999999998 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893683+0000", 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6798999999999995e-05 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893823+0000", 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014010499999999999 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.382 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179005 6.e 6:7eac465e:::benchmark_data_smithi012_182576_object73484:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e159)", 2023-10-09T10:10:35.382 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.713499+0000", 2023-10-09T10:10:35.382 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 497.63690338499998, 2023-10-09T10:10:35.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.180540097, 2023-10-09T10:10:35.382 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.382 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.382 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.382 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179005 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713499+0000", 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713499+0000", 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713499+0000", 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713502+0000", 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5119999999999998e-06 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713507+0000", 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6210000000000001e-06 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713512+0000", 2023-10-09T10:10:35.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8520000000000003e-06 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713622+0000", 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011010699999999999 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.713630+0000", 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2810000000000008e-06 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.852723+0000", 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1390927390000001 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.852856+0000", 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013302100000000001 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.852955+0000", 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8459e-05 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862454+0000", 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094993660000000004 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862530+0000", 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5827000000000004e-05 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875797+0000", 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01326738 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893869+0000", 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018071595999999999 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.893923+0000", 2023-10-09T10:10:35.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4400000000000001e-05 2023-10-09T10:10:35.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894040+0000", 2023-10-09T10:10:35.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011633600000000001 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179023 6.e 6:7174819a:::benchmark_data_smithi012_182576_object73502:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.834938+0000", 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 497.51546472400003, 2023-10-09T10:10:35.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0592905880000001, 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179023 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834938+0000", 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834938+0000", 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834936+0000", 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834945+0000", 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2689999999999999e-06 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834965+0000", 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0072e-05 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.834975+0000", 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0453e-05 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.835050+0000", 2023-10-09T10:10:35.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5319000000000002e-05 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.835065+0000", 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4058e-05 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.855671+0000", 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.020606715 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.855802+0000", 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013044 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.855936+0000", 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000134544 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862781+0000", 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0068451839999999998 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.868766+0000", 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059849960000000002 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875941+0000", 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071743689999999999 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894101+0000", 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018160477000000001 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894138+0000", 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7097000000000001e-05 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894229+0000", 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0295000000000002e-05 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179038 6.e 6:70e5e476:::benchmark_data_smithi012_182576_object73517:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.876255+0000", 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 497.47414744899999, 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0181381389999999, 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.401 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179038 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876255+0000", 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876255+0000", 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876253+0000", 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876265+0000", 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1569999999999999e-05 2023-10-09T10:10:35.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876285+0000", 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0534e-05 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876297+0000", 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2055999999999999e-05 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876381+0000", 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4178000000000006e-05 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.876395+0000", 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3522e-05 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.859167+0000", 2023-10-09T10:10:35.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98277201000000003 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.859382+0000", 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021509900000000001 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.859526+0000", 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000143683 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.868813+0000", 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092870709999999992 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875816+0000", 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0070034199999999998 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875984+0000", 2023-10-09T10:10:35.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016793600000000001 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894266+0000", 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018281822 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894295+0000", 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9096000000000001e-05 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894393+0000", 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8420999999999998e-05 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179055 6.e 6:7b811f8d:::benchmark_data_smithi012_182576_object73534:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.895097+0000", 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 497.45530602000002, 2023-10-09T10:10:35.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99952659899999996, 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179055 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895097+0000", 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895097+0000", 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895095+0000", 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895102+0000", 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1790000000000002e-06 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895119+0000", 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6711e-05 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895129+0000", 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9899999999999992e-06 2023-10-09T10:10:35.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895191+0000", 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2129000000000001e-05 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.895201+0000", 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.026e-05 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862795+0000", 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96759368599999995 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.862878+0000", 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3028000000000005e-05 2023-10-09T10:10:35.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.863006+0000", 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012810499999999999 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.868853+0000", 2023-10-09T10:10:35.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058470140000000002 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875826+0000", 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006972677 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.876204+0000", 2023-10-09T10:10:35.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037816400000000001 2023-10-09T10:10:35.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894454+0000", 2023-10-09T10:10:35.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018249656 2023-10-09T10:10:35.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894506+0000", 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2283000000000001e-05 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894623+0000", 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011736 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:179059 6.e 6:791d2037:::benchmark_data_smithi012_182576_object73538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:17.901344+0000", 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 497.44905917, 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99352547499999999, 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 179059 2023-10-09T10:10:35.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901344+0000", 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901344+0000", 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901342+0000", 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901349+0000", 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5409999999999997e-06 2023-10-09T10:10:35.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901365+0000", 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5889e-05 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901373+0000", 2023-10-09T10:10:35.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9929999999999995e-06 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901442+0000", 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8917000000000004e-05 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:17.901453+0000", 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1832e-05 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.865783+0000", 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96432974800000004 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.865913+0000", 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012940900000000001 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.866043+0000", 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013029 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.875833+0000", 2023-10-09T10:10:35.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0097904150000000002 2023-10-09T10:10:35.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.876030+0000", 2023-10-09T10:10:35.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019691000000000001 2023-10-09T10:10:35.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.876273+0000", 2023-10-09T10:10:35.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00024249800000000001 2023-10-09T10:10:35.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894688+0000", 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018415177000000001 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894747+0000", 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9382999999999997e-05 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:18.894869+0000", 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121925 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195715 6.14 6:2f73fb93:::benchmark_data_smithi012_182576_object90194:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:35.429 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 473.05319170899998, 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.606497794, 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195715 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297210+0000", 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297215+0000", 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2869999999999998e-06 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297225+0000", 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9709999999999999e-06 2023-10-09T10:10:35.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297233+0000", 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1200000000000002e-06 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.301089+0000", 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038566210000000002 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.301096+0000", 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0010000000000004e-06 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.715860+0000", 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.41476375300000001 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.715875+0000", 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5322e-05 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.886573+0000", 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1706976149999999 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.886704+0000", 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000131271 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.887173+0000", 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00046823600000000002 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.899644+0000", 2023-10-09T10:10:35.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012471365 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.899817+0000", 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000172919 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900021+0000", 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000204556 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903542+0000", 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035206030000000002 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903591+0000", 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8568e-05 2023-10-09T10:10:35.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903709+0000", 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118261 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195727 6.4 6:27a88943:::benchmark_data_smithi012_182576_object90206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 473.02154645799999, 2023-10-09T10:10:35.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5877643209999999, 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195727 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:10:35.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328855+0000", 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328861+0000", 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0430000000000004e-06 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328874+0000", 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3360000000000001e-05 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328882+0000", 2023-10-09T10:10:35.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328994+0000", 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111865 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329006+0000", 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.255e-05 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716358+0000", 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38735175199999999 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716368+0000", 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8479999999999996e-06 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900045+0000", 2023-10-09T10:10:35.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1836774349999999 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900183+0000", 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013796500000000001 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900820+0000", 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00063694999999999995 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.907584+0000", 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067633459999999999 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916106+0000", 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085223179999999992 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916306+0000", 2023-10-09T10:10:35.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020026700000000001 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916459+0000", 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000153323 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916508+0000", 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8109999999999998e-05 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916621+0000", 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000113142 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195736 6.d 6:b34830fd:::benchmark_data_smithi012_182576_object90215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 473.01301043900003, 2023-10-09T10:10:35.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6770504659999999, 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195736 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:10:35.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337391+0000", 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337398+0000", 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2910000000000002e-06 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337410+0000", 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2217e-05 2023-10-09T10:10:35.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337419+0000", 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6989999999999992e-06 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339092+0000", 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016734510000000001 2023-10-09T10:10:35.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339102+0000", 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7520000000000008e-06 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718181+0000", 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37907882999999998 2023-10-09T10:10:35.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718193+0000", 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1955e-05 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.972506+0000", 2023-10-09T10:10:35.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2543128530000001 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.975318+0000", 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028124920000000002 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.975658+0000", 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000340136 2023-10-09T10:10:35.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999408+0000", 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023749816 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999451+0000", 2023-10-09T10:10:35.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3038999999999999e-05 2023-10-09T10:10:35.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999490+0000", 2023-10-09T10:10:35.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8520999999999999e-05 2023-10-09T10:10:35.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014261+0000", 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014770873 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014326+0000", 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5018999999999995e-05 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014443+0000", 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117192 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.459 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195737 6.d 6:b631f0af:::benchmark_data_smithi012_182576_object90216:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:35.460 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:10:35.460 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 473.012958599, 2023-10-09T10:10:35.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6917381869999999, 2023-10-09T10:10:35.460 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.460 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195737 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:10:35.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337447+0000", 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5310000000000002e-06 2023-10-09T10:10:35.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337455+0000", 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8790000000000003e-06 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337462+0000", 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0650000000000001e-06 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339139+0000", 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016767990000000001 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339151+0000", 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1959e-05 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718219+0000", 2023-10-09T10:10:35.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37906844499999998 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718228+0000", 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6910000000000003e-06 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999515+0000", 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2812874459999999 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999630+0000", 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114138 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999802+0000", 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000172535 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014252+0000", 2023-10-09T10:10:35.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014449500000000001 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014454+0000", 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020259100000000001 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014691+0000", 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023644399999999999 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029029+0000", 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014338379 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029086+0000", 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.694e-05 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029182+0000", 2023-10-09T10:10:35.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6429999999999997e-05 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195738 6.d 6:b7beca9a:::benchmark_data_smithi012_182576_object90217:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 473.00711342, 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.686112541, 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.469 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195738 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343288+0000", 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343296+0000", 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6650000000000003e-06 2023-10-09T10:10:35.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343334+0000", 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8760999999999997e-05 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343345+0000", 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0872e-05 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343426+0000", 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0876000000000003e-05 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343442+0000", 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6049000000000002e-05 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718250+0000", 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37480775999999999 2023-10-09T10:10:35.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718258+0000", 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4360000000000002e-06 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.002824+0000", 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.284566216 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.002952+0000", 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127133 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.003293+0000", 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00034169000000000002 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014460+0000", 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011166931999999999 2023-10-09T10:10:35.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014643+0000", 2023-10-09T10:10:35.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018320000000000001 2023-10-09T10:10:35.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014731+0000", 2023-10-09T10:10:35.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7837999999999995e-05 2023-10-09T10:10:35.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029238+0000", 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014506865000000001 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029290+0000", 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2261e-05 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029402+0000", 2023-10-09T10:10:35.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111514 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195740 6.d 6:ba67ed77:::benchmark_data_smithi012_182576_object90219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 473.006588916, 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.685801323, 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.479 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195740 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343813+0000", 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343819+0000", 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.049e-06 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343834+0000", 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5458999999999999e-05 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343843+0000", 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7250000000000003e-06 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343908+0000", 2023-10-09T10:10:35.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4981999999999995e-05 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343924+0000", 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5843999999999999e-05 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718282+0000", 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37435805300000002 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718299+0000", 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6651999999999999e-05 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006544+0000", 2023-10-09T10:10:35.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2882459150000001 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006671+0000", 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012675699999999999 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006813+0000", 2023-10-09T10:10:35.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141285 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014467+0000", 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076547940000000004 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014774+0000", 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00030705499999999998 2023-10-09T10:10:35.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.028866+0000", 2023-10-09T10:10:35.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014091942 2023-10-09T10:10:35.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029464+0000", 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00059783999999999998 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029512+0000", 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7426000000000003e-05 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029615+0000", 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103649 2023-10-09T10:10:35.488 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195753 6.14 6:2fd4c659:::benchmark_data_smithi012_182576_object90232:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 472.98593803799997, 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5530476879999999, 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.489 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195753 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364464+0000", 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364468+0000", 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.433e-06 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364478+0000", 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0186e-05 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364484+0000", 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9349999999999999e-06 2023-10-09T10:10:35.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364551+0000", 2023-10-09T10:10:35.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7554999999999999e-05 2023-10-09T10:10:35.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364562+0000", 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0964999999999999e-05 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717712+0000", 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35315011400000001 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.721170+0000", 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003457702 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900344+0000", 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.179173823 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900467+0000", 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000122846 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900983+0000", 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051649900000000002 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.907573+0000", 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0065898750000000002 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.908808+0000", 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012348979999999999 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916252+0000", 2023-10-09T10:10:35.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074441109999999998 2023-10-09T10:10:35.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916394+0000", 2023-10-09T10:10:35.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014227099999999999 2023-10-09T10:10:35.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916845+0000", 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045041999999999999 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.917512+0000", 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00066757000000000003 2023-10-09T10:10:35.499 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195755 6.4 6:21d202b2:::benchmark_data_smithi012_182576_object90234:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 472.98202743600001, 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5484353739999999, 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.500 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195755 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:10:35.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368378+0000", 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7379999999999999e-06 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368392+0000", 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3913e-05 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368399+0000", 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6340000000000001e-06 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368462+0000", 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3113000000000004e-05 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368476+0000", 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3366e-05 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716390+0000", 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.347914734 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716399+0000", 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6989999999999992e-06 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904175+0000", 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.187776342 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904334+0000", 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015860900000000001 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904813+0000", 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00047921500000000001 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916182+0000", 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011368645 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916200+0000", 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7609e-05 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916359+0000", 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015949700000000001 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916659+0000", 2023-10-09T10:10:35.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029949099999999998 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916705+0000", 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.693e-05 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916811+0000", 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105287 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.508 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:116016 8.7 8:e6f3990b:::benchmark_data_smithi012_271002_object7333:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:32.479483+0000", 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.8709199779999999, 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.073764435000000003, 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:35.509 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 116016 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479483+0000", 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479483+0000", 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479481+0000", 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479488+0000", 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7970000000000003e-06 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479505+0000", 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6507e-05 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479513+0000", 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3620000000000006e-06 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479561+0000", 2023-10-09T10:10:35.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7917000000000002e-05 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479694+0000", 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000132736 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.479810+0000", 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116394 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485169+0000", 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053588749999999999 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.489197+0000", 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040278220000000003 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.553163+0000", 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063965771000000005 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.553204+0000", 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1838999999999997e-05 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.553247+0000", 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2901999999999997e-05 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.515 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:116021 8.7 8:edcf0a3b:::benchmark_data_smithi012_271002_object7338:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:32.485129+0000", 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.8652733120000002, 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068231516000000006, 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:35.516 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 116021 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485129+0000", 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485129+0000", 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485128+0000", 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485134+0000", 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4250000000000003e-06 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485147+0000", 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2636000000000001e-05 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485154+0000", 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7170000000000008e-06 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485196+0000", 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1635999999999999e-05 2023-10-09T10:10:35.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485335+0000", 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000139059 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.485452+0000", 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117223 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.489479+0000", 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040263130000000001 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.496250+0000", 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067713829999999997 2023-10-09T10:10:35.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.553284+0000", 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057034437 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.553327+0000", 2023-10-09T10:10:35.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2231000000000003e-05 2023-10-09T10:10:35.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:32.553361+0000", 2023-10-09T10:10:35.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4283999999999999e-05 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:117831 8.7 8:e0d06090:::benchmark_data_smithi012_271002_object9148:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:35.043350+0000", 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.30705273199999999, 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.065882416999999999, 2023-10-09T10:10:35.524 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.525 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.525 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.525 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:10:35.525 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:35.525 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117831 2023-10-09T10:10:35.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.526 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.043350+0000", 2023-10-09T10:10:35.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.043350+0000", 2023-10-09T10:10:35.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.043348+0000", 2023-10-09T10:10:35.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:10:35.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.043358+0000", 2023-10-09T10:10:35.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0370000000000001e-05 2023-10-09T10:10:35.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.043374+0000", 2023-10-09T10:10:35.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5549e-05 2023-10-09T10:10:35.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.043385+0000", 2023-10-09T10:10:35.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.134e-05 2023-10-09T10:10:35.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.065102+0000", 2023-10-09T10:10:35.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021717251999999999 2023-10-09T10:10:35.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.065272+0000", 2023-10-09T10:10:35.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016929099999999999 2023-10-09T10:10:35.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.065417+0000", 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014477399999999999 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.084407+0000", 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018990277999999999 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.084965+0000", 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00055824900000000003 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.109133+0000", 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024167884000000001 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.109186+0000", 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3130000000000001e-05 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.109232+0000", 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6391999999999999e-05 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:117973 8.7 8:e95b090a:::benchmark_data_smithi012_271002_object9290:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:10:35.534 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:35.281229+0000", 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.069174213999999998, 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060761837999999999, 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117973 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281229+0000", 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281229+0000", 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281228+0000", 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:35.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281232+0000", 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5870000000000001e-06 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281240+0000", 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3380000000000005e-06 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281248+0000", 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9880000000000004e-06 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281391+0000", 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000143017 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281521+0000", 2023-10-09T10:10:35.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012991000000000001 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.281629+0000", 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108272 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.286400+0000", 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0047705009999999999 2023-10-09T10:10:35.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.302737+0000", 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016337689999999998 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.341883+0000", 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.039145424999999998 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:10:35.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.341944+0000", 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1544999999999994e-05 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:35.341990+0000", 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6094999999999998e-05 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:10:35.541 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:10:35.542 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:35 smithi110 ceph-mon[136032]: from='client.35667 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:35.542 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:35 smithi012 ceph-mon[152263]: pgmap v779: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.7 MiB/s wr, 678 op/s; 5374/306054 objects degraded (1.756%); 4408/306054 objects misplaced (1.440%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:35.543 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:35 smithi012 ceph-mon[152263]: from='client.35667 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:35.543 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:35 smithi012 ceph-mon[147926]: pgmap v779: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.7 MiB/s wr, 678 op/s; 5374/306054 objects degraded (1.756%); 4408/306054 objects misplaced (1.440%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:35.543 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:35 smithi012 ceph-mon[147926]: from='client.35667 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:35.544 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:35.544 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:35.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:35.645 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:35.682 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:35.714 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:35.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:35.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:35.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:35.863 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:35.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:35.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:35.989 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:10:36.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:36.113 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:36.144 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:36.200 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:36.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:36.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:36.318 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:36.322 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:36.356 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:36.408 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:36.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:36.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:36.517 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:36.562 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:36.635 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:36.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:36.726 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:36.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:36.790 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:36.861 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:36.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:36.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:36.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:37.028 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:37.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:37.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:37.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:37 smithi012 ceph-mon[147926]: pgmap v780: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 371 KiB/s wr, 684 op/s; 5374/302841 objects degraded (1.775%); 4408/302841 objects misplaced (1.456%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:37.199 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:37.322 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:37.357 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:37 smithi110 ceph-mon[136032]: pgmap v780: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 371 KiB/s wr, 684 op/s; 5374/302841 objects degraded (1.775%); 4408/302841 objects misplaced (1.456%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:37.395 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:37 smithi012 ceph-mon[152263]: pgmap v780: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 371 KiB/s wr, 684 op/s; 5374/302841 objects degraded (1.775%); 4408/302841 objects misplaced (1.456%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:10:37.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:10:38.089 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:10:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:10:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:119861 8.4 8:2fd89606:::benchmark_data_smithi012_271002_object11178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:10:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:38.078483+0000", 2023-10-09T10:10:38.090 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0099755720000000003, 2023-10-09T10:10:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010006157, 2023-10-09T10:10:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:10:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:10:38.091 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 119861 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078483+0000", 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078483+0000", 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078481+0000", 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6299999999999993e-06 2023-10-09T10:10:38.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078490+0000", 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6381000000000001e-05 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078507+0000", 2023-10-09T10:10:38.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2283000000000001e-05 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078519+0000", 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3403999999999999e-05 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:38.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078572+0000", 2023-10-09T10:10:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017871099999999999 2023-10-09T10:10:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078751+0000", 2023-10-09T10:10:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018261300000000001 2023-10-09T10:10:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078934+0000", 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085466489999999999 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.087480+0000", 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043695099999999998 2023-10-09T10:10:38.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.087917+0000", 2023-10-09T10:10:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0094343900000000008 2023-10-09T10:10:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:38.098 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:119863 8.8 8:1e9bd980:::benchmark_data_smithi012_271002_object11180:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:10:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:38.078536+0000", 2023-10-09T10:10:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.009922854, 2023-10-09T10:10:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010080443, 2023-10-09T10:10:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:38.099 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:10:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:10:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 119863 2023-10-09T10:10:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078536+0000", 2023-10-09T10:10:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:38.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078536+0000", 2023-10-09T10:10:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078535+0000", 2023-10-09T10:10:38.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.985e-06 2023-10-09T10:10:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078539+0000", 2023-10-09T10:10:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3390000000000004e-06 2023-10-09T10:10:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078546+0000", 2023-10-09T10:10:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8690000000000003e-06 2023-10-09T10:10:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.078551+0000", 2023-10-09T10:10:38.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020049270000000001 2023-10-09T10:10:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.080556+0000", 2023-10-09T10:10:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011246799999999999 2023-10-09T10:10:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.080668+0000", 2023-10-09T10:10:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120356 2023-10-09T10:10:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:10:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.080789+0000", 2023-10-09T10:10:38.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067603730000000001 2023-10-09T10:10:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.087549+0000", 2023-10-09T10:10:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090135709999999997 2023-10-09T10:10:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.107 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:119868 8.2 e294/271 8:40dc6a00:::benchmark_data_smithi012_271002_object11185:head v 294'7400, mlcod=294'7398)", 2023-10-09T10:10:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:38.082902+0000", 2023-10-09T10:10:38.108 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0055568950000000001, 2023-10-09T10:10:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058049679999999998, 2023-10-09T10:10:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:10:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082902+0000", 2023-10-09T10:10:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082902+0000", 2023-10-09T10:10:38.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082901+0000", 2023-10-09T10:10:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.115e-06 2023-10-09T10:10:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082910+0000", 2023-10-09T10:10:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0389e-05 2023-10-09T10:10:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:38.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082920+0000", 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0299999999999999e-06 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082926+0000", 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9117999999999998e-05 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:38.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082995+0000", 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9896000000000001e-05 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.083015+0000", 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000113675 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:119870 8.4 8:2682688e:::benchmark_data_smithi012_271002_object11187:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:38.082974+0000", 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0054845060000000001, 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057994580000000004, 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:38.115 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 119870 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082974+0000", 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082974+0000", 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082972+0000", 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0978000000000001e-05 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.082983+0000", 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9062000000000001e-05 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.083002+0000", 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0430000000000002e-06 2023-10-09T10:10:38.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.083009+0000", 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010254799999999999 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.083112+0000", 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000182225 2023-10-09T10:10:38.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.083294+0000", 2023-10-09T10:10:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019239100000000001 2023-10-09T10:10:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:10:38.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.083487+0000", 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045818439999999998 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088068+0000", 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050945080000000002 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.122 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.45518.0:119863 8.8 e294/271 ondisk, result = 0)", 2023-10-09T10:10:38.122 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:38.088671+0000", 2023-10-09T10:10:38.122 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4294967295.9997878, 2023-10-09T10:10:38.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019373599999999999, 2023-10-09T10:10:38.122 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:38.122 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:10:38.122 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:38.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:38.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088671+0000", 2023-10-09T10:10:38.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:38.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:38.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088671+0000", 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088670+0000", 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9939999999999997e-06 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088676+0000", 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0838e-05 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088687+0000", 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.85e-06 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088695+0000", 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5537999999999994e-05 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088771+0000", 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3239999999999999e-06 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:38.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:38.088777+0000", 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010676200000000001 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 5 2023-10-09T10:10:38.127 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:10:38.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/302841 objects degraded (1.775%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:38.181 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:10:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:10:37] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T10:10:38.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/302841 objects degraded (1.775%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:38.316 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:38.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:38.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:38.428 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:38.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/302841 objects degraded (1.775%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:38.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:38.519 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:38.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:38.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:38.589 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:10:38.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:38.678 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:38.735 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:38.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:38.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:38.930 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:39.052 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:39.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:39.131 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:39.158 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:39.166 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:39 smithi110 ceph-mon[136032]: pgmap v781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5374/300990 objects degraded (1.785%); 4293/300990 objects misplaced (1.426%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:10:39.188 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:39.192 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:10:39.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:39.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:39.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:39.242 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:10:39.269 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:39.294 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:10:39.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:39.342 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:10:39.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:10:39.393 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:39.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:39 smithi012 ceph-mon[147926]: pgmap v781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5374/300990 objects degraded (1.785%); 4293/300990 objects misplaced (1.426%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:10:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:39 smithi012 ceph-mon[152263]: pgmap v781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 5374/300990 objects degraded (1.785%); 4293/300990 objects misplaced (1.426%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:10:39.437 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:10:39.492 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:10:39.514 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:10:39.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:10:39.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:10:39.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:10:39.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:10:39.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:10:39.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:10:39.544 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:10:39.544 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:10:40.001 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:40.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:40.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:40.107 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:40.182 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:40.233 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:10:40.268 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4269372598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:40.268 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1907254113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:40.268 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:40.268 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:40 smithi110 ceph-mon[136032]: pgmap v782: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5374/295893 objects degraded (1.816%); 4293/295893 objects misplaced (1.451%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:10:40.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4269372598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:40.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1907254113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:40.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:40.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:40 smithi012 ceph-mon[152263]: pgmap v782: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5374/295893 objects degraded (1.816%); 4293/295893 objects misplaced (1.451%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:10:40.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4269372598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:40.524 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1907254113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:40.524 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:10:40.524 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:40 smithi012 ceph-mon[147926]: pgmap v782: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5374/295893 objects degraded (1.816%); 4293/295893 objects misplaced (1.451%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:10:40.623 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:40.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:40.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:40.728 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:40.874 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:40.968 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:40.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:40.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:41.076 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:41.138 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:41.176 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:10:41.176 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:10:41.217 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:41.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:41.332 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:41.385 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:41.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:41.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[147926]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[147926]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:10:41.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[147926]: osdmap e295: 8 total, 8 up, 8 in 2023-10-09T10:10:41.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[147926]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:41.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[152263]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[152263]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:10:41.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[152263]: osdmap e295: 8 total, 8 up, 8 in 2023-10-09T10:10:41.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:41 smithi012 ceph-mon[152263]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.587 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:41.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:41.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:41.593 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:41 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:41.593 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:41 smithi110 ceph-mon[136032]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.593 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:41 smithi110 ceph-mon[136032]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.593 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:41 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:10:41.594 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:41 smithi110 ceph-mon[136032]: osdmap e295: 8 total, 8 up, 8 in 2023-10-09T10:10:41.594 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:41 smithi110 ceph-mon[136032]: failed to encode map e295 with expected crc 2023-10-09T10:10:41.670 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:10:41.694 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:41.752 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:41.805 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:41.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:41.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:41.923 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:41.932 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:10:42.007 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:42.113 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:42.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:42.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:42 smithi012 ceph-mon[147926]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5374/290706 objects degraded (1.849%); 4165/290706 objects misplaced (1.433%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:10:42.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:42 smithi012 ceph-mon[147926]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:42.218 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:42 smithi012 ceph-mon[152263]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5374/290706 objects degraded (1.849%); 4165/290706 objects misplaced (1.433%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:10:42.218 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:42 smithi012 ceph-mon[152263]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:42.222 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:42.268 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:10:42.294 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:42.344 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:42.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:42.431 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:42.449 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:42.475 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":784,"stamp":"2023-10-09T10:10:41.336133+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6349526968,"num_objects":96902,"num_object_clones":0,"num_object_copies":290706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":4165,"num_objects_unfound":0,"num_objects_dirty":96902,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":120744,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1187,"num_bytes_recovered":74459852,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120678,"ondisk_log_size":120678,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21189880,"kb_used_data":18786488,"kb_used_omap":167,"kb_used_meta":2403096,"kb_avail":728771336,"statfs":{"total":767960285184,"available":746261848064,"internally_reserved":0,"allocated":19237363712,"data_stored":19231081545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2460770905},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-444858368,"num_objects":-6788,"num_object_clones":0,"num_object_copies":-20364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-243,"num_objects_unfound":0,"num_objects_dirty":-6788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6788,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":243,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.263266"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":135,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.435492+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:10:22.435492+0000","last_peered":"2023-10-09T10:10:22.435492+0000","last_clean":"2023-10-09T10:10:22.435492+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:10:22.435492+0000","last_undegraded":"2023-10-09T10:10:22.435492+0000","last_fullsized":"2023-10-09T10:10:22.435492+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":252,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077690+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:10:32.077690+0000","last_peered":"2023-10-09T10:10:32.077690+0000","last_clean":"2023-10-09T10:10:32.077690+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:10:32.077690+0000","last_undegraded":"2023-10-09T10:10:32.077690+0000","last_fullsized":"2023-10-09T10:10:32.077690+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":296,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.435844+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:10:22.435844+0000","last_peered":"2023-10-09T10:10:22.435844+0000","last_clean":"2023-10-09T10:10:22.435844+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:10:22.435844+0000","last_undegraded":"2023-10-09T10:10:22.435844+0000","last_fullsized":"2023-10-09T10:10:22.435844+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":160,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092979+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:10:32.092979+0000","last_peered":"2023-10-09T10:10:32.092979+0000","last_clean":"2023-10-09T10:10:32.092979+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:10:32.092979+0000","last_undegraded":"2023-10-09T10:10:32.092979+0000","last_fullsized":"2023-10-09T10:10:32.092979+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":266,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077596+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:10:32.077596+0000","last_peered":"2023-10-09T10:10:32.077596+0000","last_clean":"2023-10-09T10:10:32.077596+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:10:32.077596+0000","last_undegraded":"2023-10-09T10:10:32.077596+0000","last_fullsized":"2023-10-09T10:10:32.077596+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":112,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445337+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:10:22.445337+0000","last_peered":"2023-10-09T10:10:22.445337+0000","last_clean":"2023-10-09T10:10:22.445337+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:10:22.445337+0000","last_undegraded":"2023-10-09T10:10:22.445337+0000","last_fullsized":"2023-10-09T10:10:22.445337+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":177,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445862+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:10:22.445862+0000","last_peered":"2023-10-09T10:10:22.445862+0000","last_clean":"2023-10-09T10:10:22.445862+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:10:22.445862+0000","last_undegraded":"2023-10-09T10:10:22.445862+0000","last_fullsized":"2023-10-09T10:10:22.445862+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":247,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077504+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:10:32.077504+0000","last_peered":"2023-10-09T10:10:32.077504+0000","last_clean":"2023-10-09T10:10:32.077504+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:10:32.077504+0000","last_undegraded":"2023-10-09T10:10:32.077504+0000","last_fullsized":"2023-10-09T10:10:32.077504+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":111,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445088+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:10:22.445088+0000","last_peered":"2023-10-09T10:10:22.445088+0000","last_clean":"2023-10-09T10:10:22.445088+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:10:22.445088+0000","last_undegraded":"2023-10-09T10:10:22.445088+0000","last_fullsized":"2023-10-09T10:10:22.445088+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":208,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.078024+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:10:32.078024+0000","last_peered":"2023-10-09T10:10:32.078024+0000","last_clean":"2023-10-09T10:10:32.078024+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:10:32.078024+0000","last_undegraded":"2023-10-09T10:10:32.078024+0000","last_fullsized":"2023-10-09T10:10:32.078024+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":332,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.442837+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:10:22.442837+0000","last_peered":"2023-10-09T10:10:22.442837+0000","last_clean":"2023-10-09T10:10:22.442837+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:10:22.442837+0000","last_undegraded":"2023-10-09T10:10:22.442837+0000","last_fullsized":"2023-10-09T10:10:22.442837+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":288,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.435297+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:10:22.435297+0000","last_peered":"2023-10-09T10:10:22.435297+0000","last_clean":"2023-10-09T10:10:22.435297+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:10:22.435297+0000","last_undegraded":"2023-10-09T10:10:22.435297+0000","last_fullsized":"2023-10-09T10:10:22.435297+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":181,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.442447+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:10:22.442447+0000","last_peered":"2023-10-09T10:10:22.442447+0000","last_clean":"2023-10-09T10:10:22.442447+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:10:22.442447+0000","last_undegraded":"2023-10-09T10:10:22.442447+0000","last_fullsized":"2023-10-09T10:10:22.442447+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434861+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:10:22.434861+0000","last_peered":"2023-10-09T10:10:22.434861+0000","last_clean":"2023-10-09T10:10:22.434861+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:10:22.434861+0000","last_undegraded":"2023-10-09T10:10:22.434861+0000","last_fullsized":"2023-10-09T10:10:22.434861+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":365,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.442207+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:10:22.442207+0000","last_peered":"2023-10-09T10:10:22.442207+0000","last_clean":"2023-10-09T10:10:22.442207+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:10:22.442207+0000","last_undegraded":"2023-10-09T10:10:22.442207+0000","last_fullsized":"2023-10-09T10:10:22.442207+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":234,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077883+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:10:32.077883+0000","last_peered":"2023-10-09T10:10:32.077883+0000","last_clean":"2023-10-09T10:10:32.077883+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:10:32.077883+0000","last_undegraded":"2023-10-09T10:10:32.077883+0000","last_fullsized":"2023-10-09T10:10:32.077883+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":159,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445746+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:10:22.445746+0000","last_peered":"2023-10-09T10:10:22.445746+0000","last_clean":"2023-10-09T10:10:22.445746+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:10:22.445746+0000","last_undegraded":"2023-10-09T10:10:22.445746+0000","last_fullsized":"2023-10-09T10:10:22.445746+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":127,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445044+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:10:22.445044+0000","last_peered":"2023-10-09T10:10:22.445044+0000","last_clean":"2023-10-09T10:10:22.445044+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:10:22.445044+0000","last_undegraded":"2023-10-09T10:10:22.445044+0000","last_fullsized":"2023-10-09T10:10:22.445044+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":162,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.441968+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:10:22.441968+0000","last_peered":"2023-10-09T10:10:22.441968+0000","last_clean":"2023-10-09T10:10:22.441968+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:10:22.441968+0000","last_undegraded":"2023-10-09T10:10:22.441968+0000","last_fullsized":"2023-10-09T10:10:22.441968+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":159,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.446079+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:10:22.446079+0000","last_peered":"2023-10-09T10:10:22.446079+0000","last_clean":"2023-10-09T10:10:22.446079+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:10:22.446079+0000","last_undegraded":"2023-10-09T10:10:22.446079+0000","last_fullsized":"2023-10-09T10:10:22.446079+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":182,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445543+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:10:22.445543+0000","last_peered":"2023-10-09T10:10:22.445543+0000","last_clean":"2023-10-09T10:10:22.445543+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:10:22.445543+0000","last_undegraded":"2023-10-09T10:10:22.445543+0000","last_fullsized":"2023-10-09T10:10:22.445543+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":247,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.078107+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:10:32.078107+0000","last_peered":"2023-10-09T10:10:32.078107+0000","last_clean":"2023-10-09T10:10:32.078107+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:10:32.078107+0000","last_undegraded":"2023-10-09T10:10:32.078107+0000","last_fullsized":"2023-10-09T10:10:32.078107+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"294'7525","reported_seq":7564,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:37.835006+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:10:37.835006+0000","last_peered":"2023-10-09T10:10:37.835006+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:10:37.835006+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7525,"log_dups_size":0,"ondisk_log_size":7525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402456576,"num_objects":6141,"num_object_clones":0,"num_object_copies":18423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7525,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":768},{"shards":"4,5,6","objects":4681}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":326,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.442659+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:10:22.442659+0000","last_peered":"2023-10-09T10:10:22.442659+0000","last_clean":"2023-10-09T10:10:22.442659+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:10:22.442659+0000","last_undegraded":"2023-10-09T10:10:22.442659+0000","last_fullsized":"2023-10-09T10:10:22.442659+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":112,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444715+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:10:22.444715+0000","last_peered":"2023-10-09T10:10:22.444715+0000","last_clean":"2023-10-09T10:10:22.444715+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:10:22.444715+0000","last_undegraded":"2023-10-09T10:10:22.444715+0000","last_fullsized":"2023-10-09T10:10:22.444715+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"294'7662","reported_seq":7693,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:39.967909+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:10:39.967909+0000","last_peered":"2023-10-09T10:10:39.967909+0000","last_clean":"2023-10-09T10:10:39.967909+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:10:39.967909+0000","last_undegraded":"2023-10-09T10:10:39.967909+0000","last_fullsized":"2023-10-09T10:10:39.967909+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7662,"log_dups_size":0,"ondisk_log_size":7662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401604608,"num_objects":6128,"num_object_clones":0,"num_object_copies":18384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7662,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":314,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.442221+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:10:22.442221+0000","last_peered":"2023-10-09T10:10:22.442221+0000","last_clean":"2023-10-09T10:10:22.442221+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:10:22.442221+0000","last_undegraded":"2023-10-09T10:10:22.442221+0000","last_fullsized":"2023-10-09T10:10:22.442221+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":205,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.099460+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:10:32.099460+0000","last_peered":"2023-10-09T10:10:32.099460+0000","last_clean":"2023-10-09T10:10:32.099460+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:10:32.099460+0000","last_undegraded":"2023-10-09T10:10:32.099460+0000","last_fullsized":"2023-10-09T10:10:32.099460+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"294'7471","reported_seq":7503,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:37.701017+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:10:37.701017+0000","last_peered":"2023-10-09T10:10:37.701017+0000","last_clean":"2023-10-09T10:10:37.701017+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:10:37.701017+0000","last_undegraded":"2023-10-09T10:10:37.701017+0000","last_fullsized":"2023-10-09T10:10:37.701017+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7471,"log_dups_size":0,"ondisk_log_size":7471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399441920,"num_objects":6095,"num_object_clones":0,"num_object_copies":18285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7471,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":116,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445127+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:10:22.445127+0000","last_peered":"2023-10-09T10:10:22.445127+0000","last_clean":"2023-10-09T10:10:22.445127+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:10:22.445127+0000","last_undegraded":"2023-10-09T10:10:22.445127+0000","last_fullsized":"2023-10-09T10:10:22.445127+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":137,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.443621+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:10:22.443621+0000","last_peered":"2023-10-09T10:10:22.443621+0000","last_clean":"2023-10-09T10:10:22.443621+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:10:22.443621+0000","last_undegraded":"2023-10-09T10:10:22.443621+0000","last_fullsized":"2023-10-09T10:10:22.443621+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"294'7456","reported_seq":7487,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:39.479985+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:10:39.479985+0000","last_peered":"2023-10-09T10:10:39.479985+0000","last_clean":"2023-10-09T10:10:39.479985+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:10:39.479985+0000","last_undegraded":"2023-10-09T10:10:39.479985+0000","last_fullsized":"2023-10-09T10:10:39.479985+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7456,"log_dups_size":0,"ondisk_log_size":7456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390201344,"num_objects":5954,"num_object_clones":0,"num_object_copies":17862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7456,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":127,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445117+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:10:22.445117+0000","last_peered":"2023-10-09T10:10:22.445117+0000","last_clean":"2023-10-09T10:10:22.445117+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:10:22.445117+0000","last_undegraded":"2023-10-09T10:10:22.445117+0000","last_fullsized":"2023-10-09T10:10:22.445117+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":160,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445981+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:10:22.445981+0000","last_peered":"2023-10-09T10:10:22.445981+0000","last_clean":"2023-10-09T10:10:22.445981+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:10:22.445981+0000","last_undegraded":"2023-10-09T10:10:22.445981+0000","last_fullsized":"2023-10-09T10:10:22.445981+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"295'7736","reported_seq":7768,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:41.200426+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:10:41.200426+0000","last_peered":"2023-10-09T10:10:41.200426+0000","last_clean":"2023-10-09T10:10:41.200426+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:10:41.200426+0000","last_undegraded":"2023-10-09T10:10:41.200426+0000","last_fullsized":"2023-10-09T10:10:41.200426+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7736,"log_dups_size":0,"ondisk_log_size":7736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400031744,"num_objects":6104,"num_object_clones":0,"num_object_copies":18312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7736,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":288,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077592+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:10:32.077592+0000","last_peered":"2023-10-09T10:10:32.077592+0000","last_clean":"2023-10-09T10:10:32.077592+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:10:32.077592+0000","last_undegraded":"2023-10-09T10:10:32.077592+0000","last_fullsized":"2023-10-09T10:10:32.077592+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077760+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:10:32.077760+0000","last_peered":"2023-10-09T10:10:32.077760+0000","last_clean":"2023-10-09T10:10:32.077760+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:10:32.077760+0000","last_undegraded":"2023-10-09T10:10:32.077760+0000","last_fullsized":"2023-10-09T10:10:32.077760+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"294'7600","reported_seq":7632,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:39.506139+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:10:39.506139+0000","last_peered":"2023-10-09T10:10:39.506139+0000","last_clean":"2023-10-09T10:10:39.506139+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:10:39.506139+0000","last_undegraded":"2023-10-09T10:10:39.506139+0000","last_fullsized":"2023-10-09T10:10:39.506139+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7600,"log_dups_size":0,"ondisk_log_size":7600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400031744,"num_objects":6104,"num_object_clones":0,"num_object_copies":18312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7600,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":219,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.099492+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:10:32.099492+0000","last_peered":"2023-10-09T10:10:32.099492+0000","last_clean":"2023-10-09T10:10:32.099492+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:10:32.099492+0000","last_undegraded":"2023-10-09T10:10:32.099492+0000","last_fullsized":"2023-10-09T10:10:32.099492+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":224,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434607+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:10:22.434607+0000","last_peered":"2023-10-09T10:10:22.434607+0000","last_clean":"2023-10-09T10:10:22.434607+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:10:22.434607+0000","last_undegraded":"2023-10-09T10:10:22.434607+0000","last_fullsized":"2023-10-09T10:10:22.434607+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"294'7384","reported_seq":7416,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:37.673644+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:10:37.673644+0000","last_peered":"2023-10-09T10:10:37.673644+0000","last_clean":"2023-10-09T10:10:37.673644+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:10:37.673644+0000","last_undegraded":"2023-10-09T10:10:37.673644+0000","last_fullsized":"2023-10-09T10:10:37.673644+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7384,"log_dups_size":0,"ondisk_log_size":7384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396361728,"num_objects":6048,"num_object_clones":0,"num_object_copies":18144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7384,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":188,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.441884+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:10:22.441884+0000","last_peered":"2023-10-09T10:10:22.441884+0000","last_clean":"2023-10-09T10:10:22.441884+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:10:22.441884+0000","last_undegraded":"2023-10-09T10:10:22.441884+0000","last_fullsized":"2023-10-09T10:10:22.441884+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":160,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434765+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:10:22.434765+0000","last_peered":"2023-10-09T10:10:22.434765+0000","last_clean":"2023-10-09T10:10:22.434765+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:10:22.434765+0000","last_undegraded":"2023-10-09T10:10:22.434765+0000","last_fullsized":"2023-10-09T10:10:22.434765+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"294'7570","reported_seq":7602,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:39.541276+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:10:39.541276+0000","last_peered":"2023-10-09T10:10:39.541276+0000","last_clean":"2023-10-09T10:10:39.541276+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:10:39.541276+0000","last_undegraded":"2023-10-09T10:10:39.541276+0000","last_fullsized":"2023-10-09T10:10:39.541276+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395051008,"num_objects":6028,"num_object_clones":0,"num_object_copies":18084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7570,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":248,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077424+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:10:32.077424+0000","last_peered":"2023-10-09T10:10:32.077424+0000","last_clean":"2023-10-09T10:10:32.077424+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:10:32.077424+0000","last_undegraded":"2023-10-09T10:10:32.077424+0000","last_fullsized":"2023-10-09T10:10:32.077424+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":297,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434952+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:10:22.434952+0000","last_peered":"2023-10-09T10:10:22.434952+0000","last_clean":"2023-10-09T10:10:22.434952+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:10:22.434952+0000","last_undegraded":"2023-10-09T10:10:22.434952+0000","last_fullsized":"2023-10-09T10:10:22.434952+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"294'7409","reported_seq":7441,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:37.673769+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:10:37.673769+0000","last_peered":"2023-10-09T10:10:37.673769+0000","last_clean":"2023-10-09T10:10:37.673769+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:10:37.673769+0000","last_undegraded":"2023-10-09T10:10:37.673769+0000","last_fullsized":"2023-10-09T10:10:37.673769+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7409,"log_dups_size":0,"ondisk_log_size":7409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397344768,"num_objects":6063,"num_object_clones":0,"num_object_copies":18189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7409,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":296,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434276+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:10:22.434276+0000","last_peered":"2023-10-09T10:10:22.434276+0000","last_clean":"2023-10-09T10:10:22.434276+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:10:22.434276+0000","last_undegraded":"2023-10-09T10:10:22.434276+0000","last_fullsized":"2023-10-09T10:10:22.434276+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445036+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:10:22.445036+0000","last_peered":"2023-10-09T10:10:22.445036+0000","last_clean":"2023-10-09T10:10:22.445036+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:10:22.445036+0000","last_undegraded":"2023-10-09T10:10:22.445036+0000","last_fullsized":"2023-10-09T10:10:22.445036+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":253,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.099763+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:10:32.099763+0000","last_peered":"2023-10-09T10:10:32.099763+0000","last_clean":"2023-10-09T10:10:32.099763+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:10:32.099763+0000","last_undegraded":"2023-10-09T10:10:32.099763+0000","last_fullsized":"2023-10-09T10:10:32.099763+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"294'7529","reported_seq":7560,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:37.893651+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:10:37.893651+0000","last_peered":"2023-10-09T10:10:37.893651+0000","last_clean":"2023-10-09T10:10:37.893651+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:10:37.893651+0000","last_undegraded":"2023-10-09T10:10:37.893651+0000","last_fullsized":"2023-10-09T10:10:37.893651+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7529,"log_dups_size":0,"ondisk_log_size":7529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396165120,"num_objects":6045,"num_object_clones":0,"num_object_copies":18135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7529,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":112,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444749+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:10:22.444749+0000","last_peered":"2023-10-09T10:10:22.444749+0000","last_clean":"2023-10-09T10:10:22.444749+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:10:22.444749+0000","last_undegraded":"2023-10-09T10:10:22.444749+0000","last_fullsized":"2023-10-09T10:10:22.444749+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":110,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:27.734802+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:10:27.734802+0000","last_peered":"2023-10-09T10:10:27.734802+0000","last_clean":"2023-10-09T10:10:27.734802+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:10:27.734802+0000","last_undegraded":"2023-10-09T10:10:27.734802+0000","last_fullsized":"2023-10-09T10:10:27.734802+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":319,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434203+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:10:22.434203+0000","last_peered":"2023-10-09T10:10:22.434203+0000","last_clean":"2023-10-09T10:10:22.434203+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:10:22.434203+0000","last_undegraded":"2023-10-09T10:10:22.434203+0000","last_fullsized":"2023-10-09T10:10:22.434203+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"295'7639","reported_seq":7672,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:41.200958+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:10:41.200958+0000","last_peered":"2023-10-09T10:10:41.200958+0000","last_clean":"2023-10-09T10:10:41.200958+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:10:41.200958+0000","last_undegraded":"2023-10-09T10:10:41.200958+0000","last_fullsized":"2023-10-09T10:10:41.200958+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7639,"log_dups_size":0,"ondisk_log_size":7639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386859008,"num_objects":5903,"num_object_clones":0,"num_object_copies":17709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7639,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":214,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.081375+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:10:32.081375+0000","last_peered":"2023-10-09T10:10:32.081375+0000","last_clean":"2023-10-09T10:10:32.081375+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:10:32.081375+0000","last_undegraded":"2023-10-09T10:10:32.081375+0000","last_fullsized":"2023-10-09T10:10:32.081375+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":288,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434214+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:10:22.434214+0000","last_peered":"2023-10-09T10:10:22.434214+0000","last_clean":"2023-10-09T10:10:22.434214+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:10:22.434214+0000","last_undegraded":"2023-10-09T10:10:22.434214+0000","last_fullsized":"2023-10-09T10:10:22.434214+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":309,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434877+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:10:22.434877+0000","last_peered":"2023-10-09T10:10:22.434877+0000","last_clean":"2023-10-09T10:10:22.434877+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:10:22.434877+0000","last_undegraded":"2023-10-09T10:10:22.434877+0000","last_fullsized":"2023-10-09T10:10:22.434877+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"294'7410","reported_seq":7441,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:37.879508+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:10:37.879508+0000","last_peered":"2023-10-09T10:10:37.879508+0000","last_clean":"2023-10-09T10:10:37.879508+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:10:37.879508+0000","last_undegraded":"2023-10-09T10:10:37.879508+0000","last_fullsized":"2023-10-09T10:10:37.879508+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7410,"log_dups_size":0,"ondisk_log_size":7410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396492800,"num_objects":6050,"num_object_clones":0,"num_object_copies":18150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7410,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":121,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444837+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:10:22.444837+0000","last_peered":"2023-10-09T10:10:22.444837+0000","last_clean":"2023-10-09T10:10:22.444837+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:10:22.444837+0000","last_undegraded":"2023-10-09T10:10:22.444837+0000","last_fullsized":"2023-10-09T10:10:22.444837+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":540,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.441944+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:10:22.441944+0000","last_peered":"2023-10-09T10:10:22.441944+0000","last_clean":"2023-10-09T10:10:22.441944+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:10:22.441944+0000","last_undegraded":"2023-10-09T10:10:22.441944+0000","last_fullsized":"2023-10-09T10:10:22.441944+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":198,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077371+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:10:32.077371+0000","last_peered":"2023-10-09T10:10:32.077371+0000","last_clean":"2023-10-09T10:10:32.077371+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:10:32.077371+0000","last_undegraded":"2023-10-09T10:10:32.077371+0000","last_fullsized":"2023-10-09T10:10:32.077371+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":209,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.442815+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:10:22.442815+0000","last_peered":"2023-10-09T10:10:22.442815+0000","last_clean":"2023-10-09T10:10:22.442815+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:10:22.442815+0000","last_undegraded":"2023-10-09T10:10:22.442815+0000","last_fullsized":"2023-10-09T10:10:22.442815+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"294'7602","reported_seq":7634,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:39.524815+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:10:39.524815+0000","last_peered":"2023-10-09T10:10:39.524815+0000","last_clean":"2023-10-09T10:10:39.524815+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:10:39.524815+0000","last_undegraded":"2023-10-09T10:10:39.524815+0000","last_fullsized":"2023-10-09T10:10:39.524815+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7602,"log_dups_size":0,"ondisk_log_size":7602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394526720,"num_objects":6020,"num_object_clones":0,"num_object_copies":18060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7602,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":269,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:24.554515+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:10:24.554515+0000","last_peered":"2023-10-09T10:10:24.554515+0000","last_clean":"2023-10-09T10:10:24.554515+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:10:24.554515+0000","last_undegraded":"2023-10-09T10:10:24.554515+0000","last_fullsized":"2023-10-09T10:10:24.554515+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":170,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445584+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:10:22.445584+0000","last_peered":"2023-10-09T10:10:22.445584+0000","last_clean":"2023-10-09T10:10:22.445584+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:10:22.445584+0000","last_undegraded":"2023-10-09T10:10:22.445584+0000","last_fullsized":"2023-10-09T10:10:22.445584+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":198,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445990+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:10:22.445990+0000","last_peered":"2023-10-09T10:10:22.445990+0000","last_clean":"2023-10-09T10:10:22.445990+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:10:22.445990+0000","last_undegraded":"2023-10-09T10:10:22.445990+0000","last_fullsized":"2023-10-09T10:10:22.445990+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"295'7635","reported_seq":9912,"reported_epoch":295,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:41.187038+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:10:41.187038+0000","last_peered":"2023-10-09T10:10:41.187038+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:10:41.187038+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7635,"log_dups_size":0,"ondisk_log_size":7635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394723328,"num_objects":6023,"num_object_clones":0,"num_object_copies":18069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":4165,"num_objects_unfound":0,"num_objects_dirty":6023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7635,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1118,"num_bytes_recovered":73269248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1857},{"shards":"1,3,4","objects":4165}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":313,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.442763+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:10:22.442763+0000","last_peered":"2023-10-09T10:10:22.442763+0000","last_clean":"2023-10-09T10:10:22.442763+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:10:22.442763+0000","last_undegraded":"2023-10-09T10:10:22.442763+0000","last_fullsized":"2023-10-09T10:10:22.442763+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":325,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434370+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:10:22.434370+0000","last_peered":"2023-10-09T10:10:22.434370+0000","last_clean":"2023-10-09T10:10:22.434370+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:10:22.434370+0000","last_undegraded":"2023-10-09T10:10:22.434370+0000","last_fullsized":"2023-10-09T10:10:22.434370+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445861+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:10:22.445861+0000","last_peered":"2023-10-09T10:10:22.445861+0000","last_clean":"2023-10-09T10:10:22.445861+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:10:22.445861+0000","last_undegraded":"2023-10-09T10:10:22.445861+0000","last_fullsized":"2023-10-09T10:10:22.445861+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"294'7452","reported_seq":7484,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:37.735915+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:10:37.735915+0000","last_peered":"2023-10-09T10:10:37.735915+0000","last_clean":"2023-10-09T10:10:37.735915+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:10:37.735915+0000","last_undegraded":"2023-10-09T10:10:37.735915+0000","last_fullsized":"2023-10-09T10:10:37.735915+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7452,"log_dups_size":0,"ondisk_log_size":7452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400162816,"num_objects":6106,"num_object_clones":0,"num_object_copies":18318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7452,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":257,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.078191+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:10:32.078191+0000","last_peered":"2023-10-09T10:10:32.078191+0000","last_clean":"2023-10-09T10:10:32.078191+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:10:32.078191+0000","last_undegraded":"2023-10-09T10:10:32.078191+0000","last_fullsized":"2023-10-09T10:10:32.078191+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":270,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434847+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:10:22.434847+0000","last_peered":"2023-10-09T10:10:22.434847+0000","last_clean":"2023-10-09T10:10:22.434847+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:10:22.434847+0000","last_undegraded":"2023-10-09T10:10:22.434847+0000","last_fullsized":"2023-10-09T10:10:22.434847+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":246,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077309+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:10:32.077309+0000","last_peered":"2023-10-09T10:10:32.077309+0000","last_clean":"2023-10-09T10:10:32.077309+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:10:32.077309+0000","last_undegraded":"2023-10-09T10:10:32.077309+0000","last_fullsized":"2023-10-09T10:10:32.077309+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"294'7404","reported_seq":7436,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:37.897052+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:10:37.897052+0000","last_peered":"2023-10-09T10:10:37.897052+0000","last_clean":"2023-10-09T10:10:37.897052+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:10:37.897052+0000","last_undegraded":"2023-10-09T10:10:37.897052+0000","last_fullsized":"2023-10-09T10:10:37.897052+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397475864,"num_objects":6066,"num_object_clones":0,"num_object_copies":18198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6066,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7404,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":202,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.443990+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:10:22.443990+0000","last_peered":"2023-10-09T10:10:22.443990+0000","last_clean":"2023-10-09T10:10:22.443990+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:10:22.443990+0000","last_undegraded":"2023-10-09T10:10:22.443990+0000","last_fullsized":"2023-10-09T10:10:22.443990+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":164,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445916+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:10:22.445916+0000","last_peered":"2023-10-09T10:10:22.445916+0000","last_clean":"2023-10-09T10:10:22.445916+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:10:22.445916+0000","last_undegraded":"2023-10-09T10:10:22.445916+0000","last_fullsized":"2023-10-09T10:10:22.445916+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":147,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444656+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:10:22.444656+0000","last_peered":"2023-10-09T10:10:22.444656+0000","last_clean":"2023-10-09T10:10:22.444656+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:10:22.444656+0000","last_undegraded":"2023-10-09T10:10:22.444656+0000","last_fullsized":"2023-10-09T10:10:22.444656+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":159,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445692+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:10:22.445692+0000","last_peered":"2023-10-09T10:10:22.445692+0000","last_clean":"2023-10-09T10:10:22.445692+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:10:22.445692+0000","last_undegraded":"2023-10-09T10:10:22.445692+0000","last_fullsized":"2023-10-09T10:10:22.445692+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":231,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434445+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:10:22.434445+0000","last_peered":"2023-10-09T10:10:22.434445+0000","last_clean":"2023-10-09T10:10:22.434445+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:10:22.434445+0000","last_undegraded":"2023-10-09T10:10:22.434445+0000","last_fullsized":"2023-10-09T10:10:22.434445+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":131,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445064+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:10:22.445064+0000","last_peered":"2023-10-09T10:10:22.445064+0000","last_clean":"2023-10-09T10:10:22.445064+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:10:22.445064+0000","last_undegraded":"2023-10-09T10:10:22.445064+0000","last_fullsized":"2023-10-09T10:10:22.445064+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":173,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077506+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:10:32.077506+0000","last_peered":"2023-10-09T10:10:32.077506+0000","last_clean":"2023-10-09T10:10:32.077506+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:10:32.077506+0000","last_undegraded":"2023-10-09T10:10:32.077506+0000","last_fullsized":"2023-10-09T10:10:32.077506+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":131,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.444954+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:10:22.444954+0000","last_peered":"2023-10-09T10:10:22.444954+0000","last_clean":"2023-10-09T10:10:22.444954+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:10:22.444954+0000","last_undegraded":"2023-10-09T10:10:22.444954+0000","last_fullsized":"2023-10-09T10:10:22.444954+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":182,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077268+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:10:32.077268+0000","last_peered":"2023-10-09T10:10:32.077268+0000","last_clean":"2023-10-09T10:10:32.077268+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:10:32.077268+0000","last_undegraded":"2023-10-09T10:10:32.077268+0000","last_fullsized":"2023-10-09T10:10:32.077268+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":311,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.077655+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:10:32.077655+0000","last_peered":"2023-10-09T10:10:32.077655+0000","last_clean":"2023-10-09T10:10:32.077655+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:10:32.077655+0000","last_undegraded":"2023-10-09T10:10:32.077655+0000","last_fullsized":"2023-10-09T10:10:32.077655+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":332,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.078142+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:10:32.078142+0000","last_peered":"2023-10-09T10:10:32.078142+0000","last_clean":"2023-10-09T10:10:32.078142+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:10:32.078142+0000","last_undegraded":"2023-10-09T10:10:32.078142+0000","last_fullsized":"2023-10-09T10:10:32.078142+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":240,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.445738+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:10:22.445738+0000","last_peered":"2023-10-09T10:10:22.445738+0000","last_clean":"2023-10-09T10:10:22.445738+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:10:22.445738+0000","last_undegraded":"2023-10-09T10:10:22.445738+0000","last_fullsized":"2023-10-09T10:10:22.445738+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":160,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T10:10:22.434010+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:10:22.434010+0000","last_peered":"2023-10-09T10:10:22.434010+0000","last_clean":"2023-10-09T10:10:22.434010+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:10:22.434010+0000","last_undegraded":"2023-10-09T10:10:22.434010+0000","last_fullsized":"2023-10-09T10:10:22.434010+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6348931096,"num_objects":96878,"num_object_clones":0,"num_object_copies":290634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":4165,"num_objects_unfound":0,"num_objects_dirty":96878,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120484,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1118,"num_bytes_recovered":73269248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19149893632,"data_stored":19149881416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120484,"ondisk_log_size":120484,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430492,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2237344,"kb_used_data":1940120,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91507808,"statfs":{"total":95995035648,"available":93703995392,"internally_reserved":0,"allocated":1986682880,"data_stored":1986145993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5940000000000001,"15min":1.5740000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":5.625,"5min":9.0690000000000008,"15min":50.591999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.548,"5min":1.8,"15min":1.6040000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":6.0670000000000002,"5min":8.2300000000000004,"15min":50.747},"last":0.30599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:10:08 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":1.8440000000000001,"15min":1.5169999999999999},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2669999999999995,"5min":16.352,"15min":25.338000000000001},"last":1.359},{"interface":"front","average":{"1min":1.53,"5min":1.8839999999999999,"15min":1.3560000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.9269999999999996,"5min":18.27,"15min":18.27},"last":0.34499999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.7190000000000001,"15min":2.266},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":3.8370000000000002,"5min":7.952,"15min":7.952},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.7310000000000001,"15min":2.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.2039999999999997,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.434}]},{"osd":3,"last update":"Mon Oct 9 10:09:47 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.0049999999999999,"15min":1.835},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.058,"5min":29.058,"15min":29.058},"last":6.3339999999999996},{"interface":"front","average":{"1min":1.214,"5min":1.7989999999999999,"15min":1.897},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.577,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":0.96599999999999997,"15min":0.93000000000000005},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.317},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.91300000000000003,"15min":0.91000000000000003},"min":{"1min":0.28599999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.7919999999999998,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.149,"15min":0.93899999999999995},"min":{"1min":0.31900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":25.277999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.4529999999999998,"5min":1.2589999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.754000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.38200000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.1299999999999999,"15min":0.88600000000000001},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5369999999999999,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.98599999999999999,"15min":0.82499999999999996},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.998,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":0.58599999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626720,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2573200,"kb_used_data":2275976,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91171952,"statfs":{"total":95995035648,"available":93360078848,"internally_reserved":0,"allocated":2330599424,"data_stored":2330079851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.9259999999999999,"15min":1.319},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":15.435,"5min":15.435,"15min":15.435},"last":1.534},{"interface":"front","average":{"1min":1.853,"5min":1.931,"15min":1.365},"min":{"1min":0.252,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.308999999999999,"5min":15.308999999999999,"15min":15.308999999999999},"last":3.3069999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:10:02 2023","interfaces":[{"interface":"back","average":{"1min":4.641,"5min":2.4300000000000002,"15min":1.7130000000000001},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":35.609999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":2.5310000000000001},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.3540000000000001,"15min":1.6319999999999999},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":32.396000000000001,"5min":32.396000000000001,"15min":32.396000000000001},"last":3.3399999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.3340000000000001,"15min":1.6919999999999999},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8149999999999999,"5min":7.6760000000000002,"15min":7.6760000000000002},"last":2.4830000000000001},{"interface":"front","average":{"1min":1.234,"5min":2.3239999999999998,"15min":3.0990000000000002},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.5600000000000005,"5min":10.26,"15min":10.26},"last":1.976}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.321,"15min":1.347},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3879999999999999,"5min":6.0999999999999996,"15min":6.0999999999999996},"last":4.3300000000000001},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.4220000000000002,"15min":2.3940000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":11.211,"5min":11.211,"15min":11.211},"last":3.3660000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.48,"5min":1.48,"15min":1.48},"last":1.456}]},{"osd":5,"last update":"Mon Oct 9 10:10:33 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.73699999999999999,"15min":0.76700000000000002},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9260000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.91900000000000004,"15min":0.79600000000000004},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":4.1529999999999996,"5min":7.508,"15min":7.508},"last":2.919}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.77000000000000002,"15min":0.755},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.5800000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":1.5049999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.81499999999999995,"15min":0.80500000000000005},"min":{"1min":0.39000000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.6080000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":1.5600000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253033,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3089196,"kb_used_data":2717796,"kb_used_omap":9,"kb_used_meta":371382,"kb_avail":90655956,"statfs":{"total":95995035648,"available":92831698944,"internally_reserved":0,"allocated":2783023104,"data_stored":2782003943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":380296169},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":2.6549999999999998,"15min":2.0299999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.185},"max":{"1min":6.3479999999999999,"5min":25.59,"15min":25.59},"last":3.3460000000000001},{"interface":"front","average":{"1min":3.7109999999999999,"5min":2.77,"15min":1.976},"min":{"1min":0.26900000000000002,"5min":0.218,"15min":0.192},"max":{"1min":11.682,"5min":14.769,"15min":16.629000000000001},"last":20.501999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":3.2160000000000002,"15min":3.8700000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.1539999999999999,"5min":17.369,"15min":17.369},"last":11.135},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.907,"15min":3.8740000000000001},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5960000000000001,"5min":16.023,"15min":16.023},"last":5.407}]},{"osd":3,"last update":"Mon Oct 9 10:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":3.1619999999999999,"15min":3.4100000000000001},"min":{"1min":0.36399999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.196999999999999,"5min":16.931000000000001,"15min":16.931000000000001},"last":6.9050000000000002},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.266,"15min":4.6900000000000004},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.8569999999999993,"5min":17.289000000000001,"15min":17.289000000000001},"last":1.419}]},{"osd":4,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.474,"15min":1.3280000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2629999999999999,"5min":7.2629999999999999,"15min":7.2629999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.879,"5min":1.639,"15min":1.4330000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.6940000000000008,"5min":10.132,"15min":10.132},"last":6.9870000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":2.0419999999999998,"15min":1.72},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.473000000000001,"5min":14.416,"15min":15.531000000000001},"last":7.3159999999999998},{"interface":"front","average":{"1min":1.782,"5min":2.0979999999999999,"15min":1.9690000000000001},"min":{"1min":0.34699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.6890000000000001,"5min":16.277999999999999,"15min":19.706},"last":6.75}]},{"osd":6,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0339999999999998,"15min":1.7230000000000001},"min":{"1min":0.50800000000000001,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.3710000000000004,"5min":14.215,"15min":16.193999999999999},"last":0.439},{"interface":"front","average":{"1min":2.75,"5min":1.9930000000000001,"15min":1.7529999999999999},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":10.161,"5min":14.471,"15min":15.481999999999999},"last":6.9530000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":2.153,"15min":1.972},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.7350000000000003,"5min":16.331,"15min":16.331},"last":6.702},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1320000000000001,"15min":1.788},"min":{"1min":0.46000000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3319999999999999,"5min":14.242000000000001,"15min":16.100999999999999},"last":1.929}]}]},{"osd":0,"up_from":79,"seq":339302416559,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2696092,"kb_used_data":2355540,"kb_used_omap":28,"kb_used_meta":340515,"kb_avail":91049060,"statfs":{"total":95995035648,"available":93234237440,"internally_reserved":0,"allocated":2412072960,"data_stored":2411067730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":348688208},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.847,"15min":1.4970000000000001},"min":{"1min":0.24099999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.2990000000000004,"5min":12.84,"15min":13.084},"last":0.97099999999999997},{"interface":"front","average":{"1min":3.9020000000000001,"5min":3.379,"15min":2.2269999999999999},"min":{"1min":0.44900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.949000000000002,"5min":22.178999999999998,"15min":22.178999999999998},"last":4.8280000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.7210000000000001,"15min":2.121},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.5579999999999998,"5min":10.034000000000001,"15min":10.034000000000001},"last":0.37},{"interface":"front","average":{"1min":1.353,"5min":2.2879999999999998,"15min":3.0910000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.4480000000000004,"5min":14.693,"15min":14.693},"last":0.28100000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4990000000000001,"5min":4.391,"15min":4.7069999999999999},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.867000000000001,"5min":25.734999999999999,"15min":25.734999999999999},"last":0.79500000000000004},{"interface":"front","average":{"1min":2.4809999999999999,"5min":4.0250000000000004,"15min":4.282},"min":{"1min":0.44,"5min":0.254,"15min":0.254},"max":{"1min":13.975,"5min":21.923999999999999,"15min":21.923999999999999},"last":4.5380000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.3760000000000003,"5min":4.3760000000000003,"15min":4.3760000000000003},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":13.789999999999999,"5min":13.789999999999999,"15min":13.789999999999999},"last":4.3049999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.484,"5min":2.8149999999999999,"15min":1.847},"min":{"1min":0.38900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.736000000000001,"5min":17.096,"15min":17.096},"last":4.3620000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.419,"15min":1.2470000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.4870000000000001,"5min":12.478,"15min":12.478},"last":0.40899999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.494,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5289999999999999,"5min":9.7260000000000009,"15min":11.699},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.6549999999999998,"15min":1.954},"min":{"1min":0.45100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.4130000000000003,"5min":14.384,"15min":15.108000000000001},"last":4.8650000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:09:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.9929999999999999,"15min":1.986},"min":{"1min":0.51400000000000001,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":13.829000000000001,"5min":14.631,"15min":26.978000000000002},"last":4.3979999999999997},{"interface":"front","average":{"1min":1.085,"5min":1.6819999999999999,"15min":1.544},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.5209999999999999,"5min":12.428000000000001,"15min":26.768000000000001},"last":0.52100000000000002}]}]},{"osd":2,"up_from":217,"seq":932007903293,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2002448,"kb_used_data":1767560,"kb_used_omap":83,"kb_used_meta":234796,"kb_avail":91742704,"statfs":{"total":95995035648,"available":93944528896,"internally_reserved":0,"allocated":1809981440,"data_stored":1808988187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":240431599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.28,"15min":1.6060000000000001},"min":{"1min":0.247,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.5139999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":5.2469999999999999},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.8849999999999998,"15min":2.923},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.891999999999999,"5min":16.309999999999999,"15min":16.309999999999999},"last":3.0569999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.0209999999999999,"15min":2.4649999999999999},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.971,"5min":11.971,"15min":11.971},"last":5.282},{"interface":"front","average":{"1min":2.8849999999999998,"5min":2.492,"15min":2.6280000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.009,"5min":15.569000000000001,"15min":15.569000000000001},"last":3.5990000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.798,"15min":1.948},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.8859999999999999,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":5.21},{"interface":"front","average":{"1min":1.1779999999999999,"5min":3.367,"15min":3.7320000000000002},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.8440000000000003,"5min":15.811,"15min":15.811},"last":3.3490000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.84499999999999997,"15min":0.753},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.29,"5min":6.29,"15min":6.29},"last":3.097},{"interface":"front","average":{"1min":2.5609999999999999,"5min":1.1759999999999999,"15min":0.94499999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":12.385,"5min":12.385,"15min":12.385},"last":2.851}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.5580000000000001,"15min":1.502},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.0870000000000002,"5min":12.423999999999999,"15min":12.423999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.21,"5min":1.2789999999999999,"15min":1.6000000000000001},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.7709999999999999,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":10.351000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2629999999999999,"15min":1.0489999999999999},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.6040000000000001,"5min":15.292999999999999,"15min":15.292999999999999},"last":3.1579999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.6499999999999999,"15min":1.712},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.984,"5min":12.394,"15min":12.394},"last":0.42599999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.2809999999999999,"15min":1.0760000000000001},"min":{"1min":0.48499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1800000000000002,"5min":15.343999999999999,"15min":15.343999999999999},"last":3.2010000000000001},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.7170000000000001,"15min":1.613},"min":{"1min":0.51200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.5010000000000003,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.46000000000000002}]}]},{"osd":3,"up_from":250,"seq":1073741824037,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2536468,"kb_used_data":2298444,"kb_used_omap":10,"kb_used_meta":237941,"kb_avail":91208684,"statfs":{"total":95995035648,"available":93397692416,"internally_reserved":0,"allocated":2353606656,"data_stored":2352441011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":243651651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":4.2489999999999997,"15min":4.5519999999999996},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.465,"5min":23.381,"15min":23.381},"last":4.0999999999999996},{"interface":"front","average":{"1min":1.853,"5min":3.7130000000000001,"15min":4.0229999999999997},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.5190000000000001,"5min":14.289999999999999,"15min":14.289999999999999},"last":0.51300000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5329999999999999,"15min":3.8130000000000002},"min":{"1min":0.28799999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.415,"5min":20.407,"15min":20.407},"last":1.1890000000000001},{"interface":"front","average":{"1min":2.702,"5min":3.2850000000000001,"15min":3.3820000000000001},"min":{"1min":0.40300000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.494,"5min":23.423999999999999,"15min":23.423999999999999},"last":4.2220000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":3.3410000000000002,"15min":3.6070000000000002},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.1020000000000003,"5min":20.356000000000002,"15min":20.356000000000002},"last":1.21},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.7290000000000001,"15min":2.819},"min":{"1min":0.34799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":9.3379999999999992,"5min":23.452000000000002,"15min":23.452000000000002},"last":4.149}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":1.6719999999999999,"15min":1.288},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.676,"5min":15.676,"15min":15.676},"last":4.1929999999999996},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5800000000000001,"15min":1.534},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.426,"5min":12.426,"15min":12.426},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.4319999999999999,"15min":2.5529999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.6199999999999992,"5min":11.836,"15min":11.836},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.274,"15min":2.327},"min":{"1min":0.55600000000000005,"5min":0.309,"15min":0.309},"max":{"1min":6.9249999999999998,"5min":14.164999999999999,"15min":14.164999999999999},"last":1.1299999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.5379999999999998,"15min":2.6720000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.532,"5min":11.887,"15min":11.887},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.496,"5min":2.3820000000000001,"15min":2.5299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.8710000000000004,"5min":14.125,"15min":14.125},"last":1.165}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.4039999999999999,"15min":1.3939999999999999},"min":{"1min":0.45400000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0609999999999999,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":4.173},{"interface":"front","average":{"1min":1.8089999999999999,"5min":2.5009999999999999,"15min":2.617},"min":{"1min":0.42199999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":8.6549999999999994,"5min":11.801,"15min":11.801},"last":0.61899999999999999}]}]},{"osd":4,"up_from":266,"seq":1142461300762,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2620436,"kb_used_data":2301772,"kb_used_omap":35,"kb_used_meta":318620,"kb_avail":91124716,"statfs":{"total":95995035648,"available":93311709184,"internally_reserved":0,"allocated":2357014528,"data_stored":2356489291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":326267118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.3799999999999999,"5min":4.3799999999999999,"15min":4.3799999999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3959999999999999,"5min":2.3959999999999999,"15min":2.3959999999999999},"last":4.3940000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":2.7160000000000002,"5min":2.7160000000000002,"15min":2.7160000000000002},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":3.101}]},{"osd":2,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.661,"5min":3.661,"15min":3.661},"last":3.3130000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6449999999999996,"5min":4.6449999999999996,"15min":4.6449999999999996},"last":0.55000000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.262,"15min":1.262},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.1259999999999999,"5min":3.1259999999999999,"15min":3.1259999999999999},"last":0.751},{"interface":"front","average":{"1min":1.014,"5min":1.014,"15min":1.014},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.4470000000000001,"5min":4.4470000000000001,"15min":4.4470000000000001},"last":0.42099999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":5.4740000000000002},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.6970000000000001,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.5609999999999999,"5min":2.5609999999999999,"15min":2.5609999999999999},"last":2.274},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.917,"5min":1.917,"15min":1.917},"last":0.38100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.5630000000000002,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":3.0680000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822947,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3434696,"kb_used_data":3129280,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90310456,"statfs":{"total":95995035648,"available":92477906944,"internally_reserved":0,"allocated":3204382720,"data_stored":3203865539,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":1.55,"15min":1.488},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.8540000000000001,"5min":7.8540000000000001,"15min":15.773999999999999},"last":1.732},{"interface":"front","average":{"1min":1.339,"5min":1.597,"15min":1.532},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.0629999999999997,"5min":11.747,"15min":15.699},"last":1.145}]},{"osd":1,"last update":"Mon Oct 9 10:10:12 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.5740000000000001,"15min":1.4430000000000001},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.7720000000000002,"5min":11.335000000000001,"15min":15.689},"last":0.626},{"interface":"front","average":{"1min":1.913,"5min":1.861,"15min":1.6180000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.8119999999999994,"5min":19.402999999999999,"15min":19.402999999999999},"last":2.1480000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.8480000000000001,"15min":2.4369999999999998},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.3819999999999997,"5min":15.640000000000001,"15min":15.640000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.302,"5min":1.8220000000000001,"15min":2.2949999999999999},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.3659999999999997,"5min":10.656000000000001,"15min":10.656000000000001},"last":1.232}]},{"osd":3,"last update":"Mon Oct 9 10:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.734,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5860000000000003,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":3.3479999999999999},{"interface":"front","average":{"1min":1.2,"5min":2.6549999999999998,"15min":2.8969999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":2.9710000000000001,"5min":17.34,"15min":17.34},"last":2.8319999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":0.71699999999999997,"15min":0.61499999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5220000000000002,"5min":5.5220000000000002,"15min":5.5220000000000002},"last":2.2400000000000002},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.752,"15min":0.746},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.823,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":1.177}]},{"osd":6,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.014,"15min":0.94499999999999995},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.25},"max":{"1min":5.4930000000000003,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.0189999999999999,"15min":0.88700000000000001},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.8099999999999996,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0589999999999999,"15min":0.88},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.282,"5min":7.4290000000000003,"15min":10.334},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.749,"15min":0.88600000000000001},"min":{"1min":0.35099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.3500000000000001,"5min":8.0210000000000008,"15min":12.356},"last":1.204}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2395471872,"data_stored":2395471872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2776236032,"data_stored":2776236032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1798836224,"data_stored":1798832152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2344484864,"data_stored":2344484864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2342977536,"data_stored":2342977536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3195801600,"data_stored":3195797528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2324103168,"data_stored":2324103168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1971982336,"data_stored":1971978264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:10:42.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:10:42.564 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:42.651 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:42.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:42.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:42.669 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:42 smithi110 ceph-mon[136032]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 5374/290706 objects degraded (1.849%); 4165/290706 objects misplaced (1.433%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:10:42.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:42 smithi110 ceph-mon[136032]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:42.761 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:42.803 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:42.855 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:42.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:42.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:42.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:10:42.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:42.996 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:43.146 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:43.149 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:43.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:43.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:43.349 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/290706 objects degraded (1.849%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:43.350 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/290706 objects degraded (1.849%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:43.353 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:43.493 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/290706 objects degraded (1.849%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:43.764 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:10:43.764 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:10:44.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:10:44.418 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:44.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:44.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:44 smithi110 ceph-mon[136032]: pgmap v785: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 5374/285789 objects degraded (1.880%); 4165/285789 objects misplaced (1.457%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:44.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:44.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:44.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:44 smithi012 ceph-mon[147926]: pgmap v785: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 5374/285789 objects degraded (1.880%); 4165/285789 objects misplaced (1.457%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:44.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:44.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:44 smithi012 ceph-mon[152263]: pgmap v785: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 5374/285789 objects degraded (1.880%); 4165/285789 objects misplaced (1.457%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:44.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:44.601 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:44.709 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:44.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:44.784 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:44.813 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:44.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:44.921 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:45.405 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:45.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:45.518 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:45.579 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:45.634 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:45.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:45.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:45.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:45.795 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:46.657 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:46 smithi012 ceph-mon[147926]: pgmap v786: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5374/282471 objects degraded (1.902%); 4165/282471 objects misplaced (1.474%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:46.657 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:46 smithi012 ceph-mon[152263]: pgmap v786: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5374/282471 objects degraded (1.902%); 4165/282471 objects misplaced (1.474%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:46.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:46 smithi110 ceph-mon[136032]: pgmap v786: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5374/282471 objects degraded (1.902%); 4165/282471 objects misplaced (1.474%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:46.696 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:46.726 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:46.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:46.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:46.928 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:47.081 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:47.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:47.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:47.119 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:10:47.119 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:10:47.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:47.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:47.505 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:47.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:47.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:47.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:10:47.611 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:47.660 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:47.718 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:47.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:47.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:47.827 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:47.860 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:47.860 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5374/280530 objects degraded (1.916%), 2 pgs degraded","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":964,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":295,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":93510,"data_bytes":6127228856,"bytes_used":21128212480,"bytes_avail":746832072704,"bytes_total":767960285184,"degraded_objects":5374,"degraded_total":280530,"degraded_ratio":0.019156596442448222,"misplaced_objects":4043,"misplaced_total":280530,"misplaced_ratio":0.014412005846077068,"recovering_objects_per_sec":24,"recovering_bytes_per_sec":1637640,"recovering_keys_per_sec":0,"num_objects_recovered":250,"num_bytes_recovered":16384000,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":681},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T10:09:33.293227+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T10:10:47.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:47.902 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:10:47.925 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:47.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:47.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:47.945 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:10:48.004 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:10:48.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:48.060 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:10:48.096 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:10:48.108 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:10:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:10:47] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-10-09T10:10:48.111 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:48.127 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:10:48.183 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:10:48.200 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:48.227 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:10:48.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:10:48.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:10:48.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:10:48.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:10:48.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:10:48.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:10:48.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:10:48.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:10:48.628 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:48.635 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:10:48.635 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:10:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:48.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:48.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:48 smithi012 ceph-mon[147926]: pgmap v787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 5374/280530 objects degraded (1.916%); 4043/280530 objects misplaced (1.441%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:10:48.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3976877636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:48.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2645998397' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:48.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/280530 objects degraded (1.916%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:48.683 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:48 smithi012 ceph-mon[152263]: pgmap v787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 5374/280530 objects degraded (1.916%); 4043/280530 objects misplaced (1.441%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:10:48.683 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3976877636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:48.683 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2645998397' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:48.683 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/280530 objects degraded (1.916%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:48.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:48 smithi110 ceph-mon[136032]: pgmap v787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 5374/280530 objects degraded (1.916%); 4043/280530 objects misplaced (1.441%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:10:48.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3976877636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:48.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2645998397' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:10:48.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/280530 objects degraded (1.916%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:10:48.738 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:48.773 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:48.939 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:48.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:48.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:49.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:49.155 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:49.246 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:49.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:49.354 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:49.437 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:49.509 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:10:49.509 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:10:49.510 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:10:49.510 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:10:49.510 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:10:49.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:49.527 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:49.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:49.628 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:49.688 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:49.729 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:49.810 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:49.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:49.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:49.919 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:49.973 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:10:49.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:50.058 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:50.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:50.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:50.209 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:50.273 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:50.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:50.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:50.473 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:50.499 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:10:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:50 smithi012 ceph-mon[147926]: pgmap v788: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5374/275508 objects degraded (1.951%); 4043/275508 objects misplaced (1.467%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:10:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1555651538' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:50.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:50 smithi012 ceph-mon[152263]: pgmap v788: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5374/275508 objects degraded (1.951%); 4043/275508 objects misplaced (1.467%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:10:50.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1555651538' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:50.681 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:50.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:50.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:50.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:50 smithi110 ceph-mon[136032]: pgmap v788: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5374/275508 objects degraded (1.951%); 4043/275508 objects misplaced (1.467%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:10:50.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1555651538' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:10:50.719 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:50.785 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:50.834 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:50.898 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:50.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:51.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:51.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:10:51.074 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:51.208 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:51.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:51.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:51.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:51.452 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:51.537 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:51.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:51.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:51.652 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:51.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:51 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:10:51.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:51 smithi012 ceph-mon[147926]: failed to encode map e296 with expected crc 2023-10-09T10:10:51.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1555651538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:51.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:51 smithi012 ceph-mon[147926]: osdmap e296: 8 total, 8 up, 8 in 2023-10-09T10:10:51.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:51 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:10:51.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:51 smithi012 ceph-mon[152263]: failed to encode map e296 with expected crc 2023-10-09T10:10:51.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1555651538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:51.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:51 smithi012 ceph-mon[152263]: osdmap e296: 8 total, 8 up, 8 in 2023-10-09T10:10:51.693 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:51.753 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:51.755 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:51 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:10:51.756 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:51 smithi110 ceph-mon[136032]: failed to encode map e296 with expected crc 2023-10-09T10:10:51.756 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1555651538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:10:51.756 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:51 smithi110 ceph-mon[136032]: osdmap e296: 8 total, 8 up, 8 in 2023-10-09T10:10:51.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:51.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:51.858 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:51.892 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:51.906 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:51.957 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:51.986 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:10:52.074 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:52.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:52.155 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:10:52.155 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:10:52.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:10:52.155 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:10:52.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.156 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190230 7.1 7:8724f90f:::benchmark_data_smithi012_221426_object82452:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.156 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.878911+0000", 2023-10-09T10:10:52.156 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27146106999999, 2023-10-09T10:10:52.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.239926774, 2023-10-09T10:10:52.156 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.156 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.156 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.156 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.157 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.157 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190230 2023-10-09T10:10:52.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.157 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878911+0000", 2023-10-09T10:10:52.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878911+0000", 2023-10-09T10:10:52.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:52.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878910+0000", 2023-10-09T10:10:52.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6420000000000007e-06 2023-10-09T10:10:52.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878917+0000", 2023-10-09T10:10:52.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3363999999999999e-05 2023-10-09T10:10:52.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878931+0000", 2023-10-09T10:10:52.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4010000000000002e-06 2023-10-09T10:10:52.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878937+0000", 2023-10-09T10:10:52.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4478999999999998e-05 2023-10-09T10:10:52.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878991+0000", 2023-10-09T10:10:52.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0682e-05 2023-10-09T10:10:52.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-10-09T10:10:52.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879002+0000", 2023-10-09T10:10:52.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080712920000000007 2023-10-09T10:10:52.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887073+0000", 2023-10-09T10:10:52.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4110000000000002e-06 2023-10-09T10:10:52.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887083+0000", 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2200019850000001 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.107085+0000", 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002097448 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.109182+0000", 2023-10-09T10:10:52.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105568 2023-10-09T10:10:52.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.109288+0000", 2023-10-09T10:10:52.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0092099759999999999 2023-10-09T10:10:52.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118498+0000", 2023-10-09T10:10:52.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8666999999999999e-05 2023-10-09T10:10:52.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118557+0000", 2023-10-09T10:10:52.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2558000000000001e-05 2023-10-09T10:10:52.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118609+0000", 2023-10-09T10:10:52.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022877999999999999 2023-10-09T10:10:52.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118838+0000", 2023-10-09T10:10:52.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.239926774 2023-10-09T10:10:52.168 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.168 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.169 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.169 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190272 7.1 7:8eae50ab:::benchmark_data_smithi012_221426_object82494:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.169 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.878946+0000", 2023-10-09T10:10:52.169 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27142600799999, 2023-10-09T10:10:52.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2401155699999999, 2023-10-09T10:10:52.169 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.170 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.170 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.170 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.170 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.170 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190272 2023-10-09T10:10:52.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.170 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878946+0000", 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878946+0000", 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878946+0000", 2023-10-09T10:10:52.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6419999999999999e-06 2023-10-09T10:10:52.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878948+0000", 2023-10-09T10:10:52.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7689999999999998e-06 2023-10-09T10:10:52.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878952+0000", 2023-10-09T10:10:52.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4310000000000001e-06 2023-10-09T10:10:52.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878955+0000", 2023-10-09T10:10:52.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010689 2023-10-09T10:10:52.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879061+0000", 2023-10-09T10:10:52.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2700000000000004e-06 2023-10-09T10:10:52.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879067+0000", 2023-10-09T10:10:52.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080352559999999993 2023-10-09T10:10:52.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887102+0000", 2023-10-09T10:10:52.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.835e-06 2023-10-09T10:10:52.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887108+0000", 2023-10-09T10:10:52.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2253334979999999 2023-10-09T10:10:52.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.112441+0000", 2023-10-09T10:10:52.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010237499999999999 2023-10-09T10:10:52.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.112544+0000", 2023-10-09T10:10:52.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6977000000000004e-05 2023-10-09T10:10:52.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.112631+0000", 2023-10-09T10:10:52.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062493150000000001 2023-10-09T10:10:52.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118880+0000", 2023-10-09T10:10:52.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1987999999999999e-05 2023-10-09T10:10:52.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118892+0000", 2023-10-09T10:10:52.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5519999999999999e-05 2023-10-09T10:10:52.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118927+0000", 2023-10-09T10:10:52.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013421700000000001 2023-10-09T10:10:52.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119062+0000", 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2401155699999999 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190277 7.1 7:82c2f525:::benchmark_data_smithi012_221426_object82499:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.878979+0000", 2023-10-09T10:10:52.181 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.271392768, 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2403365749999999, 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190277 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878979+0000", 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878979+0000", 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878979+0000", 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9800000000000001e-06 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878981+0000", 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5780000000000004e-06 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878988+0000", 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8729999999999999e-06 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878991+0000", 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011209000000000001 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879103+0000", 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7549999999999994e-06 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879110+0000", 2023-10-09T10:10:52.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080152250000000008 2023-10-09T10:10:52.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887126+0000", 2023-10-09T10:10:52.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6300000000000003e-06 2023-10-09T10:10:52.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887131+0000", 2023-10-09T10:10:52.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2282280270000001 2023-10-09T10:10:52.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.115359+0000", 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000160617 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.115520+0000", 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014887599999999999 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.115669+0000", 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003501798 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119171+0000", 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5810000000000003e-06 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119180+0000", 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0796000000000001e-05 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119211+0000", 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105062 2023-10-09T10:10:52.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119316+0000", 2023-10-09T10:10:52.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2403365749999999 2023-10-09T10:10:52.193 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.193 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.193 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.194 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190284 7.1 7:8f7a2a8f:::benchmark_data_smithi012_221426_object82506:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.194 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879000+0000", 2023-10-09T10:10:52.194 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27137174200001, 2023-10-09T10:10:52.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2740284640000001, 2023-10-09T10:10:52.194 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.194 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.194 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.195 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.195 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.195 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190284 2023-10-09T10:10:52.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.195 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879000+0000", 2023-10-09T10:10:52.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879000+0000", 2023-10-09T10:10:52.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:52.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879000+0000", 2023-10-09T10:10:52.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.689e-06 2023-10-09T10:10:52.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879002+0000", 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3459999999999998e-06 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879005+0000", 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1919999999999999e-06 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879007+0000", 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013018 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879138+0000", 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8340000000000001e-06 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879141+0000", 2023-10-09T10:10:52.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080076969999999994 2023-10-09T10:10:52.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887149+0000", 2023-10-09T10:10:52.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.173e-06 2023-10-09T10:10:52.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887155+0000", 2023-10-09T10:10:52.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.23219162 2023-10-09T10:10:52.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119347+0000", 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9079999999999995e-05 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119426+0000", 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0434999999999998e-05 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119506+0000", 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015469747000000001 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.134976+0000", 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017723145999999999 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.152699+0000", 2023-10-09T10:10:52.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9720000000000002e-05 2023-10-09T10:10:52.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.152779+0000", 2023-10-09T10:10:52.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024988099999999997 2023-10-09T10:10:52.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153029+0000", 2023-10-09T10:10:52.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2740284640000001 2023-10-09T10:10:52.205 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.205 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.205 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.206 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190290 7.1 7:8a45fb96:::benchmark_data_smithi012_221426_object82512:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.206 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879010+0000", 2023-10-09T10:10:52.206 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27136264999999, 2023-10-09T10:10:52.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.27435247, 2023-10-09T10:10:52.206 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.207 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.207 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.207 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.207 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.207 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190290 2023-10-09T10:10:52.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.207 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879010+0000", 2023-10-09T10:10:52.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879010+0000", 2023-10-09T10:10:52.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:52.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879009+0000", 2023-10-09T10:10:52.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7489999999999999e-06 2023-10-09T10:10:52.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879011+0000", 2023-10-09T10:10:52.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8449999999999999e-06 2023-10-09T10:10:52.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879014+0000", 2023-10-09T10:10:52.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1009999999999999e-06 2023-10-09T10:10:52.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879016+0000", 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015992600000000001 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879176+0000", 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2989999999999999e-06 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879183+0000", 2023-10-09T10:10:52.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079899610000000003 2023-10-09T10:10:52.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887173+0000", 2023-10-09T10:10:52.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.733e-06 2023-10-09T10:10:52.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887179+0000", 2023-10-09T10:10:52.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.235754005 2023-10-09T10:10:52.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.122933+0000", 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000144983 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.123078+0000", 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016969899999999999 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.123248+0000", 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011732963000000001 2023-10-09T10:10:52.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.134981+0000", 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018158093 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153139+0000", 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9548e-05 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153198+0000", 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016380799999999999 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153362+0000", 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.27435247 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190305 7.1 7:8f4b1bee:::benchmark_data_smithi012_221426_object82527:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879018+0000", 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27135411200001, 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2746395050000001, 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190305 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.218 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879018+0000", 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879018+0000", 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879018+0000", 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5260000000000001e-06 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879019+0000", 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9170000000000002e-06 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879022+0000", 2023-10-09T10:10:52.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9630000000000001e-06 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879024+0000", 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000183127 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879207+0000", 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.698e-06 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879211+0000", 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079855849999999999 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887197+0000", 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6919999999999999e-06 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887202+0000", 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2399458830000001 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.127148+0000", 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8029000000000006e-05 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.127246+0000", 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101876 2023-10-09T10:10:52.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.127348+0000", 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010995153000000001 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.138343+0000", 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015098489 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153442+0000", 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7228999999999997e-05 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153499+0000", 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000158574 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153658+0000", 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2746395050000001 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190320 7.1 7:8f8740a7:::benchmark_data_smithi012_221426_object82542:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879026+0000", 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.271345844, 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2749258020000001, 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.226 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190320 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879026+0000", 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879026+0000", 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879026+0000", 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9659999999999999e-06 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879028+0000", 2023-10-09T10:10:52.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8679999999999999e-06 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879031+0000", 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1040000000000001e-06 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879033+0000", 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000216457 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879249+0000", 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8610000000000004e-06 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879258+0000", 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079627389999999999 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887221+0000", 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7089999999999999e-06 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887227+0000", 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432745869999999 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.130501+0000", 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0001515 2023-10-09T10:10:52.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.130653+0000", 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000165565 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.130818+0000", 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018356846 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.149175+0000", 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045797640000000001 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153755+0000", 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7685000000000001e-05 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153813+0000", 2023-10-09T10:10:52.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139376 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153952+0000", 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2749258020000001 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190328 7.1 7:8450f8de:::benchmark_data_smithi012_221426_object82550:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.234 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879051+0000", 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27132079200001, 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2833133970000001, 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190328 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879051+0000", 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879051+0000", 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879051+0000", 2023-10-09T10:10:52.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5710000000000001e-06 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879053+0000", 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0030000000000002e-06 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879058+0000", 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2369999999999999e-06 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879060+0000", 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023918499999999999 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879299+0000", 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.952e-06 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879308+0000", 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079363869999999996 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887244+0000", 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9599999999999997e-06 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887250+0000", 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.247749566 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.135000+0000", 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3584000000000004e-05 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.135094+0000", 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103705 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.135197+0000", 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017849231 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153046+0000", 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091115929999999994 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162158+0000", 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8216999999999997e-05 2023-10-09T10:10:52.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162196+0000", 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016845900000000001 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162365+0000", 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2833133970000001 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190432 7.1 7:8d343cbd:::benchmark_data_smithi012_221426_object82654:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879062+0000", 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27130989099999, 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.283487313, 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190432 2023-10-09T10:10:52.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879062+0000", 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879062+0000", 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879062+0000", 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0451999999999999e-05 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879072+0000", 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6150000000000001e-06 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879076+0000", 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.119e-06 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879078+0000", 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000267056 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879345+0000", 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5450000000000003e-06 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879354+0000", 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079143709999999999 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887268+0000", 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.75e-06 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887274+0000", 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2511190320000001 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.138393+0000", 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012326500000000001 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.138516+0000", 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016769300000000001 2023-10-09T10:10:52.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.138684+0000", 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014373889000000001 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153058+0000", 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0093553799999999999 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162413+0000", 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2394000000000002e-05 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162446+0000", 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103983 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162550+0000", 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.283487313 2023-10-09T10:10:52.250 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190433 7.1 7:8e64efc2:::benchmark_data_smithi012_221426_object82655:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879094+0000", 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27127821600001, 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2921292529999999, 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.251 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190433 2023-10-09T10:10:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879094+0000", 2023-10-09T10:10:52.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879094+0000", 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879094+0000", 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7120000000000001e-06 2023-10-09T10:10:52.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879095+0000", 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9830000000000004e-06 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879100+0000", 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0800000000000002e-06 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879103+0000", 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030411300000000001 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879408+0000", 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0961999999999999e-05 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879419+0000", 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078758939999999996 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887294+0000", 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6760000000000001e-06 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887301+0000", 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2544338390000001 2023-10-09T10:10:52.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.141735+0000", 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012397100000000001 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.141859+0000", 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014839000000000001 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.142007+0000", 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011061447 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153069+0000", 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017947520000000002 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.171016+0000", 2023-10-09T10:10:52.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5809999999999998e-05 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.171052+0000", 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017113600000000001 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.171223+0000", 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2921292529999999 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.259 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190435 7.1 7:8ff35f65:::benchmark_data_smithi012_221426_object82657:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879106+0000", 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.271266134, 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311420099, 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.260 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190435 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879106+0000", 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879106+0000", 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879106+0000", 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4610000000000001e-06 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879107+0000", 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9289999999999998e-06 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879110+0000", 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0190000000000001e-06 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879112+0000", 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034980699999999999 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879462+0000", 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3690000000000001e-06 2023-10-09T10:10:52.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879469+0000", 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078489819999999991 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887318+0000", 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9220000000000002e-06 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887324+0000", 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2580026360000001 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.145327+0000", 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000171764 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.145499+0000", 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017433100000000001 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.145673+0000", 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080011289999999992 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153674+0000", 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036605540999999998 2023-10-09T10:10:52.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190280+0000", 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2682000000000001e-05 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190332+0000", 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019375499999999999 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190526+0000", 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311420099 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190446 7.1 7:87cf5a1e:::benchmark_data_smithi012_221426_object82668:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879114+0000", 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27125789199999, 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311691205, 2023-10-09T10:10:52.268 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190446 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879114+0000", 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879114+0000", 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879114+0000", 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3489999999999999e-06 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879115+0000", 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8270000000000002e-06 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879118+0000", 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9860000000000001e-06 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879120+0000", 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000402023 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879522+0000", 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0487e-05 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879533+0000", 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078089259999999999 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887342+0000", 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5110000000000003e-06 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887347+0000", 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.261880992 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.149228+0000", 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014189699999999999 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.149370+0000", 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017402699999999999 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.149544+0000", 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041390330000000003 2023-10-09T10:10:52.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153683+0000", 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036911868 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190595+0000", 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4880000000000002e-05 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190660+0000", 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014562900000000001 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190805+0000", 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311691205 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190476 7.1 7:8e6b68f7:::benchmark_data_smithi012_221426_object82698:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879134+0000", 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.27123767399999, 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311917888, 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190476 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879134+0000", 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879134+0000", 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879134+0000", 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.353e-06 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879136+0000", 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2619999999999999e-06 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879139+0000", 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6290000000000002e-06 2023-10-09T10:10:52.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879141+0000", 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044317599999999999 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879585+0000", 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4009999999999997e-06 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879593+0000", 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077714439999999997 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887365+0000", 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6930000000000004e-06 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887370+0000", 2023-10-09T10:10:52.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.266616481 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153987+0000", 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011042100000000001 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.154097+0000", 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9776000000000002e-05 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.154187+0000", 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081925020000000008 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162379+0000", 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02848268 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190862+0000", 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4193000000000001e-05 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190916+0000", 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013610299999999999 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.191052+0000", 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311917888 2023-10-09T10:10:52.284 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190498 7.1 7:8b87c9fd:::benchmark_data_smithi012_221426_object82720:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879144+0000", 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.271228541, 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3120772359999999, 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.285 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190498 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879144+0000", 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879144+0000", 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879143+0000", 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.46e-06 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879145+0000", 2023-10-09T10:10:52.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2200000000000001e-06 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879148+0000", 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0999999999999998e-06 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879150+0000", 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047652399999999998 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879627+0000", 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.33e-06 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879631+0000", 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077566129999999999 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887388+0000", 2023-10-09T10:10:52.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8239999999999996e-06 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887393+0000", 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.270337568 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.157731+0000", 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1216000000000002e-05 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.157822+0000", 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100985 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.157923+0000", 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069121190000000004 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.164835+0000", 2023-10-09T10:10:52.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026256166000000001 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.191092+0000", 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2458e-05 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.191124+0000", 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6877000000000001e-05 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.191221+0000", 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3120772359999999 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190516 7.1 7:83f35799:::benchmark_data_smithi012_221426_object82738:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879152+0000", 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.271219841, 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3136645630000001, 2023-10-09T10:10:52.293 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190516 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879152+0000", 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879152+0000", 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879152+0000", 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.361e-06 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879153+0000", 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.909e-06 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879156+0000", 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3750000000000001e-06 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879159+0000", 2023-10-09T10:10:52.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050572799999999999 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879664+0000", 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8779999999999998e-06 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879672+0000", 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077386989999999999 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887411+0000", 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8189999999999997e-06 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887417+0000", 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.272863077 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.160280+0000", 2023-10-09T10:10:52.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7027999999999994e-05 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.160367+0000", 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6990000000000006e-05 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.160454+0000", 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043883569999999998 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.164842+0000", 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027848528000000001 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192691+0000", 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1674000000000001e-05 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192733+0000", 2023-10-09T10:10:52.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4364999999999996e-05 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192817+0000", 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3136645630000001 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190517 7.1 7:81260c21:::benchmark_data_smithi012_221426_object82739:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:10:52.301 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879167+0000", 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 189.271205307, 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.313835233, 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190517 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879167+0000", 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879167+0000", 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879167+0000", 2023-10-09T10:10:52.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4169999999999999e-06 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879168+0000", 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2229999999999999e-06 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879171+0000", 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1289999999999999e-06 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879173+0000", 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00051873299999999995 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879692+0000", 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2479999999999998e-06 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:10:52.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879696+0000", 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077381469999999999 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887435+0000", 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7749999999999998e-06 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887440+0000", 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2751414780000001 2023-10-09T10:10:52.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162582+0000", 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1100000000000006e-05 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162663+0000", 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1507000000000004e-05 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162744+0000", 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082317209999999991 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.170976+0000", 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021878416000000001 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192855+0000", 2023-10-09T10:10:52.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4965999999999997e-05 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192889+0000", 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112611 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.193002+0000", 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.313835233 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.309 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:120633 8.3 e294/271 8:cd847d01:::benchmark_data_smithi012_271002_object11950:head v 294'7555, mlcod=294'7549)", 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:39.345158+0000", 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 12.805214563, 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021133103, 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.345158+0000", 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.345158+0000", 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.345157+0000", 2023-10-09T10:10:52.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1885999999999999e-05 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.345169+0000", 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3338e-05 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.345182+0000", 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.477e-06 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.345188+0000", 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067824360000000002 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.351971+0000", 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5686000000000001e-05 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.351997+0000", 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014250084 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.366247+0000", 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4001000000000001e-05 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:39.366291+0000", 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021133103 2023-10-09T10:10:52.314 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:127587 8.5 e295/271 8:ae4efd40:::benchmark_data_smithi012_271002_object18904:head v 295'8119, mlcod=295'8117)", 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:49.629078+0000", 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.521293934, 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021098089, 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629078+0000", 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629078+0000", 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629077+0000", 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2557e-05 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629089+0000", 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9191e-05 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629109+0000", 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0490000000000004e-06 2023-10-09T10:10:52.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629117+0000", 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038291570000000001 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.632946+0000", 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1557e-05 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.632967+0000", 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017182851999999998 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.650150+0000", 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6109e-05 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.650176+0000", 2023-10-09T10:10:52.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021098089 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:127590 8.5 e295/271 8:a261f599:::benchmark_data_smithi012_271002_object18907:head v 295'8120, mlcod=295'8117)", 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:49.629132+0000", 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.5212403249999999, 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02106661, 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.320 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629132+0000", 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629132+0000", 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629131+0000", 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3020000000000002e-06 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629137+0000", 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2879999999999998e-06 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629143+0000", 2023-10-09T10:10:52.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0600000000000001e-06 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.629147+0000", 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077495189999999999 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.636897+0000", 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1239000000000001e-05 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.636918+0000", 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013261347 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.650179+0000", 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9238000000000001e-05 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:49.650198+0000", 2023-10-09T10:10:52.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02106661 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:128833 8.3 e296/271 8:ca964581:::benchmark_data_smithi012_271002_object20150:head v 296'8102, mlcod=296'8100)", 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:10:51.614607+0000", 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.53576537300000004, 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019577653, 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:10:52.325 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.614607+0000", 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.614607+0000", 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.614606+0000", 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9279999999999999e-06 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.614615+0000", 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5133000000000001e-05 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.614630+0000", 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6240000000000002e-06 2023-10-09T10:10:52.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.614637+0000", 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8510999999999997e-05 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.614686+0000", 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6027999999999999e-05 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:10:52.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.614712+0000", 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019329802 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.634042+0000", 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014279399999999999 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:10:51.634184+0000", 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019577653 2023-10-09T10:10:52.329 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:10:52.330 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:10:52.330 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:10:52.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:52.404 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:52.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:10:52.556 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:52.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:52.577 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":790,"stamp":"2023-10-09T10:10:51.339069+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5908273080,"num_objects":90169,"num_object_clones":0,"num_object_copies":270507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":3924,"num_objects_unfound":0,"num_objects_dirty":90169,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":127477,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1428,"num_bytes_recovered":90254028,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126411,"ondisk_log_size":126411,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20012296,"kb_used_data":17531848,"kb_used_omap":167,"kb_used_meta":2480152,"kb_avail":729948920,"statfs":{"total":767960285184,"available":747467694080,"internally_reserved":0,"allocated":17952612352,"data_stored":17946313221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2539676249},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,4,2,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-441253888,"num_objects":-6733,"num_object_clones":0,"num_object_copies":-20199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-241,"num_objects_unfound":0,"num_objects_dirty":-6733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6733,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":241,"num_bytes_recovered":15794176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002936"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":136,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.099715+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:10:32.099715+0000","last_peered":"2023-10-09T10:10:32.099715+0000","last_clean":"2023-10-09T10:10:32.099715+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:10:32.099715+0000","last_undegraded":"2023-10-09T10:10:32.099715+0000","last_fullsized":"2023-10-09T10:10:32.099715+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":253,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200283+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:10:40.200283+0000","last_peered":"2023-10-09T10:10:40.200283+0000","last_clean":"2023-10-09T10:10:40.200283+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:10:40.200283+0000","last_undegraded":"2023-10-09T10:10:40.200283+0000","last_fullsized":"2023-10-09T10:10:40.200283+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":297,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.099544+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:10:32.099544+0000","last_peered":"2023-10-09T10:10:32.099544+0000","last_clean":"2023-10-09T10:10:32.099544+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:10:32.099544+0000","last_undegraded":"2023-10-09T10:10:32.099544+0000","last_fullsized":"2023-10-09T10:10:32.099544+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":161,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.739024+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:10:32.739024+0000","last_peered":"2023-10-09T10:10:32.739024+0000","last_clean":"2023-10-09T10:10:32.739024+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:10:32.739024+0000","last_undegraded":"2023-10-09T10:10:32.739024+0000","last_fullsized":"2023-10-09T10:10:32.739024+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":267,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.207944+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:10:40.207944+0000","last_peered":"2023-10-09T10:10:40.207944+0000","last_clean":"2023-10-09T10:10:40.207944+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:10:40.207944+0000","last_undegraded":"2023-10-09T10:10:40.207944+0000","last_fullsized":"2023-10-09T10:10:40.207944+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.080375+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:10:32.080375+0000","last_peered":"2023-10-09T10:10:32.080375+0000","last_clean":"2023-10-09T10:10:32.080375+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:10:32.080375+0000","last_undegraded":"2023-10-09T10:10:32.080375+0000","last_fullsized":"2023-10-09T10:10:32.080375+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":178,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092173+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:10:32.092173+0000","last_peered":"2023-10-09T10:10:32.092173+0000","last_clean":"2023-10-09T10:10:32.092173+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:10:32.092173+0000","last_undegraded":"2023-10-09T10:10:32.092173+0000","last_fullsized":"2023-10-09T10:10:32.092173+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":248,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204471+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:10:40.204471+0000","last_peered":"2023-10-09T10:10:40.204471+0000","last_clean":"2023-10-09T10:10:40.204471+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:10:40.204471+0000","last_undegraded":"2023-10-09T10:10:40.204471+0000","last_fullsized":"2023-10-09T10:10:40.204471+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":112,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.080246+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:10:32.080246+0000","last_peered":"2023-10-09T10:10:32.080246+0000","last_clean":"2023-10-09T10:10:32.080246+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:10:32.080246+0000","last_undegraded":"2023-10-09T10:10:32.080246+0000","last_fullsized":"2023-10-09T10:10:32.080246+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":209,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.199844+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:10:40.199844+0000","last_peered":"2023-10-09T10:10:40.199844+0000","last_clean":"2023-10-09T10:10:40.199844+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:10:40.199844+0000","last_undegraded":"2023-10-09T10:10:40.199844+0000","last_fullsized":"2023-10-09T10:10:40.199844+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":333,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092166+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:10:32.092166+0000","last_peered":"2023-10-09T10:10:32.092166+0000","last_clean":"2023-10-09T10:10:32.092166+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:10:32.092166+0000","last_undegraded":"2023-10-09T10:10:32.092166+0000","last_fullsized":"2023-10-09T10:10:32.092166+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":290,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204128+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:10:40.204128+0000","last_peered":"2023-10-09T10:10:40.204128+0000","last_clean":"2023-10-09T10:10:40.204128+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:10:40.204128+0000","last_undegraded":"2023-10-09T10:10:40.204128+0000","last_fullsized":"2023-10-09T10:10:40.204128+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":182,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092331+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:10:32.092331+0000","last_peered":"2023-10-09T10:10:32.092331+0000","last_clean":"2023-10-09T10:10:32.092331+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:10:32.092331+0000","last_undegraded":"2023-10-09T10:10:32.092331+0000","last_fullsized":"2023-10-09T10:10:32.092331+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":161,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.100199+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:10:32.100199+0000","last_peered":"2023-10-09T10:10:32.100199+0000","last_clean":"2023-10-09T10:10:32.100199+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:10:32.100199+0000","last_undegraded":"2023-10-09T10:10:32.100199+0000","last_fullsized":"2023-10-09T10:10:32.100199+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":366,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.091865+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:10:32.091865+0000","last_peered":"2023-10-09T10:10:32.091865+0000","last_clean":"2023-10-09T10:10:32.091865+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:10:32.091865+0000","last_undegraded":"2023-10-09T10:10:32.091865+0000","last_fullsized":"2023-10-09T10:10:32.091865+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":235,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200601+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:10:40.200601+0000","last_peered":"2023-10-09T10:10:40.200601+0000","last_clean":"2023-10-09T10:10:40.200601+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:10:40.200601+0000","last_undegraded":"2023-10-09T10:10:40.200601+0000","last_fullsized":"2023-10-09T10:10:40.200601+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":160,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.096589+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:10:32.096589+0000","last_peered":"2023-10-09T10:10:32.096589+0000","last_clean":"2023-10-09T10:10:32.096589+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:10:32.096589+0000","last_undegraded":"2023-10-09T10:10:32.096589+0000","last_fullsized":"2023-10-09T10:10:32.096589+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":128,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.080342+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:10:32.080342+0000","last_peered":"2023-10-09T10:10:32.080342+0000","last_clean":"2023-10-09T10:10:32.080342+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:10:32.080342+0000","last_undegraded":"2023-10-09T10:10:32.080342+0000","last_fullsized":"2023-10-09T10:10:32.080342+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":163,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.091753+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:10:32.091753+0000","last_peered":"2023-10-09T10:10:32.091753+0000","last_clean":"2023-10-09T10:10:32.091753+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:10:32.091753+0000","last_undegraded":"2023-10-09T10:10:32.091753+0000","last_fullsized":"2023-10-09T10:10:32.091753+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":160,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.093050+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:10:32.093050+0000","last_peered":"2023-10-09T10:10:32.093050+0000","last_clean":"2023-10-09T10:10:32.093050+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:10:32.093050+0000","last_undegraded":"2023-10-09T10:10:32.093050+0000","last_fullsized":"2023-10-09T10:10:32.093050+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":183,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.090282+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:10:32.090282+0000","last_peered":"2023-10-09T10:10:32.090282+0000","last_clean":"2023-10-09T10:10:32.090282+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:10:32.090282+0000","last_undegraded":"2023-10-09T10:10:32.090282+0000","last_fullsized":"2023-10-09T10:10:32.090282+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.199968+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:10:40.199968+0000","last_peered":"2023-10-09T10:10:40.199968+0000","last_clean":"2023-10-09T10:10:40.199968+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:10:40.199968+0000","last_undegraded":"2023-10-09T10:10:40.199968+0000","last_fullsized":"2023-10-09T10:10:40.199968+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"295'7966","reported_seq":8006,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:47.860627+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:10:47.860627+0000","last_peered":"2023-10-09T10:10:47.860627+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:10:47.860627+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7966,"log_dups_size":0,"ondisk_log_size":7966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373555200,"num_objects":5700,"num_object_clones":0,"num_object_copies":17100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7966,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":327},{"shards":"4,5,6","objects":4240}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":327,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.091986+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:10:32.091986+0000","last_peered":"2023-10-09T10:10:32.091986+0000","last_clean":"2023-10-09T10:10:32.091986+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:10:32.091986+0000","last_undegraded":"2023-10-09T10:10:32.091986+0000","last_fullsized":"2023-10-09T10:10:32.091986+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":113,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.079972+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:10:32.079972+0000","last_peered":"2023-10-09T10:10:32.079972+0000","last_clean":"2023-10-09T10:10:32.079972+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:10:32.079972+0000","last_undegraded":"2023-10-09T10:10:32.079972+0000","last_fullsized":"2023-10-09T10:10:32.079972+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"295'8130","reported_seq":8162,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:49.954254+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:10:49.954254+0000","last_peered":"2023-10-09T10:10:49.954254+0000","last_clean":"2023-10-09T10:10:49.954254+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:10:49.954254+0000","last_undegraded":"2023-10-09T10:10:49.954254+0000","last_fullsized":"2023-10-09T10:10:49.954254+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370933760,"num_objects":5660,"num_object_clones":0,"num_object_copies":16980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8130,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":315,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092686+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:10:32.092686+0000","last_peered":"2023-10-09T10:10:32.092686+0000","last_clean":"2023-10-09T10:10:32.092686+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:10:32.092686+0000","last_undegraded":"2023-10-09T10:10:32.092686+0000","last_fullsized":"2023-10-09T10:10:32.092686+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":206,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:34.560985+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:10:34.560985+0000","last_peered":"2023-10-09T10:10:34.560985+0000","last_clean":"2023-10-09T10:10:34.560985+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:10:34.560985+0000","last_undegraded":"2023-10-09T10:10:34.560985+0000","last_fullsized":"2023-10-09T10:10:34.560985+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"295'7872","reported_seq":7905,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:47.720157+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:10:47.720157+0000","last_peered":"2023-10-09T10:10:47.720157+0000","last_clean":"2023-10-09T10:10:47.720157+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:10:47.720157+0000","last_undegraded":"2023-10-09T10:10:47.720157+0000","last_fullsized":"2023-10-09T10:10:47.720157+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7872,"log_dups_size":0,"ondisk_log_size":7872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373161984,"num_objects":5694,"num_object_clones":0,"num_object_copies":17082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7872,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":117,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.080430+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:10:32.080430+0000","last_peered":"2023-10-09T10:10:32.080430+0000","last_clean":"2023-10-09T10:10:32.080430+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:10:32.080430+0000","last_undegraded":"2023-10-09T10:10:32.080430+0000","last_fullsized":"2023-10-09T10:10:32.080430+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":138,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.075958+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:10:32.075958+0000","last_peered":"2023-10-09T10:10:32.075958+0000","last_clean":"2023-10-09T10:10:32.075958+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:10:32.075958+0000","last_undegraded":"2023-10-09T10:10:32.075958+0000","last_fullsized":"2023-10-09T10:10:32.075958+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"295'7837","reported_seq":7870,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:49.564397+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:10:49.564397+0000","last_peered":"2023-10-09T10:10:49.564397+0000","last_clean":"2023-10-09T10:10:49.564397+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:10:49.564397+0000","last_undegraded":"2023-10-09T10:10:49.564397+0000","last_fullsized":"2023-10-09T10:10:49.564397+0000","mapping_epoch":271,"log_start":"275'100","ondisk_log_start":"275'100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7737,"log_dups_size":0,"ondisk_log_size":7737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365232128,"num_objects":5573,"num_object_clones":0,"num_object_copies":16719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7837,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":128,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.083463+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:10:32.083463+0000","last_peered":"2023-10-09T10:10:32.083463+0000","last_clean":"2023-10-09T10:10:32.083463+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:10:32.083463+0000","last_undegraded":"2023-10-09T10:10:32.083463+0000","last_fullsized":"2023-10-09T10:10:32.083463+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":161,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.090674+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:10:32.090674+0000","last_peered":"2023-10-09T10:10:32.090674+0000","last_clean":"2023-10-09T10:10:32.090674+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:10:32.090674+0000","last_undegraded":"2023-10-09T10:10:32.090674+0000","last_fullsized":"2023-10-09T10:10:32.090674+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"296'8129","reported_seq":8164,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:51.172718+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:10:51.172718+0000","last_peered":"2023-10-09T10:10:51.172718+0000","last_clean":"2023-10-09T10:10:51.172718+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:10:51.172718+0000","last_undegraded":"2023-10-09T10:10:51.172718+0000","last_fullsized":"2023-10-09T10:10:51.172718+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8129,"log_dups_size":0,"ondisk_log_size":8129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374276096,"num_objects":5711,"num_object_clones":0,"num_object_copies":17133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8129,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":289,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.203693+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:10:40.203693+0000","last_peered":"2023-10-09T10:10:40.203693+0000","last_clean":"2023-10-09T10:10:40.203693+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:10:40.203693+0000","last_undegraded":"2023-10-09T10:10:40.203693+0000","last_fullsized":"2023-10-09T10:10:40.203693+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":159,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200478+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:10:40.200478+0000","last_peered":"2023-10-09T10:10:40.200478+0000","last_clean":"2023-10-09T10:10:40.200478+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:10:40.200478+0000","last_undegraded":"2023-10-09T10:10:40.200478+0000","last_fullsized":"2023-10-09T10:10:40.200478+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"295'8049","reported_seq":8082,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:49.543429+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:10:49.543429+0000","last_peered":"2023-10-09T10:10:49.543429+0000","last_clean":"2023-10-09T10:10:49.543429+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:10:49.543429+0000","last_undegraded":"2023-10-09T10:10:49.543429+0000","last_fullsized":"2023-10-09T10:10:49.543429+0000","mapping_epoch":271,"log_start":"276'300","ondisk_log_start":"276'300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7749,"log_dups_size":0,"ondisk_log_size":7749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370606080,"num_objects":5655,"num_object_clones":0,"num_object_copies":16965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8049,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":220,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:34.561018+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:10:34.561018+0000","last_peered":"2023-10-09T10:10:34.561018+0000","last_clean":"2023-10-09T10:10:34.561018+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:10:34.561018+0000","last_undegraded":"2023-10-09T10:10:34.561018+0000","last_fullsized":"2023-10-09T10:10:34.561018+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":225,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.103717+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:10:32.103717+0000","last_peered":"2023-10-09T10:10:32.103717+0000","last_clean":"2023-10-09T10:10:32.103717+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:10:32.103717+0000","last_undegraded":"2023-10-09T10:10:32.103717+0000","last_fullsized":"2023-10-09T10:10:32.103717+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"295'7774","reported_seq":7807,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:47.738482+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:10:47.738482+0000","last_peered":"2023-10-09T10:10:47.738482+0000","last_clean":"2023-10-09T10:10:47.738482+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:10:47.738482+0000","last_undegraded":"2023-10-09T10:10:47.738482+0000","last_fullsized":"2023-10-09T10:10:47.738482+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7774,"log_dups_size":0,"ondisk_log_size":7774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370802688,"num_objects":5658,"num_object_clones":0,"num_object_copies":16974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7774,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":189,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.091664+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:10:32.091664+0000","last_peered":"2023-10-09T10:10:32.091664+0000","last_clean":"2023-10-09T10:10:32.091664+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:10:32.091664+0000","last_undegraded":"2023-10-09T10:10:32.091664+0000","last_fullsized":"2023-10-09T10:10:32.091664+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":162,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.206426+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:10:40.206426+0000","last_peered":"2023-10-09T10:10:40.206426+0000","last_clean":"2023-10-09T10:10:40.206426+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:10:40.206426+0000","last_undegraded":"2023-10-09T10:10:40.206426+0000","last_fullsized":"2023-10-09T10:10:40.206426+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"295'8031","reported_seq":8065,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:49.539916+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:10:49.539916+0000","last_peered":"2023-10-09T10:10:49.539916+0000","last_clean":"2023-10-09T10:10:49.539916+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:10:49.539916+0000","last_undegraded":"2023-10-09T10:10:49.539916+0000","last_fullsized":"2023-10-09T10:10:49.539916+0000","mapping_epoch":271,"log_start":"276'300","ondisk_log_start":"276'300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7731,"log_dups_size":0,"ondisk_log_size":7731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364838912,"num_objects":5567,"num_object_clones":0,"num_object_copies":16701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8031,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204403+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:10:40.204403+0000","last_peered":"2023-10-09T10:10:40.204403+0000","last_clean":"2023-10-09T10:10:40.204403+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:10:40.204403+0000","last_undegraded":"2023-10-09T10:10:40.204403+0000","last_fullsized":"2023-10-09T10:10:40.204403+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":298,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.102801+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:10:32.102801+0000","last_peered":"2023-10-09T10:10:32.102801+0000","last_clean":"2023-10-09T10:10:32.102801+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:10:32.102801+0000","last_undegraded":"2023-10-09T10:10:32.102801+0000","last_fullsized":"2023-10-09T10:10:32.102801+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"295'7835","reported_seq":7868,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:47.688452+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:10:47.688452+0000","last_peered":"2023-10-09T10:10:47.688452+0000","last_clean":"2023-10-09T10:10:47.688452+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:10:47.688452+0000","last_undegraded":"2023-10-09T10:10:47.688452+0000","last_fullsized":"2023-10-09T10:10:47.688452+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7835,"log_dups_size":0,"ondisk_log_size":7835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369426432,"num_objects":5637,"num_object_clones":0,"num_object_copies":16911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7835,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":297,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.103500+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:10:32.103500+0000","last_peered":"2023-10-09T10:10:32.103500+0000","last_clean":"2023-10-09T10:10:32.103500+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:10:32.103500+0000","last_undegraded":"2023-10-09T10:10:32.103500+0000","last_fullsized":"2023-10-09T10:10:32.103500+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":137,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.080265+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:10:32.080265+0000","last_peered":"2023-10-09T10:10:32.080265+0000","last_clean":"2023-10-09T10:10:32.080265+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:10:32.080265+0000","last_undegraded":"2023-10-09T10:10:32.080265+0000","last_fullsized":"2023-10-09T10:10:32.080265+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":254,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:39.564013+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:10:39.564013+0000","last_peered":"2023-10-09T10:10:39.564013+0000","last_clean":"2023-10-09T10:10:39.564013+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:10:39.564013+0000","last_undegraded":"2023-10-09T10:10:39.564013+0000","last_fullsized":"2023-10-09T10:10:39.564013+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"295'7942","reported_seq":7974,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:47.876727+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:10:47.876727+0000","last_peered":"2023-10-09T10:10:47.876727+0000","last_clean":"2023-10-09T10:10:47.876727+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:10:47.876727+0000","last_undegraded":"2023-10-09T10:10:47.876727+0000","last_fullsized":"2023-10-09T10:10:47.876727+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7942,"log_dups_size":0,"ondisk_log_size":7942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369098752,"num_objects":5632,"num_object_clones":0,"num_object_copies":16896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7942,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":113,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.080109+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:10:32.080109+0000","last_peered":"2023-10-09T10:10:32.080109+0000","last_clean":"2023-10-09T10:10:32.080109+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:10:32.080109+0000","last_undegraded":"2023-10-09T10:10:32.080109+0000","last_fullsized":"2023-10-09T10:10:32.080109+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.090425+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:10:32.090425+0000","last_peered":"2023-10-09T10:10:32.090425+0000","last_clean":"2023-10-09T10:10:32.090425+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:10:32.090425+0000","last_undegraded":"2023-10-09T10:10:32.090425+0000","last_fullsized":"2023-10-09T10:10:32.090425+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":320,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.103431+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:10:32.103431+0000","last_peered":"2023-10-09T10:10:32.103431+0000","last_clean":"2023-10-09T10:10:32.103431+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:10:32.103431+0000","last_undegraded":"2023-10-09T10:10:32.103431+0000","last_fullsized":"2023-10-09T10:10:32.103431+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"296'8063","reported_seq":8097,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:51.210284+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:10:51.210284+0000","last_peered":"2023-10-09T10:10:51.210284+0000","last_clean":"2023-10-09T10:10:51.210284+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:10:51.210284+0000","last_undegraded":"2023-10-09T10:10:51.210284+0000","last_fullsized":"2023-10-09T10:10:51.210284+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8063,"log_dups_size":0,"ondisk_log_size":8063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359071744,"num_objects":5479,"num_object_clones":0,"num_object_copies":16437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8063,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":215,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.199721+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:10:40.199721+0000","last_peered":"2023-10-09T10:10:40.199721+0000","last_clean":"2023-10-09T10:10:40.199721+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:10:40.199721+0000","last_undegraded":"2023-10-09T10:10:40.199721+0000","last_fullsized":"2023-10-09T10:10:40.199721+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":290,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.207729+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:10:40.207729+0000","last_peered":"2023-10-09T10:10:40.207729+0000","last_clean":"2023-10-09T10:10:40.207729+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:10:40.207729+0000","last_undegraded":"2023-10-09T10:10:40.207729+0000","last_fullsized":"2023-10-09T10:10:40.207729+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":310,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.102737+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:10:32.102737+0000","last_peered":"2023-10-09T10:10:32.102737+0000","last_clean":"2023-10-09T10:10:32.102737+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:10:32.102737+0000","last_undegraded":"2023-10-09T10:10:32.102737+0000","last_fullsized":"2023-10-09T10:10:32.102737+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"295'7846","reported_seq":7878,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:47.885231+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:10:47.885231+0000","last_peered":"2023-10-09T10:10:47.885231+0000","last_clean":"2023-10-09T10:10:47.885231+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:10:47.885231+0000","last_undegraded":"2023-10-09T10:10:47.885231+0000","last_fullsized":"2023-10-09T10:10:47.885231+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7846,"log_dups_size":0,"ondisk_log_size":7846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367919104,"num_objects":5614,"num_object_clones":0,"num_object_copies":16842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7846,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":122,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.080117+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:10:32.080117+0000","last_peered":"2023-10-09T10:10:32.080117+0000","last_clean":"2023-10-09T10:10:32.080117+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:10:32.080117+0000","last_undegraded":"2023-10-09T10:10:32.080117+0000","last_fullsized":"2023-10-09T10:10:32.080117+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":541,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.091680+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:10:32.091680+0000","last_peered":"2023-10-09T10:10:32.091680+0000","last_clean":"2023-10-09T10:10:32.091680+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:10:32.091680+0000","last_undegraded":"2023-10-09T10:10:32.091680+0000","last_fullsized":"2023-10-09T10:10:32.091680+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":199,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200138+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:10:40.200138+0000","last_peered":"2023-10-09T10:10:40.200138+0000","last_clean":"2023-10-09T10:10:40.200138+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:10:40.200138+0000","last_undegraded":"2023-10-09T10:10:40.200138+0000","last_fullsized":"2023-10-09T10:10:40.200138+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":210,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092236+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:10:32.092236+0000","last_peered":"2023-10-09T10:10:32.092236+0000","last_clean":"2023-10-09T10:10:32.092236+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:10:32.092236+0000","last_undegraded":"2023-10-09T10:10:32.092236+0000","last_fullsized":"2023-10-09T10:10:32.092236+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"295'8016","reported_seq":8049,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:49.480005+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:10:49.480005+0000","last_peered":"2023-10-09T10:10:49.480005+0000","last_clean":"2023-10-09T10:10:49.480005+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:10:49.480005+0000","last_undegraded":"2023-10-09T10:10:49.480005+0000","last_fullsized":"2023-10-09T10:10:49.480005+0000","mapping_epoch":271,"log_start":"276'300","ondisk_log_start":"276'300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7716,"log_dups_size":0,"ondisk_log_size":7716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367394816,"num_objects":5606,"num_object_clones":0,"num_object_copies":16818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8016,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":270,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.099139+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:10:32.099139+0000","last_peered":"2023-10-09T10:10:32.099139+0000","last_clean":"2023-10-09T10:10:32.099139+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:10:32.099139+0000","last_undegraded":"2023-10-09T10:10:32.099139+0000","last_fullsized":"2023-10-09T10:10:32.099139+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":171,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.090865+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:10:32.090865+0000","last_peered":"2023-10-09T10:10:32.090865+0000","last_clean":"2023-10-09T10:10:32.090865+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:10:32.090865+0000","last_undegraded":"2023-10-09T10:10:32.090865+0000","last_fullsized":"2023-10-09T10:10:32.090865+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":199,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092910+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:10:32.092910+0000","last_peered":"2023-10-09T10:10:32.092910+0000","last_clean":"2023-10-09T10:10:32.092910+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:10:32.092910+0000","last_undegraded":"2023-10-09T10:10:32.092910+0000","last_fullsized":"2023-10-09T10:10:32.092910+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"296'8065","reported_seq":10825,"reported_epoch":296,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:51.243753+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:10:51.243753+0000","last_peered":"2023-10-09T10:10:51.243753+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:10:51.243753+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8065,"log_dups_size":0,"ondisk_log_size":8065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366542848,"num_objects":5593,"num_object_clones":0,"num_object_copies":16779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":3924,"num_objects_unfound":0,"num_objects_dirty":5593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8065,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1359,"num_bytes_recovered":89063424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1668},{"shards":"1,3,4","objects":3924}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":314,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092035+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:10:32.092035+0000","last_peered":"2023-10-09T10:10:32.092035+0000","last_clean":"2023-10-09T10:10:32.092035+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:10:32.092035+0000","last_undegraded":"2023-10-09T10:10:32.092035+0000","last_fullsized":"2023-10-09T10:10:32.092035+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":326,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.103576+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:10:32.103576+0000","last_peered":"2023-10-09T10:10:32.103576+0000","last_clean":"2023-10-09T10:10:32.103576+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:10:32.103576+0000","last_undegraded":"2023-10-09T10:10:32.103576+0000","last_fullsized":"2023-10-09T10:10:32.103576+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":163,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.090508+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:10:32.090508+0000","last_peered":"2023-10-09T10:10:32.090508+0000","last_clean":"2023-10-09T10:10:32.090508+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:10:32.090508+0000","last_undegraded":"2023-10-09T10:10:32.090508+0000","last_fullsized":"2023-10-09T10:10:32.090508+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"295'7855","reported_seq":7887,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:47.696883+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:10:47.696883+0000","last_peered":"2023-10-09T10:10:47.696883+0000","last_clean":"2023-10-09T10:10:47.696883+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:10:47.696883+0000","last_undegraded":"2023-10-09T10:10:47.696883+0000","last_fullsized":"2023-10-09T10:10:47.696883+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7855,"log_dups_size":0,"ondisk_log_size":7855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373751808,"num_objects":5703,"num_object_clones":0,"num_object_copies":17109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7855,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":258,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200064+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:10:40.200064+0000","last_peered":"2023-10-09T10:10:40.200064+0000","last_clean":"2023-10-09T10:10:40.200064+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:10:40.200064+0000","last_undegraded":"2023-10-09T10:10:40.200064+0000","last_fullsized":"2023-10-09T10:10:40.200064+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":271,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.099709+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:10:32.099709+0000","last_peered":"2023-10-09T10:10:32.099709+0000","last_clean":"2023-10-09T10:10:32.099709+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:10:32.099709+0000","last_undegraded":"2023-10-09T10:10:32.099709+0000","last_fullsized":"2023-10-09T10:10:32.099709+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":247,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204322+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:10:40.204322+0000","last_peered":"2023-10-09T10:10:40.204322+0000","last_clean":"2023-10-09T10:10:40.204322+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:10:40.204322+0000","last_undegraded":"2023-10-09T10:10:40.204322+0000","last_fullsized":"2023-10-09T10:10:40.204322+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"295'7807","reported_seq":7838,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:47.871924+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:10:47.871924+0000","last_peered":"2023-10-09T10:10:47.871924+0000","last_clean":"2023-10-09T10:10:47.871924+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:10:47.871924+0000","last_undegraded":"2023-10-09T10:10:47.871924+0000","last_fullsized":"2023-10-09T10:10:47.871924+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7807,"log_dups_size":0,"ondisk_log_size":7807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371064856,"num_objects":5663,"num_object_clones":0,"num_object_copies":16989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5663,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7807,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":203,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.076421+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:10:32.076421+0000","last_peered":"2023-10-09T10:10:32.076421+0000","last_clean":"2023-10-09T10:10:32.076421+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:10:32.076421+0000","last_undegraded":"2023-10-09T10:10:32.076421+0000","last_fullsized":"2023-10-09T10:10:32.076421+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":165,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.092836+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:10:32.092836+0000","last_peered":"2023-10-09T10:10:32.092836+0000","last_clean":"2023-10-09T10:10:32.092836+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:10:32.092836+0000","last_undegraded":"2023-10-09T10:10:32.092836+0000","last_fullsized":"2023-10-09T10:10:32.092836+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":148,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.079896+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:10:32.079896+0000","last_peered":"2023-10-09T10:10:32.079896+0000","last_clean":"2023-10-09T10:10:32.079896+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:10:32.079896+0000","last_undegraded":"2023-10-09T10:10:32.079896+0000","last_fullsized":"2023-10-09T10:10:32.079896+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":160,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.090294+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:10:32.090294+0000","last_peered":"2023-10-09T10:10:32.090294+0000","last_clean":"2023-10-09T10:10:32.090294+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:10:32.090294+0000","last_undegraded":"2023-10-09T10:10:32.090294+0000","last_fullsized":"2023-10-09T10:10:32.090294+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":233,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204654+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:10:40.204654+0000","last_peered":"2023-10-09T10:10:40.204654+0000","last_clean":"2023-10-09T10:10:40.204654+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:10:40.204654+0000","last_undegraded":"2023-10-09T10:10:40.204654+0000","last_fullsized":"2023-10-09T10:10:40.204654+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":132,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.083379+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:10:32.083379+0000","last_peered":"2023-10-09T10:10:32.083379+0000","last_clean":"2023-10-09T10:10:32.083379+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:10:32.083379+0000","last_undegraded":"2023-10-09T10:10:32.083379+0000","last_fullsized":"2023-10-09T10:10:32.083379+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":174,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.203588+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:10:40.203588+0000","last_peered":"2023-10-09T10:10:40.203588+0000","last_clean":"2023-10-09T10:10:40.203588+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:10:40.203588+0000","last_undegraded":"2023-10-09T10:10:40.203588+0000","last_fullsized":"2023-10-09T10:10:40.203588+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":132,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.080180+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:10:32.080180+0000","last_peered":"2023-10-09T10:10:32.080180+0000","last_clean":"2023-10-09T10:10:32.080180+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:10:32.080180+0000","last_undegraded":"2023-10-09T10:10:32.080180+0000","last_fullsized":"2023-10-09T10:10:32.080180+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":183,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200060+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:10:40.200060+0000","last_peered":"2023-10-09T10:10:40.200060+0000","last_clean":"2023-10-09T10:10:40.200060+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:10:40.200060+0000","last_undegraded":"2023-10-09T10:10:40.200060+0000","last_fullsized":"2023-10-09T10:10:40.200060+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":312,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200645+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:10:40.200645+0000","last_peered":"2023-10-09T10:10:40.200645+0000","last_clean":"2023-10-09T10:10:40.200645+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:10:40.200645+0000","last_undegraded":"2023-10-09T10:10:40.200645+0000","last_fullsized":"2023-10-09T10:10:40.200645+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":333,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.199971+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:10:40.199971+0000","last_peered":"2023-10-09T10:10:40.199971+0000","last_clean":"2023-10-09T10:10:40.199971+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:10:40.199971+0000","last_undegraded":"2023-10-09T10:10:40.199971+0000","last_fullsized":"2023-10-09T10:10:40.199971+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":241,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.090477+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:10:32.090477+0000","last_peered":"2023-10-09T10:10:32.090477+0000","last_clean":"2023-10-09T10:10:32.090477+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:10:32.090477+0000","last_undegraded":"2023-10-09T10:10:32.090477+0000","last_fullsized":"2023-10-09T10:10:32.090477+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":161,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T10:10:32.098918+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:10:32.098918+0000","last_peered":"2023-10-09T10:10:32.098918+0000","last_clean":"2023-10-09T10:10:32.098918+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:10:32.098918+0000","last_undegraded":"2023-10-09T10:10:32.098918+0000","last_fullsized":"2023-10-09T10:10:32.098918+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5907677208,"num_objects":90145,"num_object_clones":0,"num_object_copies":270435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":3924,"num_objects_unfound":0,"num_objects_dirty":90145,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127217,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1359,"num_bytes_recovered":89063424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17867878400,"data_stored":17867866184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126217,"ondisk_log_size":126217,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430494,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2110312,"kb_used_data":1804896,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91634840,"statfs":{"total":95995035648,"available":93834076160,"internally_reserved":0,"allocated":1848213504,"data_stored":1847676379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":1.758,"15min":1.716},"min":{"1min":0.254,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":7.8419999999999996,"5min":9.0690000000000008,"15min":50.591999999999999},"last":1.681},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.6699999999999999,"15min":1.6970000000000001},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":7.9610000000000003,"5min":8.2300000000000004,"15min":50.747},"last":0.23400000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:08 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":1.8440000000000001,"15min":1.5169999999999999},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2669999999999995,"5min":16.352,"15min":25.338000000000001},"last":3.6509999999999998},{"interface":"front","average":{"1min":1.53,"5min":1.8839999999999999,"15min":1.3560000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.9269999999999996,"5min":18.27,"15min":18.27},"last":4.0449999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.7190000000000001,"15min":2.266},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":3.8370000000000002,"5min":7.952,"15min":7.952},"last":3.7440000000000002},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.7310000000000001,"15min":2.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.2039999999999997,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":3.8740000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:09:47 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.0049999999999999,"15min":1.835},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.058,"5min":29.058,"15min":29.058},"last":3.9700000000000002},{"interface":"front","average":{"1min":1.214,"5min":1.7989999999999999,"15min":1.897},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.577,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":3.7639999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":0.96599999999999997,"15min":0.93000000000000005},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":1.6200000000000001},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.91300000000000003,"15min":0.91000000000000003},"min":{"1min":0.28599999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.7919999999999998,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.27800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.149,"15min":0.93899999999999995},"min":{"1min":0.31900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":25.277999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":1.6519999999999999},{"interface":"front","average":{"1min":2.4529999999999998,"5min":1.2589999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.754000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.42699999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.1299999999999999,"15min":0.88600000000000001},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5369999999999999,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.98599999999999999,"15min":0.82499999999999996},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.998,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":1.2470000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626722,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2440600,"kb_used_data":2135184,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91304552,"statfs":{"total":95995035648,"available":93495861248,"internally_reserved":0,"allocated":2186428416,"data_stored":2185908605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.821,"15min":1.4410000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.2870000000000008,"5min":15.435,"15min":15.435},"last":6.7450000000000001},{"interface":"front","average":{"1min":2.149,"5min":2.0859999999999999,"15min":1.4750000000000001},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":7.4139999999999997,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.39300000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:10:02 2023","interfaces":[{"interface":"back","average":{"1min":4.641,"5min":2.4300000000000002,"15min":1.7130000000000001},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":35.609999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":6.2930000000000001},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.3540000000000001,"15min":1.6319999999999999},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":32.396000000000001,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.499}]},{"osd":2,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":1.4319999999999999,"15min":1.724},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.898,"5min":12.898,"15min":12.898},"last":3.2450000000000001},{"interface":"front","average":{"1min":2.569,"5min":2.141,"15min":3.0379999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5640000000000001,"5min":10.26,"15min":10.26},"last":6.3570000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.8140000000000001,"15min":1.5109999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":12.943,"5min":12.943,"15min":12.943},"last":6.0549999999999997},{"interface":"front","average":{"1min":1.653,"5min":2.2770000000000001,"15min":2.3460000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.077,"5min":11.211,"15min":11.211},"last":2.5569999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.50700000000000001,"15min":0.46200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":3.198},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.60099999999999998,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1749999999999998,"5min":4.1749999999999998,"15min":4.1749999999999998},"last":6.2370000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:10:33 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.73699999999999999,"15min":0.76700000000000002},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9260000000000002,"5min":7.9260000000000002,"15min":14.09},"last":3.222},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.91900000000000004,"15min":0.79600000000000004},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":4.1529999999999996,"5min":7.508,"15min":7.508},"last":6.3369999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.77000000000000002,"15min":0.755},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.5800000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":6.2709999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.81499999999999995,"15min":0.80500000000000005},"min":{"1min":0.39000000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.6080000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":6.3159999999999998}]}]},{"osd":1,"up_from":84,"seq":360777253035,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2923960,"kb_used_data":2526320,"kb_used_omap":9,"kb_used_meta":397622,"kb_avail":90821192,"statfs":{"total":95995035648,"available":93000900608,"internally_reserved":0,"allocated":2586951680,"data_stored":2585928516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":407165929},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":2.6549999999999998,"15min":2.0299999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.185},"max":{"1min":6.3479999999999999,"5min":25.59,"15min":25.59},"last":0.21099999999999999},{"interface":"front","average":{"1min":3.7109999999999999,"5min":2.77,"15min":1.976},"min":{"1min":0.26900000000000002,"5min":0.218,"15min":0.192},"max":{"1min":11.682,"5min":14.769,"15min":16.629000000000001},"last":6.2779999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":3.2160000000000002,"15min":3.8700000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.1539999999999999,"5min":17.369,"15min":17.369},"last":0.376},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.907,"15min":3.8740000000000001},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5960000000000001,"5min":16.023,"15min":16.023},"last":0.44800000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":3.4540000000000002,"15min":3.5070000000000001},"min":{"1min":0.39500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.587,"5min":18.587,"15min":18.587},"last":6.2240000000000002},{"interface":"front","average":{"1min":4.6959999999999997,"5min":4.2249999999999996,"15min":4.6760000000000002},"min":{"1min":1.0329999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.022,"5min":17.289000000000001,"15min":17.289000000000001},"last":1.2849999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.474,"15min":1.3280000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2629999999999999,"5min":7.2629999999999999,"15min":7.2629999999999999},"last":3.4769999999999999},{"interface":"front","average":{"1min":2.879,"5min":1.639,"15min":1.4330000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.6940000000000008,"5min":10.132,"15min":10.132},"last":6.3129999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":2.0419999999999998,"15min":1.72},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.473000000000001,"5min":14.416,"15min":15.531000000000001},"last":6.3710000000000004},{"interface":"front","average":{"1min":1.782,"5min":2.0979999999999999,"15min":1.9690000000000001},"min":{"1min":0.34699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.6890000000000001,"5min":16.277999999999999,"15min":19.706},"last":0.41899999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0339999999999998,"15min":1.7230000000000001},"min":{"1min":0.50800000000000001,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.3710000000000004,"5min":14.215,"15min":16.193999999999999},"last":6.4560000000000004},{"interface":"front","average":{"1min":2.75,"5min":1.9930000000000001,"15min":1.7529999999999999},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":10.161,"5min":14.471,"15min":15.481999999999999},"last":6.3440000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":2.153,"15min":1.972},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.7350000000000003,"5min":16.331,"15min":16.331},"last":6.0529999999999999},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1320000000000001,"15min":1.788},"min":{"1min":0.46000000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3319999999999999,"5min":14.242000000000001,"15min":16.100999999999999},"last":6.423}]}]},{"osd":0,"up_from":79,"seq":339302416561,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2559784,"kb_used_data":2192992,"kb_used_omap":28,"kb_used_meta":366755,"kb_avail":91185368,"statfs":{"total":95995035648,"available":93373816832,"internally_reserved":0,"allocated":2245623808,"data_stored":2244614575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":375557968},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.847,"15min":1.4970000000000001},"min":{"1min":0.24099999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.2990000000000004,"5min":12.84,"15min":13.084},"last":7.6459999999999999},{"interface":"front","average":{"1min":3.9020000000000001,"5min":3.379,"15min":2.2269999999999999},"min":{"1min":0.44900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.949000000000002,"5min":22.178999999999998,"15min":22.178999999999998},"last":7.9710000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.7210000000000001,"15min":2.121},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.5579999999999998,"5min":10.034000000000001,"15min":10.034000000000001},"last":5.8289999999999997},{"interface":"front","average":{"1min":1.353,"5min":2.2879999999999998,"15min":3.0910000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.4480000000000004,"5min":14.693,"15min":14.693},"last":5.9189999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:09:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4990000000000001,"5min":4.391,"15min":4.7069999999999999},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.867000000000001,"5min":25.734999999999999,"15min":25.734999999999999},"last":7.8860000000000001},{"interface":"front","average":{"1min":2.4809999999999999,"5min":4.0250000000000004,"15min":4.282},"min":{"1min":0.44,"5min":0.254,"15min":0.254},"max":{"1min":13.975,"5min":21.923999999999999,"15min":21.923999999999999},"last":8.1120000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.911,"15min":1.5649999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":4.3710000000000004},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.0459999999999998,"15min":1.9730000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.6720000000000006,"5min":13.789999999999999,"15min":13.789999999999999},"last":8.0760000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.7309999999999999,"15min":1.9630000000000001},"min":{"1min":0.40899999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.6229999999999993,"5min":17.096,"15min":17.096},"last":8.0470000000000006},{"interface":"front","average":{"1min":1.5,"5min":1.454,"15min":1.3029999999999999},"min":{"1min":0.30499999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.4260000000000002,"5min":9.4260000000000002,"15min":12.478},"last":5.8650000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.7010000000000001,"15min":1.4590000000000001},"min":{"1min":0.33700000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":9.4719999999999995,"5min":9.7260000000000009,"15min":11.699},"last":5.8920000000000003},{"interface":"front","average":{"1min":4.4729999999999999,"5min":2.6320000000000001,"15min":2.2069999999999999},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.466000000000001,"5min":20.466000000000001,"15min":20.466000000000001},"last":7.8460000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8239999999999998,"15min":2.1139999999999999},"min":{"1min":0.46400000000000002,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":9.7050000000000001,"5min":14.631,"15min":26.978000000000002},"last":8.1349999999999998},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.7190000000000001,"15min":1.605},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":9.5020000000000007,"5min":11.166,"15min":26.768000000000001},"last":5.9560000000000004}]}]},{"osd":2,"up_from":217,"seq":932007903295,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1881436,"kb_used_data":1650644,"kb_used_omap":83,"kb_used_meta":230700,"kb_avail":91863716,"statfs":{"total":95995035648,"available":94068445184,"internally_reserved":0,"allocated":1690259456,"data_stored":1689262200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":236237295},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.28,"15min":1.6060000000000001},"min":{"1min":0.247,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.5139999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":1.343},{"interface":"front","average":{"1min":2.2029999999999998,"5min":2.8849999999999998,"15min":2.923},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.891999999999999,"5min":16.309999999999999,"15min":16.309999999999999},"last":0.58699999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.0209999999999999,"15min":2.4649999999999999},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.971,"5min":11.971,"15min":11.971},"last":1.375},{"interface":"front","average":{"1min":2.8849999999999998,"5min":2.492,"15min":2.6280000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.009,"5min":15.569000000000001,"15min":15.569000000000001},"last":1.405}]},{"osd":3,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.798,"15min":1.948},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.8859999999999999,"5min":7.3289999999999997,"15min":7.3289999999999997},"last":1.6870000000000001},{"interface":"front","average":{"1min":1.1779999999999999,"5min":3.367,"15min":3.7320000000000002},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.8440000000000003,"5min":15.811,"15min":15.811},"last":1.72}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.84499999999999997,"15min":0.753},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.29,"5min":6.29,"15min":6.29},"last":0.65600000000000003},{"interface":"front","average":{"1min":2.5609999999999999,"5min":1.1759999999999999,"15min":0.94499999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":12.385,"5min":12.385,"15min":12.385},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.5580000000000001,"15min":1.502},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.0870000000000002,"5min":12.423999999999999,"15min":12.423999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.21,"5min":1.2789999999999999,"15min":1.6000000000000001},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.7709999999999999,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":0.54500000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2629999999999999,"15min":1.0489999999999999},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.6040000000000001,"5min":15.292999999999999,"15min":15.292999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.6499999999999999,"15min":1.712},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.984,"5min":12.394,"15min":12.394},"last":0.47999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.2809999999999999,"15min":1.0760000000000001},"min":{"1min":0.48499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1800000000000002,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.7170000000000001,"15min":1.613},"min":{"1min":0.51200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.5010000000000003,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.41699999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824039,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2392544,"kb_used_data":2146392,"kb_used_omap":10,"kb_used_meta":246133,"kb_avail":91352608,"statfs":{"total":95995035648,"available":93545070592,"internally_reserved":0,"allocated":2197905408,"data_stored":2196735760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":252040259},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":4.2709999999999999,"15min":4.5590000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.753,"5min":23.381,"15min":23.381},"last":2.6949999999999998},{"interface":"front","average":{"1min":6.0720000000000001,"5min":4.0919999999999996,"15min":4.1500000000000004},"min":{"1min":0.42099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":20.239999999999998,"5min":20.239999999999998,"15min":20.239999999999998},"last":0.72299999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.6150000000000002,"15min":3.8410000000000002},"min":{"1min":0.26300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":20.422999999999998,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.85399999999999998},{"interface":"front","average":{"1min":4.5039999999999996,"5min":3.4990000000000001,"15min":3.4529999999999998},"min":{"1min":0.33100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.704000000000001,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.97099999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1369999999999996,"5min":3.4199999999999999,"15min":3.6339999999999999},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":20.594999999999999,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.88400000000000001},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.8809999999999998,"15min":2.8700000000000001},"min":{"1min":0.34200000000000003,"5min":0.253,"15min":0.253},"max":{"1min":15.73,"5min":23.452000000000002,"15min":23.452000000000002},"last":2.7309999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":1.6719999999999999,"15min":1.288},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.676,"5min":15.676,"15min":15.676},"last":8.4789999999999992},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5800000000000001,"15min":1.534},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.426,"5min":12.426,"15min":12.426},"last":0.753}]},{"osd":5,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8290000000000002,"5min":2.4750000000000001,"15min":2.5670000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":12.388,"5min":12.388,"15min":12.388},"last":6.2130000000000001},{"interface":"front","average":{"1min":3.5699999999999998,"5min":2.5169999999999999,"15min":2.4079999999999999},"min":{"1min":0.443,"5min":0.309,"15min":0.309},"max":{"1min":15.992000000000001,"5min":15.992000000000001,"15min":15.992000000000001},"last":1.7090000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":2.536,"15min":2.6709999999999998},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.468,"5min":12.468,"15min":12.468},"last":0.83499999999999996},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.5609999999999999,"15min":2.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":16.16,"5min":16.16,"15min":16.16},"last":0.90700000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.825,"15min":1.534},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.635999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.4399999999999999,"15min":2.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.77900000000000003}]}]},{"osd":4,"up_from":266,"seq":1142461300764,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2489436,"kb_used_data":2162580,"kb_used_omap":35,"kb_used_meta":326812,"kb_avail":91255716,"statfs":{"total":95995035648,"available":93445853184,"internally_reserved":0,"allocated":2214481920,"data_stored":2213956445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":334655726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.98499999999999999,"15min":0.84099999999999997},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.881,"5min":10.881,"15min":10.881},"last":1.181},{"interface":"front","average":{"1min":1.923,"5min":1.1519999999999999,"15min":1.024},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":1.335}]},{"osd":1,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":1.169,"15min":0.93400000000000005},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":10.93,"5min":10.93,"15min":10.93},"last":1.532},{"interface":"front","average":{"1min":2.633,"5min":1.546,"15min":1.365},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.141,"5min":11.141,"15min":11.141},"last":1.4830000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":1.2310000000000001,"15min":1.073},"min":{"1min":0.32100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.1189999999999998,"5min":7.1189999999999998,"15min":7.1189999999999998},"last":3.677},{"interface":"front","average":{"1min":1.587,"5min":1.042,"15min":0.95099999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.1709999999999994,"5min":8.1709999999999994,"15min":8.1709999999999994},"last":1.2829999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.4610000000000001,"15min":1.3280000000000001},"min":{"1min":0.40699999999999997,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5530000000000008,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":1.7090000000000001},{"interface":"front","average":{"1min":2.4420000000000002,"5min":1.2989999999999999,"15min":1.109},"min":{"1min":0.373,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.6069999999999993,"5min":9.6069999999999993,"15min":9.6069999999999993},"last":3.3340000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.80400000000000005,"15min":0.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4740000000000002,"5min":5.4740000000000002,"15min":5.4740000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.60399999999999998,"15min":0.63400000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.36399999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.60499999999999998,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":5.5709999999999997,"5min":5.5709999999999997,"15min":5.5709999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.60999999999999999,"15min":0.59199999999999997},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.319,"5min":2.319,"15min":2.319},"last":0.38100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.61299999999999999,"15min":0.61799999999999999},"min":{"1min":0.33700000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.3460000000000001,"5min":2.3460000000000001,"15min":2.3460000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.82299999999999995,"15min":0.82699999999999996},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.0680000000000001,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.48299999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822949,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3214224,"kb_used_data":2912840,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":90530928,"statfs":{"total":95995035648,"available":92703670272,"internally_reserved":0,"allocated":2982748160,"data_stored":2982230741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":1.55,"15min":1.488},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.8540000000000001,"5min":7.8540000000000001,"15min":15.773999999999999},"last":4.5289999999999999},{"interface":"front","average":{"1min":1.339,"5min":1.597,"15min":1.532},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.0629999999999997,"5min":11.747,"15min":15.699},"last":0.39800000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:10:12 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.5740000000000001,"15min":1.4430000000000001},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.7720000000000002,"5min":11.335000000000001,"15min":15.689},"last":1.831},{"interface":"front","average":{"1min":1.913,"5min":1.861,"15min":1.6180000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.8119999999999994,"5min":19.402999999999999,"15min":19.402999999999999},"last":2.073}]},{"osd":2,"last update":"Mon Oct 9 10:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.8480000000000001,"15min":2.4369999999999998},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.3819999999999997,"5min":15.640000000000001,"15min":15.640000000000001},"last":1.853},{"interface":"front","average":{"1min":1.302,"5min":1.8220000000000001,"15min":2.2949999999999999},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.3659999999999997,"5min":10.656000000000001,"15min":10.656000000000001},"last":2.226}]},{"osd":3,"last update":"Mon Oct 9 10:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.734,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5860000000000003,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.2,"5min":2.6549999999999998,"15min":2.8969999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":2.9710000000000001,"5min":17.34,"15min":17.34},"last":0.85399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":0.71699999999999997,"15min":0.61499999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5220000000000002,"5min":5.5220000000000002,"15min":5.5220000000000002},"last":3.4449999999999998},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.752,"15min":0.746},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.823,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.41999999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.014,"15min":0.94499999999999995},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.25},"max":{"1min":5.4930000000000003,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.0189999999999999,"15min":0.88700000000000001},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.8099999999999996,"5min":18.600999999999999,"15min":18.600999999999999},"last":1.768}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0589999999999999,"15min":0.88},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.282,"5min":7.4290000000000003,"15min":10.334},"last":1.8029999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.749,"15min":0.88600000000000001},"min":{"1min":0.35099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.3500000000000001,"5min":8.0210000000000008,"15min":12.356},"last":0.497}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2230714368,"data_stored":2230714368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2581200896,"data_stored":2581200896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1675759616,"data_stored":1675755544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2185953280,"data_stored":2185953280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2202533888,"data_stored":2202533888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2976124928,"data_stored":2976120856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2181038080,"data_stored":2181038080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1834553344,"data_stored":1834549272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:10:52.677 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:52 smithi012 ceph-mon[147926]: pgmap v790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 5374/270507 objects degraded (1.987%); 3924/270507 objects misplaced (1.451%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:52 smithi012 ceph-mon[147926]: from='client.45665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:52 smithi012 ceph-mon[152263]: pgmap v790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 5374/270507 objects degraded (1.987%); 3924/270507 objects misplaced (1.451%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:52.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:52 smithi012 ceph-mon[152263]: from='client.45665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:52.758 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:52.846 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:52.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:52.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:52.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:52 smithi110 ceph-mon[136032]: pgmap v790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 5374/270507 objects degraded (1.987%); 3924/270507 objects misplaced (1.451%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:52.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:52 smithi110 ceph-mon[136032]: from='client.45665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:10:52.952 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:10:53.028 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:53.225 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:53.259 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:10:53.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:53.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:10:53.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:53.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:10:53.488 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:53.654 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:53.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:53.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:53.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:53 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/270507 objects degraded (1.987%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:10:53.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:53 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:10:53.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:53.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:53 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/270507 objects degraded (1.987%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:10:53.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:53 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:10:53.823 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:53 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/270507 objects degraded (1.987%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:10:53.823 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:53 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:10:53.825 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:54.028 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:54.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:54.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:10:54.122 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:10:54.135 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:54.165 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:10:54.188 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:54.215 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:10:54.270 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:54.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:54.286 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:10:54.330 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:10:54.366 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:54.373 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:10:54.376 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:54.414 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:10:54.417 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:54.438 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:10:54.438 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:10:54.438 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:10:54.438 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:10:54.438 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:10:54.438 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:10:54.438 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:10:54.439 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:10:54.565 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:54.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:54.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:54.656 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:10:54.656 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:10:54.673 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:54.743 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:54.871 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:54.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:54.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:54.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:55.007 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:55 smithi012 ceph-mon[147926]: pgmap v791: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 5374/265548 objects degraded (2.024%); 3924/265548 objects misplaced (1.478%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:55.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/166315582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:55.009 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:10:55.089 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:55.211 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:55.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:55.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:55.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:55.423 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:55 smithi012 ceph-mon[152263]: pgmap v791: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 5374/265548 objects degraded (2.024%); 3924/265548 objects misplaced (1.478%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:55.423 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/166315582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:55.431 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:55.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:55 smithi110 ceph-mon[136032]: pgmap v791: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 5374/265548 objects degraded (2.024%); 3924/265548 objects misplaced (1.478%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/166315582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:10:55.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:10:55.728 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:55.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:55.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:55.847 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:55.855 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:55.965 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:56.814 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:10:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:56.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:10:56.929 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:10:56.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:10:56.980 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:57.097 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:10:57.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:57.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:10:57.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:57 smithi012 ceph-mon[147926]: pgmap v792: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5374/262770 objects degraded (2.045%); 3924/262770 objects misplaced (1.493%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:57.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:57 smithi012 ceph-mon[152263]: pgmap v792: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5374/262770 objects degraded (2.045%); 3924/262770 objects misplaced (1.493%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:57.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:57 smithi110 ceph-mon[136032]: pgmap v792: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5374/262770 objects degraded (2.045%); 3924/262770 objects misplaced (1.493%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:57.201 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:57.254 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:57.380 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:57.406 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:57.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:57.481 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:10:57.481 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:10:57.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:57.597 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:57.686 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:57.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:57.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:57.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:10:57.850 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:57.919 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:10:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:10:57] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-10-09T10:10:57.946 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:10:57.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:57.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:10:58.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:10:58.084 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:58.088 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:10:58.108 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:10:58.144 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:10:58.150 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1910773465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:58.150 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5374/262770 objects degraded (2.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:10:58.152 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:10:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:58.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:10:58.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1910773465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:58.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5374/262770 objects degraded (2.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:10:58.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1910773465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:10:58.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5374/262770 objects degraded (2.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:10:58.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:10:58.355 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:58.901 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:10:58.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:58.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:10:58.965 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:10:59.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:10:59.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[147926]: pgmap v793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5374/260991 objects degraded (2.059%); 3802/260991 objects misplaced (1.457%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:59.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:59.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[147926]: failed to encode map e297 with expected crc 2023-10-09T10:10:59.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1910773465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:10:59.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[147926]: osdmap e297: 8 total, 8 up, 8 in 2023-10-09T10:10:59.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[147926]: failed to encode map e297 with expected crc 2023-10-09T10:10:59.086 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:59.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[152263]: pgmap v793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5374/260991 objects degraded (2.059%); 3802/260991 objects misplaced (1.457%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:59.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:59.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[152263]: failed to encode map e297 with expected crc 2023-10-09T10:10:59.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1910773465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:10:59.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[152263]: osdmap e297: 8 total, 8 up, 8 in 2023-10-09T10:10:59.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[152263]: failed to encode map e297 with expected crc 2023-10-09T10:10:59.087 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:10:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:59.112 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:59.195 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:10:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:59.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:10:59.305 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:10:59.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:59 smithi110 ceph-mon[136032]: pgmap v793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5374/260991 objects degraded (2.059%); 3802/260991 objects misplaced (1.457%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:10:59.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:59 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:10:59.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:59 smithi110 ceph-mon[136032]: failed to encode map e297 with expected crc 2023-10-09T10:10:59.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1910773465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:10:59.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:59 smithi110 ceph-mon[136032]: osdmap e297: 8 total, 8 up, 8 in 2023-10-09T10:10:59.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:59 smithi110 ceph-mon[136032]: failed to encode map e297 with expected crc 2023-10-09T10:10:59.343 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:10:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:10:59.343 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:59.489 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:10:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:10:59.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:10:59.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:10:59.739 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:10:59.766 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:10:59.786 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:11:00.137 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:00.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:00.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:00.290 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:00.354 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:00.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:00.440 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:11:00.440 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:11:00.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:00.540 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:11:00.563 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:00.569 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:11:00.612 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:11:00.643 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:11:00.680 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:11:00.713 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:00.715 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:11:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:00.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:00.757 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:11:00.771 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":795,"stamp":"2023-10-09T10:10:59.352083+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5599860664,"num_objects":85463,"num_object_clones":0,"num_object_copies":256389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":3802,"num_objects_unfound":0,"num_objects_dirty":85463,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":132183,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1550,"num_bytes_recovered":98249420,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129917,"ondisk_log_size":129917,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19188740,"kb_used_data":16708228,"kb_used_omap":167,"kb_used_meta":2480152,"kb_avail":730772476,"statfs":{"total":767960285184,"available":748311015424,"internally_reserved":0,"allocated":17109225472,"data_stored":17102920910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2539676249},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,7,1,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-347734016,"num_objects":-5306,"num_object_clones":0,"num_object_copies":-15918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-241,"num_objects_unfound":0,"num_objects_dirty":-5306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5306,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":241,"num_bytes_recovered":15794176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.883062"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":137,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204915+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:10:40.204915+0000","last_peered":"2023-10-09T10:10:40.204915+0000","last_clean":"2023-10-09T10:10:40.204915+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:10:40.204915+0000","last_undegraded":"2023-10-09T10:10:40.204915+0000","last_fullsized":"2023-10-09T10:10:40.204915+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":253,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200283+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:10:40.200283+0000","last_peered":"2023-10-09T10:10:40.200283+0000","last_clean":"2023-10-09T10:10:40.200283+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:10:40.200283+0000","last_undegraded":"2023-10-09T10:10:40.200283+0000","last_fullsized":"2023-10-09T10:10:40.200283+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":298,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204203+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:10:40.204203+0000","last_peered":"2023-10-09T10:10:40.204203+0000","last_clean":"2023-10-09T10:10:40.204203+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:10:40.204203+0000","last_undegraded":"2023-10-09T10:10:40.204203+0000","last_fullsized":"2023-10-09T10:10:40.204203+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":162,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216817+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:10:40.216817+0000","last_peered":"2023-10-09T10:10:40.216817+0000","last_clean":"2023-10-09T10:10:40.216817+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:10:40.216817+0000","last_undegraded":"2023-10-09T10:10:40.216817+0000","last_fullsized":"2023-10-09T10:10:40.216817+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":267,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.207944+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:10:40.207944+0000","last_peered":"2023-10-09T10:10:40.207944+0000","last_clean":"2023-10-09T10:10:40.207944+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:10:40.207944+0000","last_undegraded":"2023-10-09T10:10:40.207944+0000","last_fullsized":"2023-10-09T10:10:40.207944+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":114,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.217112+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:10:40.217112+0000","last_peered":"2023-10-09T10:10:40.217112+0000","last_clean":"2023-10-09T10:10:40.217112+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:10:40.217112+0000","last_undegraded":"2023-10-09T10:10:40.217112+0000","last_fullsized":"2023-10-09T10:10:40.217112+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":179,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216928+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:10:40.216928+0000","last_peered":"2023-10-09T10:10:40.216928+0000","last_clean":"2023-10-09T10:10:40.216928+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:10:40.216928+0000","last_undegraded":"2023-10-09T10:10:40.216928+0000","last_fullsized":"2023-10-09T10:10:40.216928+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":248,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204471+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:10:40.204471+0000","last_peered":"2023-10-09T10:10:40.204471+0000","last_clean":"2023-10-09T10:10:40.204471+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:10:40.204471+0000","last_undegraded":"2023-10-09T10:10:40.204471+0000","last_fullsized":"2023-10-09T10:10:40.204471+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":113,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216871+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:10:40.216871+0000","last_peered":"2023-10-09T10:10:40.216871+0000","last_clean":"2023-10-09T10:10:40.216871+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:10:40.216871+0000","last_undegraded":"2023-10-09T10:10:40.216871+0000","last_fullsized":"2023-10-09T10:10:40.216871+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":209,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.199844+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:10:40.199844+0000","last_peered":"2023-10-09T10:10:40.199844+0000","last_clean":"2023-10-09T10:10:40.199844+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:10:40.199844+0000","last_undegraded":"2023-10-09T10:10:40.199844+0000","last_fullsized":"2023-10-09T10:10:40.199844+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":334,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.221835+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:10:40.221835+0000","last_peered":"2023-10-09T10:10:40.221835+0000","last_clean":"2023-10-09T10:10:40.221835+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:10:40.221835+0000","last_undegraded":"2023-10-09T10:10:40.221835+0000","last_fullsized":"2023-10-09T10:10:40.221835+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":291,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:44.577323+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:10:44.577323+0000","last_peered":"2023-10-09T10:10:44.577323+0000","last_clean":"2023-10-09T10:10:44.577323+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:10:44.577323+0000","last_undegraded":"2023-10-09T10:10:44.577323+0000","last_fullsized":"2023-10-09T10:10:44.577323+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":183,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.221559+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:10:40.221559+0000","last_peered":"2023-10-09T10:10:40.221559+0000","last_clean":"2023-10-09T10:10:40.221559+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:10:40.221559+0000","last_undegraded":"2023-10-09T10:10:40.221559+0000","last_fullsized":"2023-10-09T10:10:40.221559+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":163,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:50.503069+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:10:50.503069+0000","last_peered":"2023-10-09T10:10:50.503069+0000","last_clean":"2023-10-09T10:10:50.503069+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:10:50.503069+0000","last_undegraded":"2023-10-09T10:10:50.503069+0000","last_fullsized":"2023-10-09T10:10:50.503069+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":367,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.221468+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:10:40.221468+0000","last_peered":"2023-10-09T10:10:40.221468+0000","last_clean":"2023-10-09T10:10:40.221468+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:10:40.221468+0000","last_undegraded":"2023-10-09T10:10:40.221468+0000","last_fullsized":"2023-10-09T10:10:40.221468+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":235,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200601+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:10:40.200601+0000","last_peered":"2023-10-09T10:10:40.200601+0000","last_clean":"2023-10-09T10:10:40.200601+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:10:40.200601+0000","last_undegraded":"2023-10-09T10:10:40.200601+0000","last_fullsized":"2023-10-09T10:10:40.200601+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":161,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216163+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:10:40.216163+0000","last_peered":"2023-10-09T10:10:40.216163+0000","last_clean":"2023-10-09T10:10:40.216163+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:10:40.216163+0000","last_undegraded":"2023-10-09T10:10:40.216163+0000","last_fullsized":"2023-10-09T10:10:40.216163+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216975+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:10:40.216975+0000","last_peered":"2023-10-09T10:10:40.216975+0000","last_clean":"2023-10-09T10:10:40.216975+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:10:40.216975+0000","last_undegraded":"2023-10-09T10:10:40.216975+0000","last_fullsized":"2023-10-09T10:10:40.216975+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":164,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.221245+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:10:40.221245+0000","last_peered":"2023-10-09T10:10:40.221245+0000","last_clean":"2023-10-09T10:10:40.221245+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:10:40.221245+0000","last_undegraded":"2023-10-09T10:10:40.221245+0000","last_fullsized":"2023-10-09T10:10:40.221245+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":161,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216892+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:10:40.216892+0000","last_peered":"2023-10-09T10:10:40.216892+0000","last_clean":"2023-10-09T10:10:40.216892+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:10:40.216892+0000","last_undegraded":"2023-10-09T10:10:40.216892+0000","last_fullsized":"2023-10-09T10:10:40.216892+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":184,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.211837+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:10:40.211837+0000","last_peered":"2023-10-09T10:10:40.211837+0000","last_clean":"2023-10-09T10:10:40.211837+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:10:40.211837+0000","last_undegraded":"2023-10-09T10:10:40.211837+0000","last_fullsized":"2023-10-09T10:10:40.211837+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.199968+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:10:40.199968+0000","last_peered":"2023-10-09T10:10:40.199968+0000","last_clean":"2023-10-09T10:10:40.199968+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:10:40.199968+0000","last_undegraded":"2023-10-09T10:10:40.199968+0000","last_fullsized":"2023-10-09T10:10:40.199968+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"296'8353","reported_seq":8394,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:57.843147+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:10:57.843147+0000","last_peered":"2023-10-09T10:10:57.843147+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:10:57.843147+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8353,"log_dups_size":0,"ondisk_log_size":8353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348192768,"num_objects":5313,"num_object_clones":0,"num_object_copies":15939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8353,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-60},{"shards":"4,5,6","objects":3853}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":328,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.223858+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:10:40.223858+0000","last_peered":"2023-10-09T10:10:40.223858+0000","last_clean":"2023-10-09T10:10:40.223858+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:10:40.223858+0000","last_undegraded":"2023-10-09T10:10:40.223858+0000","last_fullsized":"2023-10-09T10:10:40.223858+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":114,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.208713+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:10:40.208713+0000","last_peered":"2023-10-09T10:10:40.208713+0000","last_clean":"2023-10-09T10:10:40.208713+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:10:40.208713+0000","last_undegraded":"2023-10-09T10:10:40.208713+0000","last_fullsized":"2023-10-09T10:10:40.208713+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"296'8325","reported_seq":8358,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:55.022013+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:10:55.022013+0000","last_peered":"2023-10-09T10:10:55.022013+0000","last_clean":"2023-10-09T10:10:55.022013+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:10:55.022013+0000","last_undegraded":"2023-10-09T10:10:55.022013+0000","last_fullsized":"2023-10-09T10:10:55.022013+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8325,"log_dups_size":0,"ondisk_log_size":8325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358154240,"num_objects":5465,"num_object_clones":0,"num_object_copies":16395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8325,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":316,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.223748+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:10:40.223748+0000","last_peered":"2023-10-09T10:10:40.223748+0000","last_clean":"2023-10-09T10:10:40.223748+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:10:40.223748+0000","last_undegraded":"2023-10-09T10:10:40.223748+0000","last_fullsized":"2023-10-09T10:10:40.223748+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":207,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.203980+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:10:40.203980+0000","last_peered":"2023-10-09T10:10:40.203980+0000","last_clean":"2023-10-09T10:10:40.203980+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:10:40.203980+0000","last_undegraded":"2023-10-09T10:10:40.203980+0000","last_fullsized":"2023-10-09T10:10:40.203980+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"296'8282","reported_seq":8317,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:57.687991+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:10:57.687991+0000","last_peered":"2023-10-09T10:10:57.687991+0000","last_clean":"2023-10-09T10:10:57.687991+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:10:57.687991+0000","last_undegraded":"2023-10-09T10:10:57.687991+0000","last_fullsized":"2023-10-09T10:10:57.687991+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8282,"log_dups_size":0,"ondisk_log_size":8282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346292224,"num_objects":5284,"num_object_clones":0,"num_object_copies":15852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8282,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":118,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.217031+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:10:40.217031+0000","last_peered":"2023-10-09T10:10:40.217031+0000","last_clean":"2023-10-09T10:10:40.217031+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:10:40.217031+0000","last_undegraded":"2023-10-09T10:10:40.217031+0000","last_fullsized":"2023-10-09T10:10:40.217031+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":139,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.222978+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:10:40.222978+0000","last_peered":"2023-10-09T10:10:40.222978+0000","last_clean":"2023-10-09T10:10:40.222978+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:10:40.222978+0000","last_undegraded":"2023-10-09T10:10:40.222978+0000","last_fullsized":"2023-10-09T10:10:40.222978+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"296'7997","reported_seq":8030,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:54.563891+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:10:54.563891+0000","last_peered":"2023-10-09T10:10:54.563891+0000","last_clean":"2023-10-09T10:10:54.563891+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:10:54.563891+0000","last_undegraded":"2023-10-09T10:10:54.563891+0000","last_fullsized":"2023-10-09T10:10:54.563891+0000","mapping_epoch":271,"log_start":"276'300","ondisk_log_start":"276'300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7697,"log_dups_size":0,"ondisk_log_size":7697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354746368,"num_objects":5413,"num_object_clones":0,"num_object_copies":16239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7997,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":129,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.210355+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:10:40.210355+0000","last_peered":"2023-10-09T10:10:40.210355+0000","last_clean":"2023-10-09T10:10:40.210355+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:10:40.210355+0000","last_undegraded":"2023-10-09T10:10:40.210355+0000","last_fullsized":"2023-10-09T10:10:40.210355+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":162,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216606+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:10:40.216606+0000","last_peered":"2023-10-09T10:10:40.216606+0000","last_clean":"2023-10-09T10:10:40.216606+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:10:40.216606+0000","last_undegraded":"2023-10-09T10:10:40.216606+0000","last_fullsized":"2023-10-09T10:10:40.216606+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"296'8326","reported_seq":8361,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:56.263587+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:10:56.263587+0000","last_peered":"2023-10-09T10:10:56.263587+0000","last_clean":"2023-10-09T10:10:56.263587+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:10:56.263587+0000","last_undegraded":"2023-10-09T10:10:56.263587+0000","last_fullsized":"2023-10-09T10:10:56.263587+0000","mapping_epoch":271,"log_start":"275'200","ondisk_log_start":"275'200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361365504,"num_objects":5514,"num_object_clones":0,"num_object_copies":16542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8326,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":289,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.203693+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:10:40.203693+0000","last_peered":"2023-10-09T10:10:40.203693+0000","last_clean":"2023-10-09T10:10:40.203693+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:10:40.203693+0000","last_undegraded":"2023-10-09T10:10:40.203693+0000","last_fullsized":"2023-10-09T10:10:40.203693+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":159,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200478+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:10:40.200478+0000","last_peered":"2023-10-09T10:10:40.200478+0000","last_clean":"2023-10-09T10:10:40.200478+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:10:40.200478+0000","last_undegraded":"2023-10-09T10:10:40.200478+0000","last_fullsized":"2023-10-09T10:10:40.200478+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"296'8240","reported_seq":8274,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:54.581083+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:10:54.581083+0000","last_peered":"2023-10-09T10:10:54.581083+0000","last_clean":"2023-10-09T10:10:54.581083+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:10:54.581083+0000","last_undegraded":"2023-10-09T10:10:54.581083+0000","last_fullsized":"2023-10-09T10:10:54.581083+0000","mapping_epoch":271,"log_start":"276'500","ondisk_log_start":"276'500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7740,"log_dups_size":0,"ondisk_log_size":7740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358088704,"num_objects":5464,"num_object_clones":0,"num_object_copies":16392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8240,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":221,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204049+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:10:40.204049+0000","last_peered":"2023-10-09T10:10:40.204049+0000","last_clean":"2023-10-09T10:10:40.204049+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:10:40.204049+0000","last_undegraded":"2023-10-09T10:10:40.204049+0000","last_fullsized":"2023-10-09T10:10:40.204049+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":226,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204328+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:10:40.204328+0000","last_peered":"2023-10-09T10:10:40.204328+0000","last_clean":"2023-10-09T10:10:40.204328+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:10:40.204328+0000","last_undegraded":"2023-10-09T10:10:40.204328+0000","last_fullsized":"2023-10-09T10:10:40.204328+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"296'8196","reported_seq":8231,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:57.732578+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:10:57.732578+0000","last_peered":"2023-10-09T10:10:57.732578+0000","last_clean":"2023-10-09T10:10:57.732578+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:10:57.732578+0000","last_undegraded":"2023-10-09T10:10:57.732578+0000","last_fullsized":"2023-10-09T10:10:57.732578+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8196,"log_dups_size":0,"ondisk_log_size":8196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343146496,"num_objects":5236,"num_object_clones":0,"num_object_copies":15708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8196,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":190,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.218693+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:10:40.218693+0000","last_peered":"2023-10-09T10:10:40.218693+0000","last_clean":"2023-10-09T10:10:40.218693+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:10:40.218693+0000","last_undegraded":"2023-10-09T10:10:40.218693+0000","last_fullsized":"2023-10-09T10:10:40.218693+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:44.577409+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:10:44.577409+0000","last_peered":"2023-10-09T10:10:44.577409+0000","last_clean":"2023-10-09T10:10:44.577409+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:10:44.577409+0000","last_undegraded":"2023-10-09T10:10:44.577409+0000","last_fullsized":"2023-10-09T10:10:44.577409+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"296'8220","reported_seq":8255,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:54.554531+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:10:54.554531+0000","last_peered":"2023-10-09T10:10:54.554531+0000","last_clean":"2023-10-09T10:10:54.554531+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:10:54.554531+0000","last_undegraded":"2023-10-09T10:10:54.554531+0000","last_fullsized":"2023-10-09T10:10:54.554531+0000","mapping_epoch":271,"log_start":"276'500","ondisk_log_start":"276'500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7720,"log_dups_size":0,"ondisk_log_size":7720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352452608,"num_objects":5378,"num_object_clones":0,"num_object_copies":16134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8220,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204403+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:10:40.204403+0000","last_peered":"2023-10-09T10:10:40.204403+0000","last_clean":"2023-10-09T10:10:40.204403+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:10:40.204403+0000","last_undegraded":"2023-10-09T10:10:40.204403+0000","last_fullsized":"2023-10-09T10:10:40.204403+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":299,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204653+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:10:40.204653+0000","last_peered":"2023-10-09T10:10:40.204653+0000","last_clean":"2023-10-09T10:10:40.204653+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:10:40.204653+0000","last_undegraded":"2023-10-09T10:10:40.204653+0000","last_fullsized":"2023-10-09T10:10:40.204653+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"296'8224","reported_seq":8259,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:57.744950+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:10:57.744950+0000","last_peered":"2023-10-09T10:10:57.744950+0000","last_clean":"2023-10-09T10:10:57.744950+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:10:57.744950+0000","last_undegraded":"2023-10-09T10:10:57.744950+0000","last_fullsized":"2023-10-09T10:10:57.744950+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8224,"log_dups_size":0,"ondisk_log_size":8224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343932928,"num_objects":5248,"num_object_clones":0,"num_object_copies":15744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8224,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":298,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204488+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:10:40.204488+0000","last_peered":"2023-10-09T10:10:40.204488+0000","last_clean":"2023-10-09T10:10:40.204488+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:10:40.204488+0000","last_undegraded":"2023-10-09T10:10:40.204488+0000","last_fullsized":"2023-10-09T10:10:40.204488+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":138,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216906+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:10:40.216906+0000","last_peered":"2023-10-09T10:10:40.216906+0000","last_clean":"2023-10-09T10:10:40.216906+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:10:40.216906+0000","last_undegraded":"2023-10-09T10:10:40.216906+0000","last_fullsized":"2023-10-09T10:10:40.216906+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":255,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204957+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:10:40.204957+0000","last_peered":"2023-10-09T10:10:40.204957+0000","last_clean":"2023-10-09T10:10:40.204957+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:10:40.204957+0000","last_undegraded":"2023-10-09T10:10:40.204957+0000","last_fullsized":"2023-10-09T10:10:40.204957+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"296'8339","reported_seq":8372,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:57.778365+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:10:57.778365+0000","last_peered":"2023-10-09T10:10:57.778365+0000","last_clean":"2023-10-09T10:10:57.778365+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:10:57.778365+0000","last_undegraded":"2023-10-09T10:10:57.778365+0000","last_fullsized":"2023-10-09T10:10:57.778365+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8339,"log_dups_size":0,"ondisk_log_size":8339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343080960,"num_objects":5235,"num_object_clones":0,"num_object_copies":15705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8339,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":114,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.206681+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:10:40.206681+0000","last_peered":"2023-10-09T10:10:40.206681+0000","last_clean":"2023-10-09T10:10:40.206681+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:10:40.206681+0000","last_undegraded":"2023-10-09T10:10:40.206681+0000","last_fullsized":"2023-10-09T10:10:40.206681+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":112,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216393+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:10:40.216393+0000","last_peered":"2023-10-09T10:10:40.216393+0000","last_clean":"2023-10-09T10:10:40.216393+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:10:40.216393+0000","last_undegraded":"2023-10-09T10:10:40.216393+0000","last_fullsized":"2023-10-09T10:10:40.216393+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":321,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204091+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:10:40.204091+0000","last_peered":"2023-10-09T10:10:40.204091+0000","last_clean":"2023-10-09T10:10:40.204091+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:10:40.204091+0000","last_undegraded":"2023-10-09T10:10:40.204091+0000","last_fullsized":"2023-10-09T10:10:40.204091+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"296'8262","reported_seq":8296,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:56.259984+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:10:56.259984+0000","last_peered":"2023-10-09T10:10:56.259984+0000","last_clean":"2023-10-09T10:10:56.259984+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:10:56.259984+0000","last_undegraded":"2023-10-09T10:10:56.259984+0000","last_fullsized":"2023-10-09T10:10:56.259984+0000","mapping_epoch":271,"log_start":"275'100","ondisk_log_start":"275'100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8162,"log_dups_size":0,"ondisk_log_size":8162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346030080,"num_objects":5280,"num_object_clones":0,"num_object_copies":15840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8262,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":215,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.199721+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:10:40.199721+0000","last_peered":"2023-10-09T10:10:40.199721+0000","last_clean":"2023-10-09T10:10:40.199721+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:10:40.199721+0000","last_undegraded":"2023-10-09T10:10:40.199721+0000","last_fullsized":"2023-10-09T10:10:40.199721+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":291,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:44.577240+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:10:44.577240+0000","last_peered":"2023-10-09T10:10:44.577240+0000","last_clean":"2023-10-09T10:10:44.577240+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:10:44.577240+0000","last_undegraded":"2023-10-09T10:10:44.577240+0000","last_fullsized":"2023-10-09T10:10:44.577240+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":311,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204573+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:10:40.204573+0000","last_peered":"2023-10-09T10:10:40.204573+0000","last_clean":"2023-10-09T10:10:40.204573+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:10:40.204573+0000","last_undegraded":"2023-10-09T10:10:40.204573+0000","last_fullsized":"2023-10-09T10:10:40.204573+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"296'8247","reported_seq":8280,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:57.759489+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:10:57.759489+0000","last_peered":"2023-10-09T10:10:57.759489+0000","last_clean":"2023-10-09T10:10:57.759489+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:10:57.759489+0000","last_undegraded":"2023-10-09T10:10:57.759489+0000","last_fullsized":"2023-10-09T10:10:57.759489+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8247,"log_dups_size":0,"ondisk_log_size":8247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341639168,"num_objects":5213,"num_object_clones":0,"num_object_copies":15639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8247,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":123,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.206212+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:10:40.206212+0000","last_peered":"2023-10-09T10:10:40.206212+0000","last_clean":"2023-10-09T10:10:40.206212+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:10:40.206212+0000","last_undegraded":"2023-10-09T10:10:40.206212+0000","last_fullsized":"2023-10-09T10:10:40.206212+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":542,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.218844+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:10:40.218844+0000","last_peered":"2023-10-09T10:10:40.218844+0000","last_clean":"2023-10-09T10:10:40.218844+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:10:40.218844+0000","last_undegraded":"2023-10-09T10:10:40.218844+0000","last_fullsized":"2023-10-09T10:10:40.218844+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":199,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200138+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:10:40.200138+0000","last_peered":"2023-10-09T10:10:40.200138+0000","last_clean":"2023-10-09T10:10:40.200138+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:10:40.200138+0000","last_undegraded":"2023-10-09T10:10:40.200138+0000","last_fullsized":"2023-10-09T10:10:40.200138+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":211,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.221850+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:10:40.221850+0000","last_peered":"2023-10-09T10:10:40.221850+0000","last_clean":"2023-10-09T10:10:40.221850+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:10:40.221850+0000","last_undegraded":"2023-10-09T10:10:40.221850+0000","last_fullsized":"2023-10-09T10:10:40.221850+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"296'8207","reported_seq":8242,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:54.558198+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:10:54.558198+0000","last_peered":"2023-10-09T10:10:54.558198+0000","last_clean":"2023-10-09T10:10:54.558198+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:10:54.558198+0000","last_undegraded":"2023-10-09T10:10:54.558198+0000","last_fullsized":"2023-10-09T10:10:54.558198+0000","mapping_epoch":271,"log_start":"276'500","ondisk_log_start":"276'500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7707,"log_dups_size":0,"ondisk_log_size":7707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354877440,"num_objects":5415,"num_object_clones":0,"num_object_copies":16245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8207,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":271,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.207841+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:10:40.207841+0000","last_peered":"2023-10-09T10:10:40.207841+0000","last_clean":"2023-10-09T10:10:40.207841+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:10:40.207841+0000","last_undegraded":"2023-10-09T10:10:40.207841+0000","last_fullsized":"2023-10-09T10:10:40.207841+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":173,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:50.487622+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:10:50.487622+0000","last_peered":"2023-10-09T10:10:50.487622+0000","last_clean":"2023-10-09T10:10:50.487622+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:10:50.487622+0000","last_undegraded":"2023-10-09T10:10:50.487622+0000","last_fullsized":"2023-10-09T10:10:50.487622+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":200,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216692+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:10:40.216692+0000","last_peered":"2023-10-09T10:10:40.216692+0000","last_clean":"2023-10-09T10:10:40.216692+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:10:40.216692+0000","last_undegraded":"2023-10-09T10:10:40.216692+0000","last_fullsized":"2023-10-09T10:10:40.216692+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"296'8262","reported_seq":11266,"reported_epoch":296,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:10:56.258608+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:10:56.258608+0000","last_peered":"2023-10-09T10:10:56.258608+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:10:56.258608+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"275'100","ondisk_log_start":"275'100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8162,"log_dups_size":0,"ondisk_log_size":8162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353632256,"num_objects":5396,"num_object_clones":0,"num_object_copies":16188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":3802,"num_objects_unfound":0,"num_objects_dirty":5396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8262,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1481,"num_bytes_recovered":97058816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1593},{"shards":"1,3,4","objects":3802}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":315,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.221747+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:10:40.221747+0000","last_peered":"2023-10-09T10:10:40.221747+0000","last_clean":"2023-10-09T10:10:40.221747+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:10:40.221747+0000","last_undegraded":"2023-10-09T10:10:40.221747+0000","last_fullsized":"2023-10-09T10:10:40.221747+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":327,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204572+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:10:40.204572+0000","last_peered":"2023-10-09T10:10:40.204572+0000","last_clean":"2023-10-09T10:10:40.204572+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:10:40.204572+0000","last_undegraded":"2023-10-09T10:10:40.204572+0000","last_fullsized":"2023-10-09T10:10:40.204572+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":165,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:50.483062+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:10:50.483062+0000","last_peered":"2023-10-09T10:10:50.483062+0000","last_clean":"2023-10-09T10:10:50.483062+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:10:50.483062+0000","last_undegraded":"2023-10-09T10:10:50.483062+0000","last_fullsized":"2023-10-09T10:10:50.483062+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"296'8249","reported_seq":8284,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:57.750901+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:10:57.750901+0000","last_peered":"2023-10-09T10:10:57.750901+0000","last_clean":"2023-10-09T10:10:57.750901+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:10:57.750901+0000","last_undegraded":"2023-10-09T10:10:57.750901+0000","last_fullsized":"2023-10-09T10:10:57.750901+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8249,"log_dups_size":0,"ondisk_log_size":8249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8249,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":258,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200064+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:10:40.200064+0000","last_peered":"2023-10-09T10:10:40.200064+0000","last_clean":"2023-10-09T10:10:40.200064+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:10:40.200064+0000","last_undegraded":"2023-10-09T10:10:40.200064+0000","last_fullsized":"2023-10-09T10:10:40.200064+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":272,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.207372+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:10:40.207372+0000","last_peered":"2023-10-09T10:10:40.207372+0000","last_clean":"2023-10-09T10:10:40.207372+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:10:40.207372+0000","last_undegraded":"2023-10-09T10:10:40.207372+0000","last_fullsized":"2023-10-09T10:10:40.207372+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":247,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.204322+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:10:40.204322+0000","last_peered":"2023-10-09T10:10:40.204322+0000","last_clean":"2023-10-09T10:10:40.204322+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:10:40.204322+0000","last_undegraded":"2023-10-09T10:10:40.204322+0000","last_fullsized":"2023-10-09T10:10:40.204322+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"296'8194","reported_seq":8228,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:57.865841+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:10:57.865841+0000","last_peered":"2023-10-09T10:10:57.865841+0000","last_clean":"2023-10-09T10:10:57.865841+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:10:57.865841+0000","last_undegraded":"2023-10-09T10:10:57.865841+0000","last_fullsized":"2023-10-09T10:10:57.865841+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345702424,"num_objects":5276,"num_object_clones":0,"num_object_copies":15828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5276,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8194,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":204,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.222682+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:10:40.222682+0000","last_peered":"2023-10-09T10:10:40.222682+0000","last_clean":"2023-10-09T10:10:40.222682+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:10:40.222682+0000","last_undegraded":"2023-10-09T10:10:40.222682+0000","last_fullsized":"2023-10-09T10:10:40.222682+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":166,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216623+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:10:40.216623+0000","last_peered":"2023-10-09T10:10:40.216623+0000","last_clean":"2023-10-09T10:10:40.216623+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:10:40.216623+0000","last_undegraded":"2023-10-09T10:10:40.216623+0000","last_fullsized":"2023-10-09T10:10:40.216623+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":149,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.206118+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:10:40.206118+0000","last_peered":"2023-10-09T10:10:40.206118+0000","last_clean":"2023-10-09T10:10:40.206118+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:10:40.206118+0000","last_undegraded":"2023-10-09T10:10:40.206118+0000","last_fullsized":"2023-10-09T10:10:40.206118+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":161,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216288+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:10:40.216288+0000","last_peered":"2023-10-09T10:10:40.216288+0000","last_clean":"2023-10-09T10:10:40.216288+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:10:40.216288+0000","last_undegraded":"2023-10-09T10:10:40.216288+0000","last_fullsized":"2023-10-09T10:10:40.216288+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":234,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:49.584894+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:10:49.584894+0000","last_peered":"2023-10-09T10:10:49.584894+0000","last_clean":"2023-10-09T10:10:49.584894+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:10:49.584894+0000","last_undegraded":"2023-10-09T10:10:49.584894+0000","last_fullsized":"2023-10-09T10:10:49.584894+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":133,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.210270+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:10:40.210270+0000","last_peered":"2023-10-09T10:10:40.210270+0000","last_clean":"2023-10-09T10:10:40.210270+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:10:40.210270+0000","last_undegraded":"2023-10-09T10:10:40.210270+0000","last_fullsized":"2023-10-09T10:10:40.210270+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":174,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.203588+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:10:40.203588+0000","last_peered":"2023-10-09T10:10:40.203588+0000","last_clean":"2023-10-09T10:10:40.203588+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:10:40.203588+0000","last_undegraded":"2023-10-09T10:10:40.203588+0000","last_fullsized":"2023-10-09T10:10:40.203588+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":133,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.216844+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:10:40.216844+0000","last_peered":"2023-10-09T10:10:40.216844+0000","last_clean":"2023-10-09T10:10:40.216844+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:10:40.216844+0000","last_undegraded":"2023-10-09T10:10:40.216844+0000","last_fullsized":"2023-10-09T10:10:40.216844+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":183,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200060+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:10:40.200060+0000","last_peered":"2023-10-09T10:10:40.200060+0000","last_clean":"2023-10-09T10:10:40.200060+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:10:40.200060+0000","last_undegraded":"2023-10-09T10:10:40.200060+0000","last_fullsized":"2023-10-09T10:10:40.200060+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":312,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.200645+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:10:40.200645+0000","last_peered":"2023-10-09T10:10:40.200645+0000","last_clean":"2023-10-09T10:10:40.200645+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:10:40.200645+0000","last_undegraded":"2023-10-09T10:10:40.200645+0000","last_fullsized":"2023-10-09T10:10:40.200645+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":333,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.199971+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:10:40.199971+0000","last_peered":"2023-10-09T10:10:40.199971+0000","last_clean":"2023-10-09T10:10:40.199971+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:10:40.199971+0000","last_undegraded":"2023-10-09T10:10:40.199971+0000","last_fullsized":"2023-10-09T10:10:40.199971+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":243,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T10:10:50.483246+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:10:50.483246+0000","last_peered":"2023-10-09T10:10:50.483246+0000","last_clean":"2023-10-09T10:10:50.483246+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:10:50.483246+0000","last_undegraded":"2023-10-09T10:10:50.483246+0000","last_fullsized":"2023-10-09T10:10:50.483246+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":162,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T10:10:40.207513+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:10:40.207513+0000","last_peered":"2023-10-09T10:10:40.207513+0000","last_clean":"2023-10-09T10:10:40.207513+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:10:40.207513+0000","last_undegraded":"2023-10-09T10:10:40.207513+0000","last_fullsized":"2023-10-09T10:10:40.207513+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5599264792,"num_objects":85439,"num_object_clones":0,"num_object_copies":256317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":3802,"num_objects_unfound":0,"num_objects_dirty":85439,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131923,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1481,"num_bytes_recovered":97058816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17032556544,"data_stored":17032544328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129723,"ondisk_log_size":129723,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430495,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2049968,"kb_used_data":1740456,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91695184,"statfs":{"total":95995035648,"available":93895868416,"internally_reserved":0,"allocated":1782226944,"data_stored":1781689581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":1.758,"15min":1.716},"min":{"1min":0.254,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":7.8419999999999996,"5min":9.0690000000000008,"15min":50.591999999999999},"last":2.8450000000000002},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.6699999999999999,"15min":1.6970000000000001},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":7.9610000000000003,"5min":8.2300000000000004,"15min":50.747},"last":0.36799999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:10:08 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":1.8440000000000001,"15min":1.5169999999999999},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2669999999999995,"5min":16.352,"15min":25.338000000000001},"last":0.307},{"interface":"front","average":{"1min":1.53,"5min":1.8839999999999999,"15min":1.3560000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.9269999999999996,"5min":18.27,"15min":18.27},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.7190000000000001,"15min":2.266},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":3.8370000000000002,"5min":7.952,"15min":7.952},"last":2.2930000000000001},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.7310000000000001,"15min":2.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.2039999999999997,"5min":9.0879999999999992,"15min":9.0879999999999992},"last":0.33200000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":2.2879999999999998,"15min":1.929},"min":{"1min":0.46400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.274,"5min":29.058,"15min":29.058},"last":5.3680000000000003},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.913,"15min":1.9350000000000001},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":1.3160000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":0.96599999999999997,"15min":0.93000000000000005},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.91300000000000003,"15min":0.91000000000000003},"min":{"1min":0.28599999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.7919999999999998,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.149,"15min":0.93899999999999995},"min":{"1min":0.31900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":25.277999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.374},{"interface":"front","average":{"1min":2.4529999999999998,"5min":1.2589999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.754000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.47899999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:09:54 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.1299999999999999,"15min":0.88600000000000001},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5369999999999999,"5min":5.4820000000000002,"15min":5.4820000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.98599999999999999,"15min":0.82499999999999996},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.998,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":0.45900000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626723,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2383968,"kb_used_data":2074456,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91361184,"statfs":{"total":95995035648,"available":93553852416,"internally_reserved":0,"allocated":2124242944,"data_stored":2123722895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.821,"15min":1.4410000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.2870000000000008,"5min":15.435,"15min":15.435},"last":1.228},{"interface":"front","average":{"1min":2.149,"5min":2.0859999999999999,"15min":1.4750000000000001},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":7.4139999999999997,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.88300000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:02 2023","interfaces":[{"interface":"back","average":{"1min":4.641,"5min":2.4300000000000002,"15min":1.7130000000000001},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":35.609999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.3540000000000001,"15min":1.6319999999999999},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":32.396000000000001,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.436}]},{"osd":2,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":1.4319999999999999,"15min":1.724},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.898,"5min":12.898,"15min":12.898},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.569,"5min":2.141,"15min":3.0379999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5640000000000001,"5min":10.26,"15min":10.26},"last":0.53200000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.8140000000000001,"15min":1.5109999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":12.943,"5min":12.943,"15min":12.943},"last":4.0519999999999996},{"interface":"front","average":{"1min":1.653,"5min":2.2770000000000001,"15min":2.3460000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.077,"5min":11.211,"15min":11.211},"last":0.46899999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.50700000000000001,"15min":0.46200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.60099999999999998,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1749999999999998,"5min":4.1749999999999998,"15min":4.1749999999999998},"last":0.376}]},{"osd":5,"last update":"Mon Oct 9 10:10:33 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.73699999999999999,"15min":0.76700000000000002},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9260000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.91900000000000004,"15min":0.79600000000000004},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":4.1529999999999996,"5min":7.508,"15min":7.508},"last":0.42799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.77000000000000002,"15min":0.755},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.5800000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.81499999999999995,"15min":0.80500000000000005},"min":{"1min":0.39000000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.6080000000000001,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.48799999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253036,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2828608,"kb_used_data":2443256,"kb_used_omap":9,"kb_used_meta":385334,"kb_avail":90916544,"statfs":{"total":95995035648,"available":93098541056,"internally_reserved":0,"allocated":2501894144,"data_stored":2500870742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":394583017},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":2.6549999999999998,"15min":2.0299999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.185},"max":{"1min":6.3479999999999999,"5min":25.59,"15min":25.59},"last":0.98399999999999999},{"interface":"front","average":{"1min":3.7109999999999999,"5min":2.77,"15min":1.976},"min":{"1min":0.26900000000000002,"5min":0.218,"15min":0.192},"max":{"1min":11.682,"5min":14.769,"15min":16.629000000000001},"last":13.066000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":3.0499999999999998,"15min":3.8149999999999999},"min":{"1min":0.27200000000000002,"5min":0.217,"15min":0.217},"max":{"1min":11.135,"5min":17.369,"15min":17.369},"last":6.8150000000000004},{"interface":"front","average":{"1min":3.5550000000000002,"5min":2.7469999999999999,"15min":3.8210000000000002},"min":{"1min":0.25900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.484999999999999,"5min":16.023,"15min":16.023},"last":0.878}]},{"osd":3,"last update":"Mon Oct 9 10:10:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":3.4540000000000002,"15min":3.5070000000000001},"min":{"1min":0.39500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.587,"5min":18.587,"15min":18.587},"last":13.119},{"interface":"front","average":{"1min":4.6959999999999997,"5min":4.2249999999999996,"15min":4.6760000000000002},"min":{"1min":1.0329999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.022,"5min":17.289000000000001,"15min":17.289000000000001},"last":4.5839999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.474,"15min":1.3280000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2629999999999999,"5min":7.2629999999999999,"15min":7.2629999999999999},"last":0.91900000000000004},{"interface":"front","average":{"1min":2.879,"5min":1.639,"15min":1.4330000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.6940000000000008,"5min":10.132,"15min":10.132},"last":8.3230000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":2.0419999999999998,"15min":1.72},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.473000000000001,"5min":14.416,"15min":15.531000000000001},"last":8.4489999999999998},{"interface":"front","average":{"1min":1.782,"5min":2.0979999999999999,"15min":1.9690000000000001},"min":{"1min":0.34699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.6890000000000001,"5min":16.277999999999999,"15min":19.706},"last":6.9039999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0339999999999998,"15min":1.7230000000000001},"min":{"1min":0.50800000000000001,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.3710000000000004,"5min":14.215,"15min":16.193999999999999},"last":0.95499999999999996},{"interface":"front","average":{"1min":2.75,"5min":1.9930000000000001,"15min":1.7529999999999999},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":10.161,"5min":14.471,"15min":15.481999999999999},"last":8.3610000000000007}]},{"osd":7,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":2.153,"15min":1.972},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.7350000000000003,"5min":16.331,"15min":16.331},"last":6.8620000000000001},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1320000000000001,"15min":1.788},"min":{"1min":0.46000000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3319999999999999,"5min":14.242000000000001,"15min":16.100999999999999},"last":4.5529999999999999}]}]},{"osd":0,"up_from":79,"seq":339302416563,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2396592,"kb_used_data":2037928,"kb_used_omap":28,"kb_used_meta":358563,"kb_avail":91348560,"statfs":{"total":95995035648,"available":93540925440,"internally_reserved":0,"allocated":2086838272,"data_stored":2085828801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":367169360},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.847,"15min":1.4970000000000001},"min":{"1min":0.24099999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.2990000000000004,"5min":12.84,"15min":13.084},"last":5.1459999999999999},{"interface":"front","average":{"1min":3.9020000000000001,"5min":3.379,"15min":2.2269999999999999},"min":{"1min":0.44900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.949000000000002,"5min":22.178999999999998,"15min":22.178999999999998},"last":4.7409999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8360000000000001,"15min":2.1589999999999998},"min":{"1min":0.34599999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.9949999999999992,"5min":10.034000000000001,"15min":10.034000000000001},"last":5.21},{"interface":"front","average":{"1min":2.9340000000000002,"5min":2.177,"15min":3.0539999999999998},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":15.529,"5min":15.529,"15min":15.529},"last":0.29999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":4.5259999999999998,"15min":4.7519999999999998},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":20.419,"5min":25.734999999999999,"15min":25.734999999999999},"last":4.7750000000000004},{"interface":"front","average":{"1min":3.742,"5min":3.891,"15min":4.2370000000000001},"min":{"1min":0.40999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":10.154,"5min":21.923999999999999,"15min":21.923999999999999},"last":4.1289999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.911,"15min":1.5649999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":4.6630000000000003},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.0459999999999998,"15min":1.9730000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.6720000000000006,"5min":13.789999999999999,"15min":13.789999999999999},"last":3.6230000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.7309999999999999,"15min":1.9630000000000001},"min":{"1min":0.40899999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.6229999999999993,"5min":17.096,"15min":17.096},"last":3.577},{"interface":"front","average":{"1min":1.5,"5min":1.454,"15min":1.3029999999999999},"min":{"1min":0.30499999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.4260000000000002,"5min":9.4260000000000002,"15min":12.478},"last":5.1020000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.7010000000000001,"15min":1.4590000000000001},"min":{"1min":0.33700000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":9.4719999999999995,"5min":9.7260000000000009,"15min":11.699},"last":5.1799999999999997},{"interface":"front","average":{"1min":4.4729999999999999,"5min":2.6320000000000001,"15min":2.2069999999999999},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.466000000000001,"5min":20.466000000000001,"15min":20.466000000000001},"last":4.7080000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8239999999999998,"15min":2.1139999999999999},"min":{"1min":0.46400000000000002,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":9.7050000000000001,"5min":14.631,"15min":26.978000000000002},"last":3.6509999999999998},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.7190000000000001,"15min":1.605},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":9.5020000000000007,"5min":11.166,"15min":26.768000000000001},"last":5.2400000000000002}]}]},{"osd":2,"up_from":217,"seq":932007903296,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1831528,"kb_used_data":1596640,"kb_used_omap":83,"kb_used_meta":234796,"kb_avail":91913624,"statfs":{"total":95995035648,"available":94119550976,"internally_reserved":0,"allocated":1634959360,"data_stored":1633958101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":240431599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":1.732,"15min":1.756},"min":{"1min":0.217,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":19.372,"5min":19.372,"15min":19.372},"last":2.5819999999999999},{"interface":"front","average":{"1min":3.4420000000000002,"5min":2.9849999999999999,"15min":2.9569999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.754999999999999,"5min":16.754999999999999,"15min":16.754999999999999},"last":2.093}]},{"osd":1,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":2.258,"15min":2.544},"min":{"1min":0.254,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.797000000000001,"5min":13.797000000000001,"15min":13.797000000000001},"last":0.92600000000000005},{"interface":"front","average":{"1min":2.8399999999999999,"5min":2.5209999999999999,"15min":2.6379999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.4119999999999999,"5min":15.569000000000001,"15min":15.569000000000001},"last":5.6390000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":2.198,"15min":2.0819999999999999},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.851000000000001,"5min":13.851000000000001,"15min":13.851000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.9860000000000002,"5min":3.181,"15min":3.6699999999999999},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.7240000000000002,"5min":15.811,"15min":15.811},"last":2.1190000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.84499999999999997,"15min":0.753},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.29,"5min":6.29,"15min":6.29},"last":2.1480000000000001},{"interface":"front","average":{"1min":2.5609999999999999,"5min":1.1759999999999999,"15min":0.94499999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":12.385,"5min":12.385,"15min":12.385},"last":0.50700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.6919999999999999,"15min":1.546},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":12.473000000000001,"5min":12.473000000000001,"15min":12.473000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":3.0699999999999998,"5min":1.5409999999999999,"15min":1.6870000000000001},"min":{"1min":0.372,"5min":0.318,"15min":0.318},"max":{"1min":13.587,"5min":13.587,"15min":13.587},"last":0.42099999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.3999999999999999,"15min":1.095},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2599999999999998,"5min":15.292999999999999,"15min":15.292999999999999},"last":2.1709999999999998},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.738,"15min":1.742},"min":{"1min":0.36599999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.95999999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.431,"15min":1.1259999999999999},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.2190000000000003,"5min":15.343999999999999,"15min":15.343999999999999},"last":2.198},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.859,"15min":1.6599999999999999},"min":{"1min":0.41699999999999998,"5min":0.376,"15min":0.376},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":12.433999999999999},"last":0.53600000000000003}]}]},{"osd":3,"up_from":250,"seq":1073741824040,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2321632,"kb_used_data":2071320,"kb_used_omap":10,"kb_used_meta":250229,"kb_avail":91423520,"statfs":{"total":95995035648,"available":93617684480,"internally_reserved":0,"allocated":2121031680,"data_stored":2119862032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":256234563},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":4.2709999999999999,"15min":4.5590000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.753,"5min":23.381,"15min":23.381},"last":2.6949999999999998},{"interface":"front","average":{"1min":6.0720000000000001,"5min":4.0919999999999996,"15min":4.1500000000000004},"min":{"1min":0.42099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":20.239999999999998,"5min":20.239999999999998,"15min":20.239999999999998},"last":0.72299999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.6150000000000002,"15min":3.8410000000000002},"min":{"1min":0.26300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":20.422999999999998,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.85399999999999998},{"interface":"front","average":{"1min":4.5039999999999996,"5min":3.4990000000000001,"15min":3.4529999999999998},"min":{"1min":0.33100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.704000000000001,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.97099999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1369999999999996,"5min":3.4199999999999999,"15min":3.6339999999999999},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":20.594999999999999,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.88400000000000001},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.8809999999999998,"15min":2.8700000000000001},"min":{"1min":0.34200000000000003,"5min":0.253,"15min":0.253},"max":{"1min":15.73,"5min":23.452000000000002,"15min":23.452000000000002},"last":2.7309999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":1.6719999999999999,"15min":1.288},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.676,"5min":15.676,"15min":15.676},"last":8.4789999999999992},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5800000000000001,"15min":1.534},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.426,"5min":12.426,"15min":12.426},"last":0.753}]},{"osd":5,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8290000000000002,"5min":2.4750000000000001,"15min":2.5670000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":12.388,"5min":12.388,"15min":12.388},"last":6.2130000000000001},{"interface":"front","average":{"1min":3.5699999999999998,"5min":2.5169999999999999,"15min":2.4079999999999999},"min":{"1min":0.443,"5min":0.309,"15min":0.309},"max":{"1min":15.992000000000001,"5min":15.992000000000001,"15min":15.992000000000001},"last":1.7090000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":2.536,"15min":2.6709999999999998},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.468,"5min":12.468,"15min":12.468},"last":0.83499999999999996},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.5609999999999999,"15min":2.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":16.16,"5min":16.16,"15min":16.16},"last":0.90700000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.825,"15min":1.534},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.635999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.4399999999999999,"15min":2.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.77900000000000003}]}]},{"osd":4,"up_from":266,"seq":1142461300766,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2352356,"kb_used_data":2033692,"kb_used_omap":35,"kb_used_meta":318620,"kb_avail":91392796,"statfs":{"total":95995035648,"available":93586223104,"internally_reserved":0,"allocated":2082500608,"data_stored":2081974895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":326267118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.98499999999999999,"15min":0.84099999999999997},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.881,"5min":10.881,"15min":10.881},"last":6.4829999999999997},{"interface":"front","average":{"1min":1.923,"5min":1.1519999999999999,"15min":1.024},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":4.4290000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":1.169,"15min":0.93400000000000005},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":10.93,"5min":10.93,"15min":10.93},"last":3.7029999999999998},{"interface":"front","average":{"1min":2.633,"5min":1.546,"15min":1.365},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.141,"5min":11.141,"15min":11.141},"last":0.34999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":1.2310000000000001,"15min":1.073},"min":{"1min":0.32100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.1189999999999998,"5min":7.1189999999999998,"15min":7.1189999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.587,"5min":1.042,"15min":0.95099999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.1709999999999994,"5min":8.1709999999999994,"15min":8.1709999999999994},"last":0.496}]},{"osd":3,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.4610000000000001,"15min":1.3280000000000001},"min":{"1min":0.40699999999999997,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5530000000000008,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":6.4509999999999996},{"interface":"front","average":{"1min":2.4420000000000002,"5min":1.2989999999999999,"15min":1.109},"min":{"1min":0.373,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.6069999999999993,"5min":9.6069999999999993,"15min":9.6069999999999993},"last":4.4050000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.80400000000000005,"15min":0.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4740000000000002,"5min":5.4740000000000002,"15min":5.4740000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.60399999999999998,"15min":0.63400000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.436}]},{"osd":6,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.60499999999999998,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":5.5709999999999997,"5min":5.5709999999999997,"15min":5.5709999999999997},"last":6.4119999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.60999999999999999,"15min":0.59199999999999997},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.319,"5min":2.319,"15min":2.319},"last":2.3769999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.61299999999999999,"15min":0.61799999999999999},"min":{"1min":0.33700000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.3460000000000001,"5min":2.3460000000000001,"15min":2.3460000000000001},"last":2.3380000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.82299999999999995,"15min":0.82699999999999996},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.0680000000000001,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.41099999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822951,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3024088,"kb_used_data":2710480,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":90721064,"statfs":{"total":95995035648,"available":92898369536,"internally_reserved":0,"allocated":2775531520,"data_stored":2775013863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":1.55,"15min":1.488},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.8540000000000001,"5min":7.8540000000000001,"15min":15.773999999999999},"last":4.8179999999999996},{"interface":"front","average":{"1min":1.339,"5min":1.597,"15min":1.532},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.0629999999999997,"5min":11.747,"15min":15.699},"last":4.9409999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:10:12 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.5740000000000001,"15min":1.4430000000000001},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.7720000000000002,"5min":11.335000000000001,"15min":15.689},"last":4.8470000000000004},{"interface":"front","average":{"1min":1.913,"5min":1.861,"15min":1.6180000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.8119999999999994,"5min":19.402999999999999,"15min":19.402999999999999},"last":4.5839999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":1.7090000000000001,"15min":2.391},"min":{"1min":0.34899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7249999999999996,"5min":15.640000000000001,"15min":15.640000000000001},"last":4.9189999999999996},{"interface":"front","average":{"1min":1.663,"5min":1.6479999999999999,"15min":2.2370000000000001},"min":{"1min":0.40400000000000003,"5min":0.313,"15min":0.313},"max":{"1min":5.2789999999999999,"5min":10.656000000000001,"15min":10.656000000000001},"last":0.379}]},{"osd":3,"last update":"Mon Oct 9 10:10:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.8340000000000001,"15min":1.8440000000000001},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6660000000000004,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":7.0919999999999996},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.4380000000000002,"15min":2.8250000000000002},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.085,"5min":17.34,"15min":17.34},"last":4.6139999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":0.71699999999999997,"15min":0.61499999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5220000000000002,"5min":5.5220000000000002,"15min":5.5220000000000002},"last":7.3460000000000001},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.752,"15min":0.746},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.823,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.014,"15min":0.94499999999999995},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.25},"max":{"1min":5.4930000000000003,"5min":13.483000000000001,"15min":27.716000000000001},"last":8.5259999999999998},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.0189999999999999,"15min":0.88700000000000001},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.8099999999999996,"5min":18.600999999999999,"15min":18.600999999999999},"last":4.9889999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":1.0900000000000001,"15min":0.97099999999999997},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.891,"5min":7.891,"15min":10.334},"last":4.8810000000000002},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.72299999999999998,"15min":0.93300000000000005},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.3630000000000004,"5min":6.3630000000000004,"15min":12.356},"last":0.432}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2071986176,"data_stored":2071986176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2494234624,"data_stored":2494234624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1621561344,"data_stored":1621557272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2110062592,"data_stored":2110062592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2072838144,"data_stored":2072838144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2771390464,"data_stored":2771386392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2120286208,"data_stored":2120286208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1770196992,"data_stored":1770192920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:11:00.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:11:00.788 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:11:00.812 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:11:00.812 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:11:00.813 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:11:00.813 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:11:00.813 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:11:00.813 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:11:00.814 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:11:00.814 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:11:00.827 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:00.865 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:00.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:11:00.915 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:00.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:00.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:00.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:11:01.019 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:01.046 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:01.096 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:01.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:01.201 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:01.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:01.278 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:01.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:01.377 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[147926]: pgmap v795: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5374/256389 objects degraded (2.096%); 3802/256389 objects misplaced (1.483%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:11:01.377 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[147926]: from='client.35700 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:01.377 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:01.378 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:01.378 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3045989677' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:01.378 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[152263]: pgmap v795: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5374/256389 objects degraded (2.096%); 3802/256389 objects misplaced (1.483%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:11:01.378 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[152263]: from='client.35700 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:01.378 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:01.378 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:01.378 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3045989677' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:01.386 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:01.410 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:01 smithi110 ceph-mon[136032]: pgmap v795: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5374/256389 objects degraded (2.096%); 3802/256389 objects misplaced (1.483%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:11:01.410 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:01 smithi110 ceph-mon[136032]: from='client.35700 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:01.411 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:01.411 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:01.411 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3045989677' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:01.412 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:01.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:01.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:01.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:01.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:01.648 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:01.767 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:01.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:01.840 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:01.879 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:01.908 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:01.911 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:01.961 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:01.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:01.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:02.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:02.148 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:02.852 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:02.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:02.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:03.052 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:03.151 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:03.158 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:03 smithi012 ceph-mon[147926]: pgmap v796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5395/251955 objects degraded (2.141%); 3665/251955 objects misplaced (1.455%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:03.159 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5395/251955 objects degraded (2.141%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:03.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:11:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:03 smithi012 ceph-mon[152263]: pgmap v796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5395/251955 objects degraded (2.141%); 3665/251955 objects misplaced (1.455%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5395/251955 objects degraded (2.141%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:03.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:03 smithi110 ceph-mon[136032]: pgmap v796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5395/251955 objects degraded (2.141%); 3665/251955 objects misplaced (1.455%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5395/251955 objects degraded (2.141%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:03.908 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:03.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:03.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:04.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:04.146 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:04.296 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:04.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:04.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:04.517 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:04.959 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:04.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:04.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:05.066 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:05.101 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:05.149 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:05.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:05.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:05.253 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:11:05.281 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:05.328 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:11:05.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:05.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:11:05.423 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:05 smithi012 ceph-mon[152263]: pgmap v797: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5395/247479 objects degraded (2.180%); 3665/247479 objects misplaced (1.481%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:05.423 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:05 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:05.423 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:05 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:05.425 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:05 smithi012 ceph-mon[147926]: pgmap v797: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5395/247479 objects degraded (2.180%); 3665/247479 objects misplaced (1.481%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:05.425 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:05 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:05.425 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:05 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:05.432 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:11:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:05 smithi110 ceph-mon[136032]: pgmap v797: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5395/247479 objects degraded (2.180%); 3665/247479 objects misplaced (1.481%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:05 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:05 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:05.467 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:05.509 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:11:05.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:05.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:11:05.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:05.741 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:05.858 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:05.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:05.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:06.074 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:06.186 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:06.210 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:06.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:06.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:06.350 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:06 smithi012 ceph-mon[147926]: pgmap v798: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5395/244656 objects degraded (2.205%); 3665/244656 objects misplaced (1.498%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:06.350 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:06 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:11:06.350 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:06 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:11:06.350 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:06 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:06.351 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:06 smithi012 ceph-mon[152263]: pgmap v798: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5395/244656 objects degraded (2.205%); 3665/244656 objects misplaced (1.498%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:06.351 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:06 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:11:06.351 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:06 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:11:06.351 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:06 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:06.431 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:06.641 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:11:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:11:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:137895 8.4 e297/288 8:28c9452f:::benchmark_data_smithi012_271002_object29212:head v 297'8678, mlcod=0'0)", 2023-10-09T10:11:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:06.627413+0000", 2023-10-09T10:11:06.641 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0085551919999999997, 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085824669999999999, 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.627413+0000", 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.627413+0000", 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.627412+0000", 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.627423+0000", 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1565e-05 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.627437+0000", 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3614e-05 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:06.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.627446+0000", 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2180000000000005e-06 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.627596+0000", 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014969000000000001 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:06.645 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.627634+0000", 2023-10-09T10:11:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8325999999999999e-05 2023-10-09T10:11:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:137890 8.e e297/271 8:79594b66:::benchmark_data_smithi012_271002_object29207:head v 297'8568, mlcod=297'8566)", 2023-10-09T10:11:06.646 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:06.630426+0000", 2023-10-09T10:11:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0055423579999999998, 2023-10-09T10:11:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056310910000000004, 2023-10-09T10:11:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:11:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:06.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630426+0000", 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630426+0000", 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630424+0000", 2023-10-09T10:11:06.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630435+0000", 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0261000000000001e-05 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630448+0000", 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.36e-05 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630457+0000", 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.799e-06 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630579+0000", 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121981 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630601+0000", 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1671999999999999e-05 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:137894 8.f e297/288 8:f7900b7b:::benchmark_data_smithi012_271002_object29211:head v 297'8647, mlcod=279'1818)", 2023-10-09T10:11:06.651 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:06.630502+0000", 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0054664539999999999, 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056104630000000004, 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630502+0000", 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630502+0000", 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630501+0000", 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630506+0000", 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.865e-06 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630514+0000", 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8620000000000003e-06 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.630519+0000", 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.242e-06 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.634378+0000", 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038586470000000002 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:06.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:06.634409+0000", 2023-10-09T10:11:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1608e-05 2023-10-09T10:11:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:11:06.656 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 3 2023-10-09T10:11:06.656 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:11:06.657 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:06.657 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:11:06.657 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5395/244656 objects degraded (2.205%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":983,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":297,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":81552,"data_bytes":5343549368,"bytes_used":18790498304,"bytes_avail":749169786880,"bytes_total":767960285184,"degraded_objects":5395,"degraded_total":244656,"degraded_ratio":0.022051370086979268,"misplaced_objects":3665,"misplaced_total":244656,"misplaced_ratio":0.014980217121182395,"recovering_objects_per_sec":23,"recovering_bytes_per_sec":1558188,"recovering_keys_per_sec":0,"num_objects_recovered":238,"num_bytes_recovered":15597568,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":603},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T10:09:33.293227+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T10:11:06.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:06.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:06 smithi110 ceph-mon[136032]: pgmap v798: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5395/244656 objects degraded (2.205%); 3665/244656 objects misplaced (1.498%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:06 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:11:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:06 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:11:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:06 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:06.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:06.855 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:06.965 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:06.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:07.076 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:07.183 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:07.241 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:11:07.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1507304928' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:07.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/870324254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:07.274 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1507304928' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:07.274 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/870324254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:07.300 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:07.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:07.391 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:11:07.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:11:07.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:07.545 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:07.625 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:07.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:07.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1507304928' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/870324254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:07.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:07.846 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:08.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:11:08.100 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:11:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:11:07] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T10:11:08.108 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:08.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:08.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:08.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:08.355 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:08.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:08.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[147926]: failed to encode map e298 with expected crc 2023-10-09T10:11:08.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/870324254' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:08.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[147926]: osdmap e298: 8 total, 8 up, 8 in 2023-10-09T10:11:08.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[147926]: failed to encode map e298 with expected crc 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[147926]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 5465/242859 objects degraded (2.250%); 3474/242859 objects misplaced (1.430%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5395/244656 objects degraded (2.205%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[152263]: failed to encode map e298 with expected crc 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/870324254' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[152263]: osdmap e298: 8 total, 8 up, 8 in 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[152263]: failed to encode map e298 with expected crc 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[152263]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 5465/242859 objects degraded (2.250%); 3474/242859 objects misplaced (1.430%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:08.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5395/244656 objects degraded (2.205%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:08.576 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:08.576 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:08.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:08.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:08.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:08 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:08 smithi110 ceph-mon[136032]: failed to encode map e298 with expected crc 2023-10-09T10:11:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/870324254' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:08 smithi110 ceph-mon[136032]: osdmap e298: 8 total, 8 up, 8 in 2023-10-09T10:11:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:08 smithi110 ceph-mon[136032]: failed to encode map e298 with expected crc 2023-10-09T10:11:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:08 smithi110 ceph-mon[136032]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 5465/242859 objects degraded (2.250%); 3474/242859 objects misplaced (1.430%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5395/244656 objects degraded (2.205%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:08.722 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:08.771 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:08.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:08.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:08.876 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:08.942 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:09.014 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:09.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:09.023 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:11:09.088 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:11:09.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:09.128 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:11:09.172 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:11:09.214 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:11:09.228 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:09.281 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:11:09.540 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:11:09.540 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:11:09.541 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:11:09.541 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:11:09.541 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:11:09.541 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:11:09.541 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:11:09.541 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:11:09.541 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:11:09.541 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:11:09.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/442996767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:09.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/442996767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/442996767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:09.710 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:09.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:09.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:09.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:09.928 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:09.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:11:10.123 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:10.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:10.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:10.308 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:10 smithi012 ceph-mon[147926]: pgmap v801: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5465/238485 objects degraded (2.292%); 3474/238485 objects misplaced (1.457%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:10.311 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:10 smithi012 ceph-mon[152263]: pgmap v801: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5465/238485 objects degraded (2.292%); 3474/238485 objects misplaced (1.457%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:10.328 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:10.534 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:10.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:10.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:10.645 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:10.655 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:10.671 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:10 smithi110 ceph-mon[136032]: pgmap v801: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5465/238485 objects degraded (2.292%); 3474/238485 objects misplaced (1.457%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:10.674 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:10.716 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:10.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:10.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:10.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:10.859 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:10.897 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:10.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:11.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:11.115 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:11.308 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:11.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:11.369 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:11.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:11.507 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:11.781 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:11:11.803 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:11:11.812 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:11.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:11.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:11.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:11.964 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:12.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:12.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:12.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:12.124 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:12.192 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:12.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:12 smithi110 ceph-mon[136032]: pgmap v802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 5534/234339 objects degraded (2.362%); 3293/234339 objects misplaced (1.405%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:12.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:12 smithi110 ceph-mon[136032]: from='client.55750 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:12.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:11:12.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:12 smithi012 ceph-mon[147926]: pgmap v802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 5534/234339 objects degraded (2.362%); 3293/234339 objects misplaced (1.405%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:12.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:12 smithi012 ceph-mon[147926]: from='client.55750 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:12.689 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:12 smithi012 ceph-mon[152263]: pgmap v802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 5534/234339 objects degraded (2.362%); 3293/234339 objects misplaced (1.405%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:12.689 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:12 smithi012 ceph-mon[152263]: from='client.55750 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:12.693 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:12.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:12.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:12.720 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":802,"stamp":"2023-10-09T10:11:11.359851+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5118171064,"num_objects":78113,"num_object_clones":0,"num_object_copies":234339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5534,"num_objects_misplaced":3293,"num_objects_unfound":0,"num_objects_dirty":78113,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":139533,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1899,"num_bytes_recovered":121121484,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133267,"ondisk_log_size":133267,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17789196,"kb_used_data":15303884,"kb_used_omap":167,"kb_used_meta":2484248,"kb_avail":732172020,"statfs":{"total":767960285184,"available":749744148480,"internally_reserved":0,"allocated":15671177216,"data_stored":15664854024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2543870553},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,4,1,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-384827392,"num_objects":-5872,"num_object_clones":0,"num_object_copies":-17616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":139,"num_objects_misplaced":-372,"num_objects_unfound":0,"num_objects_dirty":-5872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5872,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":233,"num_bytes_recovered":15269888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005451"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":139,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081442+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:10:58.081442+0000","last_peered":"2023-10-09T10:10:58.081442+0000","last_clean":"2023-10-09T10:10:58.081442+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:10:58.081442+0000","last_undegraded":"2023-10-09T10:10:58.081442+0000","last_fullsized":"2023-10-09T10:10:58.081442+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":255,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.077697+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:10:58.077697+0000","last_peered":"2023-10-09T10:10:58.077697+0000","last_clean":"2023-10-09T10:10:58.077697+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:10:58.077697+0000","last_undegraded":"2023-10-09T10:10:58.077697+0000","last_fullsized":"2023-10-09T10:10:58.077697+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":300,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081015+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:10:58.081015+0000","last_peered":"2023-10-09T10:10:58.081015+0000","last_clean":"2023-10-09T10:10:58.081015+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:10:58.081015+0000","last_undegraded":"2023-10-09T10:10:58.081015+0000","last_fullsized":"2023-10-09T10:10:58.081015+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":164,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.082016+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:10:58.082016+0000","last_peered":"2023-10-09T10:10:58.082016+0000","last_clean":"2023-10-09T10:10:58.082016+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:10:58.082016+0000","last_undegraded":"2023-10-09T10:10:58.082016+0000","last_fullsized":"2023-10-09T10:10:58.082016+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":269,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.064943+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:10:58.064943+0000","last_peered":"2023-10-09T10:10:58.064943+0000","last_clean":"2023-10-09T10:10:58.064943+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:10:58.064943+0000","last_undegraded":"2023-10-09T10:10:58.064943+0000","last_fullsized":"2023-10-09T10:10:58.064943+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":116,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.072169+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:10:58.072169+0000","last_peered":"2023-10-09T10:10:58.072169+0000","last_clean":"2023-10-09T10:10:58.072169+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:10:58.072169+0000","last_undegraded":"2023-10-09T10:10:58.072169+0000","last_fullsized":"2023-10-09T10:10:58.072169+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":181,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081912+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:10:58.081912+0000","last_peered":"2023-10-09T10:10:58.081912+0000","last_clean":"2023-10-09T10:10:58.081912+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:10:58.081912+0000","last_undegraded":"2023-10-09T10:10:58.081912+0000","last_fullsized":"2023-10-09T10:10:58.081912+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":250,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.064865+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:10:58.064865+0000","last_peered":"2023-10-09T10:10:58.064865+0000","last_clean":"2023-10-09T10:10:58.064865+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:10:58.064865+0000","last_undegraded":"2023-10-09T10:10:58.064865+0000","last_fullsized":"2023-10-09T10:10:58.064865+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":115,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.071956+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:10:58.071956+0000","last_peered":"2023-10-09T10:10:58.071956+0000","last_clean":"2023-10-09T10:10:58.071956+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:10:58.071956+0000","last_undegraded":"2023-10-09T10:10:58.071956+0000","last_fullsized":"2023-10-09T10:10:58.071956+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":211,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.078010+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:10:58.078010+0000","last_peered":"2023-10-09T10:10:58.078010+0000","last_clean":"2023-10-09T10:10:58.078010+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:10:58.078010+0000","last_undegraded":"2023-10-09T10:10:58.078010+0000","last_fullsized":"2023-10-09T10:10:58.078010+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":336,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.092432+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:10:58.092432+0000","last_peered":"2023-10-09T10:10:58.092432+0000","last_clean":"2023-10-09T10:10:58.092432+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:10:58.092432+0000","last_undegraded":"2023-10-09T10:10:58.092432+0000","last_fullsized":"2023-10-09T10:10:58.092432+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":293,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081308+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:10:58.081308+0000","last_peered":"2023-10-09T10:10:58.081308+0000","last_clean":"2023-10-09T10:10:58.081308+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:10:58.081308+0000","last_undegraded":"2023-10-09T10:10:58.081308+0000","last_fullsized":"2023-10-09T10:10:58.081308+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":185,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.090243+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:10:58.090243+0000","last_peered":"2023-10-09T10:10:58.090243+0000","last_clean":"2023-10-09T10:10:58.090243+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:10:58.090243+0000","last_undegraded":"2023-10-09T10:10:58.090243+0000","last_fullsized":"2023-10-09T10:10:58.090243+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":165,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.095325+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:10:58.095325+0000","last_peered":"2023-10-09T10:10:58.095325+0000","last_clean":"2023-10-09T10:10:58.095325+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:10:58.095325+0000","last_undegraded":"2023-10-09T10:10:58.095325+0000","last_fullsized":"2023-10-09T10:10:58.095325+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":369,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.089603+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:10:58.089603+0000","last_peered":"2023-10-09T10:10:58.089603+0000","last_clean":"2023-10-09T10:10:58.089603+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:10:58.089603+0000","last_undegraded":"2023-10-09T10:10:58.089603+0000","last_fullsized":"2023-10-09T10:10:58.089603+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":237,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.080830+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:10:58.080830+0000","last_peered":"2023-10-09T10:10:58.080830+0000","last_clean":"2023-10-09T10:10:58.080830+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:10:58.080830+0000","last_undegraded":"2023-10-09T10:10:58.080830+0000","last_fullsized":"2023-10-09T10:10:58.080830+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":163,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.082063+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:10:58.082063+0000","last_peered":"2023-10-09T10:10:58.082063+0000","last_clean":"2023-10-09T10:10:58.082063+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:10:58.082063+0000","last_undegraded":"2023-10-09T10:10:58.082063+0000","last_fullsized":"2023-10-09T10:10:58.082063+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":131,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.072532+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:10:58.072532+0000","last_peered":"2023-10-09T10:10:58.072532+0000","last_clean":"2023-10-09T10:10:58.072532+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:10:58.072532+0000","last_undegraded":"2023-10-09T10:10:58.072532+0000","last_fullsized":"2023-10-09T10:10:58.072532+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":166,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.089604+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:10:58.089604+0000","last_peered":"2023-10-09T10:10:58.089604+0000","last_clean":"2023-10-09T10:10:58.089604+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:10:58.089604+0000","last_undegraded":"2023-10-09T10:10:58.089604+0000","last_fullsized":"2023-10-09T10:10:58.089604+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":163,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.082059+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:10:58.082059+0000","last_peered":"2023-10-09T10:10:58.082059+0000","last_clean":"2023-10-09T10:10:58.082059+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:10:58.082059+0000","last_undegraded":"2023-10-09T10:10:58.082059+0000","last_fullsized":"2023-10-09T10:10:58.082059+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":186,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.080502+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:10:58.080502+0000","last_peered":"2023-10-09T10:10:58.080502+0000","last_clean":"2023-10-09T10:10:58.080502+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:10:58.080502+0000","last_undegraded":"2023-10-09T10:10:58.080502+0000","last_fullsized":"2023-10-09T10:10:58.080502+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.087371+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:10:58.087371+0000","last_peered":"2023-10-09T10:10:58.087371+0000","last_clean":"2023-10-09T10:10:58.087371+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:10:58.087371+0000","last_undegraded":"2023-10-09T10:10:58.087371+0000","last_fullsized":"2023-10-09T10:10:58.087371+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"298'8725","reported_seq":8768,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:07.872164+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:11:07.872164+0000","last_peered":"2023-10-09T10:11:07.872164+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:11:07.872164+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8725,"log_dups_size":0,"ondisk_log_size":8725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323813376,"num_objects":4941,"num_object_clones":0,"num_object_copies":14823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8725,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-432},{"shards":"4,5,6","objects":3481}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":330,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.090298+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:10:58.090298+0000","last_peered":"2023-10-09T10:10:58.090298+0000","last_clean":"2023-10-09T10:10:58.090298+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:10:58.090298+0000","last_undegraded":"2023-10-09T10:10:58.090298+0000","last_fullsized":"2023-10-09T10:10:58.090298+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":116,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.071562+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:10:58.071562+0000","last_peered":"2023-10-09T10:10:58.071562+0000","last_clean":"2023-10-09T10:10:58.071562+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:10:58.071562+0000","last_undegraded":"2023-10-09T10:10:58.071562+0000","last_fullsized":"2023-10-09T10:10:58.071562+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"298'8841","reported_seq":8876,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:09.991182+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:11:09.991182+0000","last_peered":"2023-10-09T10:11:09.991182+0000","last_clean":"2023-10-09T10:11:09.991182+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:11:09.991182+0000","last_undegraded":"2023-10-09T10:11:09.991182+0000","last_fullsized":"2023-10-09T10:11:09.991182+0000","mapping_epoch":271,"log_start":"275'200","ondisk_log_start":"275'200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8641,"log_dups_size":0,"ondisk_log_size":8641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324337664,"num_objects":4949,"num_object_clones":0,"num_object_copies":14847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8841,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":318,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.090267+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:10:58.090267+0000","last_peered":"2023-10-09T10:10:58.090267+0000","last_clean":"2023-10-09T10:10:58.090267+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:10:58.090267+0000","last_undegraded":"2023-10-09T10:10:58.090267+0000","last_fullsized":"2023-10-09T10:10:58.090267+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":209,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.080936+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:10:58.080936+0000","last_peered":"2023-10-09T10:10:58.080936+0000","last_clean":"2023-10-09T10:10:58.080936+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:10:58.080936+0000","last_undegraded":"2023-10-09T10:10:58.080936+0000","last_fullsized":"2023-10-09T10:10:58.080936+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"298'8642","reported_seq":8679,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.701083+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:11:07.701083+0000","last_peered":"2023-10-09T10:11:07.701083+0000","last_clean":"2023-10-09T10:11:07.701083+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:11:07.701083+0000","last_undegraded":"2023-10-09T10:11:07.701083+0000","last_fullsized":"2023-10-09T10:11:07.701083+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8642,"log_dups_size":0,"ondisk_log_size":8642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322699264,"num_objects":4924,"num_object_clones":0,"num_object_copies":14772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8642,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":120,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.072617+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:10:58.072617+0000","last_peered":"2023-10-09T10:10:58.072617+0000","last_clean":"2023-10-09T10:10:58.072617+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:10:58.072617+0000","last_undegraded":"2023-10-09T10:10:58.072617+0000","last_fullsized":"2023-10-09T10:10:58.072617+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":141,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.098000+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:10:58.098000+0000","last_peered":"2023-10-09T10:10:58.098000+0000","last_clean":"2023-10-09T10:10:58.098000+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:10:58.098000+0000","last_undegraded":"2023-10-09T10:10:58.098000+0000","last_fullsized":"2023-10-09T10:10:58.098000+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"298'8531","reported_seq":8568,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:09.460650+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:11:09.460650+0000","last_peered":"2023-10-09T10:11:09.460650+0000","last_clean":"2023-10-09T10:11:09.460650+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:11:09.460650+0000","last_undegraded":"2023-10-09T10:11:09.460650+0000","last_fullsized":"2023-10-09T10:11:09.460650+0000","mapping_epoch":271,"log_start":"277'800","ondisk_log_start":"277'800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7731,"log_dups_size":0,"ondisk_log_size":7731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319750144,"num_objects":4879,"num_object_clones":0,"num_object_copies":14637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8531,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":131,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.072261+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:10:58.072261+0000","last_peered":"2023-10-09T10:10:58.072261+0000","last_clean":"2023-10-09T10:10:58.072261+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:10:58.072261+0000","last_undegraded":"2023-10-09T10:10:58.072261+0000","last_fullsized":"2023-10-09T10:10:58.072261+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":164,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.087103+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:10:58.087103+0000","last_peered":"2023-10-09T10:10:58.087103+0000","last_clean":"2023-10-09T10:10:58.087103+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:10:58.087103+0000","last_undegraded":"2023-10-09T10:10:58.087103+0000","last_fullsized":"2023-10-09T10:10:58.087103+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"298'8895","reported_seq":8932,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:11.322849+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:11:11.322849+0000","last_peered":"2023-10-09T10:11:11.322849+0000","last_clean":"2023-10-09T10:11:11.322849+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:11:11.322849+0000","last_undegraded":"2023-10-09T10:11:11.322849+0000","last_fullsized":"2023-10-09T10:11:11.322849+0000","mapping_epoch":271,"log_start":"277'700","ondisk_log_start":"277'700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8195,"log_dups_size":0,"ondisk_log_size":8195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324075520,"num_objects":4945,"num_object_clones":0,"num_object_copies":14835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8895,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":291,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081440+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:10:58.081440+0000","last_peered":"2023-10-09T10:10:58.081440+0000","last_clean":"2023-10-09T10:10:58.081440+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:10:58.081440+0000","last_undegraded":"2023-10-09T10:10:58.081440+0000","last_fullsized":"2023-10-09T10:10:58.081440+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":161,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.080698+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:10:58.080698+0000","last_peered":"2023-10-09T10:10:58.080698+0000","last_clean":"2023-10-09T10:10:58.080698+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:10:58.080698+0000","last_undegraded":"2023-10-09T10:10:58.080698+0000","last_fullsized":"2023-10-09T10:10:58.080698+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"298'8811","reported_seq":8848,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:09.607794+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:11:09.607794+0000","last_peered":"2023-10-09T10:11:09.607794+0000","last_clean":"2023-10-09T10:11:09.607794+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:11:09.607794+0000","last_undegraded":"2023-10-09T10:11:09.607794+0000","last_fullsized":"2023-10-09T10:11:09.607794+0000","mapping_epoch":271,"log_start":"277'1100","ondisk_log_start":"277'1100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7711,"log_dups_size":0,"ondisk_log_size":7711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320667648,"num_objects":4893,"num_object_clones":0,"num_object_copies":14679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8811,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":223,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081238+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:10:58.081238+0000","last_peered":"2023-10-09T10:10:58.081238+0000","last_clean":"2023-10-09T10:10:58.081238+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:10:58.081238+0000","last_undegraded":"2023-10-09T10:10:58.081238+0000","last_fullsized":"2023-10-09T10:10:58.081238+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":228,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.091549+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:10:58.091549+0000","last_peered":"2023-10-09T10:10:58.091549+0000","last_clean":"2023-10-09T10:10:58.091549+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:10:58.091549+0000","last_undegraded":"2023-10-09T10:10:58.091549+0000","last_fullsized":"2023-10-09T10:10:58.091549+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"298'8565","reported_seq":8602,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.749222+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:11:07.749222+0000","last_peered":"2023-10-09T10:11:07.749222+0000","last_clean":"2023-10-09T10:11:07.749222+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:11:07.749222+0000","last_undegraded":"2023-10-09T10:11:07.749222+0000","last_fullsized":"2023-10-09T10:11:07.749222+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8565,"log_dups_size":0,"ondisk_log_size":8565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318963712,"num_objects":4867,"num_object_clones":0,"num_object_copies":14601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8565,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":192,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.089602+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:10:58.089602+0000","last_peered":"2023-10-09T10:10:58.089602+0000","last_clean":"2023-10-09T10:10:58.089602+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:10:58.089602+0000","last_undegraded":"2023-10-09T10:10:58.089602+0000","last_fullsized":"2023-10-09T10:10:58.089602+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":165,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.105482+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:10:58.105482+0000","last_peered":"2023-10-09T10:10:58.105482+0000","last_clean":"2023-10-09T10:10:58.105482+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:10:58.105482+0000","last_undegraded":"2023-10-09T10:10:58.105482+0000","last_fullsized":"2023-10-09T10:10:58.105482+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"298'8766","reported_seq":8803,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:09.464931+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:11:09.464931+0000","last_peered":"2023-10-09T10:11:09.464931+0000","last_clean":"2023-10-09T10:11:09.464931+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:11:09.464931+0000","last_undegraded":"2023-10-09T10:11:09.464931+0000","last_fullsized":"2023-10-09T10:11:09.464931+0000","mapping_epoch":271,"log_start":"277'1000","ondisk_log_start":"277'1000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7766,"log_dups_size":0,"ondisk_log_size":7766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316669952,"num_objects":4832,"num_object_clones":0,"num_object_copies":14496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8766,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":251,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.064789+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:10:58.064789+0000","last_peered":"2023-10-09T10:10:58.064789+0000","last_clean":"2023-10-09T10:10:58.064789+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:10:58.064789+0000","last_undegraded":"2023-10-09T10:10:58.064789+0000","last_fullsized":"2023-10-09T10:10:58.064789+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":301,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.096772+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:10:58.096772+0000","last_peered":"2023-10-09T10:10:58.096772+0000","last_clean":"2023-10-09T10:10:58.096772+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:10:58.096772+0000","last_undegraded":"2023-10-09T10:10:58.096772+0000","last_fullsized":"2023-10-09T10:10:58.096772+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"298'8633","reported_seq":8670,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.755309+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:11:07.755309+0000","last_peered":"2023-10-09T10:11:07.755309+0000","last_clean":"2023-10-09T10:11:07.755309+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:11:07.755309+0000","last_undegraded":"2023-10-09T10:11:07.755309+0000","last_fullsized":"2023-10-09T10:11:07.755309+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8633,"log_dups_size":0,"ondisk_log_size":8633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317128704,"num_objects":4839,"num_object_clones":0,"num_object_copies":14517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8633,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":300,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.089812+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:10:58.089812+0000","last_peered":"2023-10-09T10:10:58.089812+0000","last_clean":"2023-10-09T10:10:58.089812+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:10:58.089812+0000","last_undegraded":"2023-10-09T10:10:58.089812+0000","last_fullsized":"2023-10-09T10:10:58.089812+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.072474+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:10:58.072474+0000","last_peered":"2023-10-09T10:10:58.072474+0000","last_clean":"2023-10-09T10:10:58.072474+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:10:58.072474+0000","last_undegraded":"2023-10-09T10:10:58.072474+0000","last_fullsized":"2023-10-09T10:10:58.072474+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":257,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081530+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:10:58.081530+0000","last_peered":"2023-10-09T10:10:58.081530+0000","last_clean":"2023-10-09T10:10:58.081530+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:10:58.081530+0000","last_undegraded":"2023-10-09T10:10:58.081530+0000","last_fullsized":"2023-10-09T10:10:58.081530+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"298'8722","reported_seq":8757,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.886684+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:11:07.886684+0000","last_peered":"2023-10-09T10:11:07.886684+0000","last_clean":"2023-10-09T10:11:07.886684+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:11:07.886684+0000","last_undegraded":"2023-10-09T10:11:07.886684+0000","last_fullsized":"2023-10-09T10:11:07.886684+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8722,"log_dups_size":0,"ondisk_log_size":8722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317980672,"num_objects":4852,"num_object_clones":0,"num_object_copies":14556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8722,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":116,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.071746+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:10:58.071746+0000","last_peered":"2023-10-09T10:10:58.071746+0000","last_clean":"2023-10-09T10:10:58.071746+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:10:58.071746+0000","last_undegraded":"2023-10-09T10:10:58.071746+0000","last_fullsized":"2023-10-09T10:10:58.071746+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":114,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.086907+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:10:58.086907+0000","last_peered":"2023-10-09T10:10:58.086907+0000","last_clean":"2023-10-09T10:10:58.086907+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:10:58.086907+0000","last_undegraded":"2023-10-09T10:10:58.086907+0000","last_fullsized":"2023-10-09T10:10:58.086907+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":323,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.089699+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:10:58.089699+0000","last_peered":"2023-10-09T10:10:58.089699+0000","last_clean":"2023-10-09T10:10:58.089699+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:10:58.089699+0000","last_undegraded":"2023-10-09T10:10:58.089699+0000","last_fullsized":"2023-10-09T10:10:58.089699+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"298'8821","reported_seq":8857,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:11.317496+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:11:11.317496+0000","last_peered":"2023-10-09T10:11:11.317496+0000","last_clean":"2023-10-09T10:11:11.317496+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:11:11.317496+0000","last_undegraded":"2023-10-09T10:11:11.317496+0000","last_fullsized":"2023-10-09T10:11:11.317496+0000","mapping_epoch":271,"log_start":"277'700","ondisk_log_start":"277'700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8121,"log_dups_size":0,"ondisk_log_size":8121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309395456,"num_objects":4721,"num_object_clones":0,"num_object_copies":14163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8821,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":217,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.077852+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:10:58.077852+0000","last_peered":"2023-10-09T10:10:58.077852+0000","last_clean":"2023-10-09T10:10:58.077852+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:10:58.077852+0000","last_undegraded":"2023-10-09T10:10:58.077852+0000","last_fullsized":"2023-10-09T10:10:58.077852+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":293,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.080133+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:10:58.080133+0000","last_peered":"2023-10-09T10:10:58.080133+0000","last_clean":"2023-10-09T10:10:58.080133+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:10:58.080133+0000","last_undegraded":"2023-10-09T10:10:58.080133+0000","last_fullsized":"2023-10-09T10:10:58.080133+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":313,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.096684+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:10:58.096684+0000","last_peered":"2023-10-09T10:10:58.096684+0000","last_clean":"2023-10-09T10:10:58.096684+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:10:58.096684+0000","last_undegraded":"2023-10-09T10:10:58.096684+0000","last_fullsized":"2023-10-09T10:10:58.096684+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"298'8600","reported_seq":8635,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.874676+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:11:07.874676+0000","last_peered":"2023-10-09T10:11:07.874676+0000","last_clean":"2023-10-09T10:11:07.874676+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:11:07.874676+0000","last_undegraded":"2023-10-09T10:11:07.874676+0000","last_fullsized":"2023-10-09T10:11:07.874676+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8600,"log_dups_size":0,"ondisk_log_size":8600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318504960,"num_objects":4860,"num_object_clones":0,"num_object_copies":14580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8600,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":125,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.071836+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:10:58.071836+0000","last_peered":"2023-10-09T10:10:58.071836+0000","last_clean":"2023-10-09T10:10:58.071836+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:10:58.071836+0000","last_undegraded":"2023-10-09T10:10:58.071836+0000","last_fullsized":"2023-10-09T10:10:58.071836+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":544,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.089902+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:10:58.089902+0000","last_peered":"2023-10-09T10:10:58.089902+0000","last_clean":"2023-10-09T10:10:58.089902+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:10:58.089902+0000","last_undegraded":"2023-10-09T10:10:58.089902+0000","last_fullsized":"2023-10-09T10:10:58.089902+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":201,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.077702+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:10:58.077702+0000","last_peered":"2023-10-09T10:10:58.077702+0000","last_clean":"2023-10-09T10:10:58.077702+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:10:58.077702+0000","last_undegraded":"2023-10-09T10:10:58.077702+0000","last_fullsized":"2023-10-09T10:10:58.077702+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":213,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.092523+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:10:58.092523+0000","last_peered":"2023-10-09T10:10:58.092523+0000","last_clean":"2023-10-09T10:10:58.092523+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:10:58.092523+0000","last_undegraded":"2023-10-09T10:10:58.092523+0000","last_fullsized":"2023-10-09T10:10:58.092523+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"298'8756","reported_seq":8788,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:09.412632+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:11:09.412632+0000","last_peered":"2023-10-09T10:11:09.412632+0000","last_clean":"2023-10-09T10:11:09.412632+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:11:09.412632+0000","last_undegraded":"2023-10-09T10:11:09.412632+0000","last_fullsized":"2023-10-09T10:11:09.412632+0000","mapping_epoch":271,"log_start":"277'1000","ondisk_log_start":"277'1000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7756,"log_dups_size":0,"ondisk_log_size":7756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318898176,"num_objects":4866,"num_object_clones":0,"num_object_copies":14598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8756,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":273,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.080884+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:10:58.080884+0000","last_peered":"2023-10-09T10:10:58.080884+0000","last_clean":"2023-10-09T10:10:58.080884+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:10:58.080884+0000","last_undegraded":"2023-10-09T10:10:58.080884+0000","last_fullsized":"2023-10-09T10:10:58.080884+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":175,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.080498+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:10:58.080498+0000","last_peered":"2023-10-09T10:10:58.080498+0000","last_clean":"2023-10-09T10:10:58.080498+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:10:58.080498+0000","last_undegraded":"2023-10-09T10:10:58.080498+0000","last_fullsized":"2023-10-09T10:10:58.080498+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":202,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081962+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:10:58.081962+0000","last_peered":"2023-10-09T10:10:58.081962+0000","last_clean":"2023-10-09T10:10:58.081962+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:10:58.081962+0000","last_undegraded":"2023-10-09T10:10:58.081962+0000","last_fullsized":"2023-10-09T10:10:58.081962+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"298'8818","reported_seq":12521,"reported_epoch":298,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:11.327663+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:11:11.327663+0000","last_peered":"2023-10-09T10:11:11.327663+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:11:11.327663+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"277'700","ondisk_log_start":"277'700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8118,"log_dups_size":0,"ondisk_log_size":8118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317194240,"num_objects":4840,"num_object_clones":0,"num_object_copies":14520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":161,"num_objects_misplaced":3293,"num_objects_unfound":0,"num_objects_dirty":4840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8818,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1830,"num_bytes_recovered":119930880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1386},{"shards":"1,3,4","objects":3293}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":317,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.090436+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:10:58.090436+0000","last_peered":"2023-10-09T10:10:58.090436+0000","last_clean":"2023-10-09T10:10:58.090436+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:10:58.090436+0000","last_undegraded":"2023-10-09T10:10:58.090436+0000","last_fullsized":"2023-10-09T10:10:58.090436+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":329,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.089901+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:10:58.089901+0000","last_peered":"2023-10-09T10:10:58.089901+0000","last_clean":"2023-10-09T10:10:58.089901+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:10:58.089901+0000","last_undegraded":"2023-10-09T10:10:58.089901+0000","last_fullsized":"2023-10-09T10:10:58.089901+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":167,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.086976+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:10:58.086976+0000","last_peered":"2023-10-09T10:10:58.086976+0000","last_clean":"2023-10-09T10:10:58.086976+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:10:58.086976+0000","last_undegraded":"2023-10-09T10:10:58.086976+0000","last_fullsized":"2023-10-09T10:10:58.086976+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"298'8600","reported_seq":8636,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.761782+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:11:07.761782+0000","last_peered":"2023-10-09T10:11:07.761782+0000","last_clean":"2023-10-09T10:11:07.761782+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:11:07.761782+0000","last_undegraded":"2023-10-09T10:11:07.761782+0000","last_fullsized":"2023-10-09T10:11:07.761782+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8600,"log_dups_size":0,"ondisk_log_size":8600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324927488,"num_objects":4958,"num_object_clones":0,"num_object_copies":14874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8600,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":260,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.087429+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:10:58.087429+0000","last_peered":"2023-10-09T10:10:58.087429+0000","last_clean":"2023-10-09T10:10:58.087429+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:10:58.087429+0000","last_undegraded":"2023-10-09T10:10:58.087429+0000","last_fullsized":"2023-10-09T10:10:58.087429+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":274,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.105558+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:10:58.105558+0000","last_peered":"2023-10-09T10:10:58.105558+0000","last_clean":"2023-10-09T10:10:58.105558+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:10:58.105558+0000","last_undegraded":"2023-10-09T10:10:58.105558+0000","last_fullsized":"2023-10-09T10:10:58.105558+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":249,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.064702+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:10:58.064702+0000","last_peered":"2023-10-09T10:10:58.064702+0000","last_clean":"2023-10-09T10:10:58.064702+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:10:58.064702+0000","last_undegraded":"2023-10-09T10:10:58.064702+0000","last_fullsized":"2023-10-09T10:10:58.064702+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"298'8547","reported_seq":8583,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.876026+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:11:07.876026+0000","last_peered":"2023-10-09T10:11:07.876026+0000","last_clean":"2023-10-09T10:11:07.876026+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:11:07.876026+0000","last_undegraded":"2023-10-09T10:11:07.876026+0000","last_fullsized":"2023-10-09T10:11:07.876026+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8547,"log_dups_size":0,"ondisk_log_size":8547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322568216,"num_objects":4923,"num_object_clones":0,"num_object_copies":14769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4923,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8547,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":206,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.098149+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:10:58.098149+0000","last_peered":"2023-10-09T10:10:58.098149+0000","last_clean":"2023-10-09T10:10:58.098149+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:10:58.098149+0000","last_undegraded":"2023-10-09T10:10:58.098149+0000","last_fullsized":"2023-10-09T10:10:58.098149+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":168,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081893+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:10:58.081893+0000","last_peered":"2023-10-09T10:10:58.081893+0000","last_clean":"2023-10-09T10:10:58.081893+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:10:58.081893+0000","last_undegraded":"2023-10-09T10:10:58.081893+0000","last_fullsized":"2023-10-09T10:10:58.081893+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":151,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.071630+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:10:58.071630+0000","last_peered":"2023-10-09T10:10:58.071630+0000","last_clean":"2023-10-09T10:10:58.071630+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:10:58.071630+0000","last_undegraded":"2023-10-09T10:10:58.071630+0000","last_fullsized":"2023-10-09T10:10:58.071630+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":163,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.086819+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:10:58.086819+0000","last_peered":"2023-10-09T10:10:58.086819+0000","last_clean":"2023-10-09T10:10:58.086819+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:10:58.086819+0000","last_undegraded":"2023-10-09T10:10:58.086819+0000","last_fullsized":"2023-10-09T10:10:58.086819+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":236,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.089979+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:10:58.089979+0000","last_peered":"2023-10-09T10:10:58.089979+0000","last_clean":"2023-10-09T10:10:58.089979+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:10:58.089979+0000","last_undegraded":"2023-10-09T10:10:58.089979+0000","last_fullsized":"2023-10-09T10:10:58.089979+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":135,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.072176+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:10:58.072176+0000","last_peered":"2023-10-09T10:10:58.072176+0000","last_clean":"2023-10-09T10:10:58.072176+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:10:58.072176+0000","last_undegraded":"2023-10-09T10:10:58.072176+0000","last_fullsized":"2023-10-09T10:10:58.072176+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":176,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081347+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:10:58.081347+0000","last_peered":"2023-10-09T10:10:58.081347+0000","last_clean":"2023-10-09T10:10:58.081347+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:10:58.081347+0000","last_undegraded":"2023-10-09T10:10:58.081347+0000","last_fullsized":"2023-10-09T10:10:58.081347+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":135,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.072428+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:10:58.072428+0000","last_peered":"2023-10-09T10:10:58.072428+0000","last_clean":"2023-10-09T10:10:58.072428+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:10:58.072428+0000","last_undegraded":"2023-10-09T10:10:58.072428+0000","last_fullsized":"2023-10-09T10:10:58.072428+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":185,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.077864+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:10:58.077864+0000","last_peered":"2023-10-09T10:10:58.077864+0000","last_clean":"2023-10-09T10:10:58.077864+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:10:58.077864+0000","last_undegraded":"2023-10-09T10:10:58.077864+0000","last_fullsized":"2023-10-09T10:10:58.077864+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":314,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.079537+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:10:58.079537+0000","last_peered":"2023-10-09T10:10:58.079537+0000","last_clean":"2023-10-09T10:10:58.079537+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:10:58.079537+0000","last_undegraded":"2023-10-09T10:10:58.079537+0000","last_fullsized":"2023-10-09T10:10:58.079537+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":335,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.078148+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:10:58.078148+0000","last_peered":"2023-10-09T10:10:58.078148+0000","last_clean":"2023-10-09T10:10:58.078148+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:10:58.078148+0000","last_undegraded":"2023-10-09T10:10:58.078148+0000","last_fullsized":"2023-10-09T10:10:58.078148+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":245,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:10:58.081675+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:10:58.081675+0000","last_peered":"2023-10-09T10:10:58.081675+0000","last_clean":"2023-10-09T10:10:58.081675+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:10:58.081675+0000","last_undegraded":"2023-10-09T10:10:58.081675+0000","last_fullsized":"2023-10-09T10:10:58.081675+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":165,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:11:04.611614+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:11:04.611614+0000","last_peered":"2023-10-09T10:11:04.611614+0000","last_clean":"2023-10-09T10:11:04.611614+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:11:04.611614+0000","last_undegraded":"2023-10-09T10:11:04.611614+0000","last_fullsized":"2023-10-09T10:11:04.611614+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5117575192,"num_objects":78089,"num_object_clones":0,"num_object_copies":234267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5534,"num_objects_misplaced":3293,"num_objects_unfound":0,"num_objects_dirty":78089,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139273,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1830,"num_bytes_recovered":119930880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15588667392,"data_stored":15588655176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133073,"ondisk_log_size":133073,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430498,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1874560,"kb_used_data":1569144,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91870592,"statfs":{"total":95995035648,"available":94075486208,"internally_reserved":0,"allocated":1606803456,"data_stored":1606265617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":1.758,"15min":1.716},"min":{"1min":0.254,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":7.8419999999999996,"5min":9.0690000000000008,"15min":50.591999999999999},"last":2.7509999999999999},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.6699999999999999,"15min":1.6970000000000001},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":7.9610000000000003,"5min":8.2300000000000004,"15min":50.747},"last":0.496}]},{"osd":1,"last update":"Mon Oct 9 10:11:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7029999999999998,"5min":2.117,"15min":1.728},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":14.311,"5min":14.311,"15min":25.338000000000001},"last":2.831},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.2679999999999998,"15min":1.5509999999999999},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":16.526,"5min":18.27,"15min":18.27},"last":3.6280000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.752,"15min":2.2770000000000001},"min":{"1min":0.36099999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":2.8050000000000002},{"interface":"front","average":{"1min":2.8029999999999999,"5min":1.845,"15min":2.1030000000000002},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.872,"5min":11.872,"15min":11.872},"last":2.855}]},{"osd":3,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":2.2879999999999998,"15min":1.929},"min":{"1min":0.46400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.274,"5min":29.058,"15min":29.058},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.913,"15min":1.9350000000000001},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":6.298}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":0.96599999999999997,"15min":0.93000000000000005},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":2.7829999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.91300000000000003,"15min":0.91000000000000003},"min":{"1min":0.28599999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.7919999999999998,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":4.6239999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.1519999999999999,"15min":1.006},"min":{"1min":0.30199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":5.7149999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":4.7999999999999998},{"interface":"front","average":{"1min":1.012,"5min":1.145,"15min":0.99299999999999999},"min":{"1min":0.38200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.5960000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.91600000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.93899999999999995,"15min":0.90900000000000003},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.6259999999999999,"5min":5.3129999999999997,"15min":5.4820000000000002},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.226,"5min":0.95599999999999996,"15min":0.87},"min":{"1min":0.36099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.419,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":6.4020000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626726,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2206832,"kb_used_data":1901288,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91538320,"statfs":{"total":95995035648,"available":93735239680,"internally_reserved":0,"allocated":1946918912,"data_stored":1946398387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.821,"15min":1.4410000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.2870000000000008,"5min":15.435,"15min":15.435},"last":3.6219999999999999},{"interface":"front","average":{"1min":2.149,"5min":2.0859999999999999,"15min":1.4750000000000001},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":7.4139999999999997,"5min":15.308999999999999,"15min":15.308999999999999},"last":2.4820000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:04 2023","interfaces":[{"interface":"back","average":{"1min":3.032,"5min":2.7160000000000002,"15min":1.8779999999999999},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":14.888999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":3.3159999999999998},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.3580000000000001,"15min":1.8320000000000001},"min":{"1min":0.26700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.962,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.496}]},{"osd":2,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":1.4319999999999999,"15min":1.724},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.898,"5min":12.898,"15min":12.898},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.569,"5min":2.141,"15min":3.0379999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5640000000000001,"5min":10.26,"15min":10.26},"last":0.81999999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.8140000000000001,"15min":1.5109999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":12.943,"5min":12.943,"15min":12.943},"last":3.0739999999999998},{"interface":"front","average":{"1min":1.653,"5min":2.2770000000000001,"15min":2.3460000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.077,"5min":11.211,"15min":11.211},"last":3.2669999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.50700000000000001,"15min":0.46200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.60099999999999998,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1749999999999998,"5min":4.1749999999999998,"15min":4.1749999999999998},"last":0.308}]},{"osd":5,"last update":"Mon Oct 9 10:10:33 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.73699999999999999,"15min":0.76700000000000002},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9260000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.91900000000000004,"15min":0.79600000000000004},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":4.1529999999999996,"5min":7.508,"15min":7.508},"last":0.27500000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:11:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":0.91100000000000003,"15min":0.82299999999999995},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.2709999999999999,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.538,"5min":0.95499999999999996,"15min":0.872},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.3159999999999998,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.38100000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253039,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2595028,"kb_used_data":2197196,"kb_used_omap":9,"kb_used_meta":397622,"kb_avail":91150124,"statfs":{"total":95995035648,"available":93337726976,"internally_reserved":0,"allocated":2249928704,"data_stored":2248901061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":407165929},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":2.6549999999999998,"15min":2.0299999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.185},"max":{"1min":6.3479999999999999,"5min":25.59,"15min":25.59},"last":0.52600000000000002},{"interface":"front","average":{"1min":3.7109999999999999,"5min":2.77,"15min":1.976},"min":{"1min":0.26900000000000002,"5min":0.218,"15min":0.192},"max":{"1min":11.682,"5min":14.769,"15min":16.629000000000001},"last":6.7220000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":3.0499999999999998,"15min":3.8149999999999999},"min":{"1min":0.27200000000000002,"5min":0.217,"15min":0.217},"max":{"1min":11.135,"5min":17.369,"15min":17.369},"last":6.9690000000000003},{"interface":"front","average":{"1min":3.5550000000000002,"5min":2.7469999999999999,"15min":3.8210000000000002},"min":{"1min":0.25900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.484999999999999,"5min":16.023,"15min":16.023},"last":0.39100000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":3.4540000000000002,"15min":3.5070000000000001},"min":{"1min":0.39500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.587,"5min":18.587,"15min":18.587},"last":6.9000000000000004},{"interface":"front","average":{"1min":4.6959999999999997,"5min":4.2249999999999996,"15min":4.6760000000000002},"min":{"1min":1.0329999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.022,"5min":17.289000000000001,"15min":17.289000000000001},"last":3.1589999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.474,"15min":1.3280000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2629999999999999,"5min":7.2629999999999999,"15min":7.2629999999999999},"last":0.434},{"interface":"front","average":{"1min":2.879,"5min":1.639,"15min":1.4330000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.6940000000000008,"5min":10.132,"15min":10.132},"last":5.8550000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":2.0419999999999998,"15min":1.72},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.473000000000001,"5min":14.416,"15min":15.531000000000001},"last":5.9420000000000002},{"interface":"front","average":{"1min":1.782,"5min":2.0979999999999999,"15min":1.9690000000000001},"min":{"1min":0.34699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.6890000000000001,"5min":16.277999999999999,"15min":19.706},"last":5.8380000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0339999999999998,"15min":1.7230000000000001},"min":{"1min":0.50800000000000001,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.3710000000000004,"5min":14.215,"15min":16.193999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.75,"5min":1.9930000000000001,"15min":1.7529999999999999},"min":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":10.161,"5min":14.471,"15min":15.481999999999999},"last":5.907}]},{"osd":7,"last update":"Mon Oct 9 10:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":2.153,"15min":1.972},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.7350000000000003,"5min":16.331,"15min":16.331},"last":5.9180000000000001},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1320000000000001,"15min":1.788},"min":{"1min":0.46000000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3319999999999999,"5min":14.242000000000001,"15min":16.100999999999999},"last":0.504}]}]},{"osd":0,"up_from":79,"seq":339302416565,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2262328,"kb_used_data":1895408,"kb_used_omap":28,"kb_used_meta":366755,"kb_avail":91482824,"statfs":{"total":95995035648,"available":93678411776,"internally_reserved":0,"allocated":1940897792,"data_stored":1939888083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":375557968},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.847,"15min":1.4970000000000001},"min":{"1min":0.24099999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.2990000000000004,"5min":12.84,"15min":13.084},"last":1.3520000000000001},{"interface":"front","average":{"1min":3.9020000000000001,"5min":3.379,"15min":2.2269999999999999},"min":{"1min":0.44900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.949000000000002,"5min":22.178999999999998,"15min":22.178999999999998},"last":2.4460000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8360000000000001,"15min":2.1589999999999998},"min":{"1min":0.34599999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.9949999999999992,"5min":10.034000000000001,"15min":10.034000000000001},"last":1.5509999999999999},{"interface":"front","average":{"1min":2.9340000000000002,"5min":2.177,"15min":3.0539999999999998},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":15.529,"5min":15.529,"15min":15.529},"last":2.1059999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":4.5259999999999998,"15min":4.7519999999999998},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":20.419,"5min":25.734999999999999,"15min":25.734999999999999},"last":2.5169999999999999},{"interface":"front","average":{"1min":3.742,"5min":3.891,"15min":4.2370000000000001},"min":{"1min":0.40999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":10.154,"5min":21.923999999999999,"15min":21.923999999999999},"last":3.3679999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.911,"15min":1.5649999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":2.5539999999999998},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.0459999999999998,"15min":1.9730000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.6720000000000006,"5min":13.789999999999999,"15min":13.789999999999999},"last":3.3069999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.7309999999999999,"15min":1.9630000000000001},"min":{"1min":0.40899999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.6229999999999993,"5min":17.096,"15min":17.096},"last":3.3410000000000002},{"interface":"front","average":{"1min":1.5,"5min":1.454,"15min":1.3029999999999999},"min":{"1min":0.30499999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.4260000000000002,"5min":9.4260000000000002,"15min":12.478},"last":1.4039999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.7010000000000001,"15min":1.4590000000000001},"min":{"1min":0.33700000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":9.4719999999999995,"5min":9.7260000000000009,"15min":11.699},"last":2.1480000000000001},{"interface":"front","average":{"1min":4.4729999999999999,"5min":2.6320000000000001,"15min":2.2069999999999999},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.466000000000001,"5min":20.466000000000001,"15min":20.466000000000001},"last":2.5870000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8239999999999998,"15min":2.1139999999999999},"min":{"1min":0.46400000000000002,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":9.7050000000000001,"5min":14.631,"15min":26.978000000000002},"last":7.5330000000000004},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.7190000000000001,"15min":1.605},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":9.5020000000000007,"5min":11.166,"15min":26.768000000000001},"last":1.585}]}]},{"osd":2,"up_from":217,"seq":932007903299,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1673980,"kb_used_data":1442932,"kb_used_omap":83,"kb_used_meta":230700,"kb_avail":92071172,"statfs":{"total":95995035648,"available":94280880128,"internally_reserved":0,"allocated":1477562368,"data_stored":1476556868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":236237295},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":1.732,"15min":1.756},"min":{"1min":0.217,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":19.372,"5min":19.372,"15min":19.372},"last":3.0619999999999998},{"interface":"front","average":{"1min":3.4420000000000002,"5min":2.9849999999999999,"15min":2.9569999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.754999999999999,"5min":16.754999999999999,"15min":16.754999999999999},"last":12.557}]},{"osd":1,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":2.258,"15min":2.544},"min":{"1min":0.254,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.797000000000001,"5min":13.797000000000001,"15min":13.797000000000001},"last":3.0390000000000001},{"interface":"front","average":{"1min":2.8399999999999999,"5min":2.5209999999999999,"15min":2.6379999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.4119999999999999,"5min":15.569000000000001,"15min":15.569000000000001},"last":12.534000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":2.198,"15min":2.0819999999999999},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.851000000000001,"5min":13.851000000000001,"15min":13.851000000000001},"last":3},{"interface":"front","average":{"1min":2.9860000000000002,"5min":3.181,"15min":3.6699999999999999},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.7240000000000002,"5min":15.811,"15min":15.811},"last":12.51}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.84499999999999997,"15min":0.753},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.29,"5min":6.29,"15min":6.29},"last":12.413},{"interface":"front","average":{"1min":2.5609999999999999,"5min":1.1759999999999999,"15min":0.94499999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":12.385,"5min":12.385,"15min":12.385},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.6919999999999999,"15min":1.546},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":12.473000000000001,"5min":12.473000000000001,"15min":12.473000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":3.0699999999999998,"5min":1.5409999999999999,"15min":1.6870000000000001},"min":{"1min":0.372,"5min":0.318,"15min":0.318},"max":{"1min":13.587,"5min":13.587,"15min":13.587},"last":5.923}]},{"osd":6,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.3999999999999999,"15min":1.095},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2599999999999998,"5min":15.292999999999999,"15min":15.292999999999999},"last":12.484},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.738,"15min":1.742},"min":{"1min":0.36599999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.56699999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.431,"15min":1.1259999999999999},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.2190000000000003,"5min":15.343999999999999,"15min":15.343999999999999},"last":12.457000000000001},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.859,"15min":1.6599999999999999},"min":{"1min":0.41699999999999998,"5min":0.376,"15min":0.376},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":12.433999999999999},"last":0.59499999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824043,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2107960,"kb_used_data":1861680,"kb_used_omap":10,"kb_used_meta":246133,"kb_avail":91637192,"statfs":{"total":95995035648,"available":93836484608,"internally_reserved":0,"allocated":1906360320,"data_stored":1905182666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":252040259},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":4.2709999999999999,"15min":4.5590000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.753,"5min":23.381,"15min":23.381},"last":6.4299999999999997},{"interface":"front","average":{"1min":6.0720000000000001,"5min":4.0919999999999996,"15min":4.1500000000000004},"min":{"1min":0.42099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":20.239999999999998,"5min":20.239999999999998,"15min":20.239999999999998},"last":6.9640000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.6150000000000002,"15min":3.8410000000000002},"min":{"1min":0.26300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":20.422999999999998,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":4.5039999999999996,"5min":3.4990000000000001,"15min":3.4529999999999998},"min":{"1min":0.33100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.704000000000001,"5min":23.423999999999999,"15min":23.423999999999999},"last":3.452}]},{"osd":2,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1369999999999996,"5min":3.4199999999999999,"15min":3.6339999999999999},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":20.594999999999999,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.8809999999999998,"15min":2.8700000000000001},"min":{"1min":0.34200000000000003,"5min":0.253,"15min":0.253},"max":{"1min":15.73,"5min":23.452000000000002,"15min":23.452000000000002},"last":3.4990000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":1.6719999999999999,"15min":1.288},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.676,"5min":15.676,"15min":15.676},"last":3.536},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5800000000000001,"15min":1.534},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.426,"5min":12.426,"15min":12.426},"last":3.6269999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8290000000000002,"5min":2.4750000000000001,"15min":2.5670000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":12.388,"5min":12.388,"15min":12.388},"last":5.0510000000000002},{"interface":"front","average":{"1min":3.5699999999999998,"5min":2.5169999999999999,"15min":2.4079999999999999},"min":{"1min":0.443,"5min":0.309,"15min":0.309},"max":{"1min":15.992000000000001,"5min":15.992000000000001,"15min":15.992000000000001},"last":0.499}]},{"osd":6,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":2.536,"15min":2.6709999999999998},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.468,"5min":12.468,"15min":12.468},"last":3.673},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.5609999999999999,"15min":2.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":16.16,"5min":16.16,"15min":16.16},"last":0.53100000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.825,"15min":1.534},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.635999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":3.859},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.4399999999999999,"15min":2.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":3.782}]}]},{"osd":4,"up_from":266,"seq":1142461300768,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2239924,"kb_used_data":1913068,"kb_used_omap":35,"kb_used_meta":326812,"kb_avail":91505228,"statfs":{"total":95995035648,"available":93701353472,"internally_reserved":0,"allocated":1958981632,"data_stored":1958455443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":334655726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.98499999999999999,"15min":0.84099999999999997},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.881,"5min":10.881,"15min":10.881},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.923,"5min":1.1519999999999999,"15min":1.024},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":2.0760000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":1.169,"15min":0.93400000000000005},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":10.93,"5min":10.93,"15min":10.93},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.633,"5min":1.546,"15min":1.365},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.141,"5min":11.141,"15min":11.141},"last":0.92600000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":1.2310000000000001,"15min":1.073},"min":{"1min":0.32100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.1189999999999998,"5min":7.1189999999999998,"15min":7.1189999999999998},"last":2.5670000000000002},{"interface":"front","average":{"1min":1.587,"5min":1.042,"15min":0.95099999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.1709999999999994,"5min":8.1709999999999994,"15min":8.1709999999999994},"last":1.9530000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.4610000000000001,"15min":1.3280000000000001},"min":{"1min":0.40699999999999997,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5530000000000008,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":13.065},{"interface":"front","average":{"1min":2.4420000000000002,"5min":1.2989999999999999,"15min":1.109},"min":{"1min":0.373,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.6069999999999993,"5min":9.6069999999999993,"15min":9.6069999999999993},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.80400000000000005,"15min":0.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4740000000000002,"5min":5.4740000000000002,"15min":5.4740000000000002},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.60399999999999998,"15min":0.63400000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.36699999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.60499999999999998,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":5.5709999999999997,"5min":5.5709999999999997,"15min":5.5709999999999997},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.60999999999999999,"15min":0.59199999999999997},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.319,"5min":2.319,"15min":2.319},"last":0.28599999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.61299999999999999,"15min":0.61799999999999999},"min":{"1min":0.33700000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.3460000000000001,"5min":2.3460000000000001,"15min":2.3460000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.82299999999999995,"15min":0.82699999999999996},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.0680000000000001,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.30399999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822953,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2828584,"kb_used_data":2523168,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90916568,"statfs":{"total":95995035648,"available":93098565632,"internally_reserved":0,"allocated":2583724032,"data_stored":2583205899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":1.55,"15min":1.488},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.8540000000000001,"5min":7.8540000000000001,"15min":15.773999999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.339,"5min":1.597,"15min":1.532},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.0629999999999997,"5min":11.747,"15min":15.699},"last":5.6909999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:10:12 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.5740000000000001,"15min":1.4430000000000001},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.7720000000000002,"5min":11.335000000000001,"15min":15.689},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.913,"5min":1.861,"15min":1.6180000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.8119999999999994,"5min":19.402999999999999,"15min":19.402999999999999},"last":2.77}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":1.7090000000000001,"15min":2.391},"min":{"1min":0.34899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7249999999999996,"5min":15.640000000000001,"15min":15.640000000000001},"last":5.5110000000000001},{"interface":"front","average":{"1min":1.663,"5min":1.6479999999999999,"15min":2.2370000000000001},"min":{"1min":0.40400000000000003,"5min":0.313,"15min":0.313},"max":{"1min":5.2789999999999999,"5min":10.656000000000001,"15min":10.656000000000001},"last":2.1760000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:10:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.8340000000000001,"15min":1.8440000000000001},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6660000000000004,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.4380000000000002,"15min":2.8250000000000002},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.085,"5min":17.34,"15min":17.34},"last":4.5800000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":0.71699999999999997,"15min":0.61499999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5220000000000002,"5min":5.5220000000000002,"15min":5.5220000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.752,"15min":0.746},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.823,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.31}]},{"osd":6,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.014,"15min":0.94499999999999995},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.25},"max":{"1min":5.4930000000000003,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.0189999999999999,"15min":0.88700000000000001},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.8099999999999996,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.45400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":1.0900000000000001,"15min":0.97099999999999997},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.891,"5min":7.891,"15min":10.334},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.72299999999999998,"15min":0.93300000000000005},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.3630000000000004,"5min":6.3630000000000004,"15min":12.356},"last":0.39300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1927217152,"data_stored":1927217152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2239561728,"data_stored":2239561728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1467420672,"data_stored":1467416600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1893400576,"data_stored":1893400576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1949761536,"data_stored":1949761536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2575699968,"data_stored":2575695896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1943142400,"data_stored":1943142400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1592463360,"data_stored":1592459288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:11:12.801 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:12.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:12.886 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:12.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:12.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:12.989 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:13.056 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:13.208 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:13.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:13.321 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:11:13.321 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:11:13.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:13.401 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:13.510 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:13.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:13.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:13.520 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5534/234339 objects degraded (2.362%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:13.520 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5534/234339 objects degraded (2.362%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:13.620 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:11:13.670 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:13.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5534/234339 objects degraded (2.362%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:13.738 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:11:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:13.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:11:13.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:13.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:11:13.984 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:14.076 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:14.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:14.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:14.108 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:14.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:14.323 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:14.448 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:14.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:14.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:14.493 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:11:14.562 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:14.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:14 smithi012 ceph-mon[147926]: pgmap v803: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5534/230436 objects degraded (2.402%); 3293/230436 objects misplaced (1.429%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:14.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:14.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3513228442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:14.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:14.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:14 smithi012 ceph-mon[152263]: pgmap v803: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5534/230436 objects degraded (2.402%); 3293/230436 objects misplaced (1.429%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:14.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:14.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3513228442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:14.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:14.594 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:14.639 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:14.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:14.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:14.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:14.798 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:14 smithi110 ceph-mon[136032]: pgmap v803: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5534/230436 objects degraded (2.402%); 3293/230436 objects misplaced (1.429%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:14.798 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:14.798 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3513228442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:14.798 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:14.802 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:14.859 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:14.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:14.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:15.091 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:15.251 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:15.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:15.312 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:15.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:15.448 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:15.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:15.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[147926]: failed to encode map e299 with expected crc 2023-10-09T10:11:15.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:11:15.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[147926]: osdmap e299: 8 total, 8 up, 8 in 2023-10-09T10:11:15.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[147926]: failed to encode map e299 with expected crc 2023-10-09T10:11:15.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[147926]: failed to encode map e299 with expected crc 2023-10-09T10:11:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[152263]: failed to encode map e299 with expected crc 2023-10-09T10:11:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:11:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[152263]: osdmap e299: 8 total, 8 up, 8 in 2023-10-09T10:11:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[152263]: failed to encode map e299 with expected crc 2023-10-09T10:11:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:15 smithi012 ceph-mon[152263]: failed to encode map e299 with expected crc 2023-10-09T10:11:15.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:11:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:15 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:15 smithi110 ceph-mon[136032]: failed to encode map e299 with expected crc 2023-10-09T10:11:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:11:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:15 smithi110 ceph-mon[136032]: osdmap e299: 8 total, 8 up, 8 in 2023-10-09T10:11:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:15 smithi110 ceph-mon[136032]: failed to encode map e299 with expected crc 2023-10-09T10:11:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:15 smithi110 ceph-mon[136032]: failed to encode map e299 with expected crc 2023-10-09T10:11:16.336 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:16.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:16.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:16.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:16.538 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:16 smithi012 ceph-mon[152263]: pgmap v805: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5534/227538 objects degraded (2.432%); 3293/227538 objects misplaced (1.447%); 881 KiB/s, 13 objects/s recovering 2023-10-09T10:11:16.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:16 smithi012 ceph-mon[147926]: pgmap v805: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5534/227538 objects degraded (2.432%); 3293/227538 objects misplaced (1.447%); 881 KiB/s, 13 objects/s recovering 2023-10-09T10:11:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:16 smithi110 ceph-mon[136032]: pgmap v805: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5534/227538 objects degraded (2.432%); 3293/227538 objects misplaced (1.447%); 881 KiB/s, 13 objects/s recovering 2023-10-09T10:11:17.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:11:17.313 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:17.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:17.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:17.425 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:17.458 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:18.009 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:11:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:11:17] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.33.4" 2023-10-09T10:11:18.093 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:18.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:18.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:18.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:18.259 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:18.427 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:11:18.441 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:11:18.473 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:11:18.505 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:18.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:18.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:18.527 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:11:18.586 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:11:18.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:18.625 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:11:18.654 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:18.665 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:11:18.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:18 smithi012 ceph-mon[147926]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5614/223575 objects degraded (2.511%); 3092/223575 objects misplaced (1.383%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:18.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5614/223575 objects degraded (2.511%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:18.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3290054655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:18.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:18 smithi012 ceph-mon[152263]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5614/223575 objects degraded (2.511%); 3092/223575 objects misplaced (1.383%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:18.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5614/223575 objects degraded (2.511%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:18.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3290054655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:18.693 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:11:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:11:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:11:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:11:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:11:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:11:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:11:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:11:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:11:18.742 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:18 smithi110 ceph-mon[136032]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5614/223575 objects degraded (2.511%); 3092/223575 objects misplaced (1.383%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5614/223575 objects degraded (2.511%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3290054655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:18.966 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:18.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:18.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:19.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:19.176 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:19.354 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:19.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:19.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:19.481 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:19.566 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:19.727 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:19.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:19.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:19.913 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:11:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:11:19.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154065 7.8 7.6bf60448 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.93458288099998, 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86727346999999999, 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.914 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154065 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:11:19.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969553+0000", 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969561+0000", 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7109999999999995e-06 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969572+0000", 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.115e-05 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969582+0000", 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5100000000000004e-06 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.917 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969631+0000", 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9395999999999999e-05 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969640+0000", 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8540000000000003e-06 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836819+0000", 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86717872399999996 2023-10-09T10:11:19.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836828+0000", 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6709999999999994e-06 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154067 7.e 7.7d5b394e (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.93454058899999, 2023-10-09T10:11:19.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86689926799999995, 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154067 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969600+0000", 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8399999999999999e-06 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969607+0000", 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.922e-06 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969612+0000", 2023-10-09T10:11:19.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7830000000000004e-06 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976392+0000", 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067794639999999998 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976404+0000", 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1769e-05 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836486+0000", 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008206399999998 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836497+0000", 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0947e-05 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.924 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154074 7.e 7.21f6315e (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.93451068100001, 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86689908999999998, 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.925 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154074 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:11:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:11:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.927 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969629+0000", 2023-10-09T10:11:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4509999999999999e-06 2023-10-09T10:11:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.928 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969636+0000", 2023-10-09T10:11:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.216e-06 2023-10-09T10:11:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.929 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969642+0000", 2023-10-09T10:11:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0129999999999999e-06 2023-10-09T10:11:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976425+0000", 2023-10-09T10:11:19.930 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006782649 2023-10-09T10:11:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976431+0000", 2023-10-09T10:11:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8719999999999999e-06 2023-10-09T10:11:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.931 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836518+0000", 2023-10-09T10:11:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008733299999995 2023-10-09T10:11:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836526+0000", 2023-10-09T10:11:19.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9610000000000005e-06 2023-10-09T10:11:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154080 7.5 7.d54c8f95 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:11:19.933 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:11:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.93449001900001, 2023-10-09T10:11:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.868647155, 2023-10-09T10:11:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.934 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154080 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969647+0000", 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969650+0000", 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9440000000000001e-06 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969656+0000", 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2160000000000003e-06 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969663+0000", 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5780000000000004e-06 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969744+0000", 2023-10-09T10:11:19.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0678000000000002e-05 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969751+0000", 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3719999999999999e-06 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838286+0000", 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.868534837 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838295+0000", 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0669999999999998e-06 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154083 7.e 7.33ccdeae (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.934461537, 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86687670800000005, 2023-10-09T10:11:19.939 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154083 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969679+0000", 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.892e-06 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969686+0000", 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0500000000000003e-06 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969692+0000", 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7699999999999998e-06 2023-10-09T10:11:19.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976451+0000", 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067598859999999997 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976457+0000", 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1880000000000001e-06 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836546+0000", 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008883199999997 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836553+0000", 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5619999999999998e-06 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154085 7.5 7.5fb2e7e5 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.93444151199998, 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86864703600000004, 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154085 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969698+0000", 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2050000000000001e-06 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969704+0000", 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6799999999999998e-06 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969709+0000", 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5639999999999996e-06 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969770+0000", 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0999000000000003e-05 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969775+0000", 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.95e-06 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838335+0000", 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86855936700000003 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838343+0000", 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6770000000000001e-06 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154089 7.8 7.cb7923e8 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.92505298399999, 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.8577726, 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.950 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154089 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979083+0000", 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979093+0000", 2023-10-09T10:11:19.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5319999999999995e-06 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979109+0000", 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.677e-05 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979118+0000", 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979202+0000", 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3975999999999997e-05 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979214+0000", 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1324e-05 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836849+0000", 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85763573500000001 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836858+0000", 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0099999999999995e-06 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154090 7.5 7.72a407a5 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:11:19.955 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.92500403000003, 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85923808499999998, 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154090 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979133+0000", 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979137+0000", 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.332e-06 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979144+0000", 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0809999999999996e-06 2023-10-09T10:11:19.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979149+0000", 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5770000000000001e-06 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979256+0000", 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106482 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979263+0000", 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2509999999999997e-06 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838364+0000", 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85910122099999997 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838372+0000", 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7430000000000002e-06 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.960 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154096 7.8 7.af393d68 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.90947713399999, 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84222455699999998, 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.961 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154096 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994659+0000", 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994668+0000", 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0710000000000001e-06 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994692+0000", 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3887999999999999e-05 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994702+0000", 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0111e-05 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994765+0000", 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3428000000000003e-05 2023-10-09T10:11:19.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994779+0000", 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3946e-05 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836878+0000", 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84209816000000004 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836885+0000", 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7600000000000002e-06 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154097 7.e 7.c78fc5fe (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 317.90869881899999, 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84114410299999998, 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.966 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154097 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995438+0000", 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995444+0000", 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2299999999999999e-06 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995459+0000", 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5116999999999999e-05 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995467+0000", 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5329999999999994e-06 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995527+0000", 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0043000000000001e-05 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995535+0000", 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9219999999999997e-06 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836576+0000", 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84104053400000001 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836583+0000", 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4970000000000004e-06 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160622 7.e e227/222 7:7d366766:::benchmark_data_smithi012_221426_object52844:head v 227'9935, mlcod=196'3292)", 2023-10-09T10:11:19.971 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 302.22352406300001, 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069261389000000007, 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680613+0000", 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680622+0000", 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2410000000000001e-06 2023-10-09T10:11:19.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680638+0000", 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5410999999999999e-05 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680646+0000", 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0849999999999997e-06 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680752+0000", 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105919 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680779+0000", 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7885000000000002e-05 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749829+0000", 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069049252000000005 2023-10-09T10:11:19.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749875+0000", 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6548000000000002e-05 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:189857 7.e e240/235 7:72010e1c:::benchmark_data_smithi012_221426_object82079:head v 240'11708, mlcod=0'0)", 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:09.473016+0000", 2023-10-09T10:11:19.976 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 250.431121921, 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067498027000000002, 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473016+0000", 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473016+0000", 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473015+0000", 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473021+0000", 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3619999999999997e-06 2023-10-09T10:11:19.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473029+0000", 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1899999999999995e-06 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473034+0000", 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7029999999999997e-06 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.492979+0000", 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019945086000000001 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.492994+0000", 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5645999999999999e-05 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.540489+0000", 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.047494579000000002 2023-10-09T10:11:19.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.540514+0000", 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5024999999999998e-05 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGRecoveryDeleteReply(7.3 e251,250 7:c51775e5:::benchmark_data_smithi012_221426_object58922:head,230'10428)", 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:43.116592+0000", 2023-10-09T10:11:19.981 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 216.78754620000001, 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.083055424000000003, 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116592+0000", 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116592+0000", 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116591+0000", 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116597+0000", 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8200000000000002e-06 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116609+0000", 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2516e-05 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116616+0000", 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8900000000000001e-06 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116664+0000", 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7862999999999999e-05 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199647+0000", 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082982967000000005 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.985 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(7.5 251/234 [PushOp(7:ad2a751e:::benchmark_data_smithi012_221426_object89411:head, version: 206'5575, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:ad2a751e:::benchmark_data_smithi012_221426_object89411:head@206'5575, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:43.116655+0000", 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 216.78748251600001, 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082982908999999994, 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116655+0000", 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116655+0000", 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116655+0000", 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116682+0000", 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.747e-05 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116696+0000", 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3750999999999999e-05 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116711+0000", 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4603000000000001e-05 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199489+0000", 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082778400000000002 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199497+0000", 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.028e-06 2023-10-09T10:11:19.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199638+0000", 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014106700000000001 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "pg_backfill(progress 7.5 e 251/251 lb 7:ad27a7d6:::benchmark_data_smithi012_221426_object93576:head)", 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:43.116729+0000", 2023-10-09T10:11:19.990 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 216.787409346, 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082978932000000005, 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116729+0000", 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116729+0000", 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116728+0000", 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116735+0000", 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.528e-06 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199466+0000", 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082731616999999993 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199477+0000", 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0193999999999999e-05 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199664+0000", 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018752100000000001 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199676+0000", 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2228999999999999e-05 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199707+0000", 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1142000000000001e-05 2023-10-09T10:11:19.994 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:209942 7.3 7:c41ee6b6:::benchmark_data_smithi012_221426_object102164:head [delete] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:18.370550+0000", 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 181.53358813599999, 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082343928999999996, 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:19.995 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 209942 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370550+0000", 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370550+0000", 2023-10-09T10:11:19.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370549+0000", 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370553+0000", 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.534e-06 2023-10-09T10:11:19.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370575+0000", 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2643999999999999e-05 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370593+0000", 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7706e-05 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.393648+0000", 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023055218999999998 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.393804+0000", 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000155613 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:11:19.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.393940+0000", 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013610599999999999 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.407833+0000", 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013893476 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.407842+0000", 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.867e-06 2023-10-09T10:11:20.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.452613+0000", 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044770694 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.452668+0000", 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4614999999999998e-05 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:20.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.452894+0000", 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00022602500000000001 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:95854 8.f e290/288 8:f5fc2dfd:::benchmark_data_smithi012_271002_object95853:head v 290'6024, mlcod=276'276)", 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:02.773123+0000", 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 77.131014922000006, 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066014849, 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:20.002 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773123+0000", 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773123+0000", 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773122+0000", 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773149+0000", 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7163999999999999e-05 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773163+0000", 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.362e-05 2023-10-09T10:11:20.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773170+0000", 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0040000000000002e-06 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.781520+0000", 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.008349825 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.781557+0000", 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7011999999999997e-05 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.839085+0000", 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057528982999999999 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.839138+0000", 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2321000000000003e-05 2023-10-09T10:11:20.006 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:108330 8.f e292/288 8:fca24286:::benchmark_data_smithi012_271002_object108329:head v 292'6808, mlcod=277'654)", 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 58.831815913, 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068658946999999998, 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:20.007 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:20.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072351+0000", 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9468e-05 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072361+0000", 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5270000000000004e-06 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072368+0000", 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0310000000000001e-06 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.081510+0000", 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091419859999999995 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:20.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.081536+0000", 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6675000000000001e-05 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.140944+0000", 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059407616000000003 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.140981+0000", 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7005999999999998e-05 2023-10-09T10:11:20.011 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:110180 8.4 e293/288 8:2cd3d405:::benchmark_data_smithi012_271002_object1497:head v 293'6943, mlcod=0'0)", 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 56.030616713999997, 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068331423000000002, 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:11:20.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873525+0000", 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5710000000000002e-06 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873529+0000", 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4320000000000001e-06 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873533+0000", 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6629999999999999e-06 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873743+0000", 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000210688 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:20.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873766+0000", 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2617e-05 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.941812+0000", 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068045934000000002 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.941853+0000", 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0711999999999999e-05 2023-10-09T10:11:20.016 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:136718 8.4 e297/288 8:2f021f19:::benchmark_data_smithi012_271002_object28035:head v 297'8596, mlcod=0'0)", 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.368065548000001, 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069669067000000001, 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:20.017 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536071+0000", 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536082+0000", 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1034e-05 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536091+0000", 2023-10-09T10:11:20.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1370000000000008e-06 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536097+0000", 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4620000000000004e-06 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.544788+0000", 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0086914880000000007 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.544818+0000", 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0079000000000001e-05 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.605693+0000", 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.06087447 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.605741+0000", 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8838000000000002e-05 2023-10-09T10:11:20.021 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:20.022 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:20.022 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:11:20.022 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:20.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:20 smithi012 ceph-mon[152263]: pgmap v807: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5614/221124 objects degraded (2.539%); 3092/221124 objects misplaced (1.398%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:20.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:20 smithi012 ceph-mon[147926]: pgmap v807: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5614/221124 objects degraded (2.539%); 3092/221124 objects misplaced (1.398%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:20.711 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:20.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:20.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:20.852 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:20 smithi110 ceph-mon[136032]: pgmap v807: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5614/221124 objects degraded (2.539%); 3092/221124 objects misplaced (1.398%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:11:20.854 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:20.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:11:20.904 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:20.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:21.007 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:21.038 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:21.082 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:21.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:21.195 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:21.225 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:21.262 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:21.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:21.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:21.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:21.459 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:21.508 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:11:21.541 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:21.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:21.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:21.836 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:21.900 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:11:21.978 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:21.982 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:11:21.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:21.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:22.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:22.114 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:22.201 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:22.400 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:22.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:22.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:22.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":808,"stamp":"2023-10-09T10:11:21.366495+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4764407736,"num_objects":72715,"num_object_clones":0,"num_object_copies":218145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5614,"num_objects_misplaced":3092,"num_objects_unfound":0,"num_objects_dirty":72715,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":144931,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2020,"num_bytes_recovered":129051340,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136065,"ondisk_log_size":136065,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16726040,"kb_used_data":14256024,"kb_used_omap":167,"kb_used_meta":2469528,"kb_avail":733235176,"statfs":{"total":767960285184,"available":750832820224,"internally_reserved":0,"allocated":14598168576,"data_stored":14591839477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2528797273},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,3,2,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-353763328,"num_objects":-5398,"num_object_clones":0,"num_object_copies":-16194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":80,"num_objects_misplaced":-201,"num_objects_unfound":0,"num_objects_dirty":-5398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5398,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":7929856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006643"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":141,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.474653+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:11:14.474653+0000","last_peered":"2023-10-09T10:11:14.474653+0000","last_clean":"2023-10-09T10:11:14.474653+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:11:14.474653+0000","last_undegraded":"2023-10-09T10:11:14.474653+0000","last_fullsized":"2023-10-09T10:11:14.474653+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":256,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.232834+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:11:07.232834+0000","last_peered":"2023-10-09T10:11:07.232834+0000","last_clean":"2023-10-09T10:11:07.232834+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:11:07.232834+0000","last_undegraded":"2023-10-09T10:11:07.232834+0000","last_fullsized":"2023-10-09T10:11:07.232834+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":302,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.472939+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:11:14.472939+0000","last_peered":"2023-10-09T10:11:14.472939+0000","last_clean":"2023-10-09T10:11:14.472939+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:11:14.472939+0000","last_undegraded":"2023-10-09T10:11:14.472939+0000","last_fullsized":"2023-10-09T10:11:14.472939+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":165,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.234647+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:11:07.234647+0000","last_peered":"2023-10-09T10:11:07.234647+0000","last_clean":"2023-10-09T10:11:07.234647+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:11:07.234647+0000","last_undegraded":"2023-10-09T10:11:07.234647+0000","last_fullsized":"2023-10-09T10:11:07.234647+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":270,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.235872+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:11:07.235872+0000","last_peered":"2023-10-09T10:11:07.235872+0000","last_clean":"2023-10-09T10:11:07.235872+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:11:07.235872+0000","last_undegraded":"2023-10-09T10:11:07.235872+0000","last_fullsized":"2023-10-09T10:11:07.235872+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":117,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224784+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:11:07.224784+0000","last_peered":"2023-10-09T10:11:07.224784+0000","last_clean":"2023-10-09T10:11:07.224784+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:11:07.224784+0000","last_undegraded":"2023-10-09T10:11:07.224784+0000","last_fullsized":"2023-10-09T10:11:07.224784+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":182,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.234333+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:11:07.234333+0000","last_peered":"2023-10-09T10:11:07.234333+0000","last_clean":"2023-10-09T10:11:07.234333+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:11:07.234333+0000","last_undegraded":"2023-10-09T10:11:07.234333+0000","last_fullsized":"2023-10-09T10:11:07.234333+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":251,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.235805+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:11:07.235805+0000","last_peered":"2023-10-09T10:11:07.235805+0000","last_clean":"2023-10-09T10:11:07.235805+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:11:07.235805+0000","last_undegraded":"2023-10-09T10:11:07.235805+0000","last_fullsized":"2023-10-09T10:11:07.235805+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":116,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224504+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:11:07.224504+0000","last_peered":"2023-10-09T10:11:07.224504+0000","last_clean":"2023-10-09T10:11:07.224504+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:11:07.224504+0000","last_undegraded":"2023-10-09T10:11:07.224504+0000","last_fullsized":"2023-10-09T10:11:07.224504+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":212,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.234098+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:11:07.234098+0000","last_peered":"2023-10-09T10:11:07.234098+0000","last_clean":"2023-10-09T10:11:07.234098+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:11:07.234098+0000","last_undegraded":"2023-10-09T10:11:07.234098+0000","last_fullsized":"2023-10-09T10:11:07.234098+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":337,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.226073+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:11:07.226073+0000","last_peered":"2023-10-09T10:11:07.226073+0000","last_clean":"2023-10-09T10:11:07.226073+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:11:07.226073+0000","last_undegraded":"2023-10-09T10:11:07.226073+0000","last_fullsized":"2023-10-09T10:11:07.226073+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":294,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247381+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:11:07.247381+0000","last_peered":"2023-10-09T10:11:07.247381+0000","last_clean":"2023-10-09T10:11:07.247381+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:11:07.247381+0000","last_undegraded":"2023-10-09T10:11:07.247381+0000","last_fullsized":"2023-10-09T10:11:07.247381+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":186,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.226060+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:11:07.226060+0000","last_peered":"2023-10-09T10:11:07.226060+0000","last_clean":"2023-10-09T10:11:07.226060+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:11:07.226060+0000","last_undegraded":"2023-10-09T10:11:07.226060+0000","last_fullsized":"2023-10-09T10:11:07.226060+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":166,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.246948+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:11:07.246948+0000","last_peered":"2023-10-09T10:11:07.246948+0000","last_clean":"2023-10-09T10:11:07.246948+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:11:07.246948+0000","last_undegraded":"2023-10-09T10:11:07.246948+0000","last_fullsized":"2023-10-09T10:11:07.246948+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":370,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.225911+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:11:07.225911+0000","last_peered":"2023-10-09T10:11:07.225911+0000","last_clean":"2023-10-09T10:11:07.225911+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:11:07.225911+0000","last_undegraded":"2023-10-09T10:11:07.225911+0000","last_fullsized":"2023-10-09T10:11:07.225911+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":238,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233294+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:11:07.233294+0000","last_peered":"2023-10-09T10:11:07.233294+0000","last_clean":"2023-10-09T10:11:07.233294+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:11:07.233294+0000","last_undegraded":"2023-10-09T10:11:07.233294+0000","last_fullsized":"2023-10-09T10:11:07.233294+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233777+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:11:07.233777+0000","last_peered":"2023-10-09T10:11:07.233777+0000","last_clean":"2023-10-09T10:11:07.233777+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:11:07.233777+0000","last_undegraded":"2023-10-09T10:11:07.233777+0000","last_fullsized":"2023-10-09T10:11:07.233777+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":132,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224772+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:11:07.224772+0000","last_peered":"2023-10-09T10:11:07.224772+0000","last_clean":"2023-10-09T10:11:07.224772+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:11:07.224772+0000","last_undegraded":"2023-10-09T10:11:07.224772+0000","last_fullsized":"2023-10-09T10:11:07.224772+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":167,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.225778+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:11:07.225778+0000","last_peered":"2023-10-09T10:11:07.225778+0000","last_clean":"2023-10-09T10:11:07.225778+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:11:07.225778+0000","last_undegraded":"2023-10-09T10:11:07.225778+0000","last_fullsized":"2023-10-09T10:11:07.225778+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.234715+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:11:07.234715+0000","last_peered":"2023-10-09T10:11:07.234715+0000","last_clean":"2023-10-09T10:11:07.234715+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:11:07.234715+0000","last_undegraded":"2023-10-09T10:11:07.234715+0000","last_fullsized":"2023-10-09T10:11:07.234715+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":187,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.235155+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:11:07.235155+0000","last_peered":"2023-10-09T10:11:07.235155+0000","last_clean":"2023-10-09T10:11:07.235155+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:11:07.235155+0000","last_undegraded":"2023-10-09T10:11:07.235155+0000","last_fullsized":"2023-10-09T10:11:07.235155+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":251,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.248024+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:11:07.248024+0000","last_peered":"2023-10-09T10:11:07.248024+0000","last_clean":"2023-10-09T10:11:07.248024+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:11:07.248024+0000","last_undegraded":"2023-10-09T10:11:07.248024+0000","last_fullsized":"2023-10-09T10:11:07.248024+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"299'9131","reported_seq":9175,"reported_epoch":299,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:17.886440+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:11:17.886440+0000","last_peered":"2023-10-09T10:11:17.886440+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:11:17.886440+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9131,"log_dups_size":0,"ondisk_log_size":9131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297205760,"num_objects":4535,"num_object_clones":0,"num_object_copies":13605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9131,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-838},{"shards":"4,5,6","objects":3075}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":331,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.226168+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:11:07.226168+0000","last_peered":"2023-10-09T10:11:07.226168+0000","last_clean":"2023-10-09T10:11:07.226168+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:11:07.226168+0000","last_undegraded":"2023-10-09T10:11:07.226168+0000","last_fullsized":"2023-10-09T10:11:07.226168+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":117,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224219+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:11:07.224219+0000","last_peered":"2023-10-09T10:11:07.224219+0000","last_clean":"2023-10-09T10:11:07.224219+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:11:07.224219+0000","last_undegraded":"2023-10-09T10:11:07.224219+0000","last_fullsized":"2023-10-09T10:11:07.224219+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"299'9235","reported_seq":9271,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:20.004444+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:11:20.004444+0000","last_peered":"2023-10-09T10:11:20.004444+0000","last_clean":"2023-10-09T10:11:20.004444+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:11:20.004444+0000","last_undegraded":"2023-10-09T10:11:20.004444+0000","last_fullsized":"2023-10-09T10:11:20.004444+0000","mapping_epoch":271,"log_start":"276'600","ondisk_log_start":"276'600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8635,"log_dups_size":0,"ondisk_log_size":8635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298516480,"num_objects":4555,"num_object_clones":0,"num_object_copies":13665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9235,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":319,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.226243+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:11:07.226243+0000","last_peered":"2023-10-09T10:11:07.226243+0000","last_clean":"2023-10-09T10:11:07.226243+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:11:07.226243+0000","last_undegraded":"2023-10-09T10:11:07.226243+0000","last_fullsized":"2023-10-09T10:11:07.226243+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":210,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247059+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:11:07.247059+0000","last_peered":"2023-10-09T10:11:07.247059+0000","last_clean":"2023-10-09T10:11:07.247059+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:11:07.247059+0000","last_undegraded":"2023-10-09T10:11:07.247059+0000","last_fullsized":"2023-10-09T10:11:07.247059+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"299'8969","reported_seq":9007,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:16.520458+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:11:16.520458+0000","last_peered":"2023-10-09T10:11:16.520458+0000","last_clean":"2023-10-09T10:11:16.520458+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:11:16.520458+0000","last_undegraded":"2023-10-09T10:11:16.520458+0000","last_fullsized":"2023-10-09T10:11:16.520458+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8969,"log_dups_size":0,"ondisk_log_size":8969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301268992,"num_objects":4597,"num_object_clones":0,"num_object_copies":13791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8969,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":121,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224828+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:11:07.224828+0000","last_peered":"2023-10-09T10:11:07.224828+0000","last_clean":"2023-10-09T10:11:07.224828+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:11:07.224828+0000","last_undegraded":"2023-10-09T10:11:07.224828+0000","last_fullsized":"2023-10-09T10:11:07.224828+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":142,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.241305+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:11:07.241305+0000","last_peered":"2023-10-09T10:11:07.241305+0000","last_clean":"2023-10-09T10:11:07.241305+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:11:07.241305+0000","last_undegraded":"2023-10-09T10:11:07.241305+0000","last_fullsized":"2023-10-09T10:11:07.241305+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"299'8930","reported_seq":8967,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:19.604257+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:11:19.604257+0000","last_peered":"2023-10-09T10:11:19.604257+0000","last_clean":"2023-10-09T10:11:19.604257+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:11:19.604257+0000","last_undegraded":"2023-10-09T10:11:19.604257+0000","last_fullsized":"2023-10-09T10:11:19.604257+0000","mapping_epoch":271,"log_start":"277'1200","ondisk_log_start":"277'1200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7730,"log_dups_size":0,"ondisk_log_size":7730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293601280,"num_objects":4480,"num_object_clones":0,"num_object_copies":13440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8930,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":132,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224708+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:11:07.224708+0000","last_peered":"2023-10-09T10:11:07.224708+0000","last_clean":"2023-10-09T10:11:07.224708+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:11:07.224708+0000","last_undegraded":"2023-10-09T10:11:07.224708+0000","last_fullsized":"2023-10-09T10:11:07.224708+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":165,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233361+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:11:07.233361+0000","last_peered":"2023-10-09T10:11:07.233361+0000","last_clean":"2023-10-09T10:11:07.233361+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:11:07.233361+0000","last_undegraded":"2023-10-09T10:11:07.233361+0000","last_fullsized":"2023-10-09T10:11:07.233361+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"299'9087","reported_seq":9125,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:16.339091+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:11:16.339091+0000","last_peered":"2023-10-09T10:11:16.339091+0000","last_clean":"2023-10-09T10:11:16.339091+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:11:16.339091+0000","last_undegraded":"2023-10-09T10:11:16.339091+0000","last_fullsized":"2023-10-09T10:11:16.339091+0000","mapping_epoch":271,"log_start":"277'900","ondisk_log_start":"277'900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311492608,"num_objects":4753,"num_object_clones":0,"num_object_copies":14259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9087,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":292,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233724+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:11:07.233724+0000","last_peered":"2023-10-09T10:11:07.233724+0000","last_clean":"2023-10-09T10:11:07.233724+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:11:07.233724+0000","last_undegraded":"2023-10-09T10:11:07.233724+0000","last_fullsized":"2023-10-09T10:11:07.233724+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":162,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233178+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:11:07.233178+0000","last_peered":"2023-10-09T10:11:07.233178+0000","last_clean":"2023-10-09T10:11:07.233178+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:11:07.233178+0000","last_undegraded":"2023-10-09T10:11:07.233178+0000","last_fullsized":"2023-10-09T10:11:07.233178+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"299'9233","reported_seq":9271,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:19.506076+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:11:19.506076+0000","last_peered":"2023-10-09T10:11:19.506076+0000","last_clean":"2023-10-09T10:11:19.506076+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:11:19.506076+0000","last_undegraded":"2023-10-09T10:11:19.506076+0000","last_fullsized":"2023-10-09T10:11:19.506076+0000","mapping_epoch":271,"log_start":"278'1500","ondisk_log_start":"278'1500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7733,"log_dups_size":0,"ondisk_log_size":7733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293011456,"num_objects":4471,"num_object_clones":0,"num_object_copies":13413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9233,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":224,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247309+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:11:07.247309+0000","last_peered":"2023-10-09T10:11:07.247309+0000","last_clean":"2023-10-09T10:11:07.247309+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:11:07.247309+0000","last_undegraded":"2023-10-09T10:11:07.247309+0000","last_fullsized":"2023-10-09T10:11:07.247309+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":229,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247128+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:11:07.247128+0000","last_peered":"2023-10-09T10:11:07.247128+0000","last_clean":"2023-10-09T10:11:07.247128+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:11:07.247128+0000","last_undegraded":"2023-10-09T10:11:07.247128+0000","last_fullsized":"2023-10-09T10:11:07.247128+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"299'8890","reported_seq":8928,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:16.563068+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:11:16.563068+0000","last_peered":"2023-10-09T10:11:16.563068+0000","last_clean":"2023-10-09T10:11:16.563068+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:11:16.563068+0000","last_undegraded":"2023-10-09T10:11:16.563068+0000","last_fullsized":"2023-10-09T10:11:16.563068+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8890,"log_dups_size":0,"ondisk_log_size":8890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297664512,"num_objects":4542,"num_object_clones":0,"num_object_copies":13626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8890,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":193,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.225755+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:11:07.225755+0000","last_peered":"2023-10-09T10:11:07.225755+0000","last_clean":"2023-10-09T10:11:07.225755+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:11:07.225755+0000","last_undegraded":"2023-10-09T10:11:07.225755+0000","last_fullsized":"2023-10-09T10:11:07.225755+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":166,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247209+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:11:07.247209+0000","last_peered":"2023-10-09T10:11:07.247209+0000","last_clean":"2023-10-09T10:11:07.247209+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:11:07.247209+0000","last_undegraded":"2023-10-09T10:11:07.247209+0000","last_fullsized":"2023-10-09T10:11:07.247209+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"299'9157","reported_seq":9194,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:19.609838+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:11:19.609838+0000","last_peered":"2023-10-09T10:11:19.609838+0000","last_clean":"2023-10-09T10:11:19.609838+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:11:19.609838+0000","last_undegraded":"2023-10-09T10:11:19.609838+0000","last_fullsized":"2023-10-09T10:11:19.609838+0000","mapping_epoch":271,"log_start":"277'1400","ondisk_log_start":"277'1400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7757,"log_dups_size":0,"ondisk_log_size":7757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291045376,"num_objects":4441,"num_object_clones":0,"num_object_copies":13323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9157,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":252,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.235741+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:11:07.235741+0000","last_peered":"2023-10-09T10:11:07.235741+0000","last_clean":"2023-10-09T10:11:07.235741+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:11:07.235741+0000","last_undegraded":"2023-10-09T10:11:07.235741+0000","last_fullsized":"2023-10-09T10:11:07.235741+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":302,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247613+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:11:07.247613+0000","last_peered":"2023-10-09T10:11:07.247613+0000","last_clean":"2023-10-09T10:11:07.247613+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:11:07.247613+0000","last_undegraded":"2023-10-09T10:11:07.247613+0000","last_fullsized":"2023-10-09T10:11:07.247613+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"299'8946","reported_seq":8984,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:16.562305+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:11:16.562305+0000","last_peered":"2023-10-09T10:11:16.562305+0000","last_clean":"2023-10-09T10:11:16.562305+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:11:16.562305+0000","last_undegraded":"2023-10-09T10:11:16.562305+0000","last_fullsized":"2023-10-09T10:11:16.562305+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8946,"log_dups_size":0,"ondisk_log_size":8946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296615936,"num_objects":4526,"num_object_clones":0,"num_object_copies":13578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8946,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":302,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.472961+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:11:14.472961+0000","last_peered":"2023-10-09T10:11:14.472961+0000","last_clean":"2023-10-09T10:11:14.472961+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:11:14.472961+0000","last_undegraded":"2023-10-09T10:11:14.472961+0000","last_fullsized":"2023-10-09T10:11:14.472961+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":141,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224740+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:11:07.224740+0000","last_peered":"2023-10-09T10:11:07.224740+0000","last_clean":"2023-10-09T10:11:07.224740+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:11:07.224740+0000","last_undegraded":"2023-10-09T10:11:07.224740+0000","last_fullsized":"2023-10-09T10:11:07.224740+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":258,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247596+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:11:07.247596+0000","last_peered":"2023-10-09T10:11:07.247596+0000","last_clean":"2023-10-09T10:11:07.247596+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:11:07.247596+0000","last_undegraded":"2023-10-09T10:11:07.247596+0000","last_fullsized":"2023-10-09T10:11:07.247596+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"299'9096","reported_seq":9132,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:17.842912+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:11:17.842912+0000","last_peered":"2023-10-09T10:11:17.842912+0000","last_clean":"2023-10-09T10:11:17.842912+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:11:17.842912+0000","last_undegraded":"2023-10-09T10:11:17.842912+0000","last_fullsized":"2023-10-09T10:11:17.842912+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9096,"log_dups_size":0,"ondisk_log_size":9096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293470208,"num_objects":4478,"num_object_clones":0,"num_object_copies":13434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9096,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":117,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224332+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:11:07.224332+0000","last_peered":"2023-10-09T10:11:07.224332+0000","last_clean":"2023-10-09T10:11:07.224332+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:11:07.224332+0000","last_undegraded":"2023-10-09T10:11:07.224332+0000","last_fullsized":"2023-10-09T10:11:07.224332+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":115,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233159+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:11:07.233159+0000","last_peered":"2023-10-09T10:11:07.233159+0000","last_clean":"2023-10-09T10:11:07.233159+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:11:07.233159+0000","last_undegraded":"2023-10-09T10:11:07.233159+0000","last_fullsized":"2023-10-09T10:11:07.233159+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":325,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.472878+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:11:14.472878+0000","last_peered":"2023-10-09T10:11:14.472878+0000","last_clean":"2023-10-09T10:11:14.472878+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:11:14.472878+0000","last_undegraded":"2023-10-09T10:11:14.472878+0000","last_fullsized":"2023-10-09T10:11:14.472878+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"299'9014","reported_seq":9052,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:16.345670+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:11:16.345670+0000","last_peered":"2023-10-09T10:11:16.345670+0000","last_clean":"2023-10-09T10:11:16.345670+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:11:16.345670+0000","last_undegraded":"2023-10-09T10:11:16.345670+0000","last_fullsized":"2023-10-09T10:11:16.345670+0000","mapping_epoch":271,"log_start":"277'900","ondisk_log_start":"277'900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8114,"log_dups_size":0,"ondisk_log_size":8114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296747008,"num_objects":4528,"num_object_clones":0,"num_object_copies":13584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9014,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":218,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233963+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:11:07.233963+0000","last_peered":"2023-10-09T10:11:07.233963+0000","last_clean":"2023-10-09T10:11:07.233963+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:11:07.233963+0000","last_undegraded":"2023-10-09T10:11:07.233963+0000","last_fullsized":"2023-10-09T10:11:07.233963+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":294,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.246673+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:11:07.246673+0000","last_peered":"2023-10-09T10:11:07.246673+0000","last_clean":"2023-10-09T10:11:07.246673+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:11:07.246673+0000","last_undegraded":"2023-10-09T10:11:07.246673+0000","last_fullsized":"2023-10-09T10:11:07.246673+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":314,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247518+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:11:07.247518+0000","last_peered":"2023-10-09T10:11:07.247518+0000","last_clean":"2023-10-09T10:11:07.247518+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:11:07.247518+0000","last_undegraded":"2023-10-09T10:11:07.247518+0000","last_fullsized":"2023-10-09T10:11:07.247518+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"299'8990","reported_seq":9026,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:17.823090+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:11:17.823090+0000","last_peered":"2023-10-09T10:11:17.823090+0000","last_clean":"2023-10-09T10:11:17.823090+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:11:17.823090+0000","last_undegraded":"2023-10-09T10:11:17.823090+0000","last_fullsized":"2023-10-09T10:11:17.823090+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8990,"log_dups_size":0,"ondisk_log_size":8990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292945920,"num_objects":4470,"num_object_clones":0,"num_object_copies":13410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8990,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":126,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224414+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:11:07.224414+0000","last_peered":"2023-10-09T10:11:07.224414+0000","last_clean":"2023-10-09T10:11:07.224414+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:11:07.224414+0000","last_undegraded":"2023-10-09T10:11:07.224414+0000","last_fullsized":"2023-10-09T10:11:07.224414+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":545,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.225747+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:11:07.225747+0000","last_peered":"2023-10-09T10:11:07.225747+0000","last_clean":"2023-10-09T10:11:07.225747+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:11:07.225747+0000","last_undegraded":"2023-10-09T10:11:07.225747+0000","last_fullsized":"2023-10-09T10:11:07.225747+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":202,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.235347+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:11:07.235347+0000","last_peered":"2023-10-09T10:11:07.235347+0000","last_clean":"2023-10-09T10:11:07.235347+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:11:07.235347+0000","last_undegraded":"2023-10-09T10:11:07.235347+0000","last_fullsized":"2023-10-09T10:11:07.235347+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":214,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.226118+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:11:07.226118+0000","last_peered":"2023-10-09T10:11:07.226118+0000","last_clean":"2023-10-09T10:11:07.226118+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:11:07.226118+0000","last_undegraded":"2023-10-09T10:11:07.226118+0000","last_fullsized":"2023-10-09T10:11:07.226118+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"299'9109","reported_seq":9147,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:19.575260+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:11:19.575260+0000","last_peered":"2023-10-09T10:11:19.575260+0000","last_clean":"2023-10-09T10:11:19.575260+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:11:19.575260+0000","last_undegraded":"2023-10-09T10:11:19.575260+0000","last_fullsized":"2023-10-09T10:11:19.575260+0000","mapping_epoch":271,"log_start":"277'1400","ondisk_log_start":"277'1400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7709,"log_dups_size":0,"ondisk_log_size":7709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295763968,"num_objects":4513,"num_object_clones":0,"num_object_copies":13539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9109,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":274,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.246806+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:11:07.246806+0000","last_peered":"2023-10-09T10:11:07.246806+0000","last_clean":"2023-10-09T10:11:07.246806+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:11:07.246806+0000","last_undegraded":"2023-10-09T10:11:07.246806+0000","last_fullsized":"2023-10-09T10:11:07.246806+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":176,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.231340+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:11:07.231340+0000","last_peered":"2023-10-09T10:11:07.231340+0000","last_clean":"2023-10-09T10:11:07.231340+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:11:07.231340+0000","last_undegraded":"2023-10-09T10:11:07.231340+0000","last_fullsized":"2023-10-09T10:11:07.231340+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":204,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:11.604671+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:11:11.604671+0000","last_peered":"2023-10-09T10:11:11.604671+0000","last_clean":"2023-10-09T10:11:11.604671+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:11:11.604671+0000","last_undegraded":"2023-10-09T10:11:11.604671+0000","last_fullsized":"2023-10-09T10:11:11.604671+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"299'9019","reported_seq":12967,"reported_epoch":299,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:16.339467+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:11:16.339467+0000","last_peered":"2023-10-09T10:11:16.339467+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:11:16.339467+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"277'900","ondisk_log_start":"277'900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8119,"log_dups_size":0,"ondisk_log_size":8119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304021504,"num_objects":4639,"num_object_clones":0,"num_object_copies":13917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":3092,"num_objects_unfound":0,"num_objects_dirty":4639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9019,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1306},{"shards":"1,3,4","objects":3092}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":318,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.226035+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:11:07.226035+0000","last_peered":"2023-10-09T10:11:07.226035+0000","last_clean":"2023-10-09T10:11:07.226035+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:11:07.226035+0000","last_undegraded":"2023-10-09T10:11:07.226035+0000","last_fullsized":"2023-10-09T10:11:07.226035+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":331,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.473054+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:11:14.473054+0000","last_peered":"2023-10-09T10:11:14.473054+0000","last_clean":"2023-10-09T10:11:14.473054+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:11:14.473054+0000","last_undegraded":"2023-10-09T10:11:14.473054+0000","last_fullsized":"2023-10-09T10:11:14.473054+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":168,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233225+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:11:07.233225+0000","last_peered":"2023-10-09T10:11:07.233225+0000","last_clean":"2023-10-09T10:11:07.233225+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:11:07.233225+0000","last_undegraded":"2023-10-09T10:11:07.233225+0000","last_fullsized":"2023-10-09T10:11:07.233225+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"299'8917","reported_seq":8955,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:16.581020+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:11:16.581020+0000","last_peered":"2023-10-09T10:11:16.581020+0000","last_clean":"2023-10-09T10:11:16.581020+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:11:16.581020+0000","last_undegraded":"2023-10-09T10:11:16.581020+0000","last_fullsized":"2023-10-09T10:11:16.581020+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8917,"log_dups_size":0,"ondisk_log_size":8917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304152576,"num_objects":4641,"num_object_clones":0,"num_object_copies":13923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8917,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":261,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.248133+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:11:07.248133+0000","last_peered":"2023-10-09T10:11:07.248133+0000","last_clean":"2023-10-09T10:11:07.248133+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:11:07.248133+0000","last_undegraded":"2023-10-09T10:11:07.248133+0000","last_fullsized":"2023-10-09T10:11:07.248133+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":275,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247247+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:11:07.247247+0000","last_peered":"2023-10-09T10:11:07.247247+0000","last_clean":"2023-10-09T10:11:07.247247+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:11:07.247247+0000","last_undegraded":"2023-10-09T10:11:07.247247+0000","last_fullsized":"2023-10-09T10:11:07.247247+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":250,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.235671+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:11:07.235671+0000","last_peered":"2023-10-09T10:11:07.235671+0000","last_clean":"2023-10-09T10:11:07.235671+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:11:07.235671+0000","last_undegraded":"2023-10-09T10:11:07.235671+0000","last_fullsized":"2023-10-09T10:11:07.235671+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"299'8948","reported_seq":8985,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:17.856584+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:11:17.856584+0000","last_peered":"2023-10-09T10:11:17.856584+0000","last_clean":"2023-10-09T10:11:17.856584+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:11:17.856584+0000","last_undegraded":"2023-10-09T10:11:17.856584+0000","last_fullsized":"2023-10-09T10:11:17.856584+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8948,"log_dups_size":0,"ondisk_log_size":8948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296288280,"num_objects":4522,"num_object_clones":0,"num_object_copies":13566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4522,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8948,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":207,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.243914+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:11:07.243914+0000","last_peered":"2023-10-09T10:11:07.243914+0000","last_clean":"2023-10-09T10:11:07.243914+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:11:07.243914+0000","last_undegraded":"2023-10-09T10:11:07.243914+0000","last_fullsized":"2023-10-09T10:11:07.243914+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":170,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:11.604651+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:11:11.604651+0000","last_peered":"2023-10-09T10:11:11.604651+0000","last_clean":"2023-10-09T10:11:11.604651+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:11:11.604651+0000","last_undegraded":"2023-10-09T10:11:11.604651+0000","last_fullsized":"2023-10-09T10:11:11.604651+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224263+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:11:07.224263+0000","last_peered":"2023-10-09T10:11:07.224263+0000","last_clean":"2023-10-09T10:11:07.224263+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:11:07.224263+0000","last_undegraded":"2023-10-09T10:11:07.224263+0000","last_fullsized":"2023-10-09T10:11:07.224263+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":164,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233081+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:11:07.233081+0000","last_peered":"2023-10-09T10:11:07.233081+0000","last_clean":"2023-10-09T10:11:07.233081+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:11:07.233081+0000","last_undegraded":"2023-10-09T10:11:07.233081+0000","last_fullsized":"2023-10-09T10:11:07.233081+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":237,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.247026+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:11:07.247026+0000","last_peered":"2023-10-09T10:11:07.247026+0000","last_clean":"2023-10-09T10:11:07.247026+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:11:07.247026+0000","last_undegraded":"2023-10-09T10:11:07.247026+0000","last_fullsized":"2023-10-09T10:11:07.247026+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":136,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224635+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:11:07.224635+0000","last_peered":"2023-10-09T10:11:07.224635+0000","last_clean":"2023-10-09T10:11:07.224635+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:11:07.224635+0000","last_undegraded":"2023-10-09T10:11:07.224635+0000","last_fullsized":"2023-10-09T10:11:07.224635+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":177,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.233634+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:11:07.233634+0000","last_peered":"2023-10-09T10:11:07.233634+0000","last_clean":"2023-10-09T10:11:07.233634+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:11:07.233634+0000","last_undegraded":"2023-10-09T10:11:07.233634+0000","last_fullsized":"2023-10-09T10:11:07.233634+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":136,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.224652+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:11:07.224652+0000","last_peered":"2023-10-09T10:11:07.224652+0000","last_clean":"2023-10-09T10:11:07.224652+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:11:07.224652+0000","last_undegraded":"2023-10-09T10:11:07.224652+0000","last_fullsized":"2023-10-09T10:11:07.224652+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":186,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.235256+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:11:07.235256+0000","last_peered":"2023-10-09T10:11:07.235256+0000","last_clean":"2023-10-09T10:11:07.235256+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:11:07.235256+0000","last_undegraded":"2023-10-09T10:11:07.235256+0000","last_fullsized":"2023-10-09T10:11:07.235256+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":315,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.234581+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:11:07.234581+0000","last_peered":"2023-10-09T10:11:07.234581+0000","last_clean":"2023-10-09T10:11:07.234581+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:11:07.234581+0000","last_undegraded":"2023-10-09T10:11:07.234581+0000","last_fullsized":"2023-10-09T10:11:07.234581+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":336,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.234224+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:11:07.234224+0000","last_peered":"2023-10-09T10:11:07.234224+0000","last_clean":"2023-10-09T10:11:07.234224+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:11:07.234224+0000","last_undegraded":"2023-10-09T10:11:07.234224+0000","last_fullsized":"2023-10-09T10:11:07.234224+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":246,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.234122+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:11:07.234122+0000","last_peered":"2023-10-09T10:11:07.234122+0000","last_clean":"2023-10-09T10:11:07.234122+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:11:07.234122+0000","last_undegraded":"2023-10-09T10:11:07.234122+0000","last_fullsized":"2023-10-09T10:11:07.234122+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":166,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:11:07.246567+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:11:07.246567+0000","last_peered":"2023-10-09T10:11:07.246567+0000","last_clean":"2023-10-09T10:11:07.246567+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:11:07.246567+0000","last_undegraded":"2023-10-09T10:11:07.246567+0000","last_fullsized":"2023-10-09T10:11:07.246567+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4763811864,"num_objects":72691,"num_object_clones":0,"num_object_copies":218073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5614,"num_objects_misplaced":3092,"num_objects_unfound":0,"num_objects_dirty":72691,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144671,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14522200064,"data_stored":14522187848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":135871,"ondisk_log_size":135871,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430500,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1732616,"kb_used_data":1435392,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92012536,"statfs":{"total":95995035648,"available":94220836864,"internally_reserved":0,"allocated":1469841408,"data_stored":1469303331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":1.758,"15min":1.716},"min":{"1min":0.254,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":7.8419999999999996,"5min":9.0690000000000008,"15min":50.591999999999999},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.6699999999999999,"15min":1.6970000000000001},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":7.9610000000000003,"5min":8.2300000000000004,"15min":50.747},"last":3.552}]},{"osd":1,"last update":"Mon Oct 9 10:11:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7029999999999998,"5min":2.117,"15min":1.728},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":14.311,"5min":14.311,"15min":25.338000000000001},"last":0.495},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.2679999999999998,"15min":1.5509999999999999},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":16.526,"5min":18.27,"15min":18.27},"last":0.97199999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.752,"15min":2.2770000000000001},"min":{"1min":0.36099999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.8029999999999999,"5min":1.845,"15min":2.1030000000000002},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.872,"5min":11.872,"15min":11.872},"last":0.93899999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":2.2879999999999998,"15min":1.929},"min":{"1min":0.46400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.274,"5min":29.058,"15min":29.058},"last":0.44},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.913,"15min":1.9350000000000001},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":6.9880000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":0.96599999999999997,"15min":0.93000000000000005},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":2.0550000000000002},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.91300000000000003,"15min":0.91000000000000003},"min":{"1min":0.28599999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.7919999999999998,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.46600000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.1519999999999999,"15min":1.006},"min":{"1min":0.30199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":5.7149999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":2.6240000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.145,"15min":0.99299999999999999},"min":{"1min":0.38200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.5960000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.69499999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.93899999999999995,"15min":0.90900000000000003},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.6259999999999999,"5min":5.3129999999999997,"15min":5.4820000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.226,"5min":0.95599999999999996,"15min":0.87},"min":{"1min":0.36099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.419,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":0.39800000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626728,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2074744,"kb_used_data":1777520,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91670408,"statfs":{"total":95995035648,"available":93870497792,"internally_reserved":0,"allocated":1820180480,"data_stored":1819659717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.821,"15min":1.4410000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.2870000000000008,"5min":15.435,"15min":15.435},"last":0.252},{"interface":"front","average":{"1min":2.149,"5min":2.0859999999999999,"15min":1.4750000000000001},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":7.4139999999999997,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.33900000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:04 2023","interfaces":[{"interface":"back","average":{"1min":3.032,"5min":2.7160000000000002,"15min":1.8779999999999999},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":14.888999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.3580000000000001,"15min":1.8320000000000001},"min":{"1min":0.26700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.962,"5min":32.396000000000001,"15min":32.396000000000001},"last":8.1080000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":1.4319999999999999,"15min":1.724},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.898,"5min":12.898,"15min":12.898},"last":1.722},{"interface":"front","average":{"1min":2.569,"5min":2.141,"15min":3.0379999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5640000000000001,"5min":10.26,"15min":10.26},"last":0.44600000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.8140000000000001,"15min":1.5109999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":12.943,"5min":12.943,"15min":12.943},"last":5.7309999999999999},{"interface":"front","average":{"1min":1.653,"5min":2.2770000000000001,"15min":2.3460000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.077,"5min":11.211,"15min":11.211},"last":1.256}]},{"osd":4,"last update":"Mon Oct 9 10:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.50700000000000001,"15min":0.46200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":1.47},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.60099999999999998,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1749999999999998,"5min":4.1749999999999998,"15min":4.1749999999999998},"last":0.32700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:10:33 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.73699999999999999,"15min":0.76700000000000002},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9260000000000002,"5min":7.9260000000000002,"15min":14.09},"last":3.141},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.91900000000000004,"15min":0.79600000000000004},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":4.1529999999999996,"5min":7.508,"15min":7.508},"last":0.43099999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:11:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":0.91100000000000003,"15min":0.82299999999999995},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.2709999999999999,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.538,"5min":0.95499999999999996,"15min":0.872},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.3159999999999998,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.41299999999999998}]}]},{"osd":1,"up_from":84,"seq":360777253041,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2436636,"kb_used_data":2025044,"kb_used_omap":9,"kb_used_meta":411574,"kb_avail":91308516,"statfs":{"total":95995035648,"available":93499920384,"internally_reserved":0,"allocated":2073645056,"data_stored":2072617175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":421452777},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.3980000000000001,"15min":2.1800000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":7.3609999999999998,"5min":25.59,"15min":25.59},"last":3.1829999999999998},{"interface":"front","average":{"1min":6.2140000000000004,"5min":3.206,"15min":2.3639999999999999},"min":{"1min":0.314,"5min":0.245,"15min":0.192},"max":{"1min":20.501999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":3.4780000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":3.0499999999999998,"15min":3.8149999999999999},"min":{"1min":0.27200000000000002,"5min":0.217,"15min":0.217},"max":{"1min":11.135,"5min":17.369,"15min":17.369},"last":6.3259999999999996},{"interface":"front","average":{"1min":3.5550000000000002,"5min":2.7469999999999999,"15min":3.8210000000000002},"min":{"1min":0.25900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.484999999999999,"5min":16.023,"15min":16.023},"last":3.6640000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":3.4540000000000002,"15min":3.5070000000000001},"min":{"1min":0.39500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.587,"5min":18.587,"15min":18.587},"last":2.9340000000000002},{"interface":"front","average":{"1min":4.6959999999999997,"5min":4.2249999999999996,"15min":4.6760000000000002},"min":{"1min":1.0329999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.022,"5min":17.289000000000001,"15min":17.289000000000001},"last":6.5839999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.474,"15min":1.3280000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2629999999999999,"5min":7.2629999999999999,"15min":7.2629999999999999},"last":2.7250000000000001},{"interface":"front","average":{"1min":2.879,"5min":1.639,"15min":1.4330000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.6940000000000008,"5min":10.132,"15min":10.132},"last":2.8300000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":2.3519999999999999,"15min":1.9330000000000001},"min":{"1min":0.41999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.811,"5min":14.416,"15min":15.531000000000001},"last":2.8740000000000001},{"interface":"front","average":{"1min":2.742,"5min":2.1859999999999999,"15min":2.1080000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.9039999999999999,"5min":16.277999999999999,"15min":19.706},"last":6.2050000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.9450000000000001,"15min":1.825},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":7.3049999999999997,"5min":14.215,"15min":16.193999999999999},"last":2.794},{"interface":"front","average":{"1min":4.1509999999999998,"5min":2.343,"15min":1.984},"min":{"1min":0.46200000000000002,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.868,"5min":14.471,"15min":15.481999999999999},"last":2.9009999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.306,"15min":2.1360000000000001},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.8620000000000001,"5min":16.331,"15min":16.331},"last":6.2610000000000001},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.0489999999999999,"15min":1.901},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3339999999999996,"5min":14.242000000000001,"15min":16.100999999999999},"last":2.7639999999999998}]}]},{"osd":0,"up_from":79,"seq":339302416567,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2119624,"kb_used_data":1765120,"kb_used_omap":28,"kb_used_meta":354467,"kb_avail":91625528,"statfs":{"total":95995035648,"available":93824540672,"internally_reserved":0,"allocated":1807482880,"data_stored":1806472695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":362975056},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:10:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.847,"15min":1.4970000000000001},"min":{"1min":0.24099999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.2990000000000004,"5min":12.84,"15min":13.084},"last":0.60699999999999998},{"interface":"front","average":{"1min":3.9020000000000001,"5min":3.379,"15min":2.2269999999999999},"min":{"1min":0.44900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.949000000000002,"5min":22.178999999999998,"15min":22.178999999999998},"last":9.6989999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8360000000000001,"15min":2.1589999999999998},"min":{"1min":0.34599999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.9949999999999992,"5min":10.034000000000001,"15min":10.034000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.9340000000000002,"5min":2.177,"15min":3.0539999999999998},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":15.529,"5min":15.529,"15min":15.529},"last":0.53500000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":4.5259999999999998,"15min":4.7519999999999998},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":20.419,"5min":25.734999999999999,"15min":25.734999999999999},"last":11.050000000000001},{"interface":"front","average":{"1min":3.742,"5min":3.891,"15min":4.2370000000000001},"min":{"1min":0.40999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":10.154,"5min":21.923999999999999,"15min":21.923999999999999},"last":2.1560000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.911,"15min":1.5649999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":6.226},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.0459999999999998,"15min":1.9730000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.6720000000000006,"5min":13.789999999999999,"15min":13.789999999999999},"last":0.496}]},{"osd":5,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.7309999999999999,"15min":1.9630000000000001},"min":{"1min":0.40899999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.6229999999999993,"5min":17.096,"15min":17.096},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.5,"5min":1.454,"15min":1.3029999999999999},"min":{"1min":0.30499999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.4260000000000002,"5min":9.4260000000000002,"15min":12.478},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.7010000000000001,"15min":1.4590000000000001},"min":{"1min":0.33700000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":9.4719999999999995,"5min":9.7260000000000009,"15min":11.699},"last":0.47899999999999998},{"interface":"front","average":{"1min":4.4729999999999999,"5min":2.6320000000000001,"15min":2.2069999999999999},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.466000000000001,"5min":20.466000000000001,"15min":20.466000000000001},"last":6.282}]},{"osd":7,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8239999999999998,"15min":2.1139999999999999},"min":{"1min":0.46400000000000002,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":9.7050000000000001,"5min":14.631,"15min":26.978000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.7190000000000001,"15min":1.605},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":9.5020000000000007,"5min":11.166,"15min":26.768000000000001},"last":0.50700000000000001}]}]},{"osd":2,"up_from":217,"seq":932007903301,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1584708,"kb_used_data":1345660,"kb_used_omap":83,"kb_used_meta":238892,"kb_avail":92160444,"statfs":{"total":95995035648,"available":94372294656,"internally_reserved":0,"allocated":1377955840,"data_stored":1376950102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":244625903},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,2,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":1.732,"15min":1.756},"min":{"1min":0.217,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":19.372,"5min":19.372,"15min":19.372},"last":5.3300000000000001},{"interface":"front","average":{"1min":3.4420000000000002,"5min":2.9849999999999999,"15min":2.9569999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.754999999999999,"5min":16.754999999999999,"15min":16.754999999999999},"last":5.5549999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":2.258,"15min":2.544},"min":{"1min":0.254,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.797000000000001,"5min":13.797000000000001,"15min":13.797000000000001},"last":5.3819999999999997},{"interface":"front","average":{"1min":2.8399999999999999,"5min":2.5209999999999999,"15min":2.6379999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.4119999999999999,"5min":15.569000000000001,"15min":15.569000000000001},"last":7.3860000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":2.198,"15min":2.0819999999999999},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.851000000000001,"5min":13.851000000000001,"15min":13.851000000000001},"last":1.429},{"interface":"front","average":{"1min":2.9860000000000002,"5min":3.181,"15min":3.6699999999999999},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.7240000000000002,"5min":15.811,"15min":15.811},"last":5.5129999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.84499999999999997,"15min":0.753},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.29,"5min":6.29,"15min":6.29},"last":5.4119999999999999},{"interface":"front","average":{"1min":2.5609999999999999,"5min":1.1759999999999999,"15min":0.94499999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":12.385,"5min":12.385,"15min":12.385},"last":7.0810000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.6919999999999999,"15min":1.546},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":12.473000000000001,"5min":12.473000000000001,"15min":12.473000000000001},"last":7.1230000000000002},{"interface":"front","average":{"1min":3.0699999999999998,"5min":1.5409999999999999,"15min":1.6870000000000001},"min":{"1min":0.372,"5min":0.318,"15min":0.318},"max":{"1min":13.587,"5min":13.587,"15min":13.587},"last":1.3939999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.3999999999999999,"15min":1.095},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2599999999999998,"5min":15.292999999999999,"15min":15.292999999999999},"last":5.4560000000000004},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.738,"15min":1.742},"min":{"1min":0.36599999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":7.1550000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.431,"15min":1.1259999999999999},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.2190000000000003,"5min":15.343999999999999,"15min":15.343999999999999},"last":5.4859999999999998},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.859,"15min":1.6599999999999999},"min":{"1min":0.41699999999999998,"5min":0.376,"15min":0.376},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":12.433999999999999},"last":7.1829999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824044,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2036228,"kb_used_data":1785980,"kb_used_omap":10,"kb_used_meta":250229,"kb_avail":91708924,"statfs":{"total":95995035648,"available":93909938176,"internally_reserved":0,"allocated":1828843520,"data_stored":1827661863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":256234563},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":4.2709999999999999,"15min":4.5590000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.753,"5min":23.381,"15min":23.381},"last":0.53700000000000003},{"interface":"front","average":{"1min":6.0720000000000001,"5min":4.0919999999999996,"15min":4.1500000000000004},"min":{"1min":0.42099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":20.239999999999998,"5min":20.239999999999998,"15min":20.239999999999998},"last":8.0660000000000007}]},{"osd":1,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.6150000000000002,"15min":3.8410000000000002},"min":{"1min":0.26300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":20.422999999999998,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":4.5039999999999996,"5min":3.4990000000000001,"15min":3.4529999999999998},"min":{"1min":0.33100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.704000000000001,"5min":23.423999999999999,"15min":23.423999999999999},"last":3.1080000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1369999999999996,"5min":3.4199999999999999,"15min":3.6339999999999999},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":20.594999999999999,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.8809999999999998,"15min":2.8700000000000001},"min":{"1min":0.34200000000000003,"5min":0.253,"15min":0.253},"max":{"1min":15.73,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.32800000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":1.6719999999999999,"15min":1.288},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.676,"5min":15.676,"15min":15.676},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5800000000000001,"15min":1.534},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.426,"5min":12.426,"15min":12.426},"last":4.3879999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8290000000000002,"5min":2.4750000000000001,"15min":2.5670000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":12.388,"5min":12.388,"15min":12.388},"last":4.4329999999999998},{"interface":"front","average":{"1min":3.5699999999999998,"5min":2.5169999999999999,"15min":2.4079999999999999},"min":{"1min":0.443,"5min":0.309,"15min":0.309},"max":{"1min":15.992000000000001,"5min":15.992000000000001,"15min":15.992000000000001},"last":0.58299999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":2.536,"15min":2.6709999999999998},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.468,"5min":12.468,"15min":12.468},"last":4.4960000000000004},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.5609999999999999,"15min":2.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":16.16,"5min":16.16,"15min":16.16},"last":0.61799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.825,"15min":1.534},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.635999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.4399999999999999,"15min":2.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":4.46}]}]},{"osd":4,"up_from":266,"seq":1142461300770,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2111164,"kb_used_data":1792500,"kb_used_omap":35,"kb_used_meta":318620,"kb_avail":91633988,"statfs":{"total":95995035648,"available":93833203712,"internally_reserved":0,"allocated":1835520000,"data_stored":1834993573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":326267118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.98499999999999999,"15min":0.84099999999999997},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.881,"5min":10.881,"15min":10.881},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.923,"5min":1.1519999999999999,"15min":1.024},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":5.4870000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":1.169,"15min":0.93400000000000005},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":10.93,"5min":10.93,"15min":10.93},"last":5.2469999999999999},{"interface":"front","average":{"1min":2.633,"5min":1.546,"15min":1.365},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.141,"5min":11.141,"15min":11.141},"last":5.3529999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":1.2310000000000001,"15min":1.073},"min":{"1min":0.32100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.1189999999999998,"5min":7.1189999999999998,"15min":7.1189999999999998},"last":3.6480000000000001},{"interface":"front","average":{"1min":1.587,"5min":1.042,"15min":0.95099999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.1709999999999994,"5min":8.1709999999999994,"15min":8.1709999999999994},"last":3.7080000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.4610000000000001,"15min":1.3280000000000001},"min":{"1min":0.40699999999999997,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5530000000000008,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":4.6920000000000002},{"interface":"front","average":{"1min":2.4420000000000002,"5min":1.2989999999999999,"15min":1.109},"min":{"1min":0.373,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.6069999999999993,"5min":9.6069999999999993,"15min":9.6069999999999993},"last":0.45300000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.80400000000000005,"15min":0.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4740000000000002,"5min":5.4740000000000002,"15min":5.4740000000000002},"last":0.376},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.60399999999999998,"15min":0.63400000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.439}]},{"osd":6,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.60499999999999998,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":5.5709999999999997,"5min":5.5709999999999997,"15min":5.5709999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.60999999999999999,"15min":0.59199999999999997},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.319,"5min":2.319,"15min":2.319},"last":0.38500000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.61299999999999999,"15min":0.61799999999999999},"min":{"1min":0.33700000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.3460000000000001,"5min":2.3460000000000001,"15min":2.3460000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.82299999999999995,"15min":0.82699999999999996},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.0680000000000001,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.47099999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822955,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2630320,"kb_used_data":2328808,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91114832,"statfs":{"total":95995035648,"available":93301587968,"internally_reserved":0,"allocated":2384699392,"data_stored":2384181021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":1.55,"15min":1.488},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.8540000000000001,"5min":7.8540000000000001,"15min":15.773999999999999},"last":3.1019999999999999},{"interface":"front","average":{"1min":1.339,"5min":1.597,"15min":1.532},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.0629999999999997,"5min":11.747,"15min":15.699},"last":0.33900000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":1.748,"15min":1.5629999999999999},"min":{"1min":0.40300000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.4250000000000007,"5min":11.335000000000001,"15min":15.689},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.903,"15min":1.7110000000000001},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5979999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.376}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":1.7090000000000001,"15min":2.391},"min":{"1min":0.34899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7249999999999996,"5min":15.640000000000001,"15min":15.640000000000001},"last":2.8650000000000002},{"interface":"front","average":{"1min":1.663,"5min":1.6479999999999999,"15min":2.2370000000000001},"min":{"1min":0.40400000000000003,"5min":0.313,"15min":0.313},"max":{"1min":5.2789999999999999,"5min":10.656000000000001,"15min":10.656000000000001},"last":0.44}]},{"osd":3,"last update":"Mon Oct 9 10:10:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.8340000000000001,"15min":1.8440000000000001},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6660000000000004,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.4380000000000002,"15min":2.8250000000000002},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.085,"5min":17.34,"15min":17.34},"last":5.9960000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":0.71699999999999997,"15min":0.61499999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5220000000000002,"5min":5.5220000000000002,"15min":5.5220000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.752,"15min":0.746},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.823,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.88600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.014,"15min":0.94499999999999995},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.25},"max":{"1min":5.4930000000000003,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.503},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.0189999999999999,"15min":0.88700000000000001},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.8099999999999996,"5min":18.600999999999999,"15min":18.600999999999999},"last":3.3239999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":1.0900000000000001,"15min":0.97099999999999997},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.891,"5min":7.891,"15min":10.334},"last":2.831},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.72299999999999998,"15min":0.93300000000000005},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.3630000000000004,"5min":6.3630000000000004,"15min":12.356},"last":6.0250000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1796014080,"data_stored":1796014080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2059993088,"data_stored":2059993088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1366757376,"data_stored":1366753304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1816526848,"data_stored":1816526848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1823932416,"data_stored":1823932416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2375684096,"data_stored":2375680024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1816788992,"data_stored":1816788992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1466503168,"data_stored":1466499096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:11:22.531 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:11:22.608 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:22.617 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:11:22.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:22 smithi012 ceph-mon[152263]: pgmap v808: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 5614/218145 objects degraded (2.574%); 3092/218145 objects misplaced (1.417%); 774 KiB/s, 12 objects/s recovering 2023-10-09T10:11:22.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:22 smithi012 ceph-mon[152263]: from='client.45698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:22.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1561743520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:22.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:22.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:22 smithi012 ceph-mon[147926]: pgmap v808: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 5614/218145 objects degraded (2.574%); 3092/218145 objects misplaced (1.417%); 774 KiB/s, 12 objects/s recovering 2023-10-09T10:11:22.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:22 smithi012 ceph-mon[147926]: from='client.45698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:22.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1561743520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:22.667 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:22.712 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:22.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:22.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:22.825 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:22.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:22 smithi110 ceph-mon[136032]: pgmap v808: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 5614/218145 objects degraded (2.574%); 3092/218145 objects misplaced (1.417%); 774 KiB/s, 12 objects/s recovering 2023-10-09T10:11:22.864 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:22 smithi110 ceph-mon[136032]: from='client.45698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:22.864 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1561743520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:22.864 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:22.866 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:22.911 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:22.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:22.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:23.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:23.049 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:23.097 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:23.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:23.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:23.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:23.314 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:23.507 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:23.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:23.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:23.569 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:23.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:23.726 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:23.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:11:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[147926]: failed to encode map e300 with expected crc 2023-10-09T10:11:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[147926]: osdmap e300: 8 total, 8 up, 8 in 2023-10-09T10:11:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[147926]: failed to encode map e300 with expected crc 2023-10-09T10:11:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5614/218145 objects degraded (2.574%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:23.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:11:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[152263]: failed to encode map e300 with expected crc 2023-10-09T10:11:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[152263]: osdmap e300: 8 total, 8 up, 8 in 2023-10-09T10:11:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[152263]: failed to encode map e300 with expected crc 2023-10-09T10:11:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5614/218145 objects degraded (2.574%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:23.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:23 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:11:23.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:23 smithi110 ceph-mon[136032]: failed to encode map e300 with expected crc 2023-10-09T10:11:23.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:23.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:23 smithi110 ceph-mon[136032]: osdmap e300: 8 total, 8 up, 8 in 2023-10-09T10:11:23.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:23 smithi110 ceph-mon[136032]: failed to encode map e300 with expected crc 2023-10-09T10:11:23.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5614/218145 objects degraded (2.574%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:24.458 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:24.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:24.463 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:24.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:24.686 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:24.926 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:24.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:25.016 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:11:25.016 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:11:25.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:149010 8.0 8:08b3b588:::benchmark_data_smithi012_271002_object40327:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:24.974210+0000", 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.024269841, 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024301228000000001, 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 149010 2023-10-09T10:11:25.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974210+0000", 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974210+0000", 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974209+0000", 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974212+0000", 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4599999999999999e-06 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974220+0000", 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8490000000000006e-06 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974226+0000", 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8560000000000003e-06 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:25.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974276+0000", 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9885000000000001e-05 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974425+0000", 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000148706 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.974540+0000", 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115263 2023-10-09T10:11:25.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.978306+0000", 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037655229999999998 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.980716+0000", 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024103890000000002 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:25.022 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:149012 8.f 8:f36c3ab8:::benchmark_data_smithi012_271002_object40329:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:24.978124+0000", 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.020354984, 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020484203999999999, 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:25.023 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 149012 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.978124+0000", 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.978124+0000", 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.978123+0000", 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.978131+0000", 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2889999999999998e-06 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:25.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.978145+0000", 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3793e-05 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.978153+0000", 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3829999999999992e-06 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:25.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.981294+0000", 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031407079999999999 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.981399+0000", 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105284 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.981499+0000", 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100136 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.997041+0000", 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015541585 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.997075+0000", 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4440000000000002e-05 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:149014 8.f 8:f6df065d:::benchmark_data_smithi012_271002_object40331:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T10:11:25.029 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:24.985304+0000", 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.013175489, 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013393548, 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 149014 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.030 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.985304+0000", 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.985304+0000", 2023-10-09T10:11:25.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.985302+0000", 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:25.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.985325+0000", 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3155000000000001e-05 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.985343+0000", 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7495000000000001e-05 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:25.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.985353+0000", 2023-10-09T10:11:25.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0366999999999999e-05 2023-10-09T10:11:25.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:25.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.997097+0000", 2023-10-09T10:11:25.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011744342 2023-10-09T10:11:25.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:25.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.997168+0000", 2023-10-09T10:11:25.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0739000000000002e-05 2023-10-09T10:11:25.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:25.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.997284+0000", 2023-10-09T10:11:25.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116039 2023-10-09T10:11:25.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.036 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:25.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.037 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:149013 8.8 e300/271 8:141604f5:::benchmark_data_smithi012_271002_object40330:head v 300'9351, mlcod=300'9349)", 2023-10-09T10:11:25.037 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:24.988359+0000", 2023-10-09T10:11:25.037 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.010120023000000001, 2023-10-09T10:11:25.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010420492999999999, 2023-10-09T10:11:25.037 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:25.037 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:11:25.037 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:25.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:25.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.988359+0000", 2023-10-09T10:11:25.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:25.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.988359+0000", 2023-10-09T10:11:25.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:25.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.988359+0000", 2023-10-09T10:11:25.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:11:25.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.988369+0000", 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0773e-05 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.988379+0000", 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6010000000000001e-06 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:25.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.988385+0000", 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3910000000000003e-06 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.988427+0000", 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1480000000000003e-05 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.988441+0000", 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3893999999999999e-05 2023-10-09T10:11:25.041 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:149016 8.b 8:ddb3b317:::benchmark_data_smithi012_271002_object40333:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:24.989432+0000", 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.009047856, 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094245100000000005, 2023-10-09T10:11:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:25.043 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:25.043 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:25.043 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:25.043 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:25.043 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 149016 2023-10-09T10:11:25.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.043 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989432+0000", 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989432+0000", 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989430+0000", 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989437+0000", 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3289999999999996e-06 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989456+0000", 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8692999999999999e-05 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989466+0000", 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6739999999999992e-06 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989511+0000", 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5599000000000002e-05 2023-10-09T10:11:25.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989654+0000", 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014237000000000001 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.989775+0000", 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121463 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:25.047 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:149024 8.7 e300/271)", 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:24.998105+0000", 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00037459199999999998, 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000838545, 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:25.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998105+0000", 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998105+0000", 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998104+0000", 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:11:25.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998112+0000", 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4200000000000007e-06 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998124+0000", 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1770000000000001e-05 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998132+0000", 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2789999999999998e-06 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998934+0000", 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00080141900000000005 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45518.0:149010 8.0 e300/271)", 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:24.998153+0000", 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00032672499999999999, 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00084785299999999997, 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998153+0000", 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998153+0000", 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998152+0000", 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998155+0000", 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.033e-06 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998161+0000", 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0129999999999999e-06 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:25.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998166+0000", 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2209999999999999e-06 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998989+0000", 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082352499999999997 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998996+0000", 2023-10-09T10:11:25.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2830000000000004e-06 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45518.0:149016 8.b e300/271)", 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:24.998173+0000", 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00030637399999999999, 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00089255300000000003, 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:11:25.056 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998173+0000", 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998173+0000", 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998173+0000", 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998175+0000", 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.063e-06 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998180+0000", 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.079e-06 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:24.998184+0000", 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8099999999999999e-06 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.059 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:25.060 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.060 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:25.060 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:11:25.060 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 8 2023-10-09T10:11:25.060 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:11:25.060 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:25.062 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:25 smithi012 ceph-mon[152263]: pgmap v810: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 5682/211860 objects degraded (2.682%); 2892/211860 objects misplaced (1.365%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:25.090 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:25.133 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:25.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:25.240 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:25.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:11:25.296 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:25 smithi110 ceph-mon[136032]: pgmap v810: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 5682/211860 objects degraded (2.682%); 2892/211860 objects misplaced (1.365%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:25.298 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:25.346 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:25.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:25.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:25.377 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:25 smithi012 ceph-mon[147926]: pgmap v810: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 5682/211860 objects degraded (2.682%); 2892/211860 objects misplaced (1.365%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:25.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:25.496 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:25.540 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:25.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:25.644 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:11:25.683 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:25.726 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:11:25.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:25.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:11:25.838 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:25.877 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:25.937 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:25.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:25.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:26.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:11:26.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:26.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:26.378 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:26.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:26.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:26.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:26.588 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:26.730 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:26.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:26.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:26.790 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:26.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:26.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:11:26.946 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:26.974 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:11:27.013 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:11:27.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:11:27.122 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:27 smithi012 ceph-mon[147926]: pgmap v811: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5682/209106 objects degraded (2.717%); 2892/209106 objects misplaced (1.383%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:27.122 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2315378895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:27.122 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:11:27.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:27 smithi012 ceph-mon[152263]: pgmap v811: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5682/209106 objects degraded (2.717%); 2892/209106 objects misplaced (1.383%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:27.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2315378895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:27.131 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:27.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:27.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:27.180 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:11:27.232 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:11:27.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:27.273 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:11:27.300 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:11:27.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:11:27.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:11:27.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:11:27.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:11:27.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:11:27.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:11:27.301 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:11:27.341 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:27 smithi110 ceph-mon[136032]: pgmap v811: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5682/209106 objects degraded (2.717%); 2892/209106 objects misplaced (1.383%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2315378895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:27.535 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:27.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:27.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:27.712 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:11:27.713 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5682/209106 objects degraded (2.717%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1004,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":300,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":69702,"data_bytes":4566947768,"bytes_used":16568418304,"bytes_avail":751391866880,"bytes_total":767960285184,"degraded_objects":5682,"degraded_total":209106,"degraded_ratio":0.027172821439843908,"misplaced_objects":2892,"misplaced_total":209106,"misplaced_ratio":0.013830306160511894,"recovering_objects_per_sec":25,"recovering_bytes_per_sec":1656895,"recovering_keys_per_sec":0,"num_objects_recovered":253,"num_bytes_recovered":16580608,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":613},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T10:09:33.293227+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T10:11:27.717 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:27.915 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:28.086 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:11:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:11:27] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.33.4" 2023-10-09T10:11:28.260 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:28.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:28.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:28.330 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:11:28.331 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:11:28.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:11:28.407 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1198084357' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:28.407 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5682/209106 objects degraded (2.717%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:28.408 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1198084357' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:28.408 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5682/209106 objects degraded (2.717%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:28.408 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1198084357' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:28.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5682/209106 objects degraded (2.717%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:28.417 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:28.465 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:11:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:28.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:11:28.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:28.694 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:28.807 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:28.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:28.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:28.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:29.015 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:29.137 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:29.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:29.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:29.246 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:29.290 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:29.348 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:29.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:29 smithi012 ceph-mon[147926]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 5749/205056 objects degraded (2.804%); 2698/205056 objects misplaced (1.316%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:29 smithi012 ceph-mon[152263]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 5749/205056 objects degraded (2.804%); 2698/205056 objects misplaced (1.316%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:29 smithi110 ceph-mon[136032]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 5749/205056 objects degraded (2.804%); 2698/205056 objects misplaced (1.316%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:29.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:29.561 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:29.576 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:29.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:29.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:29.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:29.850 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:29.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:11:30.147 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:11:30.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/535973793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:11:30.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:11:30.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/535973793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:11:30.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:11:30.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/535973793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:11:30.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:11:30.492 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:30.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:30.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:30.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:30.629 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:31.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[147926]: pgmap v813: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 5749/202905 objects degraded (2.833%); 2698/202905 objects misplaced (1.330%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:31.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:31.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[147926]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:11:31.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[147926]: osdmap e301: 8 total, 8 up, 8 in 2023-10-09T10:11:31.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[147926]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[147926]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[152263]: pgmap v813: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 5749/202905 objects degraded (2.833%); 2698/202905 objects misplaced (1.330%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:31.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:31.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[152263]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:11:31.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[152263]: osdmap e301: 8 total, 8 up, 8 in 2023-10-09T10:11:31.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[152263]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:31 smithi012 ceph-mon[152263]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:31 smithi110 ceph-mon[136032]: pgmap v813: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 5749/202905 objects degraded (2.833%); 2698/202905 objects misplaced (1.330%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:11:31.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:31 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:31.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:31 smithi110 ceph-mon[136032]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:31 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:11:31.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:31 smithi110 ceph-mon[136032]: osdmap e301: 8 total, 8 up, 8 in 2023-10-09T10:11:31.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:31 smithi110 ceph-mon[136032]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:31 smithi110 ceph-mon[136032]: failed to encode map e301 with expected crc 2023-10-09T10:11:31.915 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:31.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:31.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:32.028 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:32.073 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:32.125 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:32.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:32.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:32.232 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:11:32.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:32.317 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:11:32.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:32.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:11:32.422 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:32.463 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:32.509 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:32.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:32.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:32.614 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:32.705 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:32.943 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:11:32.944 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:11:32.964 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:32.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:32.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:33.074 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:33.103 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:33.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:33.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:33.254 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:33.309 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:33.371 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:33.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:33 smithi110 ceph-mon[136032]: pgmap v815: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 5749/200109 objects degraded (2.873%); 2698/200109 objects misplaced (1.348%); 922 KiB/s, 14 objects/s recovering 2023-10-09T10:11:33.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5749/200109 objects degraded (2.873%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:33.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:33.424 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:33.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:33 smithi012 ceph-mon[147926]: pgmap v815: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 5749/200109 objects degraded (2.873%); 2698/200109 objects misplaced (1.348%); 922 KiB/s, 14 objects/s recovering 2023-10-09T10:11:33.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5749/200109 objects degraded (2.873%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:33.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:33 smithi012 ceph-mon[152263]: pgmap v815: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 5749/200109 objects degraded (2.873%); 2698/200109 objects misplaced (1.348%); 922 KiB/s, 14 objects/s recovering 2023-10-09T10:11:33.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5749/200109 objects degraded (2.873%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:33.479 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:33.539 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:33.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:11:33.590 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:33.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:33.697 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:33.749 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:33.792 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:33.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:33.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:34.029 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:34.106 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:34.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:34.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:34.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:34.311 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:34.466 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:34.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:34.556 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:11:34.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:34.574 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:11:34.652 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:34.727 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:11:35.135 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:35.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:35.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:35.218 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:35.240 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:35.264 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":816,"stamp":"2023-10-09T10:11:33.374776+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4232648632,"num_objects":64601,"num_object_clones":0,"num_object_copies":193803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5800,"num_objects_misplaced":2510,"num_objects_unfound":0,"num_objects_dirty":64601,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":153045,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2416,"num_bytes_recovered":155003596,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140379,"ondisk_log_size":140379,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15372224,"kb_used_data":12847424,"kb_used_omap":167,"kb_used_meta":2524568,"kb_avail":734588992,"statfs":{"total":767960285184,"available":752219127808,"internally_reserved":0,"allocated":13155762176,"data_stored":13149422691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2585158233},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,1,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-394461184,"num_objects":-6019,"num_object_clones":0,"num_object_copies":-18057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":118,"num_objects_misplaced":-382,"num_objects_unfound":0,"num_objects_dirty":-6019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6019,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":264,"num_bytes_recovered":17301504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007077"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":142,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.616141+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:11:14.616141+0000","last_peered":"2023-10-09T10:11:14.616141+0000","last_clean":"2023-10-09T10:11:14.616141+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:11:14.616141+0000","last_undegraded":"2023-10-09T10:11:14.616141+0000","last_fullsized":"2023-10-09T10:11:14.616141+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":258,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571829+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:11:22.571829+0000","last_peered":"2023-10-09T10:11:22.571829+0000","last_clean":"2023-10-09T10:11:22.571829+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:11:22.571829+0000","last_undegraded":"2023-10-09T10:11:22.571829+0000","last_fullsized":"2023-10-09T10:11:22.571829+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":303,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.616091+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:11:14.616091+0000","last_peered":"2023-10-09T10:11:14.616091+0000","last_clean":"2023-10-09T10:11:14.616091+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:11:14.616091+0000","last_undegraded":"2023-10-09T10:11:14.616091+0000","last_fullsized":"2023-10-09T10:11:14.616091+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":167,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.573725+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:11:22.573725+0000","last_peered":"2023-10-09T10:11:22.573725+0000","last_clean":"2023-10-09T10:11:22.573725+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:11:22.573725+0000","last_undegraded":"2023-10-09T10:11:22.573725+0000","last_fullsized":"2023-10-09T10:11:22.573725+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":272,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580089+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:11:22.580089+0000","last_peered":"2023-10-09T10:11:22.580089+0000","last_clean":"2023-10-09T10:11:22.580089+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:11:22.580089+0000","last_undegraded":"2023-10-09T10:11:22.580089+0000","last_fullsized":"2023-10-09T10:11:22.580089+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.583464+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:11:22.583464+0000","last_peered":"2023-10-09T10:11:22.583464+0000","last_clean":"2023-10-09T10:11:22.583464+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:11:22.583464+0000","last_undegraded":"2023-10-09T10:11:22.583464+0000","last_fullsized":"2023-10-09T10:11:22.583464+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":184,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567609+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:11:22.567609+0000","last_peered":"2023-10-09T10:11:22.567609+0000","last_clean":"2023-10-09T10:11:22.567609+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:11:22.567609+0000","last_undegraded":"2023-10-09T10:11:22.567609+0000","last_fullsized":"2023-10-09T10:11:22.567609+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.579970+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:11:22.579970+0000","last_peered":"2023-10-09T10:11:22.579970+0000","last_clean":"2023-10-09T10:11:22.579970+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:11:22.579970+0000","last_undegraded":"2023-10-09T10:11:22.579970+0000","last_fullsized":"2023-10-09T10:11:22.579970+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.584172+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:11:22.584172+0000","last_peered":"2023-10-09T10:11:22.584172+0000","last_clean":"2023-10-09T10:11:22.584172+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:11:22.584172+0000","last_undegraded":"2023-10-09T10:11:22.584172+0000","last_fullsized":"2023-10-09T10:11:22.584172+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":214,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571742+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:11:22.571742+0000","last_peered":"2023-10-09T10:11:22.571742+0000","last_clean":"2023-10-09T10:11:22.571742+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:11:22.571742+0000","last_undegraded":"2023-10-09T10:11:22.571742+0000","last_fullsized":"2023-10-09T10:11:22.571742+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":338,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.465561+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:11:14.465561+0000","last_peered":"2023-10-09T10:11:14.465561+0000","last_clean":"2023-10-09T10:11:14.465561+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:11:14.465561+0000","last_undegraded":"2023-10-09T10:11:14.465561+0000","last_fullsized":"2023-10-09T10:11:14.465561+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":295,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.474525+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:11:14.474525+0000","last_peered":"2023-10-09T10:11:14.474525+0000","last_clean":"2023-10-09T10:11:14.474525+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:11:14.474525+0000","last_undegraded":"2023-10-09T10:11:14.474525+0000","last_fullsized":"2023-10-09T10:11:14.474525+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":187,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.466704+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:11:14.466704+0000","last_peered":"2023-10-09T10:11:14.466704+0000","last_clean":"2023-10-09T10:11:14.466704+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:11:14.466704+0000","last_undegraded":"2023-10-09T10:11:14.466704+0000","last_fullsized":"2023-10-09T10:11:14.466704+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":167,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.475915+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:11:14.475915+0000","last_peered":"2023-10-09T10:11:14.475915+0000","last_clean":"2023-10-09T10:11:14.475915+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:11:14.475915+0000","last_undegraded":"2023-10-09T10:11:14.475915+0000","last_fullsized":"2023-10-09T10:11:14.475915+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":371,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.465664+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:11:14.465664+0000","last_peered":"2023-10-09T10:11:14.465664+0000","last_clean":"2023-10-09T10:11:14.465664+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:11:14.465664+0000","last_undegraded":"2023-10-09T10:11:14.465664+0000","last_fullsized":"2023-10-09T10:11:14.465664+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":240,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572934+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:11:22.572934+0000","last_peered":"2023-10-09T10:11:22.572934+0000","last_clean":"2023-10-09T10:11:22.572934+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:11:22.572934+0000","last_undegraded":"2023-10-09T10:11:22.572934+0000","last_fullsized":"2023-10-09T10:11:22.572934+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567543+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:11:22.567543+0000","last_peered":"2023-10-09T10:11:22.567543+0000","last_clean":"2023-10-09T10:11:22.567543+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:11:22.567543+0000","last_undegraded":"2023-10-09T10:11:22.567543+0000","last_fullsized":"2023-10-09T10:11:22.567543+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":134,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580673+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:11:22.580673+0000","last_peered":"2023-10-09T10:11:22.580673+0000","last_clean":"2023-10-09T10:11:22.580673+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:11:22.580673+0000","last_undegraded":"2023-10-09T10:11:22.580673+0000","last_fullsized":"2023-10-09T10:11:22.580673+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":168,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.465378+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:11:14.465378+0000","last_peered":"2023-10-09T10:11:14.465378+0000","last_clean":"2023-10-09T10:11:14.465378+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:11:14.465378+0000","last_undegraded":"2023-10-09T10:11:14.465378+0000","last_fullsized":"2023-10-09T10:11:14.465378+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":166,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.573791+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:11:22.573791+0000","last_peered":"2023-10-09T10:11:22.573791+0000","last_clean":"2023-10-09T10:11:22.573791+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:11:22.573791+0000","last_undegraded":"2023-10-09T10:11:22.573791+0000","last_fullsized":"2023-10-09T10:11:22.573791+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":189,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.566784+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:11:22.566784+0000","last_peered":"2023-10-09T10:11:22.566784+0000","last_clean":"2023-10-09T10:11:22.566784+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:11:22.566784+0000","last_undegraded":"2023-10-09T10:11:22.566784+0000","last_fullsized":"2023-10-09T10:11:22.566784+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":253,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.574878+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:11:22.574878+0000","last_peered":"2023-10-09T10:11:22.574878+0000","last_clean":"2023-10-09T10:11:22.574878+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:11:22.574878+0000","last_undegraded":"2023-10-09T10:11:22.574878+0000","last_fullsized":"2023-10-09T10:11:22.574878+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"301'9726","reported_seq":9772,"reported_epoch":301,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:32.886892+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:11:32.886892+0000","last_peered":"2023-10-09T10:11:32.886892+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:11:32.886892+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9726,"log_dups_size":0,"ondisk_log_size":9726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258211840,"num_objects":3940,"num_object_clones":0,"num_object_copies":11820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9726,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-1433},{"shards":"4,5,6","objects":2480}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":332,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.466089+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:11:14.466089+0000","last_peered":"2023-10-09T10:11:14.466089+0000","last_clean":"2023-10-09T10:11:14.466089+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:11:14.466089+0000","last_undegraded":"2023-10-09T10:11:14.466089+0000","last_fullsized":"2023-10-09T10:11:14.466089+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":119,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580403+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:11:22.580403+0000","last_peered":"2023-10-09T10:11:22.580403+0000","last_clean":"2023-10-09T10:11:22.580403+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:11:22.580403+0000","last_undegraded":"2023-10-09T10:11:22.580403+0000","last_fullsized":"2023-10-09T10:11:22.580403+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"300'9616","reported_seq":9653,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:30.020954+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:11:30.020954+0000","last_peered":"2023-10-09T10:11:30.020954+0000","last_clean":"2023-10-09T10:11:30.020954+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:11:30.020954+0000","last_undegraded":"2023-10-09T10:11:30.020954+0000","last_fullsized":"2023-10-09T10:11:30.020954+0000","mapping_epoch":271,"log_start":"277'1000","ondisk_log_start":"277'1000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273547264,"num_objects":4174,"num_object_clones":0,"num_object_copies":12522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9616,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":320,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.466275+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:11:14.466275+0000","last_peered":"2023-10-09T10:11:14.466275+0000","last_clean":"2023-10-09T10:11:14.466275+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:11:14.466275+0000","last_undegraded":"2023-10-09T10:11:14.466275+0000","last_fullsized":"2023-10-09T10:11:14.466275+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":211,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.472848+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:11:14.472848+0000","last_peered":"2023-10-09T10:11:14.472848+0000","last_clean":"2023-10-09T10:11:14.472848+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:11:14.472848+0000","last_undegraded":"2023-10-09T10:11:14.472848+0000","last_fullsized":"2023-10-09T10:11:14.472848+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"301'9507","reported_seq":9547,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:31.610686+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:11:31.610686+0000","last_peered":"2023-10-09T10:11:31.610686+0000","last_clean":"2023-10-09T10:11:31.610686+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:11:31.610686+0000","last_undegraded":"2023-10-09T10:11:31.610686+0000","last_fullsized":"2023-10-09T10:11:31.610686+0000","mapping_epoch":271,"log_start":"275'100","ondisk_log_start":"275'100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9407,"log_dups_size":0,"ondisk_log_size":9407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266010624,"num_objects":4059,"num_object_clones":0,"num_object_copies":12177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9507,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":123,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580746+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:11:22.580746+0000","last_peered":"2023-10-09T10:11:22.580746+0000","last_clean":"2023-10-09T10:11:22.580746+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:11:22.580746+0000","last_undegraded":"2023-10-09T10:11:22.580746+0000","last_fullsized":"2023-10-09T10:11:22.580746+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":143,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.469455+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:11:14.469455+0000","last_peered":"2023-10-09T10:11:14.469455+0000","last_clean":"2023-10-09T10:11:14.469455+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:11:14.469455+0000","last_undegraded":"2023-10-09T10:11:14.469455+0000","last_fullsized":"2023-10-09T10:11:14.469455+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"300'9309","reported_seq":9348,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:29.596563+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:11:29.596563+0000","last_peered":"2023-10-09T10:11:29.596563+0000","last_clean":"2023-10-09T10:11:29.596563+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:11:29.596563+0000","last_undegraded":"2023-10-09T10:11:29.596563+0000","last_fullsized":"2023-10-09T10:11:29.596563+0000","mapping_epoch":271,"log_start":"279'1600","ondisk_log_start":"279'1600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7709,"log_dups_size":0,"ondisk_log_size":7709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268763136,"num_objects":4101,"num_object_clones":0,"num_object_copies":12303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9309,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.592168+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:11:22.592168+0000","last_peered":"2023-10-09T10:11:22.592168+0000","last_clean":"2023-10-09T10:11:22.592168+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:11:22.592168+0000","last_undegraded":"2023-10-09T10:11:22.592168+0000","last_fullsized":"2023-10-09T10:11:22.592168+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":168,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:30.155878+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:11:30.155878+0000","last_peered":"2023-10-09T10:11:30.155878+0000","last_clean":"2023-10-09T10:11:30.155878+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:11:30.155878+0000","last_undegraded":"2023-10-09T10:11:30.155878+0000","last_fullsized":"2023-10-09T10:11:30.155878+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"301'9701","reported_seq":9741,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:31.321948+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:11:31.321948+0000","last_peered":"2023-10-09T10:11:31.321948+0000","last_clean":"2023-10-09T10:11:31.321948+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:11:31.321948+0000","last_undegraded":"2023-10-09T10:11:31.321948+0000","last_fullsized":"2023-10-09T10:11:31.321948+0000","mapping_epoch":271,"log_start":"279'1500","ondisk_log_start":"279'1500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8201,"log_dups_size":0,"ondisk_log_size":8201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271253504,"num_objects":4139,"num_object_clones":0,"num_object_copies":12417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9701,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":294,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572432+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:11:22.572432+0000","last_peered":"2023-10-09T10:11:22.572432+0000","last_clean":"2023-10-09T10:11:22.572432+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:11:22.572432+0000","last_undegraded":"2023-10-09T10:11:22.572432+0000","last_fullsized":"2023-10-09T10:11:22.572432+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":164,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572548+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:11:22.572548+0000","last_peered":"2023-10-09T10:11:22.572548+0000","last_clean":"2023-10-09T10:11:22.572548+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:11:22.572548+0000","last_undegraded":"2023-10-09T10:11:22.572548+0000","last_fullsized":"2023-10-09T10:11:22.572548+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"300'9604","reported_seq":9643,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:29.529653+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:11:29.529653+0000","last_peered":"2023-10-09T10:11:29.529653+0000","last_clean":"2023-10-09T10:11:29.529653+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:11:29.529653+0000","last_undegraded":"2023-10-09T10:11:29.529653+0000","last_fullsized":"2023-10-09T10:11:29.529653+0000","mapping_epoch":271,"log_start":"279'1900","ondisk_log_start":"279'1900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7704,"log_dups_size":0,"ondisk_log_size":7704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268697600,"num_objects":4100,"num_object_clones":0,"num_object_copies":12300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9604,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":225,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.474458+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:11:14.474458+0000","last_peered":"2023-10-09T10:11:14.474458+0000","last_clean":"2023-10-09T10:11:14.474458+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:11:14.474458+0000","last_undegraded":"2023-10-09T10:11:14.474458+0000","last_fullsized":"2023-10-09T10:11:14.474458+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":230,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.473294+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:11:14.473294+0000","last_peered":"2023-10-09T10:11:14.473294+0000","last_clean":"2023-10-09T10:11:14.473294+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:11:14.473294+0000","last_undegraded":"2023-10-09T10:11:14.473294+0000","last_fullsized":"2023-10-09T10:11:14.473294+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"301'9447","reported_seq":9487,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:31.603568+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:11:31.603568+0000","last_peered":"2023-10-09T10:11:31.603568+0000","last_clean":"2023-10-09T10:11:31.603568+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:11:31.603568+0000","last_undegraded":"2023-10-09T10:11:31.603568+0000","last_fullsized":"2023-10-09T10:11:31.603568+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9447,"log_dups_size":0,"ondisk_log_size":9447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261160960,"num_objects":3985,"num_object_clones":0,"num_object_copies":11955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9447,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":194,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.465296+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:11:14.465296+0000","last_peered":"2023-10-09T10:11:14.465296+0000","last_clean":"2023-10-09T10:11:14.465296+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:11:14.465296+0000","last_undegraded":"2023-10-09T10:11:14.465296+0000","last_fullsized":"2023-10-09T10:11:14.465296+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":167,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.473044+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:11:14.473044+0000","last_peered":"2023-10-09T10:11:14.473044+0000","last_clean":"2023-10-09T10:11:14.473044+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:11:14.473044+0000","last_undegraded":"2023-10-09T10:11:14.473044+0000","last_fullsized":"2023-10-09T10:11:14.473044+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"300'9533","reported_seq":9572,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:29.558076+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:11:29.558076+0000","last_peered":"2023-10-09T10:11:29.558076+0000","last_clean":"2023-10-09T10:11:29.558076+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:11:29.558076+0000","last_undegraded":"2023-10-09T10:11:29.558076+0000","last_fullsized":"2023-10-09T10:11:29.558076+0000","mapping_epoch":271,"log_start":"279'1800","ondisk_log_start":"279'1800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7733,"log_dups_size":0,"ondisk_log_size":7733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266403840,"num_objects":4065,"num_object_clones":0,"num_object_copies":12195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9533,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":254,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.579906+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:11:22.579906+0000","last_peered":"2023-10-09T10:11:22.579906+0000","last_clean":"2023-10-09T10:11:22.579906+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:11:22.579906+0000","last_undegraded":"2023-10-09T10:11:22.579906+0000","last_fullsized":"2023-10-09T10:11:22.579906+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":303,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.473659+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:11:14.473659+0000","last_peered":"2023-10-09T10:11:14.473659+0000","last_clean":"2023-10-09T10:11:14.473659+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:11:14.473659+0000","last_undegraded":"2023-10-09T10:11:14.473659+0000","last_fullsized":"2023-10-09T10:11:14.473659+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"301'9513","reported_seq":9553,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:31.616649+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:11:31.616649+0000","last_peered":"2023-10-09T10:11:31.616649+0000","last_clean":"2023-10-09T10:11:31.616649+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:11:31.616649+0000","last_undegraded":"2023-10-09T10:11:31.616649+0000","last_fullsized":"2023-10-09T10:11:31.616649+0000","mapping_epoch":271,"log_start":"275'100","ondisk_log_start":"275'100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9413,"log_dups_size":0,"ondisk_log_size":9413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259457024,"num_objects":3959,"num_object_clones":0,"num_object_copies":11877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9513,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":303,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:19.616773+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:11:19.616773+0000","last_peered":"2023-10-09T10:11:19.616773+0000","last_clean":"2023-10-09T10:11:19.616773+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:11:19.616773+0000","last_undegraded":"2023-10-09T10:11:19.616773+0000","last_fullsized":"2023-10-09T10:11:19.616773+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580608+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:11:22.580608+0000","last_peered":"2023-10-09T10:11:22.580608+0000","last_clean":"2023-10-09T10:11:22.580608+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:11:22.580608+0000","last_undegraded":"2023-10-09T10:11:22.580608+0000","last_fullsized":"2023-10-09T10:11:22.580608+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":259,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.474722+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:11:14.474722+0000","last_peered":"2023-10-09T10:11:14.474722+0000","last_clean":"2023-10-09T10:11:14.474722+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:11:14.474722+0000","last_undegraded":"2023-10-09T10:11:14.474722+0000","last_fullsized":"2023-10-09T10:11:14.474722+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"301'9627","reported_seq":9665,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:32.922485+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:11:32.922485+0000","last_peered":"2023-10-09T10:11:32.922485+0000","last_clean":"2023-10-09T10:11:32.922485+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:11:32.922485+0000","last_undegraded":"2023-10-09T10:11:32.922485+0000","last_fullsized":"2023-10-09T10:11:32.922485+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9627,"log_dups_size":0,"ondisk_log_size":9627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258670592,"num_objects":3947,"num_object_clones":0,"num_object_copies":11841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9627,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":119,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.583380+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:11:22.583380+0000","last_peered":"2023-10-09T10:11:22.583380+0000","last_clean":"2023-10-09T10:11:22.583380+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:11:22.583380+0000","last_undegraded":"2023-10-09T10:11:22.583380+0000","last_fullsized":"2023-10-09T10:11:22.583380+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":117,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.566986+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:11:22.566986+0000","last_peered":"2023-10-09T10:11:22.566986+0000","last_clean":"2023-10-09T10:11:22.566986+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:11:22.566986+0000","last_undegraded":"2023-10-09T10:11:22.566986+0000","last_fullsized":"2023-10-09T10:11:22.566986+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":326,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:19.616755+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:11:19.616755+0000","last_peered":"2023-10-09T10:11:19.616755+0000","last_clean":"2023-10-09T10:11:19.616755+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:11:19.616755+0000","last_undegraded":"2023-10-09T10:11:19.616755+0000","last_fullsized":"2023-10-09T10:11:19.616755+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"301'9581","reported_seq":9621,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:31.359966+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:11:31.359966+0000","last_peered":"2023-10-09T10:11:31.359966+0000","last_clean":"2023-10-09T10:11:31.359966+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:11:31.359966+0000","last_undegraded":"2023-10-09T10:11:31.359966+0000","last_fullsized":"2023-10-09T10:11:31.359966+0000","mapping_epoch":271,"log_start":"277'1400","ondisk_log_start":"277'1400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8181,"log_dups_size":0,"ondisk_log_size":8181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259588096,"num_objects":3961,"num_object_clones":0,"num_object_copies":11883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9581,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":220,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571557+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:11:22.571557+0000","last_peered":"2023-10-09T10:11:22.571557+0000","last_clean":"2023-10-09T10:11:22.571557+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:11:22.571557+0000","last_undegraded":"2023-10-09T10:11:22.571557+0000","last_fullsized":"2023-10-09T10:11:22.571557+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":295,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.474993+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:11:14.474993+0000","last_peered":"2023-10-09T10:11:14.474993+0000","last_clean":"2023-10-09T10:11:14.474993+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:11:14.474993+0000","last_undegraded":"2023-10-09T10:11:14.474993+0000","last_fullsized":"2023-10-09T10:11:14.474993+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":315,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.473596+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:11:14.473596+0000","last_peered":"2023-10-09T10:11:14.473596+0000","last_clean":"2023-10-09T10:11:14.473596+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:11:14.473596+0000","last_undegraded":"2023-10-09T10:11:14.473596+0000","last_fullsized":"2023-10-09T10:11:14.473596+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"301'9564","reported_seq":9602,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:32.909515+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:11:32.909515+0000","last_peered":"2023-10-09T10:11:32.909515+0000","last_clean":"2023-10-09T10:11:32.909515+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:11:32.909515+0000","last_undegraded":"2023-10-09T10:11:32.909515+0000","last_fullsized":"2023-10-09T10:11:32.909515+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9564,"log_dups_size":0,"ondisk_log_size":9564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255328256,"num_objects":3896,"num_object_clones":0,"num_object_copies":11688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9564,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":128,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.582706+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:11:22.582706+0000","last_peered":"2023-10-09T10:11:22.582706+0000","last_clean":"2023-10-09T10:11:22.582706+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:11:22.582706+0000","last_undegraded":"2023-10-09T10:11:22.582706+0000","last_fullsized":"2023-10-09T10:11:22.582706+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":546,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.465065+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:11:14.465065+0000","last_peered":"2023-10-09T10:11:14.465065+0000","last_clean":"2023-10-09T10:11:14.465065+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:11:14.465065+0000","last_undegraded":"2023-10-09T10:11:14.465065+0000","last_fullsized":"2023-10-09T10:11:14.465065+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":204,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571685+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:11:22.571685+0000","last_peered":"2023-10-09T10:11:22.571685+0000","last_clean":"2023-10-09T10:11:22.571685+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:11:22.571685+0000","last_undegraded":"2023-10-09T10:11:22.571685+0000","last_fullsized":"2023-10-09T10:11:22.571685+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":215,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.465630+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:11:14.465630+0000","last_peered":"2023-10-09T10:11:14.465630+0000","last_clean":"2023-10-09T10:11:14.465630+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:11:14.465630+0000","last_undegraded":"2023-10-09T10:11:14.465630+0000","last_fullsized":"2023-10-09T10:11:14.465630+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"300'9452","reported_seq":9491,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:29.626523+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:11:29.626523+0000","last_peered":"2023-10-09T10:11:29.626523+0000","last_clean":"2023-10-09T10:11:29.626523+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:11:29.626523+0000","last_undegraded":"2023-10-09T10:11:29.626523+0000","last_fullsized":"2023-10-09T10:11:29.626523+0000","mapping_epoch":271,"log_start":"279'1700","ondisk_log_start":"279'1700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7752,"log_dups_size":0,"ondisk_log_size":7752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273285120,"num_objects":4170,"num_object_clones":0,"num_object_copies":12510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9452,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":275,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.475105+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:11:14.475105+0000","last_peered":"2023-10-09T10:11:14.475105+0000","last_clean":"2023-10-09T10:11:14.475105+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:11:14.475105+0000","last_undegraded":"2023-10-09T10:11:14.475105+0000","last_fullsized":"2023-10-09T10:11:14.475105+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":178,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567749+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:11:22.567749+0000","last_peered":"2023-10-09T10:11:22.567749+0000","last_clean":"2023-10-09T10:11:22.567749+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:11:22.567749+0000","last_undegraded":"2023-10-09T10:11:22.567749+0000","last_fullsized":"2023-10-09T10:11:22.567749+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":206,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.573658+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:11:22.573658+0000","last_peered":"2023-10-09T10:11:22.573658+0000","last_clean":"2023-10-09T10:11:22.573658+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:11:22.573658+0000","last_undegraded":"2023-10-09T10:11:22.573658+0000","last_fullsized":"2023-10-09T10:11:22.573658+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"301'9598","reported_seq":14340,"reported_epoch":301,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:31.386804+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:11:31.386804+0000","last_peered":"2023-10-09T10:11:31.386804+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:11:31.386804+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"278'1400","ondisk_log_start":"278'1400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8198,"log_dups_size":0,"ondisk_log_size":8198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266076160,"num_objects":4060,"num_object_clones":0,"num_object_copies":12180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":427,"num_objects_misplaced":2510,"num_objects_unfound":0,"num_objects_dirty":4060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9598,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2347,"num_bytes_recovered":153812992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1123},{"shards":"1,3,4","objects":2510}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":319,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.465421+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:11:14.465421+0000","last_peered":"2023-10-09T10:11:14.465421+0000","last_clean":"2023-10-09T10:11:14.465421+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:11:14.465421+0000","last_undegraded":"2023-10-09T10:11:14.465421+0000","last_fullsized":"2023-10-09T10:11:14.465421+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":332,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:19.616789+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:11:19.616789+0000","last_peered":"2023-10-09T10:11:19.616789+0000","last_clean":"2023-10-09T10:11:19.616789+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:11:19.616789+0000","last_undegraded":"2023-10-09T10:11:19.616789+0000","last_fullsized":"2023-10-09T10:11:19.616789+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":170,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567061+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:11:22.567061+0000","last_peered":"2023-10-09T10:11:22.567061+0000","last_clean":"2023-10-09T10:11:22.567061+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:11:22.567061+0000","last_undegraded":"2023-10-09T10:11:22.567061+0000","last_fullsized":"2023-10-09T10:11:22.567061+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"301'9495","reported_seq":9535,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:31.552275+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:11:31.552275+0000","last_peered":"2023-10-09T10:11:31.552275+0000","last_clean":"2023-10-09T10:11:31.552275+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:11:31.552275+0000","last_undegraded":"2023-10-09T10:11:31.552275+0000","last_fullsized":"2023-10-09T10:11:31.552275+0000","mapping_epoch":271,"log_start":"275'100","ondisk_log_start":"275'100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9395,"log_dups_size":0,"ondisk_log_size":9395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266272768,"num_objects":4063,"num_object_clones":0,"num_object_copies":12189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9495,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":263,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.574958+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:11:22.574958+0000","last_peered":"2023-10-09T10:11:22.574958+0000","last_clean":"2023-10-09T10:11:22.574958+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:11:22.574958+0000","last_undegraded":"2023-10-09T10:11:22.574958+0000","last_fullsized":"2023-10-09T10:11:22.574958+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":276,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.473123+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:11:14.473123+0000","last_peered":"2023-10-09T10:11:14.473123+0000","last_clean":"2023-10-09T10:11:14.473123+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:11:14.473123+0000","last_undegraded":"2023-10-09T10:11:14.473123+0000","last_fullsized":"2023-10-09T10:11:14.473123+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":252,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.579817+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:11:22.579817+0000","last_peered":"2023-10-09T10:11:22.579817+0000","last_clean":"2023-10-09T10:11:22.579817+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:11:22.579817+0000","last_undegraded":"2023-10-09T10:11:22.579817+0000","last_fullsized":"2023-10-09T10:11:22.579817+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"301'9512","reported_seq":9551,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:32.922413+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:11:32.922413+0000","last_peered":"2023-10-09T10:11:32.922413+0000","last_clean":"2023-10-09T10:11:32.922413+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:11:32.922413+0000","last_undegraded":"2023-10-09T10:11:32.922413+0000","last_fullsized":"2023-10-09T10:11:32.922413+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9512,"log_dups_size":0,"ondisk_log_size":9512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259325976,"num_objects":3958,"num_object_clones":0,"num_object_copies":11874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3958,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9512,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":208,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.469467+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:11:14.469467+0000","last_peered":"2023-10-09T10:11:14.469467+0000","last_clean":"2023-10-09T10:11:14.469467+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:11:14.469467+0000","last_undegraded":"2023-10-09T10:11:14.469467+0000","last_fullsized":"2023-10-09T10:11:14.469467+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":172,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.573573+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:11:22.573573+0000","last_peered":"2023-10-09T10:11:22.573573+0000","last_clean":"2023-10-09T10:11:22.573573+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:11:22.573573+0000","last_undegraded":"2023-10-09T10:11:22.573573+0000","last_fullsized":"2023-10-09T10:11:22.573573+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":154,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580903+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:11:22.580903+0000","last_peered":"2023-10-09T10:11:22.580903+0000","last_clean":"2023-10-09T10:11:22.580903+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:11:22.580903+0000","last_undegraded":"2023-10-09T10:11:22.580903+0000","last_fullsized":"2023-10-09T10:11:22.580903+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":166,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.566904+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:11:22.566904+0000","last_peered":"2023-10-09T10:11:22.566904+0000","last_clean":"2023-10-09T10:11:22.566904+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:11:22.566904+0000","last_undegraded":"2023-10-09T10:11:22.566904+0000","last_fullsized":"2023-10-09T10:11:22.566904+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":238,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.473130+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:11:14.473130+0000","last_peered":"2023-10-09T10:11:14.473130+0000","last_clean":"2023-10-09T10:11:14.473130+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:11:14.473130+0000","last_undegraded":"2023-10-09T10:11:14.473130+0000","last_fullsized":"2023-10-09T10:11:14.473130+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":138,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.592159+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:11:22.592159+0000","last_peered":"2023-10-09T10:11:22.592159+0000","last_clean":"2023-10-09T10:11:22.592159+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:11:22.592159+0000","last_undegraded":"2023-10-09T10:11:22.592159+0000","last_fullsized":"2023-10-09T10:11:22.592159+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572348+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:11:22.572348+0000","last_peered":"2023-10-09T10:11:22.572348+0000","last_clean":"2023-10-09T10:11:22.572348+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:11:22.572348+0000","last_undegraded":"2023-10-09T10:11:22.572348+0000","last_fullsized":"2023-10-09T10:11:22.572348+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":138,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580536+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:11:22.580536+0000","last_peered":"2023-10-09T10:11:22.580536+0000","last_clean":"2023-10-09T10:11:22.580536+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:11:22.580536+0000","last_undegraded":"2023-10-09T10:11:22.580536+0000","last_fullsized":"2023-10-09T10:11:22.580536+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":188,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571602+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:11:22.571602+0000","last_peered":"2023-10-09T10:11:22.571602+0000","last_clean":"2023-10-09T10:11:22.571602+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:11:22.571602+0000","last_undegraded":"2023-10-09T10:11:22.571602+0000","last_fullsized":"2023-10-09T10:11:22.571602+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":317,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572174+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:11:22.572174+0000","last_peered":"2023-10-09T10:11:22.572174+0000","last_clean":"2023-10-09T10:11:22.572174+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:11:22.572174+0000","last_undegraded":"2023-10-09T10:11:22.572174+0000","last_fullsized":"2023-10-09T10:11:22.572174+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":338,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571883+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:11:22.571883+0000","last_peered":"2023-10-09T10:11:22.571883+0000","last_clean":"2023-10-09T10:11:22.571883+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:11:22.571883+0000","last_undegraded":"2023-10-09T10:11:22.571883+0000","last_fullsized":"2023-10-09T10:11:22.571883+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":248,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567394+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:11:22.567394+0000","last_peered":"2023-10-09T10:11:22.567394+0000","last_clean":"2023-10-09T10:11:22.567394+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:11:22.567394+0000","last_undegraded":"2023-10-09T10:11:22.567394+0000","last_fullsized":"2023-10-09T10:11:22.567394+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":167,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:11:14.474823+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:11:14.474823+0000","last_peered":"2023-10-09T10:11:14.474823+0000","last_clean":"2023-10-09T10:11:14.474823+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:11:14.474823+0000","last_undegraded":"2023-10-09T10:11:14.474823+0000","last_fullsized":"2023-10-09T10:11:14.474823+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4232052760,"num_objects":64577,"num_object_clones":0,"num_object_copies":193731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5800,"num_objects_misplaced":2510,"num_objects_unfound":0,"num_objects_dirty":64577,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152785,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2347,"num_bytes_recovered":153812992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13081587712,"data_stored":13081575496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140185,"ondisk_log_size":140185,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430502,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1622672,"kb_used_data":1321352,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92122480,"statfs":{"total":95995035648,"available":94333419520,"internally_reserved":0,"allocated":1353064448,"data_stored":1352526133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":1.758,"15min":1.716},"min":{"1min":0.254,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":7.8419999999999996,"5min":9.0690000000000008,"15min":50.591999999999999},"last":3.516},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.6699999999999999,"15min":1.6970000000000001},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":7.9610000000000003,"5min":8.2300000000000004,"15min":50.747},"last":3.7080000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7029999999999998,"5min":2.117,"15min":1.728},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":14.311,"5min":14.311,"15min":25.338000000000001},"last":6.9859999999999998},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.2679999999999998,"15min":1.5509999999999999},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":16.526,"5min":18.27,"15min":18.27},"last":7.141}]},{"osd":2,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.752,"15min":2.2770000000000001},"min":{"1min":0.36099999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.372},{"interface":"front","average":{"1min":2.8029999999999999,"5min":1.845,"15min":2.1030000000000002},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.872,"5min":11.872,"15min":11.872},"last":7.0979999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":2.2879999999999998,"15min":1.929},"min":{"1min":0.46400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.274,"5min":29.058,"15min":29.058},"last":6.7149999999999999},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.913,"15min":1.9350000000000001},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":4.4409999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":0.96599999999999997,"15min":0.93000000000000005},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.91300000000000003,"15min":0.91000000000000003},"min":{"1min":0.28599999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.7919999999999998,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":3.7509999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.1519999999999999,"15min":1.006},"min":{"1min":0.30199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":5.7149999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":3.8650000000000002},{"interface":"front","average":{"1min":1.012,"5min":1.145,"15min":0.99299999999999999},"min":{"1min":0.38200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.5960000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":3.7650000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.93899999999999995,"15min":0.90900000000000003},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.6259999999999999,"5min":5.3129999999999997,"15min":5.4820000000000002},"last":3.7349999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.95599999999999996,"15min":0.87},"min":{"1min":0.36099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.419,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":3.492}]}]},{"osd":6,"up_from":42,"seq":180388626730,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1963840,"kb_used_data":1658424,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91781312,"statfs":{"total":95995035648,"available":93984063488,"internally_reserved":0,"allocated":1698226176,"data_stored":1697705175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.821,"15min":1.4410000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.2870000000000008,"5min":15.435,"15min":15.435},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.149,"5min":2.0859999999999999,"15min":1.4750000000000001},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":7.4139999999999997,"5min":15.308999999999999,"15min":15.308999999999999},"last":7.46}]},{"osd":1,"last update":"Mon Oct 9 10:11:04 2023","interfaces":[{"interface":"back","average":{"1min":3.032,"5min":2.7160000000000002,"15min":1.8779999999999999},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":14.888999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.3580000000000001,"15min":1.8320000000000001},"min":{"1min":0.26700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.962,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":1.4319999999999999,"15min":1.724},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.898,"5min":12.898,"15min":12.898},"last":6.2069999999999999},{"interface":"front","average":{"1min":2.569,"5min":2.141,"15min":3.0379999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5640000000000001,"5min":10.26,"15min":10.26},"last":2.4750000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.8140000000000001,"15min":1.5109999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":12.943,"5min":12.943,"15min":12.943},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.653,"5min":2.2770000000000001,"15min":2.3460000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.077,"5min":11.211,"15min":11.211},"last":6.0049999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.50700000000000001,"15min":0.46200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.60099999999999998,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1749999999999998,"5min":4.1749999999999998,"15min":4.1749999999999998},"last":0.33300000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:10:33 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.73699999999999999,"15min":0.76700000000000002},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9260000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.91900000000000004,"15min":0.79600000000000004},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":4.1529999999999996,"5min":7.508,"15min":7.508},"last":0.376}]},{"osd":7,"last update":"Mon Oct 9 10:11:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":0.91100000000000003,"15min":0.82299999999999995},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.2709999999999999,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.538,"5min":0.95499999999999996,"15min":0.872},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.3159999999999998,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.52900000000000003}]}]},{"osd":1,"up_from":84,"seq":360777253043,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2281444,"kb_used_data":1857436,"kb_used_omap":9,"kb_used_meta":423990,"kb_avail":91463708,"statfs":{"total":95995035648,"available":93658836992,"internally_reserved":0,"allocated":1902014464,"data_stored":1900986345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":434166761},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,1,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.3980000000000001,"15min":2.1800000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":7.3609999999999998,"5min":25.59,"15min":25.59},"last":12.457000000000001},{"interface":"front","average":{"1min":6.2140000000000004,"5min":3.206,"15min":2.3639999999999999},"min":{"1min":0.314,"5min":0.245,"15min":0.192},"max":{"1min":20.501999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.51900000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":3.0499999999999998,"15min":3.8149999999999999},"min":{"1min":0.27200000000000002,"5min":0.217,"15min":0.217},"max":{"1min":11.135,"5min":17.369,"15min":17.369},"last":10.573},{"interface":"front","average":{"1min":3.5550000000000002,"5min":2.7469999999999999,"15min":3.8210000000000002},"min":{"1min":0.25900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.484999999999999,"5min":16.023,"15min":16.023},"last":0.60999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:10:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":3.4540000000000002,"15min":3.5070000000000001},"min":{"1min":0.39500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.587,"5min":18.587,"15min":18.587},"last":0.79200000000000004},{"interface":"front","average":{"1min":4.6959999999999997,"5min":4.2249999999999996,"15min":4.6760000000000002},"min":{"1min":1.0329999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.022,"5min":17.289000000000001,"15min":17.289000000000001},"last":4.9560000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.474,"15min":1.3280000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2629999999999999,"5min":7.2629999999999999,"15min":7.2629999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":2.879,"5min":1.639,"15min":1.4330000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.6940000000000008,"5min":10.132,"15min":10.132},"last":0.54700000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":2.3519999999999999,"15min":1.9330000000000001},"min":{"1min":0.41999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.811,"5min":14.416,"15min":15.531000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.742,"5min":2.1859999999999999,"15min":2.1080000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.9039999999999999,"5min":16.277999999999999,"15min":19.706},"last":10.855}]},{"osd":6,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.9450000000000001,"15min":1.825},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":7.3049999999999997,"5min":14.215,"15min":16.193999999999999},"last":0.625},{"interface":"front","average":{"1min":4.1509999999999998,"5min":2.343,"15min":1.984},"min":{"1min":0.46200000000000002,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.868,"5min":14.471,"15min":15.481999999999999},"last":0.57599999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.306,"15min":2.1360000000000001},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.8620000000000001,"5min":16.331,"15min":16.331},"last":10.723000000000001},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.0489999999999999,"15min":1.901},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3339999999999996,"5min":14.242000000000001,"15min":16.100999999999999},"last":0.65700000000000003}]}]},{"osd":0,"up_from":79,"seq":339302416570,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1931928,"kb_used_data":1547088,"kb_used_omap":28,"kb_used_meta":384803,"kb_avail":91813224,"statfs":{"total":95995035648,"available":94016741376,"internally_reserved":0,"allocated":1584218112,"data_stored":1583207451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":394039120},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:11:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":2.0259999999999998,"15min":1.6559999999999999},"min":{"1min":0.216,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":12.975,"5min":12.975,"15min":13.084},"last":16.797000000000001},{"interface":"front","average":{"1min":5.8680000000000003,"5min":3.653,"15min":2.5920000000000001},"min":{"1min":0.29099999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":22.934999999999999,"5min":22.934999999999999,"15min":22.934999999999999},"last":1.1599999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8360000000000001,"15min":2.1589999999999998},"min":{"1min":0.34599999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.9949999999999992,"5min":10.034000000000001,"15min":10.034000000000001},"last":3.4420000000000002},{"interface":"front","average":{"1min":2.9340000000000002,"5min":2.177,"15min":3.0539999999999998},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":15.529,"5min":15.529,"15min":15.529},"last":7.5389999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":4.5259999999999998,"15min":4.7519999999999998},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":20.419,"5min":25.734999999999999,"15min":25.734999999999999},"last":1.071},{"interface":"front","average":{"1min":3.742,"5min":3.891,"15min":4.2370000000000001},"min":{"1min":0.40999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":10.154,"5min":21.923999999999999,"15min":21.923999999999999},"last":6.5499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.911,"15min":1.5649999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":1.133},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.0459999999999998,"15min":1.9730000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.6720000000000006,"5min":13.789999999999999,"15min":13.789999999999999},"last":4.532}]},{"osd":5,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.7309999999999999,"15min":1.9630000000000001},"min":{"1min":0.40899999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.6229999999999993,"5min":17.096,"15min":17.096},"last":5.5540000000000003},{"interface":"front","average":{"1min":1.5,"5min":1.454,"15min":1.3029999999999999},"min":{"1min":0.30499999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.4260000000000002,"5min":9.4260000000000002,"15min":12.478},"last":7.6029999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.7010000000000001,"15min":1.4590000000000001},"min":{"1min":0.33700000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":9.4719999999999995,"5min":9.7260000000000009,"15min":11.699},"last":3.508},{"interface":"front","average":{"1min":4.4729999999999999,"5min":2.6320000000000001,"15min":2.2069999999999999},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.466000000000001,"5min":20.466000000000001,"15min":20.466000000000001},"last":1.107}]},{"osd":7,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8239999999999998,"15min":2.1139999999999999},"min":{"1min":0.46400000000000002,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":9.7050000000000001,"5min":14.631,"15min":26.978000000000002},"last":4.7430000000000003},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.7190000000000001,"15min":1.605},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":9.5020000000000007,"5min":11.166,"15min":26.768000000000001},"last":3.556}]}]},{"osd":2,"up_from":217,"seq":932007903303,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1482380,"kb_used_data":1251588,"kb_used_omap":83,"kb_used_meta":230700,"kb_avail":92262772,"statfs":{"total":95995035648,"available":94477078528,"internally_reserved":0,"allocated":1281626112,"data_stored":1280620136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":236237295},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":1.732,"15min":1.756},"min":{"1min":0.217,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":19.372,"5min":19.372,"15min":19.372},"last":5.1260000000000003},{"interface":"front","average":{"1min":3.4420000000000002,"5min":2.9849999999999999,"15min":2.9569999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.754999999999999,"5min":16.754999999999999,"15min":16.754999999999999},"last":0.40699999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":2.258,"15min":2.544},"min":{"1min":0.254,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.797000000000001,"5min":13.797000000000001,"15min":13.797000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.8399999999999999,"5min":2.5209999999999999,"15min":2.6379999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.4119999999999999,"5min":15.569000000000001,"15min":15.569000000000001},"last":0.88}]},{"osd":3,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":2.198,"15min":2.0819999999999999},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.851000000000001,"5min":13.851000000000001,"15min":13.851000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":2.9860000000000002,"5min":3.181,"15min":3.6699999999999999},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.7240000000000002,"5min":15.811,"15min":15.811},"last":0.91100000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.84499999999999997,"15min":0.753},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.29,"5min":6.29,"15min":6.29},"last":0.83199999999999996},{"interface":"front","average":{"1min":2.5609999999999999,"5min":1.1759999999999999,"15min":0.94499999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":12.385,"5min":12.385,"15min":12.385},"last":2.4319999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.6919999999999999,"15min":1.546},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":12.473000000000001,"5min":12.473000000000001,"15min":12.473000000000001},"last":2.3879999999999999},{"interface":"front","average":{"1min":3.0699999999999998,"5min":1.5409999999999999,"15min":1.6870000000000001},"min":{"1min":0.372,"5min":0.318,"15min":0.318},"max":{"1min":13.587,"5min":13.587,"15min":13.587},"last":4.7510000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.3999999999999999,"15min":1.095},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2599999999999998,"5min":15.292999999999999,"15min":15.292999999999999},"last":0.79000000000000004},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.738,"15min":1.742},"min":{"1min":0.36599999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":4.484}]},{"osd":7,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.431,"15min":1.1259999999999999},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.2190000000000003,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.859,"15min":1.6599999999999999},"min":{"1min":0.41699999999999998,"5min":0.376,"15min":0.376},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":12.433999999999999},"last":4.4400000000000004}]}]},{"osd":3,"up_from":250,"seq":1073741824047,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810396,"kb_used_data":1564244,"kb_used_omap":10,"kb_used_meta":246133,"kb_avail":91934756,"statfs":{"total":95995035648,"available":94141190144,"internally_reserved":0,"allocated":1601785856,"data_stored":1600596193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":252040259},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":4.2709999999999999,"15min":4.5590000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.753,"5min":23.381,"15min":23.381},"last":1.615},{"interface":"front","average":{"1min":6.0720000000000001,"5min":4.0919999999999996,"15min":4.1500000000000004},"min":{"1min":0.42099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":20.239999999999998,"5min":20.239999999999998,"15min":20.239999999999998},"last":1.1619999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.6150000000000002,"15min":3.8410000000000002},"min":{"1min":0.26300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":20.422999999999998,"5min":20.422999999999998,"15min":20.422999999999998},"last":4.8680000000000003},{"interface":"front","average":{"1min":4.5039999999999996,"5min":3.4990000000000001,"15min":3.4529999999999998},"min":{"1min":0.33100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.704000000000001,"5min":23.423999999999999,"15min":23.423999999999999},"last":1.6539999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1369999999999996,"5min":3.4199999999999999,"15min":3.6339999999999999},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":20.594999999999999,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.24099999999999999},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.8809999999999998,"15min":2.8700000000000001},"min":{"1min":0.34200000000000003,"5min":0.253,"15min":0.253},"max":{"1min":15.73,"5min":23.452000000000002,"15min":23.452000000000002},"last":1.7490000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":1.6719999999999999,"15min":1.288},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.676,"5min":15.676,"15min":15.676},"last":1.6839999999999999},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5800000000000001,"15min":1.534},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.426,"5min":12.426,"15min":12.426},"last":1.024}]},{"osd":5,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8290000000000002,"5min":2.4750000000000001,"15min":2.5670000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":12.388,"5min":12.388,"15min":12.388},"last":1.0609999999999999},{"interface":"front","average":{"1min":3.5699999999999998,"5min":2.5169999999999999,"15min":2.4079999999999999},"min":{"1min":0.443,"5min":0.309,"15min":0.309},"max":{"1min":15.992000000000001,"5min":15.992000000000001,"15min":15.992000000000001},"last":4.7910000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":2.536,"15min":2.6709999999999998},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.468,"5min":12.468,"15min":12.468},"last":1.085},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.5609999999999999,"15min":2.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":16.16,"5min":16.16,"15min":16.16},"last":4.8360000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.825,"15min":1.534},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.635999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":1.712},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.4399999999999999,"15min":2.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":1.105}]}]},{"osd":4,"up_from":266,"seq":1142461300773,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1940236,"kb_used_data":1609284,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":91804916,"statfs":{"total":95995035648,"available":94008233984,"internally_reserved":0,"allocated":1647906816,"data_stored":1647379913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.98499999999999999,"15min":0.84099999999999997},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.881,"5min":10.881,"15min":10.881},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.923,"5min":1.1519999999999999,"15min":1.024},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":0.27900000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":1.169,"15min":0.93400000000000005},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":10.93,"5min":10.93,"15min":10.93},"last":1.929},{"interface":"front","average":{"1min":2.633,"5min":1.546,"15min":1.365},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.141,"5min":11.141,"15min":11.141},"last":0.82599999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":1.2310000000000001,"15min":1.073},"min":{"1min":0.32100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.1189999999999998,"5min":7.1189999999999998,"15min":7.1189999999999998},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.587,"5min":1.042,"15min":0.95099999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.1709999999999994,"5min":8.1709999999999994,"15min":8.1709999999999994},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.4610000000000001,"15min":1.3280000000000001},"min":{"1min":0.40699999999999997,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5530000000000008,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.4420000000000002,"5min":1.2989999999999999,"15min":1.109},"min":{"1min":0.373,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.6069999999999993,"5min":9.6069999999999993,"15min":9.6069999999999993},"last":1.913}]},{"osd":5,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.80400000000000005,"15min":0.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4740000000000002,"5min":5.4740000000000002,"15min":5.4740000000000002},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.60399999999999998,"15min":0.63400000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.40300000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.60499999999999998,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":5.5709999999999997,"5min":5.5709999999999997,"15min":5.5709999999999997},"last":0.437},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.60999999999999999,"15min":0.59199999999999997},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.319,"5min":2.319,"15min":2.319},"last":1.875}]},{"osd":7,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.61299999999999999,"15min":0.61799999999999999},"min":{"1min":0.33700000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.3460000000000001,"5min":2.3460000000000001,"15min":2.3460000000000001},"last":1.8959999999999999},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.82299999999999995,"15min":0.82699999999999996},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.0680000000000001,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.70899999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822958,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2339328,"kb_used_data":2038008,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91405824,"statfs":{"total":95995035648,"available":93599563776,"internally_reserved":0,"allocated":2086920192,"data_stored":2086401345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":1.55,"15min":1.488},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.8540000000000001,"5min":7.8540000000000001,"15min":15.773999999999999},"last":2.3530000000000002},{"interface":"front","average":{"1min":1.339,"5min":1.597,"15min":1.532},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.0629999999999997,"5min":11.747,"15min":15.699},"last":0.248}]},{"osd":1,"last update":"Mon Oct 9 10:11:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":1.748,"15min":1.5629999999999999},"min":{"1min":0.40300000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.4250000000000007,"5min":11.335000000000001,"15min":15.689},"last":2.6859999999999999},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.903,"15min":1.7110000000000001},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5979999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":5.5880000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":1.7090000000000001,"15min":2.391},"min":{"1min":0.34899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7249999999999996,"5min":15.640000000000001,"15min":15.640000000000001},"last":2.2000000000000002},{"interface":"front","average":{"1min":1.663,"5min":1.6479999999999999,"15min":2.2370000000000001},"min":{"1min":0.40400000000000003,"5min":0.313,"15min":0.313},"max":{"1min":5.2789999999999999,"5min":10.656000000000001,"15min":10.656000000000001},"last":0.42699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:10:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.8340000000000001,"15min":1.8440000000000001},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6660000000000004,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":2.875},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.4380000000000002,"15min":2.8250000000000002},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.085,"5min":17.34,"15min":17.34},"last":5.5099999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:35 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":0.71699999999999997,"15min":0.61499999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5220000000000002,"5min":5.5220000000000002,"15min":5.5220000000000002},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.752,"15min":0.746},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.823,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.313}]},{"osd":6,"last update":"Mon Oct 9 10:10:32 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.014,"15min":0.94499999999999995},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.25},"max":{"1min":5.4930000000000003,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.6299999999999999,"5min":1.0189999999999999,"15min":0.88700000000000001},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.8099999999999996,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.436}]},{"osd":7,"last update":"Mon Oct 9 10:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":1.0900000000000001,"15min":0.97099999999999997},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.891,"5min":7.891,"15min":10.334},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.72299999999999998,"15min":0.93300000000000005},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.3630000000000004,"5min":6.3630000000000004,"15min":12.356},"last":0.38600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1575092224,"data_stored":1575092224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1887830016,"data_stored":1887830016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1271533568,"data_stored":1271529496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1589772288,"data_stored":1589772288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1637023744,"data_stored":1637023744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2078216192,"data_stored":2078212120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1694826496,"data_stored":1694826496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1347293184,"data_stored":1347289112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:11:35.276 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:35.323 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:35.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:35.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:35.399 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:35 smithi012 ceph-mon[147926]: pgmap v816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5800/193803 objects degraded (2.993%); 2510/193803 objects misplaced (1.295%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:11:35.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:35 smithi012 ceph-mon[152263]: pgmap v816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5800/193803 objects degraded (2.993%); 2510/193803 objects misplaced (1.295%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:11:35.427 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:35.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:35 smithi110 ceph-mon[136032]: pgmap v816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5800/193803 objects degraded (2.993%); 2510/193803 objects misplaced (1.295%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:11:35.498 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:35.554 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:35.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:35.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:35.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:35.753 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:11:35.793 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:11:35.841 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:11:35.899 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:35.901 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:11:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:35.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:35.934 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:11:35.979 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:11:36.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:36.015 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:36.019 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:11:36.061 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:11:36.086 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:36.089 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:11:36.089 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:11:36.089 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:11:36.090 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:11:36.090 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:11:36.090 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:11:36.090 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:11:36.090 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:11:36.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:36 smithi012 ceph-mon[147926]: from='client.55783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:36.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2141410726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:36.328 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:36 smithi012 ceph-mon[152263]: from='client.55783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:36.328 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2141410726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:36.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:36 smithi110 ceph-mon[136032]: from='client.55783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2141410726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:36.553 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:36.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:36.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:36.659 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:36.741 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:36.824 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:36.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:36.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:36.930 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:11:36.958 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:37.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:37.059 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:11:37.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:11:37.160 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:37.194 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:37.249 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:37.250 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:37 smithi110 ceph-mon[136032]: pgmap v817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 5800/190779 objects degraded (3.040%); 2510/190779 objects misplaced (1.316%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:11:37.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:37.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:37.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:37.454 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:37.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:37 smithi012 ceph-mon[147926]: pgmap v817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 5800/190779 objects degraded (3.040%); 2510/190779 objects misplaced (1.316%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:11:37.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:37 smithi012 ceph-mon[152263]: pgmap v817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 5800/190779 objects degraded (3.040%); 2510/190779 objects misplaced (1.316%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:11:37.996 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:38.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:38.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:38.165 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:38.180 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:11:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:11:37] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T10:11:38.210 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:38.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:38.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:38.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:11:38.318 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:38.358 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:38.406 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:38.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5800/190779 objects degraded (3.040%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:38.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:38.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:38.552 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:38.593 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:38.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:38.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:38.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:11:38.668 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5800/190779 objects degraded (3.040%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:38.669 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5800/190779 objects degraded (3.040%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:38.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:38.822 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:39.079 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:39.103 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:11:39.103 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190290 7.1 7.69dfa251 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944018+0000", 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14242595299999, 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93132975699999998, 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.104 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190290 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944018+0000", 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944018+0000", 2023-10-09T10:11:39.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944018+0000", 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944020+0000", 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9e-06 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944025+0000", 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9930000000000003e-06 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944029+0000", 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4569999999999998e-06 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944394+0000", 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036461600000000001 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944400+0000", 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6969999999999998e-06 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875339+0000", 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93093981999999997 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875348+0000", 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5730000000000008e-06 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190305 7.1 7.77d8d2f1 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944033+0000", 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.142411237, 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93134264200000005, 2023-10-09T10:11:39.109 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190305 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944033+0000", 2023-10-09T10:11:39.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944033+0000", 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944033+0000", 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944034+0000", 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.767e-06 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944040+0000", 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1379999999999998e-06 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944051+0000", 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1425e-05 2023-10-09T10:11:39.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944417+0000", 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036612400000000001 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944422+0000", 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2719999999999997e-06 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875367+0000", 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93094488799999997 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875376+0000", 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2849999999999995e-06 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.114 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190320 7.1 7.e502e1f1 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944056+0000", 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14238850800001, 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93134817800000003, 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190320 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944056+0000", 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944056+0000", 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944055+0000", 2023-10-09T10:11:39.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944058+0000", 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2180000000000002e-06 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944063+0000", 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2939999999999996e-06 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944068+0000", 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8500000000000002e-06 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944442+0000", 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037445300000000001 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944448+0000", 2023-10-09T10:11:39.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3319999999999999e-06 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875395+0000", 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93094789200000005 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875404+0000", 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4249999999999998e-06 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.119 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190328 7.1 7.7b1f0a21 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944071+0000", 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14237285499999, 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93135938699999998, 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.120 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190328 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944071+0000", 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944071+0000", 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944071+0000", 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944073+0000", 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0640000000000001e-06 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944078+0000", 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8679999999999998e-06 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944083+0000", 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7480000000000001e-06 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944466+0000", 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038282300000000002 2023-10-09T10:11:39.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944471+0000", 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5860000000000004e-06 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875422+0000", 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93095124200000001 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875431+0000", 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3489999999999992e-06 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190432 7.1 7.bd3c2cb1 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944087+0000", 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.142357686, 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93137576099999997, 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.125 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190432 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944087+0000", 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944087+0000", 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944086+0000", 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944095+0000", 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1279999999999997e-06 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944101+0000", 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6269999999999997e-06 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944106+0000", 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6720000000000003e-06 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944490+0000", 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038417699999999998 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944495+0000", 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5940000000000001e-06 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875454+0000", 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93095862699999998 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875462+0000", 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2230000000000007e-06 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.130 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190433 7.1 7.43f72671 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944109+0000", 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14233473799999, 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93138109899999999, 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190433 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944109+0000", 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944109+0000", 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944109+0000", 2023-10-09T10:11:39.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944111+0000", 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6789999999999999e-06 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944116+0000", 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1599999999999997e-06 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944121+0000", 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5900000000000001e-06 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944514+0000", 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039366999999999999 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944519+0000", 2023-10-09T10:11:39.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0989999999999998e-06 2023-10-09T10:11:39.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875482+0000", 2023-10-09T10:11:39.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93096299100000002 2023-10-09T10:11:39.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875491+0000", 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1980000000000001e-06 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190435 7.1 7.a6facff1 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944124+0000", 2023-10-09T10:11:39.137 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.142319958, 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93139482200000001, 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190435 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944124+0000", 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944124+0000", 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944124+0000", 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944126+0000", 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7010000000000001e-06 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944131+0000", 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3140000000000003e-06 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944136+0000", 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0150000000000003e-06 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944537+0000", 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040121800000000001 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944542+0000", 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2440000000000001e-06 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875511+0000", 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.930968452 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875519+0000", 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1480000000000006e-06 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190446 7.1 7.785af3e1 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944140+0000", 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14230431799999, 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.931407384, 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190446 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.143 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944140+0000", 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944140+0000", 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944140+0000", 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944145+0000", 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5180000000000004e-06 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944151+0000", 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6969999999999998e-06 2023-10-09T10:11:39.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944155+0000", 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2660000000000003e-06 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944560+0000", 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040515700000000001 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944566+0000", 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3600000000000004e-06 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875539+0000", 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93097330700000003 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875547+0000", 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3650000000000004e-06 2023-10-09T10:11:39.147 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190476 7.1 7.ef16d671 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944159+0000", 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14228488800001, 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93141608300000001, 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190476 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944159+0000", 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944159+0000", 2023-10-09T10:11:39.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944159+0000", 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944161+0000", 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.809e-06 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944166+0000", 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2299999999999999e-06 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944170+0000", 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1189999999999999e-06 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944583+0000", 2023-10-09T10:11:39.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041306900000000001 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944589+0000", 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2410000000000003e-06 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875566+0000", 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.930977159 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875575+0000", 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7550000000000006e-06 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190498 7.1 7.bf93e1d1 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944174+0000", 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14227057799999, 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93143571300000005, 2023-10-09T10:11:39.153 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190498 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944174+0000", 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944174+0000", 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944173+0000", 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944175+0000", 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8840000000000001e-06 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944180+0000", 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0200000000000002e-06 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944185+0000", 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2989999999999998e-06 2023-10-09T10:11:39.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944606+0000", 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042150899999999998 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944611+0000", 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3249999999999998e-06 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875598+0000", 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.930986702 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875609+0000", 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1233000000000001e-05 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.158 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190516 7.1 7.99eacfc1 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944188+0000", 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14225617700001, 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93144974800000002, 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190516 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944188+0000", 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944188+0000", 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944188+0000", 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944190+0000", 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9259999999999999e-06 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944195+0000", 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.48e-06 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944200+0000", 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5680000000000001e-06 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944629+0000", 2023-10-09T10:11:39.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042957999999999999 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944635+0000", 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3059999999999997e-06 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875629+0000", 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93099482200000006 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875638+0000", 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3489999999999992e-06 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:190517 7.1 7.84306481 (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:41.944204+0000", 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 237.14223976400001, 2023-10-09T10:11:39.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93146199299999999, 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 190517 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944204+0000", 2023-10-09T10:11:39.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944204+0000", 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944204+0000", 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944206+0000", 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7880000000000001e-06 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944211+0000", 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2700000000000004e-06 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944216+0000", 2023-10-09T10:11:39.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4220000000000002e-06 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944652+0000", 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043648100000000001 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:41.944657+0000", 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2490000000000001e-06 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875656+0000", 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.93099900199999996 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:42.875666+0000", 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0084e-05 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:84973 8.f 8:f4bd00a8:::benchmark_data_smithi012_271002_object84972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e286)", 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:09:45.993208+0000", 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 113.093235865, 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4152056869999998, 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 84973 2023-10-09T10:11:39.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993208+0000", 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993208+0000", 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993208+0000", 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993236+0000", 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8710000000000001e-05 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993246+0000", 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5599999999999999e-06 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993254+0000", 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7500000000000003e-06 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993511+0000", 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00025703 2023-10-09T10:11:39.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993627+0000", 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000115876 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:45.993758+0000", 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013131500000000001 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.000420+0000", 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066615759999999998 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.000437+0000", 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7323999999999999e-05 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.997961+0000", 2023-10-09T10:11:39.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99752421499999999 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.997979+0000", 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7433999999999999e-05 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.408301+0000", 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.410322882 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.408414+0000", 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112625 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:85012 8.f 8:ff65f247:::benchmark_data_smithi012_271002_object85011:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:09:46.051278+0000", 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 113.03516574299999, 2023-10-09T10:11:39.177 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3811075690000001, 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85012 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.051278+0000", 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.051278+0000", 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.051277+0000", 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.051303+0000", 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6349000000000001e-05 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.051321+0000", 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7606999999999999e-05 2023-10-09T10:11:39.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.051329+0000", 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4300000000000006e-06 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.051423+0000", 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4412000000000003e-05 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.051435+0000", 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1823e-05 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998007+0000", 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94657208500000001 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998029+0000", 2023-10-09T10:11:39.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1620999999999998e-05 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.410874+0000", 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.412845167 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.411176+0000", 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00030159300000000001 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.411325+0000", 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000148985 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.420996+0000", 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0096710159999999993 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.427709+0000", 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067128480000000004 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.427723+0000", 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4056000000000001e-05 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.427785+0000", 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2398999999999997e-05 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432386+0000", 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046010410000000002 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:85014 8.f 8:f02b8c01:::benchmark_data_smithi012_271002_object85013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:09:46.053721+0000", 2023-10-09T10:11:39.186 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 113.03272328, 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3904110460000001, 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85014 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.053721+0000", 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.053721+0000", 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.053719+0000", 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.053748+0000", 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8552999999999999e-05 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.053759+0000", 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1426e-05 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.053767+0000", 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8110000000000003e-06 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.053841+0000", 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4182000000000002e-05 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.053853+0000", 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1627000000000001e-05 2023-10-09T10:11:39.190 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998048+0000", 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94419483500000001 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998055+0000", 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9859999999999997e-06 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.414305+0000", 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4162499829999999 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.414434+0000", 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012908200000000001 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.414563+0000", 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012949700000000001 2023-10-09T10:11:39.192 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432452+0000", 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017888257000000001 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432511+0000", 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9024000000000003e-05 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.193 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.439662+0000", 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071514530000000003 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.439721+0000", 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8335000000000003e-05 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.444132+0000", 2023-10-09T10:11:39.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044114760000000001 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:85036 8.f 8:f655735c:::benchmark_data_smithi012_271002_object85035:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:09:46.078654+0000", 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 113.007790149, 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3702127100000001, 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:39.195 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85036 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.078654+0000", 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.196 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.078654+0000", 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.078653+0000", 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.078678+0000", 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5054000000000001e-05 2023-10-09T10:11:39.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.078688+0000", 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9550000000000004e-06 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.078695+0000", 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4139999999999997e-06 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.198 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.078756+0000", 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0600999999999998e-05 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.078767+0000", 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1664e-05 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998074+0000", 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.91930672999999996 2023-10-09T10:11:39.199 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998081+0000", 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6089999999999997e-06 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.417560+0000", 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4194797270000001 2023-10-09T10:11:39.200 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.417706+0000", 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014530399999999999 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.417879+0000", 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017359499999999999 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432496+0000", 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014617005000000001 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432521+0000", 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4332999999999999e-05 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.444215+0000", 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011694072999999999 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.444267+0000", 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2290999999999998e-05 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.448867+0000", 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045998039999999999 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.203 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:85044 8.f 8:fdf257b8:::benchmark_data_smithi012_271002_object85043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:09:46.090101+0000", 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 112.99634289799999, 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3635246309999998, 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:39.204 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85044 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.090101+0000", 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.090101+0000", 2023-10-09T10:11:39.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.090100+0000", 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.090127+0000", 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7399e-05 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.090146+0000", 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9032999999999999e-05 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.090160+0000", 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3920999999999999e-05 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.207 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.090220+0000", 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0189999999999998e-05 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.090232+0000", 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1838000000000001e-05 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998099+0000", 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.90786671200000002 2023-10-09T10:11:39.208 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998105+0000", 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4760000000000003e-06 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.421026+0000", 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4229202359999999 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.209 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.421183+0000", 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000157737 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.421362+0000", 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017838700000000001 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.210 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432532+0000", 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011170225000000001 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432589+0000", 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6898000000000003e-05 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.448925+0000", 2023-10-09T10:11:39.211 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016336639 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.448989+0000", 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3238000000000007e-05 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.453626+0000", 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046372439999999996 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.212 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:85055 8.f 8:f93d92e5:::benchmark_data_smithi012_271002_object85054:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:09:46.104491+0000", 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 112.981953398, 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3539993570000002, 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:39.213 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85055 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.104491+0000", 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.104491+0000", 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.214 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.104489+0000", 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.104518+0000", 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9570999999999999e-05 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.215 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.104549+0000", 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0411000000000001e-05 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.104560+0000", 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1606999999999999e-05 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.216 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.104631+0000", 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0611000000000005e-05 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.104643+0000", 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2247e-05 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998123+0000", 2023-10-09T10:11:39.217 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.893479366 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998129+0000", 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6590000000000001e-06 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.424273+0000", 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.426143597 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.218 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.424468+0000", 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019531799999999999 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.424669+0000", 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020094000000000001 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:39.219 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432544+0000", 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0078749949999999992 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432736+0000", 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019178700000000001 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.453862+0000", 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021126431000000001 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.220 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.453930+0000", 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7974999999999999e-05 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.458490+0000", 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004559853 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.221 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:85070 8.f 8:f5f89b6d:::benchmark_data_smithi012_271002_object85069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:09:46.143706+0000", 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 112.942738564, 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.319572215, 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85070 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.222 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.143706+0000", 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.143706+0000", 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.223 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.143703+0000", 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.143741+0000", 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7564000000000002e-05 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.143767+0000", 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6695000000000001e-05 2023-10-09T10:11:39.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.143782+0000", 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5012e-05 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.143868+0000", 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.53e-05 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.225 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.143882+0000", 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4687e-05 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998148+0000", 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85426575800000004 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998155+0000", 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0910000000000004e-06 2023-10-09T10:11:39.226 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432762+0000", 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.434606321 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.432915+0000", 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015333 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:39.227 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.433091+0000", 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017573299999999999 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.453700+0000", 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020608956000000001 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.453718+0000", 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8746e-05 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.228 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.458566+0000", 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048471699999999996 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.458638+0000", 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2471999999999993e-05 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.463278+0000", 2023-10-09T10:11:39.229 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046399010000000001 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:85075 8.f 8:f6aa2a9a:::benchmark_data_smithi012_271002_object85074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:09:46.153628+0000", 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 112.932816458, 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3141922579999998, 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:11:39.230 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 85075 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.153628+0000", 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.231 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.153628+0000", 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.153626+0000", 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.153653+0000", 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6981000000000001e-05 2023-10-09T10:11:39.232 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.153672+0000", 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8929000000000001e-05 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.153680+0000", 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4640000000000006e-06 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.233 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.153762+0000", 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1258e-05 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.153779+0000", 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7272999999999999e-05 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998174+0000", 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84439531099999998 2023-10-09T10:11:39.234 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:46.998182+0000", 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6590000000000007e-06 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.436291+0000", 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.438109509 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.235 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.436446+0000", 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015474099999999999 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.436620+0000", 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017419899999999999 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.453774+0000", 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017153442000000001 2023-10-09T10:11:39.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.453792+0000", 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8181000000000001e-05 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.463345+0000", 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095527570000000003 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:11:39.237 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.463415+0000", 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0580999999999994e-05 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:09:48.467820+0000", 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044047469999999997 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:39.238 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:11:39.239 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:39.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:39.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:39.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:39.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:39.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:39 smithi012 ceph-mon[147926]: pgmap v818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 5848/186699 objects degraded (3.132%); 2313/186699 objects misplaced (1.239%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:39 smithi012 ceph-mon[152263]: pgmap v818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 5848/186699 objects degraded (3.132%); 2313/186699 objects misplaced (1.239%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:39.477 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:39.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:39.536 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:39 smithi110 ceph-mon[136032]: pgmap v818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 5848/186699 objects degraded (3.132%); 2313/186699 objects misplaced (1.239%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:39.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:39.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:39.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:39.688 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:39.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:11:39.748 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:39.841 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:39.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:39.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:39.941 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:39.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:40.011 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:40.154 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:40.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:40.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:40.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:40.273 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:11:40.344 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:40.376 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:40 smithi012 ceph-mon[147926]: pgmap v819: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5848/184434 objects degraded (3.171%); 2313/184434 objects misplaced (1.254%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:40.376 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1636286435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:40.376 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:40.376 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:40 smithi012 ceph-mon[152263]: pgmap v819: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5848/184434 objects degraded (3.171%); 2313/184434 objects misplaced (1.254%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:40.377 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1636286435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:40.377 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:40.423 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:40.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:40.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:40.445 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:11:40.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:40.623 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:40.648 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:40 smithi110 ceph-mon[136032]: pgmap v819: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5848/184434 objects degraded (3.171%); 2313/184434 objects misplaced (1.254%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:40.648 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1636286435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:40.648 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:40.704 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:40.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:40.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:40.813 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:40.865 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:41.175 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:41.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:41.250 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:41 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:41.251 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:41 smithi110 ceph-mon[136032]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.251 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:41 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:41.251 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:41 smithi110 ceph-mon[136032]: osdmap e302: 8 total, 8 up, 8 in 2023-10-09T10:11:41.251 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:41 smithi110 ceph-mon[136032]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.251 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:41 smithi110 ceph-mon[136032]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:41.381 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:41.381 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[152263]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.381 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:41.381 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[152263]: osdmap e302: 8 total, 8 up, 8 in 2023-10-09T10:11:41.381 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[152263]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.382 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[152263]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[147926]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[147926]: osdmap e302: 8 total, 8 up, 8 in 2023-10-09T10:11:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[147926]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.383 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:41 smithi012 ceph-mon[147926]: failed to encode map e302 with expected crc 2023-10-09T10:11:41.392 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:41.485 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:41.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:41.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:41.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:41.662 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:41.768 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:11:41.768 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:11:41.769 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:11:41.769 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:11:41.793 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:41.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:41.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:41.981 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:42.086 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:42.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:42.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:42.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:42.227 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:11:42.239 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:11:42.269 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:42 smithi012 ceph-mon[147926]: pgmap v821: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5848/181581 objects degraded (3.221%); 2313/181581 objects misplaced (1.274%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:42.269 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/395938760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:42.300 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:42.320 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:11:42.362 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:11:42.390 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:11:42.396 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:42.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:42.414 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:11:42.424 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:11:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:42 smithi110 ceph-mon[136032]: pgmap v821: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5848/181581 objects degraded (3.221%); 2313/181581 objects misplaced (1.274%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/395938760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:42.454 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:11:42.504 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:11:42.507 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:42.557 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:11:42.564 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:42.574 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:42 smithi012 ceph-mon[152263]: pgmap v821: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5848/181581 objects degraded (3.221%); 2313/181581 objects misplaced (1.274%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:11:42.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/395938760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:42.592 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:11:42.605 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:42.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:42.624 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:11:42.624 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:11:42.624 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:11:42.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:11:42.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:11:42.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:11:42.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:11:42.625 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:11:42.710 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:42.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:11:42.783 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:42.983 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:11:43.109 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:43.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:43.187 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":821,"stamp":"2023-10-09T10:11:41.381120+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3965654968,"num_objects":60527,"num_object_clones":0,"num_object_copies":181581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5848,"num_objects_misplaced":2313,"num_objects_unfound":0,"num_objects_dirty":60527,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":157119,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2565,"num_bytes_recovered":164768460,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141153,"ondisk_log_size":141153,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14575080,"kb_used_data":12023976,"kb_used_omap":167,"kb_used_meta":2550680,"kb_avail":735386136,"statfs":{"total":767960285184,"available":753035403264,"internally_reserved":0,"allocated":12312551424,"data_stored":12306203457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2611896921},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,3,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-404750336,"num_objects":-6176,"num_object_clones":0,"num_object_copies":-18528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":99,"num_objects_misplaced":-385,"num_objects_unfound":0,"num_objects_dirty":-6176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6176,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":18743296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006838"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":143,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.587736+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:11:22.587736+0000","last_peered":"2023-10-09T10:11:22.587736+0000","last_clean":"2023-10-09T10:11:22.587736+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:11:22.587736+0000","last_undegraded":"2023-10-09T10:11:22.587736+0000","last_fullsized":"2023-10-09T10:11:22.587736+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":258,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571829+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:11:22.571829+0000","last_peered":"2023-10-09T10:11:22.571829+0000","last_clean":"2023-10-09T10:11:22.571829+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:11:22.571829+0000","last_undegraded":"2023-10-09T10:11:22.571829+0000","last_fullsized":"2023-10-09T10:11:22.571829+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":304,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.587385+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:11:22.587385+0000","last_peered":"2023-10-09T10:11:22.587385+0000","last_clean":"2023-10-09T10:11:22.587385+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:11:22.587385+0000","last_undegraded":"2023-10-09T10:11:22.587385+0000","last_fullsized":"2023-10-09T10:11:22.587385+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":167,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.573725+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:11:22.573725+0000","last_peered":"2023-10-09T10:11:22.573725+0000","last_clean":"2023-10-09T10:11:22.573725+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:11:22.573725+0000","last_undegraded":"2023-10-09T10:11:22.573725+0000","last_fullsized":"2023-10-09T10:11:22.573725+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":272,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580089+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:11:22.580089+0000","last_peered":"2023-10-09T10:11:22.580089+0000","last_clean":"2023-10-09T10:11:22.580089+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:11:22.580089+0000","last_undegraded":"2023-10-09T10:11:22.580089+0000","last_fullsized":"2023-10-09T10:11:22.580089+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.583464+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:11:22.583464+0000","last_peered":"2023-10-09T10:11:22.583464+0000","last_clean":"2023-10-09T10:11:22.583464+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:11:22.583464+0000","last_undegraded":"2023-10-09T10:11:22.583464+0000","last_fullsized":"2023-10-09T10:11:22.583464+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":184,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567609+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:11:22.567609+0000","last_peered":"2023-10-09T10:11:22.567609+0000","last_clean":"2023-10-09T10:11:22.567609+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:11:22.567609+0000","last_undegraded":"2023-10-09T10:11:22.567609+0000","last_fullsized":"2023-10-09T10:11:22.567609+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.579970+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:11:22.579970+0000","last_peered":"2023-10-09T10:11:22.579970+0000","last_clean":"2023-10-09T10:11:22.579970+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:11:22.579970+0000","last_undegraded":"2023-10-09T10:11:22.579970+0000","last_fullsized":"2023-10-09T10:11:22.579970+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.584172+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:11:22.584172+0000","last_peered":"2023-10-09T10:11:22.584172+0000","last_clean":"2023-10-09T10:11:22.584172+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:11:22.584172+0000","last_undegraded":"2023-10-09T10:11:22.584172+0000","last_fullsized":"2023-10-09T10:11:22.584172+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":214,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571742+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:11:22.571742+0000","last_peered":"2023-10-09T10:11:22.571742+0000","last_clean":"2023-10-09T10:11:22.571742+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:11:22.571742+0000","last_undegraded":"2023-10-09T10:11:22.571742+0000","last_fullsized":"2023-10-09T10:11:22.571742+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":339,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571900+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:11:22.571900+0000","last_peered":"2023-10-09T10:11:22.571900+0000","last_clean":"2023-10-09T10:11:22.571900+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:11:22.571900+0000","last_undegraded":"2023-10-09T10:11:22.571900+0000","last_fullsized":"2023-10-09T10:11:22.571900+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":296,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.587606+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:11:22.587606+0000","last_peered":"2023-10-09T10:11:22.587606+0000","last_clean":"2023-10-09T10:11:22.587606+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:11:22.587606+0000","last_undegraded":"2023-10-09T10:11:22.587606+0000","last_fullsized":"2023-10-09T10:11:22.587606+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":188,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572045+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:11:22.572045+0000","last_peered":"2023-10-09T10:11:22.572045+0000","last_clean":"2023-10-09T10:11:22.572045+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:11:22.572045+0000","last_undegraded":"2023-10-09T10:11:22.572045+0000","last_fullsized":"2023-10-09T10:11:22.572045+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":168,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589780+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:11:22.589780+0000","last_peered":"2023-10-09T10:11:22.589780+0000","last_clean":"2023-10-09T10:11:22.589780+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:11:22.589780+0000","last_undegraded":"2023-10-09T10:11:22.589780+0000","last_fullsized":"2023-10-09T10:11:22.589780+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":372,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571305+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:11:22.571305+0000","last_peered":"2023-10-09T10:11:22.571305+0000","last_clean":"2023-10-09T10:11:22.571305+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:11:22.571305+0000","last_undegraded":"2023-10-09T10:11:22.571305+0000","last_fullsized":"2023-10-09T10:11:22.571305+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":240,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572934+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:11:22.572934+0000","last_peered":"2023-10-09T10:11:22.572934+0000","last_clean":"2023-10-09T10:11:22.572934+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:11:22.572934+0000","last_undegraded":"2023-10-09T10:11:22.572934+0000","last_fullsized":"2023-10-09T10:11:22.572934+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567543+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:11:22.567543+0000","last_peered":"2023-10-09T10:11:22.567543+0000","last_clean":"2023-10-09T10:11:22.567543+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:11:22.567543+0000","last_undegraded":"2023-10-09T10:11:22.567543+0000","last_fullsized":"2023-10-09T10:11:22.567543+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":134,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580673+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:11:22.580673+0000","last_peered":"2023-10-09T10:11:22.580673+0000","last_clean":"2023-10-09T10:11:22.580673+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:11:22.580673+0000","last_undegraded":"2023-10-09T10:11:22.580673+0000","last_fullsized":"2023-10-09T10:11:22.580673+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":169,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571224+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:11:22.571224+0000","last_peered":"2023-10-09T10:11:22.571224+0000","last_clean":"2023-10-09T10:11:22.571224+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:11:22.571224+0000","last_undegraded":"2023-10-09T10:11:22.571224+0000","last_fullsized":"2023-10-09T10:11:22.571224+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":166,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.573791+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:11:22.573791+0000","last_peered":"2023-10-09T10:11:22.573791+0000","last_clean":"2023-10-09T10:11:22.573791+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:11:22.573791+0000","last_undegraded":"2023-10-09T10:11:22.573791+0000","last_fullsized":"2023-10-09T10:11:22.573791+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":189,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.566784+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:11:22.566784+0000","last_peered":"2023-10-09T10:11:22.566784+0000","last_clean":"2023-10-09T10:11:22.566784+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:11:22.566784+0000","last_undegraded":"2023-10-09T10:11:22.566784+0000","last_fullsized":"2023-10-09T10:11:22.566784+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":253,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.574878+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:11:22.574878+0000","last_peered":"2023-10-09T10:11:22.574878+0000","last_clean":"2023-10-09T10:11:22.574878+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:11:22.574878+0000","last_undegraded":"2023-10-09T10:11:22.574878+0000","last_fullsized":"2023-10-09T10:11:22.574878+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"301'9928","reported_seq":9970,"reported_epoch":301,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:37.894808+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:11:37.894808+0000","last_peered":"2023-10-09T10:11:37.894808+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:11:37.894808+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9928,"log_dups_size":0,"ondisk_log_size":9928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244973568,"num_objects":3738,"num_object_clones":0,"num_object_copies":11214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9928,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-1635},{"shards":"4,5,6","objects":2278}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":333,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571761+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:11:22.571761+0000","last_peered":"2023-10-09T10:11:22.571761+0000","last_clean":"2023-10-09T10:11:22.571761+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:11:22.571761+0000","last_undegraded":"2023-10-09T10:11:22.571761+0000","last_fullsized":"2023-10-09T10:11:22.571761+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":119,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580403+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:11:22.580403+0000","last_peered":"2023-10-09T10:11:22.580403+0000","last_clean":"2023-10-09T10:11:22.580403+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:11:22.580403+0000","last_undegraded":"2023-10-09T10:11:22.580403+0000","last_fullsized":"2023-10-09T10:11:22.580403+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"301'10016","reported_seq":10054,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.040986+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:11:40.040986+0000","last_peered":"2023-10-09T10:11:40.040986+0000","last_clean":"2023-10-09T10:11:40.040986+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:11:40.040986+0000","last_undegraded":"2023-10-09T10:11:40.040986+0000","last_fullsized":"2023-10-09T10:11:40.040986+0000","mapping_epoch":271,"log_start":"277'1400","ondisk_log_start":"277'1400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247332864,"num_objects":3774,"num_object_clones":0,"num_object_copies":11322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10016,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":321,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.578278+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:11:22.578278+0000","last_peered":"2023-10-09T10:11:22.578278+0000","last_clean":"2023-10-09T10:11:22.578278+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:11:22.578278+0000","last_undegraded":"2023-10-09T10:11:22.578278+0000","last_fullsized":"2023-10-09T10:11:22.578278+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":212,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.587269+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:11:22.587269+0000","last_peered":"2023-10-09T10:11:22.587269+0000","last_clean":"2023-10-09T10:11:22.587269+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:11:22.587269+0000","last_undegraded":"2023-10-09T10:11:22.587269+0000","last_fullsized":"2023-10-09T10:11:22.587269+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"301'9700","reported_seq":9740,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:36.543273+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:11:36.543273+0000","last_peered":"2023-10-09T10:11:36.543273+0000","last_clean":"2023-10-09T10:11:36.543273+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:11:36.543273+0000","last_undegraded":"2023-10-09T10:11:36.543273+0000","last_fullsized":"2023-10-09T10:11:36.543273+0000","mapping_epoch":271,"log_start":"276'300","ondisk_log_start":"276'300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9400,"log_dups_size":0,"ondisk_log_size":9400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253362176,"num_objects":3866,"num_object_clones":0,"num_object_copies":11598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9700,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":123,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580746+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:11:22.580746+0000","last_peered":"2023-10-09T10:11:22.580746+0000","last_clean":"2023-10-09T10:11:22.580746+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:11:22.580746+0000","last_undegraded":"2023-10-09T10:11:22.580746+0000","last_fullsized":"2023-10-09T10:11:22.580746+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":144,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571218+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:11:22.571218+0000","last_peered":"2023-10-09T10:11:22.571218+0000","last_clean":"2023-10-09T10:11:22.571218+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:11:22.571218+0000","last_undegraded":"2023-10-09T10:11:22.571218+0000","last_fullsized":"2023-10-09T10:11:22.571218+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"301'9679","reported_seq":9718,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:39.657310+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:11:39.657310+0000","last_peered":"2023-10-09T10:11:39.657310+0000","last_clean":"2023-10-09T10:11:39.657310+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:11:39.657310+0000","last_undegraded":"2023-10-09T10:11:39.657310+0000","last_fullsized":"2023-10-09T10:11:39.657310+0000","mapping_epoch":271,"log_start":"279'1900","ondisk_log_start":"279'1900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7779,"log_dups_size":0,"ondisk_log_size":7779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244514816,"num_objects":3731,"num_object_clones":0,"num_object_copies":11193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9679,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.592168+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:11:22.592168+0000","last_peered":"2023-10-09T10:11:22.592168+0000","last_clean":"2023-10-09T10:11:22.592168+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:11:22.592168+0000","last_undegraded":"2023-10-09T10:11:22.592168+0000","last_fullsized":"2023-10-09T10:11:22.592168+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":168,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:30.155878+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:11:30.155878+0000","last_peered":"2023-10-09T10:11:30.155878+0000","last_clean":"2023-10-09T10:11:30.155878+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:11:30.155878+0000","last_undegraded":"2023-10-09T10:11:30.155878+0000","last_fullsized":"2023-10-09T10:11:30.155878+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"301'9895","reported_seq":9935,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:36.365407+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:11:36.365407+0000","last_peered":"2023-10-09T10:11:36.365407+0000","last_clean":"2023-10-09T10:11:36.365407+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:11:36.365407+0000","last_undegraded":"2023-10-09T10:11:36.365407+0000","last_fullsized":"2023-10-09T10:11:36.365407+0000","mapping_epoch":271,"log_start":"279'1700","ondisk_log_start":"279'1700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8195,"log_dups_size":0,"ondisk_log_size":8195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258539520,"num_objects":3945,"num_object_clones":0,"num_object_copies":11835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9895,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":294,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572432+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:11:22.572432+0000","last_peered":"2023-10-09T10:11:22.572432+0000","last_clean":"2023-10-09T10:11:22.572432+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:11:22.572432+0000","last_undegraded":"2023-10-09T10:11:22.572432+0000","last_fullsized":"2023-10-09T10:11:22.572432+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":164,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572548+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:11:22.572548+0000","last_peered":"2023-10-09T10:11:22.572548+0000","last_clean":"2023-10-09T10:11:22.572548+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:11:22.572548+0000","last_undegraded":"2023-10-09T10:11:22.572548+0000","last_fullsized":"2023-10-09T10:11:22.572548+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"301'9997","reported_seq":10037,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:39.595643+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:11:39.595643+0000","last_peered":"2023-10-09T10:11:39.595643+0000","last_clean":"2023-10-09T10:11:39.595643+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:11:39.595643+0000","last_undegraded":"2023-10-09T10:11:39.595643+0000","last_fullsized":"2023-10-09T10:11:39.595643+0000","mapping_epoch":271,"log_start":"280'2300","ondisk_log_start":"280'2300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7697,"log_dups_size":0,"ondisk_log_size":7697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242941952,"num_objects":3707,"num_object_clones":0,"num_object_copies":11121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9997,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":226,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.587534+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:11:22.587534+0000","last_peered":"2023-10-09T10:11:22.587534+0000","last_clean":"2023-10-09T10:11:22.587534+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:11:22.587534+0000","last_undegraded":"2023-10-09T10:11:22.587534+0000","last_fullsized":"2023-10-09T10:11:22.587534+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":232,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:30.146573+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:11:30.146573+0000","last_peered":"2023-10-09T10:11:30.146573+0000","last_clean":"2023-10-09T10:11:30.146573+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:11:30.146573+0000","last_undegraded":"2023-10-09T10:11:30.146573+0000","last_fullsized":"2023-10-09T10:11:30.146573+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"301'9630","reported_seq":9670,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:36.618321+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:11:36.618321+0000","last_peered":"2023-10-09T10:11:36.618321+0000","last_clean":"2023-10-09T10:11:36.618321+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:11:36.618321+0000","last_undegraded":"2023-10-09T10:11:36.618321+0000","last_fullsized":"2023-10-09T10:11:36.618321+0000","mapping_epoch":271,"log_start":"275'200","ondisk_log_start":"275'200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9430,"log_dups_size":0,"ondisk_log_size":9430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249167872,"num_objects":3802,"num_object_clones":0,"num_object_copies":11406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9630,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":195,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.570890+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:11:22.570890+0000","last_peered":"2023-10-09T10:11:22.570890+0000","last_clean":"2023-10-09T10:11:22.570890+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:11:22.570890+0000","last_undegraded":"2023-10-09T10:11:22.570890+0000","last_fullsized":"2023-10-09T10:11:22.570890+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":168,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.588182+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:11:22.588182+0000","last_peered":"2023-10-09T10:11:22.588182+0000","last_clean":"2023-10-09T10:11:22.588182+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:11:22.588182+0000","last_undegraded":"2023-10-09T10:11:22.588182+0000","last_fullsized":"2023-10-09T10:11:22.588182+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"301'9935","reported_seq":9975,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:39.635558+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:11:39.635558+0000","last_peered":"2023-10-09T10:11:39.635558+0000","last_clean":"2023-10-09T10:11:39.635558+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:11:39.635558+0000","last_undegraded":"2023-10-09T10:11:39.635558+0000","last_fullsized":"2023-10-09T10:11:39.635558+0000","mapping_epoch":271,"log_start":"280'2200","ondisk_log_start":"280'2200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7735,"log_dups_size":0,"ondisk_log_size":7735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240058368,"num_objects":3663,"num_object_clones":0,"num_object_copies":10989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9935,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":254,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.579906+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:11:22.579906+0000","last_peered":"2023-10-09T10:11:22.579906+0000","last_clean":"2023-10-09T10:11:22.579906+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:11:22.579906+0000","last_undegraded":"2023-10-09T10:11:22.579906+0000","last_fullsized":"2023-10-09T10:11:22.579906+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":304,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589420+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:11:22.589420+0000","last_peered":"2023-10-09T10:11:22.589420+0000","last_clean":"2023-10-09T10:11:22.589420+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:11:22.589420+0000","last_undegraded":"2023-10-09T10:11:22.589420+0000","last_fullsized":"2023-10-09T10:11:22.589420+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"301'9690","reported_seq":9730,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:36.489189+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:11:36.489189+0000","last_peered":"2023-10-09T10:11:36.489189+0000","last_clean":"2023-10-09T10:11:36.489189+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:11:36.489189+0000","last_undegraded":"2023-10-09T10:11:36.489189+0000","last_fullsized":"2023-10-09T10:11:36.489189+0000","mapping_epoch":271,"log_start":"276'300","ondisk_log_start":"276'300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9390,"log_dups_size":0,"ondisk_log_size":9390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247857152,"num_objects":3782,"num_object_clones":0,"num_object_copies":11346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9690,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":304,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589051+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:11:22.589051+0000","last_peered":"2023-10-09T10:11:22.589051+0000","last_clean":"2023-10-09T10:11:22.589051+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:11:22.589051+0000","last_undegraded":"2023-10-09T10:11:22.589051+0000","last_fullsized":"2023-10-09T10:11:22.589051+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580608+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:11:22.580608+0000","last_peered":"2023-10-09T10:11:22.580608+0000","last_clean":"2023-10-09T10:11:22.580608+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:11:22.580608+0000","last_undegraded":"2023-10-09T10:11:22.580608+0000","last_fullsized":"2023-10-09T10:11:22.580608+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":260,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.587806+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:11:22.587806+0000","last_peered":"2023-10-09T10:11:22.587806+0000","last_clean":"2023-10-09T10:11:22.587806+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:11:22.587806+0000","last_undegraded":"2023-10-09T10:11:22.587806+0000","last_fullsized":"2023-10-09T10:11:22.587806+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"301'9812","reported_seq":9850,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:37.897951+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:11:37.897951+0000","last_peered":"2023-10-09T10:11:37.897951+0000","last_clean":"2023-10-09T10:11:37.897951+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:11:37.897951+0000","last_undegraded":"2023-10-09T10:11:37.897951+0000","last_fullsized":"2023-10-09T10:11:37.897951+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9812,"log_dups_size":0,"ondisk_log_size":9812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246546432,"num_objects":3762,"num_object_clones":0,"num_object_copies":11286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9812,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":119,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.583380+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:11:22.583380+0000","last_peered":"2023-10-09T10:11:22.583380+0000","last_clean":"2023-10-09T10:11:22.583380+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:11:22.583380+0000","last_undegraded":"2023-10-09T10:11:22.583380+0000","last_fullsized":"2023-10-09T10:11:22.583380+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":117,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.566986+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:11:22.566986+0000","last_peered":"2023-10-09T10:11:22.566986+0000","last_clean":"2023-10-09T10:11:22.566986+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:11:22.566986+0000","last_undegraded":"2023-10-09T10:11:22.566986+0000","last_fullsized":"2023-10-09T10:11:22.566986+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":327,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.587518+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:11:22.587518+0000","last_peered":"2023-10-09T10:11:22.587518+0000","last_clean":"2023-10-09T10:11:22.587518+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:11:22.587518+0000","last_undegraded":"2023-10-09T10:11:22.587518+0000","last_fullsized":"2023-10-09T10:11:22.587518+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"301'9785","reported_seq":9825,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:36.284996+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:11:36.284996+0000","last_peered":"2023-10-09T10:11:36.284996+0000","last_clean":"2023-10-09T10:11:36.284996+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:11:36.284996+0000","last_undegraded":"2023-10-09T10:11:36.284996+0000","last_fullsized":"2023-10-09T10:11:36.284996+0000","mapping_epoch":271,"log_start":"279'1600","ondisk_log_start":"279'1600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8185,"log_dups_size":0,"ondisk_log_size":8185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246218752,"num_objects":3757,"num_object_clones":0,"num_object_copies":11271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9785,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":220,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571557+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:11:22.571557+0000","last_peered":"2023-10-09T10:11:22.571557+0000","last_clean":"2023-10-09T10:11:22.571557+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:11:22.571557+0000","last_undegraded":"2023-10-09T10:11:22.571557+0000","last_fullsized":"2023-10-09T10:11:22.571557+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":296,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589797+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:11:22.589797+0000","last_peered":"2023-10-09T10:11:22.589797+0000","last_clean":"2023-10-09T10:11:22.589797+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:11:22.589797+0000","last_undegraded":"2023-10-09T10:11:22.589797+0000","last_fullsized":"2023-10-09T10:11:22.589797+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":316,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.588299+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:11:22.588299+0000","last_peered":"2023-10-09T10:11:22.588299+0000","last_clean":"2023-10-09T10:11:22.588299+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:11:22.588299+0000","last_undegraded":"2023-10-09T10:11:22.588299+0000","last_fullsized":"2023-10-09T10:11:22.588299+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"301'9751","reported_seq":9789,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:37.922778+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:11:37.922778+0000","last_peered":"2023-10-09T10:11:37.922778+0000","last_clean":"2023-10-09T10:11:37.922778+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:11:37.922778+0000","last_undegraded":"2023-10-09T10:11:37.922778+0000","last_fullsized":"2023-10-09T10:11:37.922778+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9751,"log_dups_size":0,"ondisk_log_size":9751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243073024,"num_objects":3709,"num_object_clones":0,"num_object_copies":11127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9751,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":128,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.582706+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:11:22.582706+0000","last_peered":"2023-10-09T10:11:22.582706+0000","last_clean":"2023-10-09T10:11:22.582706+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:11:22.582706+0000","last_undegraded":"2023-10-09T10:11:22.582706+0000","last_fullsized":"2023-10-09T10:11:22.582706+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":547,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.570992+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:11:22.570992+0000","last_peered":"2023-10-09T10:11:22.570992+0000","last_clean":"2023-10-09T10:11:22.570992+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:11:22.570992+0000","last_undegraded":"2023-10-09T10:11:22.570992+0000","last_fullsized":"2023-10-09T10:11:22.570992+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":204,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571685+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:11:22.571685+0000","last_peered":"2023-10-09T10:11:22.571685+0000","last_clean":"2023-10-09T10:11:22.571685+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:11:22.571685+0000","last_undegraded":"2023-10-09T10:11:22.571685+0000","last_fullsized":"2023-10-09T10:11:22.571685+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":216,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571971+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:11:22.571971+0000","last_peered":"2023-10-09T10:11:22.571971+0000","last_clean":"2023-10-09T10:11:22.571971+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:11:22.571971+0000","last_undegraded":"2023-10-09T10:11:22.571971+0000","last_fullsized":"2023-10-09T10:11:22.571971+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"301'9846","reported_seq":9886,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:39.642125+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:11:39.642125+0000","last_peered":"2023-10-09T10:11:39.642125+0000","last_clean":"2023-10-09T10:11:39.642125+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:11:39.642125+0000","last_undegraded":"2023-10-09T10:11:39.642125+0000","last_fullsized":"2023-10-09T10:11:39.642125+0000","mapping_epoch":271,"log_start":"280'2100","ondisk_log_start":"280'2100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7746,"log_dups_size":0,"ondisk_log_size":7746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247463936,"num_objects":3776,"num_object_clones":0,"num_object_copies":11328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9846,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":276,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589909+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:11:22.589909+0000","last_peered":"2023-10-09T10:11:22.589909+0000","last_clean":"2023-10-09T10:11:22.589909+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:11:22.589909+0000","last_undegraded":"2023-10-09T10:11:22.589909+0000","last_fullsized":"2023-10-09T10:11:22.589909+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":178,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567749+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:11:22.567749+0000","last_peered":"2023-10-09T10:11:22.567749+0000","last_clean":"2023-10-09T10:11:22.567749+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:11:22.567749+0000","last_undegraded":"2023-10-09T10:11:22.567749+0000","last_fullsized":"2023-10-09T10:11:22.567749+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":206,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.573658+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:11:22.573658+0000","last_peered":"2023-10-09T10:11:22.573658+0000","last_clean":"2023-10-09T10:11:22.573658+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:11:22.573658+0000","last_undegraded":"2023-10-09T10:11:22.573658+0000","last_fullsized":"2023-10-09T10:11:22.573658+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"301'9787","reported_seq":14827,"reported_epoch":301,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:36.407770+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:11:36.407770+0000","last_peered":"2023-10-09T10:11:36.407770+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:11:36.407770+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"279'1600","ondisk_log_start":"279'1600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253689856,"num_objects":3871,"num_object_clones":0,"num_object_copies":11613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":475,"num_objects_misplaced":2313,"num_objects_unfound":0,"num_objects_dirty":3871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9787,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2496,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1083},{"shards":"1,3,4","objects":2313}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":320,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571767+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:11:22.571767+0000","last_peered":"2023-10-09T10:11:22.571767+0000","last_clean":"2023-10-09T10:11:22.571767+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:11:22.571767+0000","last_undegraded":"2023-10-09T10:11:22.571767+0000","last_fullsized":"2023-10-09T10:11:22.571767+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":333,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589124+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:11:22.589124+0000","last_peered":"2023-10-09T10:11:22.589124+0000","last_clean":"2023-10-09T10:11:22.589124+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:11:22.589124+0000","last_undegraded":"2023-10-09T10:11:22.589124+0000","last_fullsized":"2023-10-09T10:11:22.589124+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":170,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567061+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:11:22.567061+0000","last_peered":"2023-10-09T10:11:22.567061+0000","last_clean":"2023-10-09T10:11:22.567061+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:11:22.567061+0000","last_undegraded":"2023-10-09T10:11:22.567061+0000","last_fullsized":"2023-10-09T10:11:22.567061+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"301'9715","reported_seq":9755,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:36.612692+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:11:36.612692+0000","last_peered":"2023-10-09T10:11:36.612692+0000","last_clean":"2023-10-09T10:11:36.612692+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:11:36.612692+0000","last_undegraded":"2023-10-09T10:11:36.612692+0000","last_fullsized":"2023-10-09T10:11:36.612692+0000","mapping_epoch":271,"log_start":"276'300","ondisk_log_start":"276'300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9415,"log_dups_size":0,"ondisk_log_size":9415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251854848,"num_objects":3843,"num_object_clones":0,"num_object_copies":11529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9715,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":263,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.574958+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:11:22.574958+0000","last_peered":"2023-10-09T10:11:22.574958+0000","last_clean":"2023-10-09T10:11:22.574958+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:11:22.574958+0000","last_undegraded":"2023-10-09T10:11:22.574958+0000","last_fullsized":"2023-10-09T10:11:22.574958+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":277,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589285+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:11:22.589285+0000","last_peered":"2023-10-09T10:11:22.589285+0000","last_clean":"2023-10-09T10:11:22.589285+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:11:22.589285+0000","last_undegraded":"2023-10-09T10:11:22.589285+0000","last_fullsized":"2023-10-09T10:11:22.589285+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":252,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.579817+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:11:22.579817+0000","last_peered":"2023-10-09T10:11:22.579817+0000","last_clean":"2023-10-09T10:11:22.579817+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:11:22.579817+0000","last_undegraded":"2023-10-09T10:11:22.579817+0000","last_fullsized":"2023-10-09T10:11:22.579817+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"301'9693","reported_seq":9731,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T10:11:37.883167+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:11:37.883167+0000","last_peered":"2023-10-09T10:11:37.883167+0000","last_clean":"2023-10-09T10:11:37.883167+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:11:37.883167+0000","last_undegraded":"2023-10-09T10:11:37.883167+0000","last_fullsized":"2023-10-09T10:11:37.883167+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9693,"log_dups_size":0,"ondisk_log_size":9693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247463960,"num_objects":3777,"num_object_clones":0,"num_object_copies":11331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3777,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9693,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":209,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572728+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:11:22.572728+0000","last_peered":"2023-10-09T10:11:22.572728+0000","last_clean":"2023-10-09T10:11:22.572728+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:11:22.572728+0000","last_undegraded":"2023-10-09T10:11:22.572728+0000","last_fullsized":"2023-10-09T10:11:22.572728+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":172,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.573573+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:11:22.573573+0000","last_peered":"2023-10-09T10:11:22.573573+0000","last_clean":"2023-10-09T10:11:22.573573+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:11:22.573573+0000","last_undegraded":"2023-10-09T10:11:22.573573+0000","last_fullsized":"2023-10-09T10:11:22.573573+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":154,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580903+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:11:22.580903+0000","last_peered":"2023-10-09T10:11:22.580903+0000","last_clean":"2023-10-09T10:11:22.580903+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:11:22.580903+0000","last_undegraded":"2023-10-09T10:11:22.580903+0000","last_fullsized":"2023-10-09T10:11:22.580903+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":166,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.566904+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:11:22.566904+0000","last_peered":"2023-10-09T10:11:22.566904+0000","last_clean":"2023-10-09T10:11:22.566904+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:11:22.566904+0000","last_undegraded":"2023-10-09T10:11:22.566904+0000","last_fullsized":"2023-10-09T10:11:22.566904+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":239,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589192+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:11:22.589192+0000","last_peered":"2023-10-09T10:11:22.589192+0000","last_clean":"2023-10-09T10:11:22.589192+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:11:22.589192+0000","last_undegraded":"2023-10-09T10:11:22.589192+0000","last_fullsized":"2023-10-09T10:11:22.589192+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":138,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.592159+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:11:22.592159+0000","last_peered":"2023-10-09T10:11:22.592159+0000","last_clean":"2023-10-09T10:11:22.592159+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:11:22.592159+0000","last_undegraded":"2023-10-09T10:11:22.592159+0000","last_fullsized":"2023-10-09T10:11:22.592159+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572348+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:11:22.572348+0000","last_peered":"2023-10-09T10:11:22.572348+0000","last_clean":"2023-10-09T10:11:22.572348+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:11:22.572348+0000","last_undegraded":"2023-10-09T10:11:22.572348+0000","last_fullsized":"2023-10-09T10:11:22.572348+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":138,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.580536+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:11:22.580536+0000","last_peered":"2023-10-09T10:11:22.580536+0000","last_clean":"2023-10-09T10:11:22.580536+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:11:22.580536+0000","last_undegraded":"2023-10-09T10:11:22.580536+0000","last_fullsized":"2023-10-09T10:11:22.580536+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":188,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571602+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:11:22.571602+0000","last_peered":"2023-10-09T10:11:22.571602+0000","last_clean":"2023-10-09T10:11:22.571602+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:11:22.571602+0000","last_undegraded":"2023-10-09T10:11:22.571602+0000","last_fullsized":"2023-10-09T10:11:22.571602+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":317,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.572174+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:11:22.572174+0000","last_peered":"2023-10-09T10:11:22.572174+0000","last_clean":"2023-10-09T10:11:22.572174+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:11:22.572174+0000","last_undegraded":"2023-10-09T10:11:22.572174+0000","last_fullsized":"2023-10-09T10:11:22.572174+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":338,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.571883+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:11:22.571883+0000","last_peered":"2023-10-09T10:11:22.571883+0000","last_clean":"2023-10-09T10:11:22.571883+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:11:22.571883+0000","last_undegraded":"2023-10-09T10:11:22.571883+0000","last_fullsized":"2023-10-09T10:11:22.571883+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":248,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.567394+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:11:22.567394+0000","last_peered":"2023-10-09T10:11:22.567394+0000","last_clean":"2023-10-09T10:11:22.567394+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:11:22.567394+0000","last_undegraded":"2023-10-09T10:11:22.567394+0000","last_fullsized":"2023-10-09T10:11:22.567394+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":168,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T10:11:22.589614+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:11:22.589614+0000","last_peered":"2023-10-09T10:11:22.589614+0000","last_clean":"2023-10-09T10:11:22.589614+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:11:22.589614+0000","last_undegraded":"2023-10-09T10:11:22.589614+0000","last_fullsized":"2023-10-09T10:11:22.589614+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3965059096,"num_objects":60503,"num_object_clones":0,"num_object_copies":181509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5848,"num_objects_misplaced":2313,"num_objects_unfound":0,"num_objects_dirty":60503,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":156859,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2496,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12228702208,"data_stored":12228689992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140959,"ondisk_log_size":140959,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430504,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1504664,"kb_used_data":1195152,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92240488,"statfs":{"total":95995035648,"available":94454259712,"internally_reserved":0,"allocated":1223835648,"data_stored":1223297095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":1.758,"15min":1.716},"min":{"1min":0.254,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":7.8419999999999996,"5min":9.0690000000000008,"15min":50.591999999999999},"last":9.9870000000000001},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.6699999999999999,"15min":1.6970000000000001},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":7.9610000000000003,"5min":8.2300000000000004,"15min":50.747},"last":15.974}]},{"osd":1,"last update":"Mon Oct 9 10:11:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7029999999999998,"5min":2.117,"15min":1.728},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":14.311,"5min":14.311,"15min":25.338000000000001},"last":1.0469999999999999},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.2679999999999998,"15min":1.5509999999999999},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":16.526,"5min":18.27,"15min":18.27},"last":1.147}]},{"osd":2,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.752,"15min":2.2770000000000001},"min":{"1min":0.36099999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":1.012},{"interface":"front","average":{"1min":2.8029999999999999,"5min":1.845,"15min":2.1030000000000002},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.872,"5min":11.872,"15min":11.872},"last":0.503}]},{"osd":3,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":2.2879999999999998,"15min":1.929},"min":{"1min":0.46400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.274,"5min":29.058,"15min":29.058},"last":4.4950000000000001},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.913,"15min":1.9350000000000001},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":2.1829999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.0469999999999999,"15min":0.95699999999999996},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.9820000000000002,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":3.1349999999999998},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.97599999999999998,"15min":0.93100000000000005},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.6239999999999997,"5min":4.6239999999999997,"15min":4.6239999999999997},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.1519999999999999,"15min":1.006},"min":{"1min":0.30199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":5.7149999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":4.9640000000000004},{"interface":"front","average":{"1min":1.012,"5min":1.145,"15min":0.99299999999999999},"min":{"1min":0.38200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.5960000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":3.298}]},{"osd":6,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.93899999999999995,"15min":0.90900000000000003},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.6259999999999999,"5min":5.3129999999999997,"15min":5.4820000000000002},"last":2.1259999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.95599999999999996,"15min":0.87},"min":{"1min":0.36099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.419,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":2.1459999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626732,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1853384,"kb_used_data":1543872,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91891768,"statfs":{"total":95995035648,"available":94097170432,"internally_reserved":0,"allocated":1580924928,"data_stored":1580403689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.821,"15min":1.4410000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.2870000000000008,"5min":15.435,"15min":15.435},"last":0.317},{"interface":"front","average":{"1min":2.149,"5min":2.0859999999999999,"15min":1.4750000000000001},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":7.4139999999999997,"5min":15.308999999999999,"15min":15.308999999999999},"last":1.7250000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:11:04 2023","interfaces":[{"interface":"back","average":{"1min":3.032,"5min":2.7160000000000002,"15min":1.8779999999999999},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":14.888999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.3580000000000001,"15min":1.8320000000000001},"min":{"1min":0.26700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.962,"5min":32.396000000000001,"15min":32.396000000000001},"last":1.861}]},{"osd":2,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":1.4319999999999999,"15min":1.724},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.898,"5min":12.898,"15min":12.898},"last":3.6930000000000001},{"interface":"front","average":{"1min":2.569,"5min":2.141,"15min":3.0379999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5640000000000001,"5min":10.26,"15min":10.26},"last":3.7930000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.8140000000000001,"15min":1.5109999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":12.943,"5min":12.943,"15min":12.943},"last":1.738},{"interface":"front","average":{"1min":1.653,"5min":2.2770000000000001,"15min":2.3460000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.077,"5min":11.211,"15min":11.211},"last":2.1629999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.50700000000000001,"15min":0.46200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.60099999999999998,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1749999999999998,"5min":4.1749999999999998,"15min":4.1749999999999998},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:37 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.81599999999999995},"min":{"1min":0.29399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.4580000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.042,"15min":0.872},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":6.3369999999999997,"5min":7.508,"15min":7.508},"last":0.47899999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:11:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":0.91100000000000003,"15min":0.82299999999999995},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.2709999999999999,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.505},{"interface":"front","average":{"1min":1.538,"5min":0.95499999999999996,"15min":0.872},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.3159999999999998,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":0.52800000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253045,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2093488,"kb_used_data":1681896,"kb_used_omap":9,"kb_used_meta":411574,"kb_avail":91651664,"statfs":{"total":95995035648,"available":93851303936,"internally_reserved":0,"allocated":1722261504,"data_stored":1721229382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":421452777},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,3,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.3980000000000001,"15min":2.1800000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":7.3609999999999998,"5min":25.59,"15min":25.59},"last":1.605},{"interface":"front","average":{"1min":6.2140000000000004,"5min":3.206,"15min":2.3639999999999999},"min":{"1min":0.314,"5min":0.245,"15min":0.192},"max":{"1min":20.501999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.48899999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":3.0499999999999998,"15min":3.8149999999999999},"min":{"1min":0.27200000000000002,"5min":0.217,"15min":0.217},"max":{"1min":11.135,"5min":17.369,"15min":17.369},"last":0.91100000000000003},{"interface":"front","average":{"1min":3.5550000000000002,"5min":2.7469999999999999,"15min":3.8210000000000002},"min":{"1min":0.25900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.484999999999999,"5min":16.023,"15min":16.023},"last":1.639}]},{"osd":3,"last update":"Mon Oct 9 10:10:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":3.4540000000000002,"15min":3.5070000000000001},"min":{"1min":0.39500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.587,"5min":18.587,"15min":18.587},"last":0.64500000000000002},{"interface":"front","average":{"1min":4.6959999999999997,"5min":4.2249999999999996,"15min":4.6760000000000002},"min":{"1min":1.0329999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.022,"5min":17.289000000000001,"15min":17.289000000000001},"last":0.97399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.474,"15min":1.3280000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2629999999999999,"5min":7.2629999999999999,"15min":7.2629999999999999},"last":1.0509999999999999},{"interface":"front","average":{"1min":2.879,"5min":1.639,"15min":1.4330000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.6940000000000008,"5min":10.132,"15min":10.132},"last":0.53100000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":2.3519999999999999,"15min":1.9330000000000001},"min":{"1min":0.41999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.811,"5min":14.416,"15min":15.531000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.742,"5min":2.1859999999999999,"15min":2.1080000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.9039999999999999,"5min":16.277999999999999,"15min":19.706},"last":0.84999999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.9450000000000001,"15min":1.825},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":7.3049999999999997,"5min":14.215,"15min":16.193999999999999},"last":1.0109999999999999},{"interface":"front","average":{"1min":4.1509999999999998,"5min":2.343,"15min":1.984},"min":{"1min":0.46200000000000002,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.868,"5min":14.471,"15min":15.481999999999999},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.306,"15min":2.1360000000000001},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.8620000000000001,"5min":16.331,"15min":16.331},"last":0.93999999999999995},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.0489999999999999,"15min":1.901},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3339999999999996,"5min":14.242000000000001,"15min":16.100999999999999},"last":1.0840000000000001}]}]},{"osd":0,"up_from":79,"seq":339302416571,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1845592,"kb_used_data":1473040,"kb_used_omap":28,"kb_used_meta":372515,"kb_avail":91899560,"statfs":{"total":95995035648,"available":94105149440,"internally_reserved":0,"allocated":1508392960,"data_stored":1507382299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":381456208},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:11:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":2.0259999999999998,"15min":1.6559999999999999},"min":{"1min":0.216,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":12.975,"5min":12.975,"15min":13.084},"last":7.048},{"interface":"front","average":{"1min":5.8680000000000003,"5min":3.653,"15min":2.5920000000000001},"min":{"1min":0.29099999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":22.934999999999999,"5min":22.934999999999999,"15min":22.934999999999999},"last":2.8050000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8360000000000001,"15min":2.1589999999999998},"min":{"1min":0.34599999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.9949999999999992,"5min":10.034000000000001,"15min":10.034000000000001},"last":7.1559999999999997},{"interface":"front","average":{"1min":2.9340000000000002,"5min":2.177,"15min":3.0539999999999998},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":15.529,"5min":15.529,"15min":15.529},"last":7.0030000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":4.5259999999999998,"15min":4.7519999999999998},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":20.419,"5min":25.734999999999999,"15min":25.734999999999999},"last":3.9540000000000002},{"interface":"front","average":{"1min":3.742,"5min":3.891,"15min":4.2370000000000001},"min":{"1min":0.40999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":10.154,"5min":21.923999999999999,"15min":21.923999999999999},"last":16.045000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.911,"15min":1.5649999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":8.0869999999999997},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.0459999999999998,"15min":1.9730000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.6720000000000006,"5min":13.789999999999999,"15min":13.789999999999999},"last":15.907}]},{"osd":5,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.7309999999999999,"15min":1.9630000000000001},"min":{"1min":0.40899999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.6229999999999993,"5min":17.096,"15min":17.096},"last":15.868},{"interface":"front","average":{"1min":1.5,"5min":1.454,"15min":1.3029999999999999},"min":{"1min":0.30499999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.4260000000000002,"5min":9.4260000000000002,"15min":12.478},"last":10.371}]},{"osd":6,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.7010000000000001,"15min":1.4590000000000001},"min":{"1min":0.33700000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":9.4719999999999995,"5min":9.7260000000000009,"15min":11.699},"last":7.9800000000000004},{"interface":"front","average":{"1min":4.4729999999999999,"5min":2.6320000000000001,"15min":2.2069999999999999},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.466000000000001,"5min":20.466000000000001,"15min":20.466000000000001},"last":8.0150000000000006}]},{"osd":7,"last update":"Mon Oct 9 10:10:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8239999999999998,"15min":2.1139999999999999},"min":{"1min":0.46400000000000002,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":9.7050000000000001,"5min":14.631,"15min":26.978000000000002},"last":15.776},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.7190000000000001,"15min":1.605},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":9.5020000000000007,"5min":11.166,"15min":26.768000000000001},"last":7.093}]}]},{"osd":2,"up_from":217,"seq":932007903305,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1410264,"kb_used_data":1153168,"kb_used_omap":83,"kb_used_meta":256940,"kb_avail":92334888,"statfs":{"total":95995035648,"available":94550925312,"internally_reserved":0,"allocated":1180844032,"data_stored":1179834053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":263107055},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":1.732,"15min":1.756},"min":{"1min":0.217,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":19.372,"5min":19.372,"15min":19.372},"last":0.245},{"interface":"front","average":{"1min":3.4420000000000002,"5min":2.9849999999999999,"15min":2.9569999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.754999999999999,"5min":16.754999999999999,"15min":16.754999999999999},"last":7.9260000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":2.258,"15min":2.544},"min":{"1min":0.254,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.797000000000001,"5min":13.797000000000001,"15min":13.797000000000001},"last":5.0460000000000003},{"interface":"front","average":{"1min":2.8399999999999999,"5min":2.5209999999999999,"15min":2.6379999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.4119999999999999,"5min":15.569000000000001,"15min":15.569000000000001},"last":8.0860000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":2.198,"15min":2.0819999999999999},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.851000000000001,"5min":13.851000000000001,"15min":13.851000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.9860000000000002,"5min":3.181,"15min":3.6699999999999999},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.7240000000000002,"5min":15.811,"15min":15.811},"last":7.9550000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.84499999999999997,"15min":0.753},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.29,"5min":6.29,"15min":6.29},"last":7.8769999999999998},{"interface":"front","average":{"1min":2.5609999999999999,"5min":1.1759999999999999,"15min":0.94499999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":12.385,"5min":12.385,"15min":12.385},"last":0.442}]},{"osd":5,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.6919999999999999,"15min":1.546},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":12.473000000000001,"5min":12.473000000000001,"15min":12.473000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":3.0699999999999998,"5min":1.5409999999999999,"15min":1.6870000000000001},"min":{"1min":0.372,"5min":0.318,"15min":0.318},"max":{"1min":13.587,"5min":13.587,"15min":13.587},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.3999999999999999,"15min":1.095},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2599999999999998,"5min":15.292999999999999,"15min":15.292999999999999},"last":8.0069999999999997},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.738,"15min":1.742},"min":{"1min":0.36599999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.41099999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.431,"15min":1.1259999999999999},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.2190000000000003,"5min":15.343999999999999,"15min":15.343999999999999},"last":7.9800000000000004},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.859,"15min":1.6599999999999999},"min":{"1min":0.41699999999999998,"5min":0.376,"15min":0.376},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":12.433999999999999},"last":0.38600000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824048,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1739996,"kb_used_data":1489684,"kb_used_omap":10,"kb_used_meta":250229,"kb_avail":92005156,"statfs":{"total":95995035648,"available":94213279744,"internally_reserved":0,"allocated":1525436416,"data_stored":1524246753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":256234563},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":4.2709999999999999,"15min":4.5590000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.753,"5min":23.381,"15min":23.381},"last":0.27500000000000002},{"interface":"front","average":{"1min":6.0720000000000001,"5min":4.0919999999999996,"15min":4.1500000000000004},"min":{"1min":0.42099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":20.239999999999998,"5min":20.239999999999998,"15min":20.239999999999998},"last":0.22900000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.6150000000000002,"15min":3.8410000000000002},"min":{"1min":0.26300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":20.422999999999998,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.68799999999999994},{"interface":"front","average":{"1min":4.5039999999999996,"5min":3.4990000000000001,"15min":3.4529999999999998},"min":{"1min":0.33100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.704000000000001,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.46700000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1369999999999996,"5min":3.4199999999999999,"15min":3.6339999999999999},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":20.594999999999999,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.8809999999999998,"15min":2.8700000000000001},"min":{"1min":0.34200000000000003,"5min":0.253,"15min":0.253},"max":{"1min":15.73,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.53100000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:10:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":1.6719999999999999,"15min":1.288},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.676,"5min":15.676,"15min":15.676},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5800000000000001,"15min":1.534},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.426,"5min":12.426,"15min":12.426},"last":0.30299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8290000000000002,"5min":2.4750000000000001,"15min":2.5670000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":12.388,"5min":12.388,"15min":12.388},"last":0.33300000000000002},{"interface":"front","average":{"1min":3.5699999999999998,"5min":2.5169999999999999,"15min":2.4079999999999999},"min":{"1min":0.443,"5min":0.309,"15min":0.309},"max":{"1min":15.992000000000001,"5min":15.992000000000001,"15min":15.992000000000001},"last":1.5089999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":2.536,"15min":2.6709999999999998},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.468,"5min":12.468,"15min":12.468},"last":0.42099999999999999},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.5609999999999999,"15min":2.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":16.16,"5min":16.16,"15min":16.16},"last":0.38800000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.825,"15min":1.534},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.635999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.4399999999999999,"15min":2.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.44600000000000001}]}]},{"osd":4,"up_from":266,"seq":1142461300774,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1879756,"kb_used_data":1548804,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":91865396,"statfs":{"total":95995035648,"available":94070165504,"internally_reserved":0,"allocated":1585975296,"data_stored":1585448393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":0.98499999999999999,"15min":0.84099999999999997},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.881,"5min":10.881,"15min":10.881},"last":0.432},{"interface":"front","average":{"1min":1.923,"5min":1.1519999999999999,"15min":1.024},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.847,"5min":10.847,"15min":10.847},"last":1.8660000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":1.169,"15min":0.93400000000000005},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":10.93,"5min":10.93,"15min":10.93},"last":0.438},{"interface":"front","average":{"1min":2.633,"5min":1.546,"15min":1.365},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.141,"5min":11.141,"15min":11.141},"last":0.29899999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":1.2310000000000001,"15min":1.073},"min":{"1min":0.32100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.1189999999999998,"5min":7.1189999999999998,"15min":7.1189999999999998},"last":0.81000000000000005},{"interface":"front","average":{"1min":1.587,"5min":1.042,"15min":0.95099999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.1709999999999994,"5min":8.1709999999999994,"15min":8.1709999999999994},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.4610000000000001,"15min":1.3280000000000001},"min":{"1min":0.40699999999999997,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5530000000000008,"5min":9.5530000000000008,"15min":9.5530000000000008},"last":7.6219999999999999},{"interface":"front","average":{"1min":2.4420000000000002,"5min":1.2989999999999999,"15min":1.109},"min":{"1min":0.373,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.6069999999999993,"5min":9.6069999999999993,"15min":9.6069999999999993},"last":0.40799999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.80400000000000005,"15min":0.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4740000000000002,"5min":5.4740000000000002,"15min":5.4740000000000002},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.60399999999999998,"15min":0.63400000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.38400000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.60499999999999998,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":5.5709999999999997,"5min":5.5709999999999997,"15min":5.5709999999999997},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.60999999999999999,"15min":0.59199999999999997},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.319,"5min":2.319,"15min":2.319},"last":0.376}]},{"osd":7,"last update":"Mon Oct 9 10:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.61299999999999999,"15min":0.61799999999999999},"min":{"1min":0.33700000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.3460000000000001,"5min":2.3460000000000001,"15min":2.3460000000000001},"last":3.6190000000000002},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.82299999999999995,"15min":0.82699999999999996},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.0680000000000001,"5min":3.5630000000000002,"15min":3.5630000000000002},"last":0.42399999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822959,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2247936,"kb_used_data":1938360,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91497216,"statfs":{"total":95995035648,"available":93693149184,"internally_reserved":0,"allocated":1984880640,"data_stored":1984361793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4710000000000001,"15min":1.5720000000000001},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.8179999999999996,"5min":7.8540000000000001,"15min":15.773999999999999},"last":6.1859999999999999},{"interface":"front","average":{"1min":2.7770000000000001,"5min":1.6879999999999999,"15min":1.6819999999999999},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.9950000000000001,"5min":10.244999999999999,"15min":15.699},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:11:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":1.748,"15min":1.5629999999999999},"min":{"1min":0.40300000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.4250000000000007,"5min":11.335000000000001,"15min":15.689},"last":5.2859999999999996},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.903,"15min":1.7110000000000001},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5979999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.45600000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":1.7090000000000001,"15min":2.391},"min":{"1min":0.34899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7249999999999996,"5min":15.640000000000001,"15min":15.640000000000001},"last":5.3390000000000004},{"interface":"front","average":{"1min":1.663,"5min":1.6479999999999999,"15min":2.2370000000000001},"min":{"1min":0.40400000000000003,"5min":0.313,"15min":0.313},"max":{"1min":5.2789999999999999,"5min":10.656000000000001,"15min":10.656000000000001},"last":0.47899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:10:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.8340000000000001,"15min":1.8440000000000001},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6660000000000004,"5min":6.2830000000000004,"15min":6.2830000000000004},"last":7.2770000000000001},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.4380000000000002,"15min":2.8250000000000002},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.085,"5min":17.34,"15min":17.34},"last":1.3520000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.84899999999999998,"15min":0.65900000000000003},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3460000000000001,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":7.3419999999999996},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.70499999999999996,"15min":0.73099999999999998},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.462,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.56399999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":0.97599999999999998,"15min":0.996},"min":{"1min":0.26200000000000001,"5min":0.25,"15min":0.25},"max":{"1min":8.5259999999999998,"5min":13.483000000000001,"15min":27.716000000000001},"last":7.3129999999999997},{"interface":"front","average":{"1min":1.198,"5min":1.075,"15min":0.92600000000000005},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.9889999999999999,"5min":18.600999999999999,"15min":18.600999999999999},"last":6.4550000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":1.0900000000000001,"15min":0.97099999999999997},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.891,"5min":7.891,"15min":10.334},"last":6.2240000000000002},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.72299999999999998,"15min":0.93300000000000005},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.3630000000000004,"5min":6.3630000000000004,"15min":12.356},"last":0.504}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1499398144,"data_stored":1499398144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1708326912,"data_stored":1708326912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1171001344,"data_stored":1170997272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1513816064,"data_stored":1513816064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1574502400,"data_stored":1574502400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1977618432,"data_stored":1977614360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1566441472,"data_stored":1566441472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1217597440,"data_stored":1217593368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:11:43.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:11:43.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:43.311 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:43.424 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:43.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:43 smithi012 ceph-mon[152263]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:43.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5848/181581 objects degraded (3.221%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:43.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:43 smithi012 ceph-mon[147926]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:43.433 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5848/181581 objects degraded (3.221%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:43.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:43.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:43.525 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:43 smithi110 ceph-mon[136032]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:43.526 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5848/181581 objects degraded (3.221%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:43.538 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:43.593 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:43.668 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:43.775 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:43.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:43.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:43.943 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:44.046 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:44.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:44.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:44.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:44.215 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:44.469 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:11:44.470 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:11:44.741 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:44.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:44.847 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:44.894 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:44.897 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:11:44.973 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:44.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:44.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:45.084 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:45.137 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:44 smithi110 ceph-mon[136032]: pgmap v822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 5900/175422 objects degraded (3.363%); 2115/175422 objects misplaced (1.206%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T10:11:45.138 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:45.138 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:45.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:45.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:44 smithi012 ceph-mon[152263]: pgmap v822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 5900/175422 objects degraded (3.363%); 2115/175422 objects misplaced (1.206%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T10:11:45.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:45.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:45.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:44 smithi012 ceph-mon[147926]: pgmap v822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 5900/175422 objects degraded (3.363%); 2115/175422 objects misplaced (1.206%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T10:11:45.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:45.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:45.242 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:45.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:45.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:45.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:11:45.349 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:45.391 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:45.460 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:45.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:45.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:45.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:45.652 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:45.701 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:11:45.858 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:45.877 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:45.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:45.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:45.991 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:46.054 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:46.570 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:46.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:46.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:46.726 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:46.933 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:46.936 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:46.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:46.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:47.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:47.156 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:47.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:46 smithi110 ceph-mon[136032]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 5900/172959 objects degraded (3.411%); 2115/172959 objects misplaced (1.223%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T10:11:47.207 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:46 smithi012 ceph-mon[152263]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 5900/172959 objects degraded (3.411%); 2115/172959 objects misplaced (1.223%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T10:11:47.212 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:46 smithi012 ceph-mon[147926]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 5900/172959 objects degraded (3.411%); 2115/172959 objects misplaced (1.223%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T10:11:47.260 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:47.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:47.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:47.369 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:47.407 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:47.446 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:47.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:47.502 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:11:47.502 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:11:47.554 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:47.663 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:47.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:11:48.000 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:48.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:48.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:48.034 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:11:48.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/980567254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:48.181 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:11:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:11:47] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T10:11:48.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/980567254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:48.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/980567254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:11:48.234 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:48.366 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:48.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:48.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:48.630 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:48.738 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:48.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:48.794 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:11:48.794 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5928/168798 objects degraded (3.512%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1025,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":303,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":56266,"data_bytes":3686406072,"bytes_used":14150848512,"bytes_avail":753809436672,"bytes_total":767960285184,"degraded_objects":5928,"degraded_total":168798,"degraded_ratio":0.035118899513027407,"misplaced_objects":1918,"misplaced_total":168798,"misplaced_ratio":0.01136269387078046,"recovering_objects_per_sec":31,"recovering_bytes_per_sec":2063079,"recovering_keys_per_sec":0,"num_objects_recovered":315,"num_bytes_recovered":20643840,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":596},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T10:09:33.293227+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T10:11:48.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:48.885 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:48.929 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:48.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:49.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:49.123 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: pgmap v824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5928/168798 objects degraded (3.512%); 1918/168798 objects misplaced (1.136%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/980567254' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: osdmap e303: 8 total, 8 up, 8 in 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/168798 objects degraded (3.512%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2680874549' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:49.124 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: pgmap v824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5928/168798 objects degraded (3.512%); 1918/168798 objects misplaced (1.136%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:11:49.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:49.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/980567254' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:11:49.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: osdmap e303: 8 total, 8 up, 8 in 2023-10-09T10:11:49.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/168798 objects degraded (3.512%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:49.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2680874549' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:49.127 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:49.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: pgmap v824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5928/168798 objects degraded (3.512%); 1918/168798 objects misplaced (1.136%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:11:49.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:11:49.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/980567254' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:11:49.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: osdmap e303: 8 total, 8 up, 8 in 2023-10-09T10:11:49.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: failed to encode map e303 with expected crc 2023-10-09T10:11:49.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/168798 objects degraded (3.512%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:49.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2680874549' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:11:49.254 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:11:49.305 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:11:49.345 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:49.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:49.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:49.357 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:11:49.373 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:49.401 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:11:49.421 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:11:49.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:11:49.450 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:11:49.453 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:49.485 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:11:49.523 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:11:49.524 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:49.564 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:11:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:11:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:11:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:11:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:11:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:11:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:11:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:11:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:11:49.784 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:49.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:49.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:49.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:49.989 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:50.092 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1776551540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:50.098 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1776551540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:50.433 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:50.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:50.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1776551540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:50.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:50.676 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:50.723 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:50.740 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:50.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:50.784 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:11:50.853 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:50.889 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:50.925 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:50.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:50.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:51.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:50 smithi012 ceph-mon[147926]: pgmap v826: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5928/164142 objects degraded (3.612%); 1918/164142 objects misplaced (1.169%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:51.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:50 smithi012 ceph-mon[152263]: pgmap v826: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5928/164142 objects degraded (3.612%); 1918/164142 objects misplaced (1.169%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:51.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:50 smithi110 ceph-mon[136032]: pgmap v826: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5928/164142 objects degraded (3.612%); 1918/164142 objects misplaced (1.169%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:51.234 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:164823 8.c 8:3c3557ca:::benchmark_data_smithi012_271002_object56140:head [delete] snapc 0=[] ondisk+write+known_if_redirected e303)", 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:51.202585+0000", 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.030870613000000002, 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030907318999999999, 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:51.235 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 164823 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202585+0000", 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:51.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202585+0000", 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202583+0000", 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202592+0000", 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6929999999999996e-06 2023-10-09T10:11:51.237 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.238 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.238 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:51.238 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202615+0000", 2023-10-09T10:11:51.238 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2974999999999999e-05 2023-10-09T10:11:51.238 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.238 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.239 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:51.239 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202625+0000", 2023-10-09T10:11:51.239 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0341e-05 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202777+0000", 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015140999999999999 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202911+0000", 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000134151 2023-10-09T10:11:51.240 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.203040+0000", 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128541 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.230150+0000", 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027110088000000001 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.241 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.230178+0000", 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8690000000000001e-05 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:164827 8.e 8:7b024ea5:::benchmark_data_smithi012_271002_object56144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e303)", 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:51.202641+0000", 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.030814502000000001, 2023-10-09T10:11:51.242 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030912325000000001, 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 164827 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202641+0000", 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.243 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202641+0000", 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202641+0000", 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.244 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202644+0000", 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1099999999999999e-06 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202650+0000", 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2840000000000003e-06 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.245 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.202655+0000", 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2360000000000004e-06 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.206082+0000", 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034268839999999998 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:51.246 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.206219+0000", 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013730799999999999 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.206353+0000", 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013384000000000001 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.212938+0000", 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0065851140000000004 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.247 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.221490+0000", 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085514079999999999 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:164832 8.6 8:62f31395:::benchmark_data_smithi012_271002_object56149:head [delete] snapc 0=[] ondisk+write+known_if_redirected e303)", 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:51.207603+0000", 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.025852238, 2023-10-09T10:11:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026002560000000001, 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 164832 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207603+0000", 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.249 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207603+0000", 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207602+0000", 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.250 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207608+0000", 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9510000000000003e-06 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207620+0000", 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2539e-05 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207629+0000", 2023-10-09T10:11:51.251 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3180000000000007e-06 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.211286+0000", 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0036577229999999999 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.216153+0000", 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048661390000000002 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.252 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.216286+0000", 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013369099999999999 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:164833 8.c 8:349ab495:::benchmark_data_smithi012_271002_object56150:head [delete] snapc 0=[] ondisk+write+known_if_redirected e303)", 2023-10-09T10:11:51.253 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:51.207642+0000", 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.025813546, 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026009002999999999, 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 164833 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.254 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207642+0000", 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207642+0000", 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.255 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207642+0000", 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207644+0000", 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7989999999999998e-06 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.256 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207651+0000", 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0469999999999999e-06 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207656+0000", 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.079e-06 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.257 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.208601+0000", 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094560900000000003 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.208737+0000", 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135344 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.258 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.208868+0000", 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000131469 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.230197+0000", 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021328836 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:51.259 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:164834 8.2 8:478e226e:::benchmark_data_smithi012_271002_object56151:head [delete] snapc 0=[] ondisk+write+known_if_redirected e303)", 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:51.207661+0000", 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.025794632000000001, 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026037075, 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:51.260 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 164834 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207661+0000", 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207661+0000", 2023-10-09T10:11:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207661+0000", 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207663+0000", 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.357e-06 2023-10-09T10:11:51.262 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207669+0000", 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4589999999999997e-06 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207673+0000", 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5859999999999998e-06 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.263 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207719+0000", 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6164e-05 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207851+0000", 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013166400000000001 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.207969+0000", 2023-10-09T10:11:51.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117868 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.221193+0000", 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01322425 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.230103+0000", 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0089103350000000001 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.265 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:164837 8.6 8:69e6a562:::benchmark_data_smithi012_271002_object56154:head [delete] snapc 0=[] ondisk+write+known_if_redirected e303)", 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:51.229582+0000", 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0038741539999999999, 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041672310000000004, 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:11:51.266 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 164837 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229582+0000", 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.267 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229582+0000", 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229580+0000", 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:11:51.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229585+0000", 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.972e-06 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229594+0000", 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6230000000000003e-06 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229600+0000", 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7259999999999996e-06 2023-10-09T10:11:51.269 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229701+0000", 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100324 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229840+0000", 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013940600000000001 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:11:51.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:51.229972+0000", 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000131898 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 6 2023-10-09T10:11:51.271 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:11:51.272 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:51.444 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:51.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:51.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:51.771 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:51.814 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:51.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:51.973 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:52.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:52.075 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:52.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:52.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:52.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:52.218 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:52.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:11:52.268 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:52.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:52.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:52.422 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:52.471 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:52.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:52.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:52.575 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:52.615 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:52.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:52.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:52.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:52.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:52.871 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:53 smithi012 ceph-mon[147926]: pgmap v827: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5928/163653 objects degraded (3.622%); 1918/163653 objects misplaced (1.172%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:11:53.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:53 smithi012 ceph-mon[152263]: pgmap v827: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5928/163653 objects degraded (3.622%); 1918/163653 objects misplaced (1.172%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:11:53.200 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:53.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:53.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:53.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:11:53.309 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:53.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:53 smithi110 ceph-mon[136032]: pgmap v827: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5928/163653 objects degraded (3.622%); 1918/163653 objects misplaced (1.172%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T10:11:53.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:53.403 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:53.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:53.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:53.509 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:53.648 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:53.669 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:53.736 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:53.741 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:53.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:53.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:53.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:53.960 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:54.110 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:54.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:54.219 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:11:54.298 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:54 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/163653 objects degraded (3.622%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:54.298 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:54 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/163653 objects degraded (3.622%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:54.317 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:54.439 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:11:54.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:54 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/163653 objects degraded (3.622%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:54.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:11:54.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:54.638 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:54.778 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:54.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:54.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:54.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:54.982 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:55.001 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:11:55.290 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:55.295 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:55 smithi012 ceph-mon[147926]: pgmap v828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5928/157647 objects degraded (3.760%); 1740/157647 objects misplaced (1.104%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:55.295 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:55 smithi012 ceph-mon[152263]: pgmap v828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5928/157647 objects degraded (3.760%); 1740/157647 objects misplaced (1.104%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:55.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:55.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:55.390 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:11:55.393 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:11:55.402 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:11:55.439 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:55 smithi110 ceph-mon[136032]: pgmap v828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5928/157647 objects degraded (3.760%); 1740/157647 objects misplaced (1.104%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:55.441 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:55.491 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:11:55.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:55.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:11:55.595 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:11:55.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:55.708 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:11:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:55.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:11:55.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:11:55.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:55.931 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:11:55.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:55.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:11:56.004 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":828,"stamp":"2023-10-09T10:11:53.389951+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3442808760,"num_objects":52549,"num_object_clones":0,"num_object_copies":157647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5928,"num_objects_misplaced":1740,"num_objects_unfound":0,"num_objects_dirty":52549,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":165097,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3058,"num_bytes_recovered":197077708,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142031,"ondisk_log_size":142031,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13143996,"kb_used_data":10625276,"kb_used_omap":167,"kb_used_meta":2517912,"kb_avail":736817220,"statfs":{"total":767960285184,"available":754500833280,"internally_reserved":0,"allocated":10880282624,"data_stored":10873912024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2578342489},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,1,4,3,2,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-388300800,"num_objects":-5925,"num_object_clones":0,"num_object_copies":-17775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28,"num_objects_misplaced":-375,"num_objects_unfound":0,"num_objects_dirty":-5925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5925,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":347,"num_bytes_recovered":22740992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008331"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":145,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.270951+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:11:40.270951+0000","last_peered":"2023-10-09T10:11:40.270951+0000","last_clean":"2023-10-09T10:11:40.270951+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:11:40.270951+0000","last_undegraded":"2023-10-09T10:11:40.270951+0000","last_fullsized":"2023-10-09T10:11:40.270951+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":260,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.279885+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:11:40.279885+0000","last_peered":"2023-10-09T10:11:40.279885+0000","last_clean":"2023-10-09T10:11:40.279885+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:11:40.279885+0000","last_undegraded":"2023-10-09T10:11:40.279885+0000","last_fullsized":"2023-10-09T10:11:40.279885+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":306,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.257063+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:11:40.257063+0000","last_peered":"2023-10-09T10:11:40.257063+0000","last_clean":"2023-10-09T10:11:40.257063+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:11:40.257063+0000","last_undegraded":"2023-10-09T10:11:40.257063+0000","last_fullsized":"2023-10-09T10:11:40.257063+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":169,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.271684+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:11:40.271684+0000","last_peered":"2023-10-09T10:11:40.271684+0000","last_clean":"2023-10-09T10:11:40.271684+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:11:40.271684+0000","last_undegraded":"2023-10-09T10:11:40.271684+0000","last_fullsized":"2023-10-09T10:11:40.271684+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":274,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.272734+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:11:40.272734+0000","last_peered":"2023-10-09T10:11:40.272734+0000","last_clean":"2023-10-09T10:11:40.272734+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:11:40.272734+0000","last_undegraded":"2023-10-09T10:11:40.272734+0000","last_fullsized":"2023-10-09T10:11:40.272734+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":121,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266868+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:11:40.266868+0000","last_peered":"2023-10-09T10:11:40.266868+0000","last_clean":"2023-10-09T10:11:40.266868+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:11:40.266868+0000","last_undegraded":"2023-10-09T10:11:40.266868+0000","last_fullsized":"2023-10-09T10:11:40.266868+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":186,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.259275+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:11:40.259275+0000","last_peered":"2023-10-09T10:11:40.259275+0000","last_clean":"2023-10-09T10:11:40.259275+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:11:40.259275+0000","last_undegraded":"2023-10-09T10:11:40.259275+0000","last_fullsized":"2023-10-09T10:11:40.259275+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":255,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.272656+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:11:40.272656+0000","last_peered":"2023-10-09T10:11:40.272656+0000","last_clean":"2023-10-09T10:11:40.272656+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:11:40.272656+0000","last_undegraded":"2023-10-09T10:11:40.272656+0000","last_fullsized":"2023-10-09T10:11:40.272656+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266678+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:11:40.266678+0000","last_peered":"2023-10-09T10:11:40.266678+0000","last_clean":"2023-10-09T10:11:40.266678+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:11:40.266678+0000","last_undegraded":"2023-10-09T10:11:40.266678+0000","last_fullsized":"2023-10-09T10:11:40.266678+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":216,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.272617+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:11:40.272617+0000","last_peered":"2023-10-09T10:11:40.272617+0000","last_clean":"2023-10-09T10:11:40.272617+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:11:40.272617+0000","last_undegraded":"2023-10-09T10:11:40.272617+0000","last_fullsized":"2023-10-09T10:11:40.272617+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":341,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.313966+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:11:40.313966+0000","last_peered":"2023-10-09T10:11:40.313966+0000","last_clean":"2023-10-09T10:11:40.313966+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:11:40.313966+0000","last_undegraded":"2023-10-09T10:11:40.313966+0000","last_fullsized":"2023-10-09T10:11:40.313966+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":298,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266290+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:11:40.266290+0000","last_peered":"2023-10-09T10:11:40.266290+0000","last_clean":"2023-10-09T10:11:40.266290+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:11:40.266290+0000","last_undegraded":"2023-10-09T10:11:40.266290+0000","last_fullsized":"2023-10-09T10:11:40.266290+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":190,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.310645+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:11:40.310645+0000","last_peered":"2023-10-09T10:11:40.310645+0000","last_clean":"2023-10-09T10:11:40.310645+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:11:40.310645+0000","last_undegraded":"2023-10-09T10:11:40.310645+0000","last_fullsized":"2023-10-09T10:11:40.310645+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":170,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266769+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:11:40.266769+0000","last_peered":"2023-10-09T10:11:40.266769+0000","last_clean":"2023-10-09T10:11:40.266769+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:11:40.266769+0000","last_undegraded":"2023-10-09T10:11:40.266769+0000","last_fullsized":"2023-10-09T10:11:40.266769+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":374,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.310988+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:11:40.310988+0000","last_peered":"2023-10-09T10:11:40.310988+0000","last_clean":"2023-10-09T10:11:40.310988+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:11:40.310988+0000","last_undegraded":"2023-10-09T10:11:40.310988+0000","last_fullsized":"2023-10-09T10:11:40.310988+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":242,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.279743+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:11:40.279743+0000","last_peered":"2023-10-09T10:11:40.279743+0000","last_clean":"2023-10-09T10:11:40.279743+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:11:40.279743+0000","last_undegraded":"2023-10-09T10:11:40.279743+0000","last_fullsized":"2023-10-09T10:11:40.279743+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":169,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.983864+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:11:47.983864+0000","last_peered":"2023-10-09T10:11:47.983864+0000","last_clean":"2023-10-09T10:11:47.983864+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:11:47.983864+0000","last_undegraded":"2023-10-09T10:11:47.983864+0000","last_fullsized":"2023-10-09T10:11:47.983864+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":136,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266706+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:11:40.266706+0000","last_peered":"2023-10-09T10:11:40.266706+0000","last_clean":"2023-10-09T10:11:40.266706+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:11:40.266706+0000","last_undegraded":"2023-10-09T10:11:40.266706+0000","last_fullsized":"2023-10-09T10:11:40.266706+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":171,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.310685+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:11:40.310685+0000","last_peered":"2023-10-09T10:11:40.310685+0000","last_clean":"2023-10-09T10:11:40.310685+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:11:40.310685+0000","last_undegraded":"2023-10-09T10:11:40.310685+0000","last_fullsized":"2023-10-09T10:11:40.310685+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":168,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.271749+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:11:40.271749+0000","last_peered":"2023-10-09T10:11:40.271749+0000","last_clean":"2023-10-09T10:11:40.271749+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:11:40.271749+0000","last_undegraded":"2023-10-09T10:11:40.271749+0000","last_fullsized":"2023-10-09T10:11:40.271749+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":192,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.983937+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:11:47.983937+0000","last_peered":"2023-10-09T10:11:47.983937+0000","last_clean":"2023-10-09T10:11:47.983937+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:11:47.983937+0000","last_undegraded":"2023-10-09T10:11:47.983937+0000","last_fullsized":"2023-10-09T10:11:47.983937+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":255,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.279649+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:11:40.279649+0000","last_peered":"2023-10-09T10:11:40.279649+0000","last_clean":"2023-10-09T10:11:40.279649+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:11:40.279649+0000","last_undegraded":"2023-10-09T10:11:40.279649+0000","last_fullsized":"2023-10-09T10:11:40.279649+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"303'10505","reported_seq":10552,"reported_epoch":303,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:52.938270+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:11:52.938270+0000","last_peered":"2023-10-09T10:11:52.938270+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:11:52.938270+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"276'500","ondisk_log_start":"276'500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10005,"log_dups_size":0,"ondisk_log_size":10005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207159296,"num_objects":3161,"num_object_clones":0,"num_object_copies":9483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10505,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-2212},{"shards":"4,5,6","objects":1701}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":335,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.311451+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:11:40.311451+0000","last_peered":"2023-10-09T10:11:40.311451+0000","last_clean":"2023-10-09T10:11:40.311451+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:11:40.311451+0000","last_undegraded":"2023-10-09T10:11:40.311451+0000","last_fullsized":"2023-10-09T10:11:40.311451+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":121,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.264390+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:11:40.264390+0000","last_peered":"2023-10-09T10:11:40.264390+0000","last_clean":"2023-10-09T10:11:40.264390+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:11:40.264390+0000","last_undegraded":"2023-10-09T10:11:40.264390+0000","last_fullsized":"2023-10-09T10:11:40.264390+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"303'10388","reported_seq":10428,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:50.058823+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:11:50.058823+0000","last_peered":"2023-10-09T10:11:50.058823+0000","last_clean":"2023-10-09T10:11:50.058823+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:11:50.058823+0000","last_undegraded":"2023-10-09T10:11:50.058823+0000","last_fullsized":"2023-10-09T10:11:50.058823+0000","mapping_epoch":271,"log_start":"279'1800","ondisk_log_start":"279'1800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8588,"log_dups_size":0,"ondisk_log_size":8588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222953472,"num_objects":3402,"num_object_clones":0,"num_object_copies":10206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10388,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":323,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.313427+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:11:40.313427+0000","last_peered":"2023-10-09T10:11:40.313427+0000","last_clean":"2023-10-09T10:11:40.313427+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:11:40.313427+0000","last_undegraded":"2023-10-09T10:11:40.313427+0000","last_fullsized":"2023-10-09T10:11:40.313427+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":214,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.256984+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:11:40.256984+0000","last_peered":"2023-10-09T10:11:40.256984+0000","last_clean":"2023-10-09T10:11:40.256984+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:11:40.256984+0000","last_undegraded":"2023-10-09T10:11:40.256984+0000","last_fullsized":"2023-10-09T10:11:40.256984+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"303'10279","reported_seq":10321,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.638566+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:11:51.638566+0000","last_peered":"2023-10-09T10:11:51.638566+0000","last_clean":"2023-10-09T10:11:51.638566+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:11:51.638566+0000","last_undegraded":"2023-10-09T10:11:51.638566+0000","last_fullsized":"2023-10-09T10:11:51.638566+0000","mapping_epoch":271,"log_start":"277'900","ondisk_log_start":"277'900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9379,"log_dups_size":0,"ondisk_log_size":9379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215416832,"num_objects":3287,"num_object_clones":0,"num_object_copies":9861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10279,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":125,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266737+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:11:40.266737+0000","last_peered":"2023-10-09T10:11:40.266737+0000","last_clean":"2023-10-09T10:11:40.266737+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:11:40.266737+0000","last_undegraded":"2023-10-09T10:11:40.266737+0000","last_fullsized":"2023-10-09T10:11:40.266737+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":146,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.269368+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:11:40.269368+0000","last_peered":"2023-10-09T10:11:40.269368+0000","last_clean":"2023-10-09T10:11:40.269368+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:11:40.269368+0000","last_undegraded":"2023-10-09T10:11:40.269368+0000","last_fullsized":"2023-10-09T10:11:40.269368+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"303'10022","reported_seq":10064,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:48.560194+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:11:48.560194+0000","last_peered":"2023-10-09T10:11:48.560194+0000","last_clean":"2023-10-09T10:11:48.560194+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:11:48.560194+0000","last_undegraded":"2023-10-09T10:11:48.560194+0000","last_fullsized":"2023-10-09T10:11:48.560194+0000","mapping_epoch":271,"log_start":"280'2300","ondisk_log_start":"280'2300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7722,"log_dups_size":0,"ondisk_log_size":7722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222035968,"num_objects":3388,"num_object_clones":0,"num_object_copies":10164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10022,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":136,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.265130+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:11:40.265130+0000","last_peered":"2023-10-09T10:11:40.265130+0000","last_clean":"2023-10-09T10:11:40.265130+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:11:40.265130+0000","last_undegraded":"2023-10-09T10:11:40.265130+0000","last_fullsized":"2023-10-09T10:11:40.265130+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":170,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.259421+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:11:40.259421+0000","last_peered":"2023-10-09T10:11:40.259421+0000","last_clean":"2023-10-09T10:11:40.259421+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:11:40.259421+0000","last_undegraded":"2023-10-09T10:11:40.259421+0000","last_fullsized":"2023-10-09T10:11:40.259421+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"303'10487","reported_seq":10529,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.455509+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:11:51.455509+0000","last_peered":"2023-10-09T10:11:51.455509+0000","last_clean":"2023-10-09T10:11:51.455509+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:11:51.455509+0000","last_undegraded":"2023-10-09T10:11:51.455509+0000","last_fullsized":"2023-10-09T10:11:51.455509+0000","mapping_epoch":271,"log_start":"280'2300","ondisk_log_start":"280'2300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219742208,"num_objects":3353,"num_object_clones":0,"num_object_copies":10059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10487,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":296,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.279381+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:11:40.279381+0000","last_peered":"2023-10-09T10:11:40.279381+0000","last_clean":"2023-10-09T10:11:40.279381+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:11:40.279381+0000","last_undegraded":"2023-10-09T10:11:40.279381+0000","last_fullsized":"2023-10-09T10:11:40.279381+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":166,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.279615+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:11:40.279615+0000","last_peered":"2023-10-09T10:11:40.279615+0000","last_clean":"2023-10-09T10:11:40.279615+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:11:40.279615+0000","last_undegraded":"2023-10-09T10:11:40.279615+0000","last_fullsized":"2023-10-09T10:11:40.279615+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"303'10345","reported_seq":10387,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:48.496333+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:11:48.496333+0000","last_peered":"2023-10-09T10:11:48.496333+0000","last_clean":"2023-10-09T10:11:48.496333+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:11:48.496333+0000","last_undegraded":"2023-10-09T10:11:48.496333+0000","last_fullsized":"2023-10-09T10:11:48.496333+0000","mapping_epoch":271,"log_start":"280'2600","ondisk_log_start":"280'2600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220135424,"num_objects":3359,"num_object_clones":0,"num_object_copies":10077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10345,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":228,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266206+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:11:40.266206+0000","last_peered":"2023-10-09T10:11:40.266206+0000","last_clean":"2023-10-09T10:11:40.266206+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:11:40.266206+0000","last_undegraded":"2023-10-09T10:11:40.266206+0000","last_fullsized":"2023-10-09T10:11:40.266206+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":234,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.261251+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:11:40.261251+0000","last_peered":"2023-10-09T10:11:40.261251+0000","last_clean":"2023-10-09T10:11:40.261251+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:11:40.261251+0000","last_undegraded":"2023-10-09T10:11:40.261251+0000","last_fullsized":"2023-10-09T10:11:40.261251+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"303'10200","reported_seq":10242,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.613383+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:11:51.613383+0000","last_peered":"2023-10-09T10:11:51.613383+0000","last_clean":"2023-10-09T10:11:51.613383+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:11:51.613383+0000","last_undegraded":"2023-10-09T10:11:51.613383+0000","last_fullsized":"2023-10-09T10:11:51.613383+0000","mapping_epoch":271,"log_start":"277'800","ondisk_log_start":"277'800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9400,"log_dups_size":0,"ondisk_log_size":9400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211812352,"num_objects":3232,"num_object_clones":0,"num_object_copies":9696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10200,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":197,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.310600+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:11:40.310600+0000","last_peered":"2023-10-09T10:11:40.310600+0000","last_clean":"2023-10-09T10:11:40.310600+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:11:40.310600+0000","last_undegraded":"2023-10-09T10:11:40.310600+0000","last_fullsized":"2023-10-09T10:11:40.310600+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":170,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.261318+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:11:40.261318+0000","last_peered":"2023-10-09T10:11:40.261318+0000","last_clean":"2023-10-09T10:11:40.261318+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:11:40.261318+0000","last_undegraded":"2023-10-09T10:11:40.261318+0000","last_fullsized":"2023-10-09T10:11:40.261318+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"303'10269","reported_seq":10311,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:48.519931+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:11:48.519931+0000","last_peered":"2023-10-09T10:11:48.519931+0000","last_clean":"2023-10-09T10:11:48.519931+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:11:48.519931+0000","last_undegraded":"2023-10-09T10:11:48.519931+0000","last_fullsized":"2023-10-09T10:11:48.519931+0000","mapping_epoch":271,"log_start":"280'2500","ondisk_log_start":"280'2500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7769,"log_dups_size":0,"ondisk_log_size":7769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218169344,"num_objects":3329,"num_object_clones":0,"num_object_copies":9987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10269,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":256,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.272577+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:11:40.272577+0000","last_peered":"2023-10-09T10:11:40.272577+0000","last_clean":"2023-10-09T10:11:40.272577+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:11:40.272577+0000","last_undegraded":"2023-10-09T10:11:40.272577+0000","last_fullsized":"2023-10-09T10:11:40.272577+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":306,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.262517+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:11:40.262517+0000","last_peered":"2023-10-09T10:11:40.262517+0000","last_clean":"2023-10-09T10:11:40.262517+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:11:40.262517+0000","last_undegraded":"2023-10-09T10:11:40.262517+0000","last_fullsized":"2023-10-09T10:11:40.262517+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"303'10243","reported_seq":10285,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.568161+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:11:51.568161+0000","last_peered":"2023-10-09T10:11:51.568161+0000","last_clean":"2023-10-09T10:11:51.568161+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:11:51.568161+0000","last_undegraded":"2023-10-09T10:11:51.568161+0000","last_fullsized":"2023-10-09T10:11:51.568161+0000","mapping_epoch":271,"log_start":"277'800","ondisk_log_start":"277'800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9443,"log_dups_size":0,"ondisk_log_size":9443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211615744,"num_objects":3229,"num_object_clones":0,"num_object_copies":9687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10243,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":306,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.261420+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:11:40.261420+0000","last_peered":"2023-10-09T10:11:40.261420+0000","last_clean":"2023-10-09T10:11:40.261420+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:11:40.261420+0000","last_undegraded":"2023-10-09T10:11:40.261420+0000","last_fullsized":"2023-10-09T10:11:40.261420+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":145,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266636+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:11:40.266636+0000","last_peered":"2023-10-09T10:11:40.266636+0000","last_clean":"2023-10-09T10:11:40.266636+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:11:40.266636+0000","last_undegraded":"2023-10-09T10:11:40.266636+0000","last_fullsized":"2023-10-09T10:11:40.266636+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":262,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.270930+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:11:40.270930+0000","last_peered":"2023-10-09T10:11:40.270930+0000","last_clean":"2023-10-09T10:11:40.270930+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:11:40.270930+0000","last_undegraded":"2023-10-09T10:11:40.270930+0000","last_fullsized":"2023-10-09T10:11:40.270930+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"303'10374","reported_seq":10414,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:52.834733+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:11:52.834733+0000","last_peered":"2023-10-09T10:11:52.834733+0000","last_clean":"2023-10-09T10:11:52.834733+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:11:52.834733+0000","last_undegraded":"2023-10-09T10:11:52.834733+0000","last_fullsized":"2023-10-09T10:11:52.834733+0000","mapping_epoch":271,"log_start":"276'300","ondisk_log_start":"276'300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209715200,"num_objects":3200,"num_object_clones":0,"num_object_copies":9600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10374,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":121,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.265636+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:11:40.265636+0000","last_peered":"2023-10-09T10:11:40.265636+0000","last_clean":"2023-10-09T10:11:40.265636+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:11:40.265636+0000","last_undegraded":"2023-10-09T10:11:40.265636+0000","last_fullsized":"2023-10-09T10:11:40.265636+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":119,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.259048+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:11:40.259048+0000","last_peered":"2023-10-09T10:11:40.259048+0000","last_clean":"2023-10-09T10:11:40.259048+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:11:40.259048+0000","last_undegraded":"2023-10-09T10:11:40.259048+0000","last_fullsized":"2023-10-09T10:11:40.259048+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":329,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.261627+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:11:40.261627+0000","last_peered":"2023-10-09T10:11:40.261627+0000","last_clean":"2023-10-09T10:11:40.261627+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:11:40.261627+0000","last_undegraded":"2023-10-09T10:11:40.261627+0000","last_fullsized":"2023-10-09T10:11:40.261627+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"303'10332","reported_seq":10374,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.394362+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:11:51.394362+0000","last_peered":"2023-10-09T10:11:51.394362+0000","last_clean":"2023-10-09T10:11:51.394362+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:11:51.394362+0000","last_undegraded":"2023-10-09T10:11:51.394362+0000","last_fullsized":"2023-10-09T10:11:51.394362+0000","mapping_epoch":271,"log_start":"280'2200","ondisk_log_start":"280'2200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8132,"log_dups_size":0,"ondisk_log_size":8132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210370560,"num_objects":3210,"num_object_clones":0,"num_object_copies":9630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10332,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":222,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.272500+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:11:40.272500+0000","last_peered":"2023-10-09T10:11:40.272500+0000","last_clean":"2023-10-09T10:11:40.272500+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:11:40.272500+0000","last_undegraded":"2023-10-09T10:11:40.272500+0000","last_fullsized":"2023-10-09T10:11:40.272500+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":298,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.258592+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:11:40.258592+0000","last_peered":"2023-10-09T10:11:40.258592+0000","last_clean":"2023-10-09T10:11:40.258592+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:11:40.258592+0000","last_undegraded":"2023-10-09T10:11:40.258592+0000","last_fullsized":"2023-10-09T10:11:40.258592+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":318,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.261591+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:11:40.261591+0000","last_peered":"2023-10-09T10:11:40.261591+0000","last_clean":"2023-10-09T10:11:40.261591+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:11:40.261591+0000","last_undegraded":"2023-10-09T10:11:40.261591+0000","last_fullsized":"2023-10-09T10:11:40.261591+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"303'10292","reported_seq":10332,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:52.949421+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:11:52.949421+0000","last_peered":"2023-10-09T10:11:52.949421+0000","last_clean":"2023-10-09T10:11:52.949421+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:11:52.949421+0000","last_undegraded":"2023-10-09T10:11:52.949421+0000","last_fullsized":"2023-10-09T10:11:52.949421+0000","mapping_epoch":271,"log_start":"275'200","ondisk_log_start":"275'200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10092,"log_dups_size":0,"ondisk_log_size":10092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207618048,"num_objects":3168,"num_object_clones":0,"num_object_copies":9504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10292,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":130,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.264610+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:11:40.264610+0000","last_peered":"2023-10-09T10:11:40.264610+0000","last_clean":"2023-10-09T10:11:40.264610+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:11:40.264610+0000","last_undegraded":"2023-10-09T10:11:40.264610+0000","last_fullsized":"2023-10-09T10:11:40.264610+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":549,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.310267+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:11:40.310267+0000","last_peered":"2023-10-09T10:11:40.310267+0000","last_clean":"2023-10-09T10:11:40.310267+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:11:40.310267+0000","last_undegraded":"2023-10-09T10:11:40.310267+0000","last_fullsized":"2023-10-09T10:11:40.310267+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":207,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977533+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:11:47.977533+0000","last_peered":"2023-10-09T10:11:47.977533+0000","last_clean":"2023-10-09T10:11:47.977533+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:11:47.977533+0000","last_undegraded":"2023-10-09T10:11:47.977533+0000","last_fullsized":"2023-10-09T10:11:47.977533+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":218,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.314049+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:11:40.314049+0000","last_peered":"2023-10-09T10:11:40.314049+0000","last_clean":"2023-10-09T10:11:40.314049+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:11:40.314049+0000","last_undegraded":"2023-10-09T10:11:40.314049+0000","last_fullsized":"2023-10-09T10:11:40.314049+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"303'10224","reported_seq":10265,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:48.545089+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:11:48.545089+0000","last_peered":"2023-10-09T10:11:48.545089+0000","last_clean":"2023-10-09T10:11:48.545089+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:11:48.545089+0000","last_undegraded":"2023-10-09T10:11:48.545089+0000","last_fullsized":"2023-10-09T10:11:48.545089+0000","mapping_epoch":271,"log_start":"280'2500","ondisk_log_start":"280'2500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7724,"log_dups_size":0,"ondisk_log_size":7724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222691328,"num_objects":3398,"num_object_clones":0,"num_object_copies":10194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10224,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":278,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.258730+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:11:40.258730+0000","last_peered":"2023-10-09T10:11:40.258730+0000","last_clean":"2023-10-09T10:11:40.258730+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:11:40.258730+0000","last_undegraded":"2023-10-09T10:11:40.258730+0000","last_fullsized":"2023-10-09T10:11:40.258730+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":180,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.258909+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:11:40.258909+0000","last_peered":"2023-10-09T10:11:40.258909+0000","last_clean":"2023-10-09T10:11:40.258909+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:11:40.258909+0000","last_undegraded":"2023-10-09T10:11:40.258909+0000","last_fullsized":"2023-10-09T10:11:40.258909+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":208,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.271618+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:11:40.271618+0000","last_peered":"2023-10-09T10:11:40.271618+0000","last_clean":"2023-10-09T10:11:40.271618+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:11:40.271618+0000","last_undegraded":"2023-10-09T10:11:40.271618+0000","last_fullsized":"2023-10-09T10:11:40.271618+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"303'10345","reported_seq":16373,"reported_epoch":303,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:11:51.448855+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:11:51.448855+0000","last_peered":"2023-10-09T10:11:51.448855+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:11:51.448855+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"280'2200","ondisk_log_start":"280'2200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217120768,"num_objects":3313,"num_object_clones":0,"num_object_copies":9939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":555,"num_objects_misplaced":1740,"num_objects_unfound":0,"num_objects_dirty":3313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10345,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2989,"num_bytes_recovered":195887104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1018},{"shards":"1,3,4","objects":1740}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":322,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.313805+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:11:40.313805+0000","last_peered":"2023-10-09T10:11:40.313805+0000","last_clean":"2023-10-09T10:11:40.313805+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:11:40.313805+0000","last_undegraded":"2023-10-09T10:11:40.313805+0000","last_fullsized":"2023-10-09T10:11:40.313805+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":335,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.261488+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:11:40.261488+0000","last_peered":"2023-10-09T10:11:40.261488+0000","last_clean":"2023-10-09T10:11:40.261488+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:11:40.261488+0000","last_undegraded":"2023-10-09T10:11:40.261488+0000","last_fullsized":"2023-10-09T10:11:40.261488+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":172,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.259289+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:11:40.259289+0000","last_peered":"2023-10-09T10:11:40.259289+0000","last_clean":"2023-10-09T10:11:40.259289+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:11:40.259289+0000","last_undegraded":"2023-10-09T10:11:40.259289+0000","last_fullsized":"2023-10-09T10:11:40.259289+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"303'10296","reported_seq":10335,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.688248+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:11:51.688248+0000","last_peered":"2023-10-09T10:11:51.688248+0000","last_clean":"2023-10-09T10:11:51.688248+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:11:51.688248+0000","last_undegraded":"2023-10-09T10:11:51.688248+0000","last_fullsized":"2023-10-09T10:11:51.688248+0000","mapping_epoch":271,"log_start":"277'900","ondisk_log_start":"277'900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9396,"log_dups_size":0,"ondisk_log_size":9396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213778432,"num_objects":3262,"num_object_clones":0,"num_object_copies":9786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10296,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":265,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.279732+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:11:40.279732+0000","last_peered":"2023-10-09T10:11:40.279732+0000","last_clean":"2023-10-09T10:11:40.279732+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:11:40.279732+0000","last_undegraded":"2023-10-09T10:11:40.279732+0000","last_fullsized":"2023-10-09T10:11:40.279732+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":279,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.261394+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:11:40.261394+0000","last_peered":"2023-10-09T10:11:40.261394+0000","last_clean":"2023-10-09T10:11:40.261394+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:11:40.261394+0000","last_undegraded":"2023-10-09T10:11:40.261394+0000","last_fullsized":"2023-10-09T10:11:40.261394+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":255,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:41.441672+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:11:41.441672+0000","last_peered":"2023-10-09T10:11:41.441672+0000","last_clean":"2023-10-09T10:11:41.441672+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:11:41.441672+0000","last_undegraded":"2023-10-09T10:11:41.441672+0000","last_fullsized":"2023-10-09T10:11:41.441672+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"303'10236","reported_seq":10277,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:52.893480+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:11:52.893480+0000","last_peered":"2023-10-09T10:11:52.893480+0000","last_clean":"2023-10-09T10:11:52.893480+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:11:52.893480+0000","last_undegraded":"2023-10-09T10:11:52.893480+0000","last_fullsized":"2023-10-09T10:11:52.893480+0000","mapping_epoch":271,"log_start":"275'200","ondisk_log_start":"275'200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10036,"log_dups_size":0,"ondisk_log_size":10036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211877912,"num_objects":3234,"num_object_clones":0,"num_object_copies":9702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3234,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10236,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":211,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.269728+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:11:40.269728+0000","last_peered":"2023-10-09T10:11:40.269728+0000","last_clean":"2023-10-09T10:11:40.269728+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:11:40.269728+0000","last_undegraded":"2023-10-09T10:11:40.269728+0000","last_fullsized":"2023-10-09T10:11:40.269728+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":174,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.271546+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:11:40.271546+0000","last_peered":"2023-10-09T10:11:40.271546+0000","last_clean":"2023-10-09T10:11:40.271546+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:11:40.271546+0000","last_undegraded":"2023-10-09T10:11:40.271546+0000","last_fullsized":"2023-10-09T10:11:40.271546+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":156,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.264415+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:11:40.264415+0000","last_peered":"2023-10-09T10:11:40.264415+0000","last_clean":"2023-10-09T10:11:40.264415+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:11:40.264415+0000","last_undegraded":"2023-10-09T10:11:40.264415+0000","last_fullsized":"2023-10-09T10:11:40.264415+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":168,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.258953+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:11:40.258953+0000","last_peered":"2023-10-09T10:11:40.258953+0000","last_clean":"2023-10-09T10:11:40.258953+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:11:40.258953+0000","last_undegraded":"2023-10-09T10:11:40.258953+0000","last_fullsized":"2023-10-09T10:11:40.258953+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":241,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.261535+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:11:40.261535+0000","last_peered":"2023-10-09T10:11:40.261535+0000","last_clean":"2023-10-09T10:11:40.261535+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:11:40.261535+0000","last_undegraded":"2023-10-09T10:11:40.261535+0000","last_fullsized":"2023-10-09T10:11:40.261535+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":140,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.265043+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:11:40.265043+0000","last_peered":"2023-10-09T10:11:40.265043+0000","last_clean":"2023-10-09T10:11:40.265043+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:11:40.265043+0000","last_undegraded":"2023-10-09T10:11:40.265043+0000","last_fullsized":"2023-10-09T10:11:40.265043+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":181,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.279376+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:11:40.279376+0000","last_peered":"2023-10-09T10:11:40.279376+0000","last_clean":"2023-10-09T10:11:40.279376+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:11:40.279376+0000","last_undegraded":"2023-10-09T10:11:40.279376+0000","last_fullsized":"2023-10-09T10:11:40.279376+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":140,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.266617+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:11:40.266617+0000","last_peered":"2023-10-09T10:11:40.266617+0000","last_clean":"2023-10-09T10:11:40.266617+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:11:40.266617+0000","last_undegraded":"2023-10-09T10:11:40.266617+0000","last_fullsized":"2023-10-09T10:11:40.266617+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":190,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.272814+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:11:40.272814+0000","last_peered":"2023-10-09T10:11:40.272814+0000","last_clean":"2023-10-09T10:11:40.272814+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:11:40.272814+0000","last_undegraded":"2023-10-09T10:11:40.272814+0000","last_fullsized":"2023-10-09T10:11:40.272814+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":320,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:41.441837+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:11:41.441837+0000","last_peered":"2023-10-09T10:11:41.441837+0000","last_clean":"2023-10-09T10:11:41.441837+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:11:41.441837+0000","last_undegraded":"2023-10-09T10:11:41.441837+0000","last_fullsized":"2023-10-09T10:11:41.441837+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":340,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.272729+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:11:40.272729+0000","last_peered":"2023-10-09T10:11:40.272729+0000","last_clean":"2023-10-09T10:11:40.272729+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:11:40.272729+0000","last_undegraded":"2023-10-09T10:11:40.272729+0000","last_fullsized":"2023-10-09T10:11:40.272729+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":250,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.259036+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:11:40.259036+0000","last_peered":"2023-10-09T10:11:40.259036+0000","last_clean":"2023-10-09T10:11:40.259036+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:11:40.259036+0000","last_undegraded":"2023-10-09T10:11:40.259036+0000","last_fullsized":"2023-10-09T10:11:40.259036+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T10:11:40.258372+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:11:40.258372+0000","last_peered":"2023-10-09T10:11:40.258372+0000","last_clean":"2023-10-09T10:11:40.258372+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:11:40.258372+0000","last_undegraded":"2023-10-09T10:11:40.258372+0000","last_fullsized":"2023-10-09T10:11:40.258372+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3442212888,"num_objects":52525,"num_object_clones":0,"num_object_copies":157575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5928,"num_objects_misplaced":1740,"num_objects_unfound":0,"num_objects_dirty":52525,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":164837,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2989,"num_bytes_recovered":195887104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10806505472,"data_stored":10806493256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141837,"ondisk_log_size":141837,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430506,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379112,"kb_used_data":1077792,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92366040,"statfs":{"total":95995035648,"available":94582824960,"internally_reserved":0,"allocated":1103659008,"data_stored":1103119979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:43 2023","interfaces":[{"interface":"back","average":{"1min":3.508,"5min":1.986,"15min":1.911},"min":{"1min":0.90100000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":9.9870000000000001,"5min":9.9870000000000001,"15min":50.591999999999999},"last":2.762},{"interface":"front","average":{"1min":3.661,"5min":2.133,"15min":1.905},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.974,"5min":15.974,"15min":50.747},"last":4.7880000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:11:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7029999999999998,"5min":2.117,"15min":1.728},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":14.311,"5min":14.311,"15min":25.338000000000001},"last":1.913},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.2679999999999998,"15min":1.5509999999999999},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":16.526,"5min":18.27,"15min":18.27},"last":0.504}]},{"osd":2,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.752,"15min":2.2770000000000001},"min":{"1min":0.36099999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":4.5739999999999998},{"interface":"front","average":{"1min":2.8029999999999999,"5min":1.845,"15min":2.1030000000000002},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.872,"5min":11.872,"15min":11.872},"last":5.5140000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":2.2879999999999998,"15min":1.929},"min":{"1min":0.46400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.274,"5min":29.058,"15min":29.058},"last":4.484},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.913,"15min":1.9350000000000001},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":4.5300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.0469999999999999,"15min":0.95699999999999996},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.9820000000000002,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.875},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.97599999999999998,"15min":0.93100000000000005},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.6239999999999997,"5min":4.6239999999999997,"15min":4.6239999999999997},"last":0.52900000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.1519999999999999,"15min":1.006},"min":{"1min":0.30199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":5.7149999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.012,"5min":1.145,"15min":0.99299999999999999},"min":{"1min":0.38200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.5960000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":1.0569999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.93899999999999995,"15min":0.90900000000000003},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.6259999999999999,"5min":5.3129999999999997,"15min":5.4820000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.226,"5min":0.95599999999999996,"15min":0.87},"min":{"1min":0.36099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.419,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":1.1259999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626734,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717080,"kb_used_data":1415760,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92028072,"statfs":{"total":95995035648,"available":94236745728,"internally_reserved":0,"allocated":1449738240,"data_stored":1449216525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.821,"15min":1.4410000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.2870000000000008,"5min":15.435,"15min":15.435},"last":3.7850000000000001},{"interface":"front","average":{"1min":2.149,"5min":2.0859999999999999,"15min":1.4750000000000001},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":7.4139999999999997,"5min":15.308999999999999,"15min":15.308999999999999},"last":1.8220000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:11:04 2023","interfaces":[{"interface":"back","average":{"1min":3.032,"5min":2.7160000000000002,"15min":1.8779999999999999},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":14.888999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":3.8069999999999999},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.3580000000000001,"15min":1.8320000000000001},"min":{"1min":0.26700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.962,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.41999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":1.4319999999999999,"15min":1.724},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.898,"5min":12.898,"15min":12.898},"last":2.242},{"interface":"front","average":{"1min":2.569,"5min":2.141,"15min":3.0379999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5640000000000001,"5min":10.26,"15min":10.26},"last":3.694}]},{"osd":3,"last update":"Mon Oct 9 10:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.8140000000000001,"15min":1.5109999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":12.943,"5min":12.943,"15min":12.943},"last":2.1640000000000001},{"interface":"front","average":{"1min":1.653,"5min":2.2770000000000001,"15min":2.3460000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.077,"5min":11.211,"15min":11.211},"last":4.0019999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.67100000000000004,"15min":0.51700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.8970000000000002,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":2.2130000000000001},{"interface":"front","average":{"1min":1.284,"5min":0.75800000000000001,"15min":0.58499999999999996},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.75700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:37 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.81599999999999995},"min":{"1min":0.29399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.4580000000000002,"5min":7.9260000000000002,"15min":14.09},"last":2.1920000000000002},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.042,"15min":0.872},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":6.3369999999999997,"5min":7.508,"15min":7.508},"last":3.7610000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:11:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":0.91100000000000003,"15min":0.82299999999999995},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.2709999999999999,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":3.8290000000000002},{"interface":"front","average":{"1min":1.538,"5min":0.95499999999999996,"15min":0.872},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.3159999999999998,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":3.8479999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253047,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1928712,"kb_used_data":1525184,"kb_used_omap":9,"kb_used_meta":403382,"kb_avail":91816440,"statfs":{"total":95995035648,"available":94020034560,"internally_reserved":0,"allocated":1561788416,"data_stored":1560748288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":413064169},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.3980000000000001,"15min":2.1800000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":7.3609999999999998,"5min":25.59,"15min":25.59},"last":4.9779999999999998},{"interface":"front","average":{"1min":6.2140000000000004,"5min":3.206,"15min":2.3639999999999999},"min":{"1min":0.314,"5min":0.245,"15min":0.192},"max":{"1min":20.501999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.60399999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":3.0499999999999998,"15min":3.8149999999999999},"min":{"1min":0.27200000000000002,"5min":0.217,"15min":0.217},"max":{"1min":11.135,"5min":17.369,"15min":17.369},"last":3.8620000000000001},{"interface":"front","average":{"1min":3.5550000000000002,"5min":2.7469999999999999,"15min":3.8210000000000002},"min":{"1min":0.25900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.484999999999999,"5min":16.023,"15min":16.023},"last":5.0300000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:10:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":3.4540000000000002,"15min":3.5070000000000001},"min":{"1min":0.39500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.587,"5min":18.587,"15min":18.587},"last":4.6479999999999997},{"interface":"front","average":{"1min":4.6959999999999997,"5min":4.2249999999999996,"15min":4.6760000000000002},"min":{"1min":1.0329999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.022,"5min":17.289000000000001,"15min":17.289000000000001},"last":5.0039999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.536,"15min":1.3480000000000001},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.976000000000001,"5min":11.976000000000001,"15min":11.976000000000001},"last":4.9470000000000001},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.177,"15min":1.6120000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.904,"5min":12.904,"15min":12.904},"last":3.0379999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":2.3519999999999999,"15min":1.9330000000000001},"min":{"1min":0.41999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.811,"5min":14.416,"15min":15.531000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":2.742,"5min":2.1859999999999999,"15min":2.1080000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.9039999999999999,"5min":16.277999999999999,"15min":19.706},"last":3.5680000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.9450000000000001,"15min":1.825},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":7.3049999999999997,"5min":14.215,"15min":16.193999999999999},"last":4.9169999999999998},{"interface":"front","average":{"1min":4.1509999999999998,"5min":2.343,"15min":1.984},"min":{"1min":0.46200000000000002,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.868,"5min":14.471,"15min":15.481999999999999},"last":0.63500000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.306,"15min":2.1360000000000001},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.8620000000000001,"5min":16.331,"15min":16.331},"last":3.6120000000000001},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.0489999999999999,"15min":1.901},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3339999999999996,"5min":14.242000000000001,"15min":16.100999999999999},"last":4.8730000000000002}]}]},{"osd":0,"up_from":79,"seq":339302416574,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1640360,"kb_used_data":1255264,"kb_used_omap":28,"kb_used_meta":384803,"kb_avail":92104792,"statfs":{"total":95995035648,"available":94315307008,"internally_reserved":0,"allocated":1285390336,"data_stored":1284379199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":394039120},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:11:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":2.0259999999999998,"15min":1.6559999999999999},"min":{"1min":0.216,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":12.975,"5min":12.975,"15min":13.084},"last":16.776},{"interface":"front","average":{"1min":5.8680000000000003,"5min":3.653,"15min":2.5920000000000001},"min":{"1min":0.29099999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":22.934999999999999,"5min":22.934999999999999,"15min":22.934999999999999},"last":11.811}]},{"osd":2,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8360000000000001,"15min":2.1589999999999998},"min":{"1min":0.34599999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.9949999999999992,"5min":10.034000000000001,"15min":10.034000000000001},"last":4.6639999999999997},{"interface":"front","average":{"1min":2.9340000000000002,"5min":2.177,"15min":3.0539999999999998},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":15.529,"5min":15.529,"15min":15.529},"last":11.278}]},{"osd":3,"last update":"Mon Oct 9 10:10:50 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":4.5259999999999998,"15min":4.7519999999999998},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":20.419,"5min":25.734999999999999,"15min":25.734999999999999},"last":5.0339999999999998},{"interface":"front","average":{"1min":3.742,"5min":3.891,"15min":4.2370000000000001},"min":{"1min":0.40999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":10.154,"5min":21.923999999999999,"15min":21.923999999999999},"last":0.627}]},{"osd":4,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3780000000000001,"5min":2.3079999999999998,"15min":1.6970000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.989000000000001,"5min":20.494,"15min":20.494},"last":4.9770000000000003},{"interface":"front","average":{"1min":3.6819999999999999,"5min":2.395,"15min":2.089},"min":{"1min":0.40200000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.907,"5min":15.907,"15min":15.907},"last":0.65800000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5739999999999998,"15min":2.1869999999999998},"min":{"1min":0.36099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.868,"5min":17.096,"15min":17.096},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.52,"15min":1.4299999999999999},"min":{"1min":0.371,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.371,"5min":10.371,"15min":12.478},"last":4.7690000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.734,"15min":1.5860000000000001},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":8.827,"5min":9.4719999999999995,"15min":11.699},"last":4.7949999999999999},{"interface":"front","average":{"1min":4.5800000000000001,"5min":2.9740000000000002,"15min":2.4660000000000002},"min":{"1min":0.52100000000000002,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":23.106999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":5.0129999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2679999999999998,"5min":2.6699999999999999,"15min":2.3540000000000001},"min":{"1min":0.437,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":16.460000000000001,"5min":16.460000000000001,"15min":26.978000000000002},"last":0.64200000000000002},{"interface":"front","average":{"1min":2.677,"5min":1.7490000000000001,"15min":1.738},"min":{"1min":0.45100000000000001,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":8.8040000000000003,"5min":10.122,"15min":26.768000000000001},"last":4.7249999999999996}]}]},{"osd":2,"up_from":217,"seq":932007903307,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1307052,"kb_used_data":1058212,"kb_used_omap":83,"kb_used_meta":248748,"kb_avail":92438100,"statfs":{"total":95995035648,"available":94656614400,"internally_reserved":0,"allocated":1083609088,"data_stored":1082594868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":254718447},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0300000000000002,"5min":1.732,"15min":1.756},"min":{"1min":0.217,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":19.372,"5min":19.372,"15min":19.372},"last":0.60099999999999998},{"interface":"front","average":{"1min":3.4420000000000002,"5min":2.9849999999999999,"15min":2.9569999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.754999999999999,"5min":16.754999999999999,"15min":16.754999999999999},"last":6.4299999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":2.258,"15min":2.544},"min":{"1min":0.254,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.797000000000001,"5min":13.797000000000001,"15min":13.797000000000001},"last":9.5950000000000006},{"interface":"front","average":{"1min":2.8399999999999999,"5min":2.5209999999999999,"15min":2.6379999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.4119999999999999,"5min":15.569000000000001,"15min":15.569000000000001},"last":7.6109999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":2.198,"15min":2.0819999999999999},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.851000000000001,"5min":13.851000000000001,"15min":13.851000000000001},"last":0.96199999999999997},{"interface":"front","average":{"1min":2.9860000000000002,"5min":3.181,"15min":3.6699999999999999},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.7240000000000002,"5min":15.811,"15min":15.811},"last":1.0389999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":1.3,"15min":0.90400000000000003},"min":{"1min":0.29399999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.413,"5min":12.413,"15min":12.413},"last":1.006},{"interface":"front","average":{"1min":1.597,"5min":1.329,"15min":0.996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.314},"max":{"1min":7.0810000000000004,"5min":12.385,"15min":12.385},"last":8.391}]},{"osd":5,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.6919999999999999,"15min":1.546},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":12.473000000000001,"5min":12.473000000000001,"15min":12.473000000000001},"last":6.9039999999999999},{"interface":"front","average":{"1min":3.0699999999999998,"5min":1.5409999999999999,"15min":1.6870000000000001},"min":{"1min":0.372,"5min":0.318,"15min":0.318},"max":{"1min":13.587,"5min":13.587,"15min":13.587},"last":4.9059999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.3999999999999999,"15min":1.095},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2599999999999998,"5min":15.292999999999999,"15min":15.292999999999999},"last":6.5289999999999999},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.738,"15min":1.742},"min":{"1min":0.36599999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":7.6879999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.431,"15min":1.1259999999999999},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.2190000000000003,"5min":15.343999999999999,"15min":15.343999999999999},"last":6.4930000000000003},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.859,"15min":1.6599999999999999},"min":{"1min":0.41699999999999998,"5min":0.376,"15min":0.376},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":12.433999999999999},"last":6.7009999999999996}]}]},{"osd":3,"up_from":250,"seq":1073741824051,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1517172,"kb_used_data":1275052,"kb_used_omap":10,"kb_used_meta":242037,"kb_avail":92227980,"statfs":{"total":95995035648,"available":94441451520,"internally_reserved":0,"allocated":1305653248,"data_stored":1304455579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":247845955},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":4.2709999999999999,"15min":4.5590000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.753,"5min":23.381,"15min":23.381},"last":0.38800000000000001},{"interface":"front","average":{"1min":6.0720000000000001,"5min":4.0919999999999996,"15min":4.1500000000000004},"min":{"1min":0.42099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":20.239999999999998,"5min":20.239999999999998,"15min":20.239999999999998},"last":1.0529999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.6150000000000002,"15min":3.8410000000000002},"min":{"1min":0.26300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":20.422999999999998,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.94999999999999996},{"interface":"front","average":{"1min":4.5039999999999996,"5min":3.4990000000000001,"15min":3.4529999999999998},"min":{"1min":0.33100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.704000000000001,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.76100000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1369999999999996,"5min":3.4199999999999999,"15min":3.6339999999999999},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":20.594999999999999,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.623},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.8809999999999998,"15min":2.8700000000000001},"min":{"1min":0.34200000000000003,"5min":0.253,"15min":0.253},"max":{"1min":15.73,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.44800000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":2.048,"15min":1.413},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.696999999999999,"5min":15.676,"15min":15.676},"last":0.35999999999999999},{"interface":"front","average":{"1min":4.7910000000000004,"5min":2.2360000000000002,"15min":1.752},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.478999999999999,"5min":16.478999999999999,"15min":16.478999999999999},"last":1.119}]},{"osd":5,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8290000000000002,"5min":2.4750000000000001,"15min":2.5670000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":12.388,"5min":12.388,"15min":12.388},"last":1.141},{"interface":"front","average":{"1min":3.5699999999999998,"5min":2.5169999999999999,"15min":2.4079999999999999},"min":{"1min":0.443,"5min":0.309,"15min":0.309},"max":{"1min":15.992000000000001,"5min":15.992000000000001,"15min":15.992000000000001},"last":0.55700000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":2.536,"15min":2.6709999999999998},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.468,"5min":12.468,"15min":12.468},"last":1.159},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.5609999999999999,"15min":2.589},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":16.16,"5min":16.16,"15min":16.16},"last":0.59299999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.825,"15min":1.534},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.635999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.4399999999999999,"15min":2.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":1.1739999999999999}]}]},{"osd":4,"up_from":266,"seq":1142461300777,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1694364,"kb_used_data":1367444,"kb_used_omap":35,"kb_used_meta":326812,"kb_avail":92050788,"statfs":{"total":95995035648,"available":94260006912,"internally_reserved":0,"allocated":1400262656,"data_stored":1399735277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":334655726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.0109999999999999,"15min":0.84899999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.4829999999999997,"5min":10.881,"15min":10.881},"last":4.4889999999999999},{"interface":"front","average":{"1min":3.0150000000000001,"5min":1.5629999999999999,"15min":1.161},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":12.246,"5min":12.246,"15min":12.246},"last":4.5599999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2429999999999999,"5min":1.655,"15min":1.0960000000000001},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":12.161,"5min":12.161,"15min":12.161},"last":4.3959999999999999},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.7290000000000001,"15min":1.4259999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.504,"5min":11.141,"15min":11.141},"last":0.92300000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.4430000000000001,"15min":1.143},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.2469999999999999,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":4.774},{"interface":"front","average":{"1min":1.905,"5min":1.242,"15min":1.018},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":4.4130000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":1.9490000000000001,"15min":1.4910000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.065,"5min":13.065,"15min":13.065},"last":0.433},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.766,"15min":1.264},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.218,"5min":12.218,"15min":12.218},"last":9.3499999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":0.93200000000000005,"15min":0.79600000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.6869999999999994,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":4.3550000000000004},{"interface":"front","average":{"1min":0.436,"5min":0.56100000000000005,"15min":0.61899999999999999},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0609999999999999,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.40100000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.69799999999999995,"15min":0.58299999999999996},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.311},"max":{"1min":6.4119999999999999,"5min":6.4119999999999999,"15min":6.4119999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.7270000000000001,"5min":0.83899999999999997,"15min":0.66900000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.255000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":4.3239999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":0.89100000000000001,"15min":0.71099999999999997},"min":{"1min":0.38800000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":12.237,"5min":12.237,"15min":12.237},"last":4.4320000000000004},{"interface":"front","average":{"1min":1.244,"5min":0.90600000000000003,"15min":0.85399999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.7590000000000003,"5min":9.7590000000000003,"15min":9.7590000000000003},"last":0.36199999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822962,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1960144,"kb_used_data":1650568,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91785008,"statfs":{"total":95995035648,"available":93987848192,"internally_reserved":0,"allocated":1690181632,"data_stored":1689662309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,1,2,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4710000000000001,"15min":1.5720000000000001},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.8179999999999996,"5min":7.8540000000000001,"15min":15.773999999999999},"last":0.26400000000000001},{"interface":"front","average":{"1min":2.7770000000000001,"5min":1.6879999999999999,"15min":1.6819999999999999},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.9950000000000001,"5min":10.244999999999999,"15min":15.699},"last":0.91700000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:11:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":1.748,"15min":1.5629999999999999},"min":{"1min":0.40300000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.4250000000000007,"5min":11.335000000000001,"15min":15.689},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.903,"15min":1.7110000000000001},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5979999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.434}]},{"osd":2,"last update":"Mon Oct 9 10:10:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0350000000000001,"5min":1.7090000000000001,"15min":2.391},"min":{"1min":0.34899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7249999999999996,"5min":15.640000000000001,"15min":15.640000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.663,"5min":1.6479999999999999,"15min":2.2370000000000001},"min":{"1min":0.40400000000000003,"5min":0.313,"15min":0.313},"max":{"1min":5.2789999999999999,"5min":10.656000000000001,"15min":10.656000000000001},"last":0.51000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":2.1829999999999998,"15min":1.96},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2260000000000009,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.2370000000000001,"5min":2.4820000000000002,"15min":2.8399999999999999},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.9900000000000002,"5min":17.34,"15min":17.34},"last":3.258}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.84899999999999998,"15min":0.65900000000000003},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3460000000000001,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.70499999999999996,"15min":0.73099999999999998},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.462,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.40200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":0.97599999999999998,"15min":0.996},"min":{"1min":0.26200000000000001,"5min":0.25,"15min":0.25},"max":{"1min":8.5259999999999998,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.198,"5min":1.075,"15min":0.92600000000000005},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.9889999999999999,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.41799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":1.0900000000000001,"15min":0.97099999999999997},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.891,"5min":7.891,"15min":10.334},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.72299999999999998,"15min":0.93300000000000005},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.3630000000000004,"5min":6.3630000000000004,"15min":12.356},"last":0.47299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1276313600,"data_stored":1276313600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1550581760,"data_stored":1550581760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1073090560,"data_stored":1073086488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1291583488,"data_stored":1291583488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1390018560,"data_stored":1390018560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1681002496,"data_stored":1680998424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1446379520,"data_stored":1446379520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1097535488,"data_stored":1097531416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:11:56.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:56.142 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:56.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:56.565 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:56.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:56.566 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:11:56.566 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:11:56.571 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:11:56.969 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:11:56.971 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:11:56.971 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:11:56.971 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:11:56.971 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:11:56.971 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:11:56.971 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:11:56.972 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:11:56.972 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:11:56.972 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:11:56.972 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:11:56.972 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:11:56.972 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:11:56.974 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:11:57.007 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:57.050 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:11:57.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:57.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:11:57.104 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:11:57.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:57.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-mon[152263]: from='client.45749 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:57.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-mon[152263]: pgmap v829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5928/154806 objects degraded (3.829%); 1740/154806 objects misplaced (1.124%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:57.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3050899750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:57.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2905191936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:57.223 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-mon[147926]: from='client.45749 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:57.223 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-mon[147926]: pgmap v829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5928/154806 objects degraded (3.829%); 1740/154806 objects misplaced (1.124%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:57.223 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3050899750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:57.223 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2905191936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:57.234 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:57 smithi110 ceph-mon[136032]: from='client.45749 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:11:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:57 smithi110 ceph-mon[136032]: pgmap v829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5928/154806 objects degraded (3.829%); 1740/154806 objects misplaced (1.124%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:11:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3050899750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:11:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2905191936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:11:57.479 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:11:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:57.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:11:57.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:11:57.678 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:57.749 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:11:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:57.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:11:57.855 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:11:57.947 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:58.051 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:11:58.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:11:58.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:11:58.066 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:11:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:11:57] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T10:11:58.080 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:11:58.080 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:11:58.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:11:58.275 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:11:58.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:11:58.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2905191936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:58.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[147926]: osdmap e304: 8 total, 8 up, 8 in 2023-10-09T10:11:58.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[147926]: failed to encode map e304 with expected crc 2023-10-09T10:11:58.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[147926]: failed to encode map e304 with expected crc 2023-10-09T10:11:58.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/154806 objects degraded (3.829%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:58.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:11:58.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2905191936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:58.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[152263]: osdmap e304: 8 total, 8 up, 8 in 2023-10-09T10:11:58.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[152263]: failed to encode map e304 with expected crc 2023-10-09T10:11:58.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[152263]: failed to encode map e304 with expected crc 2023-10-09T10:11:58.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/154806 objects degraded (3.829%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:58 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:11:58.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2905191936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:11:58.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:58 smithi110 ceph-mon[136032]: osdmap e304: 8 total, 8 up, 8 in 2023-10-09T10:11:58.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:58 smithi110 ceph-mon[136032]: failed to encode map e304 with expected crc 2023-10-09T10:11:58.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:58 smithi110 ceph-mon[136032]: failed to encode map e304 with expected crc 2023-10-09T10:11:58.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/154806 objects degraded (3.829%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:11:59.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:11:59.363 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:11:59.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:59 smithi012 ceph-mon[152263]: pgmap v831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 5928/150915 objects degraded (3.928%); 1566/150915 objects misplaced (1.038%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:11:59.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:59.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:11:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:59.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:59 smithi012 ceph-mon[147926]: pgmap v831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 5928/150915 objects degraded (3.928%); 1566/150915 objects misplaced (1.038%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:11:59.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:59.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:11:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:59 smithi110 ceph-mon[136032]: pgmap v831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 5928/150915 objects degraded (3.928%); 1566/150915 objects misplaced (1.038%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:11:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:11:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:11:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:11:59.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:12:00.141 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:00.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:00.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:12:00.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:00.349 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:00.460 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:00.496 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:00.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:00.553 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:00.605 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:00.607 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:00.640 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:00.677 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:00.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:00.781 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:00.829 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:00.894 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:00.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:01.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:01.053 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:01.097 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:01.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:01.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:01.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:01 smithi110 ceph-mon[136032]: pgmap v832: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5928/146358 objects degraded (4.050%); 1566/146358 objects misplaced (1.070%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T10:12:01.201 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:01.240 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:01.294 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:01.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:01.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:01.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:01 smithi012 ceph-mon[147926]: pgmap v832: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5928/146358 objects degraded (4.050%); 1566/146358 objects misplaced (1.070%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T10:12:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:01 smithi012 ceph-mon[152263]: pgmap v832: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5928/146358 objects degraded (4.050%); 1566/146358 objects misplaced (1.070%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T10:12:01.468 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:01.752 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:01.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:01.857 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:01.952 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:02.757 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:02.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:02.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:02.940 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:03.275 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:03.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:03.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:03.289 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:03 smithi012 ceph-mon[147926]: pgmap v833: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5928/145740 objects degraded (4.068%); 1566/145740 objects misplaced (1.075%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T10:12:03.289 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/145740 objects degraded (4.068%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:03.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:03 smithi012 ceph-mon[152263]: pgmap v833: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5928/145740 objects degraded (4.068%); 1566/145740 objects misplaced (1.075%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T10:12:03.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/145740 objects degraded (4.068%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:03.390 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:03.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:03 smithi110 ceph-mon[136032]: pgmap v833: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 5928/145740 objects degraded (4.068%); 1566/145740 objects misplaced (1.075%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T10:12:03.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/145740 objects degraded (4.068%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:03.448 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:03.508 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:03.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:03.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:03.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:03.699 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:03.795 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:03.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:03.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:03.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:03.984 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:04.124 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:04.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:04.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:04.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:04.325 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:04.338 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:04.357 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:12:04.583 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:04.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:04.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:04.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:04.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:04.832 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:05.155 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":834,"stamp":"2023-10-09T10:12:03.396747+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3042973624,"num_objects":46448,"num_object_clones":0,"num_object_copies":139344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5928,"num_objects_misplaced":1369,"num_objects_unfound":0,"num_objects_dirty":46448,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":171198,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3429,"num_bytes_recovered":221391564,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142232,"ondisk_log_size":142232,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12036492,"kb_used_data":9493708,"kb_used_omap":167,"kb_used_meta":2542488,"kb_avail":737924724,"statfs":{"total":767960285184,"available":755634917376,"internally_reserved":0,"allocated":9721556992,"data_stored":9715169428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2603508313},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-399835136,"num_objects":-6101,"num_object_clones":0,"num_object_copies":-18303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-371,"num_objects_unfound":0,"num_objects_dirty":-6101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6101,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":371,"num_bytes_recovered":24313856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006795"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":146,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987400+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:11:47.987400+0000","last_peered":"2023-10-09T10:11:47.987400+0000","last_clean":"2023-10-09T10:11:47.987400+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:11:47.987400+0000","last_undegraded":"2023-10-09T10:11:47.987400+0000","last_fullsized":"2023-10-09T10:11:47.987400+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":261,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977707+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:11:47.977707+0000","last_peered":"2023-10-09T10:11:47.977707+0000","last_clean":"2023-10-09T10:11:47.977707+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:11:47.977707+0000","last_undegraded":"2023-10-09T10:11:47.977707+0000","last_fullsized":"2023-10-09T10:11:47.977707+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":307,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987003+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:11:47.987003+0000","last_peered":"2023-10-09T10:11:47.987003+0000","last_clean":"2023-10-09T10:11:47.987003+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:11:47.987003+0000","last_undegraded":"2023-10-09T10:11:47.987003+0000","last_fullsized":"2023-10-09T10:11:47.987003+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":170,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.984686+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:11:47.984686+0000","last_peered":"2023-10-09T10:11:47.984686+0000","last_clean":"2023-10-09T10:11:47.984686+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:11:47.984686+0000","last_undegraded":"2023-10-09T10:11:47.984686+0000","last_fullsized":"2023-10-09T10:11:47.984686+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":275,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.978203+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:11:47.978203+0000","last_peered":"2023-10-09T10:11:47.978203+0000","last_clean":"2023-10-09T10:11:47.978203+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:11:47.978203+0000","last_undegraded":"2023-10-09T10:11:47.978203+0000","last_fullsized":"2023-10-09T10:11:47.978203+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.980083+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:11:47.980083+0000","last_peered":"2023-10-09T10:11:47.980083+0000","last_clean":"2023-10-09T10:11:47.980083+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:11:47.980083+0000","last_undegraded":"2023-10-09T10:11:47.980083+0000","last_fullsized":"2023-10-09T10:11:47.980083+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":187,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.984553+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:11:47.984553+0000","last_peered":"2023-10-09T10:11:47.984553+0000","last_clean":"2023-10-09T10:11:47.984553+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:11:47.984553+0000","last_undegraded":"2023-10-09T10:11:47.984553+0000","last_fullsized":"2023-10-09T10:11:47.984553+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":256,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.978135+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:11:47.978135+0000","last_peered":"2023-10-09T10:11:47.978135+0000","last_clean":"2023-10-09T10:11:47.978135+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:11:47.978135+0000","last_undegraded":"2023-10-09T10:11:47.978135+0000","last_fullsized":"2023-10-09T10:11:47.978135+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":121,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.979853+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:11:47.979853+0000","last_peered":"2023-10-09T10:11:47.979853+0000","last_clean":"2023-10-09T10:11:47.979853+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:11:47.979853+0000","last_undegraded":"2023-10-09T10:11:47.979853+0000","last_fullsized":"2023-10-09T10:11:47.979853+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":217,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977599+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:11:47.977599+0000","last_peered":"2023-10-09T10:11:47.977599+0000","last_clean":"2023-10-09T10:11:47.977599+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:11:47.977599+0000","last_undegraded":"2023-10-09T10:11:47.977599+0000","last_fullsized":"2023-10-09T10:11:47.977599+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":342,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.986159+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:11:47.986159+0000","last_peered":"2023-10-09T10:11:47.986159+0000","last_clean":"2023-10-09T10:11:47.986159+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:11:47.986159+0000","last_undegraded":"2023-10-09T10:11:47.986159+0000","last_fullsized":"2023-10-09T10:11:47.986159+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":299,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987243+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:11:47.987243+0000","last_peered":"2023-10-09T10:11:47.987243+0000","last_clean":"2023-10-09T10:11:47.987243+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:11:47.987243+0000","last_undegraded":"2023-10-09T10:11:47.987243+0000","last_fullsized":"2023-10-09T10:11:47.987243+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":191,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.990482+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:11:47.990482+0000","last_peered":"2023-10-09T10:11:47.990482+0000","last_clean":"2023-10-09T10:11:47.990482+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:11:47.990482+0000","last_undegraded":"2023-10-09T10:11:47.990482+0000","last_fullsized":"2023-10-09T10:11:47.990482+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":171,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.990341+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:11:47.990341+0000","last_peered":"2023-10-09T10:11:47.990341+0000","last_clean":"2023-10-09T10:11:47.990341+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:11:47.990341+0000","last_undegraded":"2023-10-09T10:11:47.990341+0000","last_fullsized":"2023-10-09T10:11:47.990341+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":375,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.985889+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:11:47.985889+0000","last_peered":"2023-10-09T10:11:47.985889+0000","last_clean":"2023-10-09T10:11:47.985889+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:11:47.985889+0000","last_undegraded":"2023-10-09T10:11:47.985889+0000","last_fullsized":"2023-10-09T10:11:47.985889+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":243,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.991184+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:11:47.991184+0000","last_peered":"2023-10-09T10:11:47.991184+0000","last_clean":"2023-10-09T10:11:47.991184+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:11:47.991184+0000","last_undegraded":"2023-10-09T10:11:47.991184+0000","last_fullsized":"2023-10-09T10:11:47.991184+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.685840+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:11:51.685840+0000","last_peered":"2023-10-09T10:11:51.685840+0000","last_clean":"2023-10-09T10:11:51.685840+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:11:51.685840+0000","last_undegraded":"2023-10-09T10:11:51.685840+0000","last_fullsized":"2023-10-09T10:11:51.685840+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":137,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.979815+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:11:47.979815+0000","last_peered":"2023-10-09T10:11:47.979815+0000","last_clean":"2023-10-09T10:11:47.979815+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:11:47.979815+0000","last_undegraded":"2023-10-09T10:11:47.979815+0000","last_fullsized":"2023-10-09T10:11:47.979815+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":172,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.983886+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:11:47.983886+0000","last_peered":"2023-10-09T10:11:47.983886+0000","last_clean":"2023-10-09T10:11:47.983886+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:11:47.983886+0000","last_undegraded":"2023-10-09T10:11:47.983886+0000","last_fullsized":"2023-10-09T10:11:47.983886+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":169,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.984755+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:11:47.984755+0000","last_peered":"2023-10-09T10:11:47.984755+0000","last_clean":"2023-10-09T10:11:47.984755+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:11:47.984755+0000","last_undegraded":"2023-10-09T10:11:47.984755+0000","last_fullsized":"2023-10-09T10:11:47.984755+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":193,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.685803+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:11:51.685803+0000","last_peered":"2023-10-09T10:11:51.685803+0000","last_clean":"2023-10-09T10:11:51.685803+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:11:51.685803+0000","last_undegraded":"2023-10-09T10:11:51.685803+0000","last_fullsized":"2023-10-09T10:11:51.685803+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":256,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.988231+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:11:47.988231+0000","last_peered":"2023-10-09T10:11:47.988231+0000","last_clean":"2023-10-09T10:11:47.988231+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:11:47.988231+0000","last_undegraded":"2023-10-09T10:11:47.988231+0000","last_fullsized":"2023-10-09T10:11:47.988231+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"304'10883","reported_seq":10932,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:02.937639+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:12:02.937639+0000","last_peered":"2023-10-09T10:12:02.937639+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:12:02.937639+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"277'800","ondisk_log_start":"277'800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10083,"log_dups_size":0,"ondisk_log_size":10083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182386688,"num_objects":2783,"num_object_clones":0,"num_object_copies":8349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10883,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-2590},{"shards":"4,5,6","objects":1323}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":336,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.986235+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:11:47.986235+0000","last_peered":"2023-10-09T10:11:47.986235+0000","last_clean":"2023-10-09T10:11:47.986235+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:11:47.986235+0000","last_undegraded":"2023-10-09T10:11:47.986235+0000","last_fullsized":"2023-10-09T10:11:47.986235+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":122,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.978498+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:11:47.978498+0000","last_peered":"2023-10-09T10:11:47.978498+0000","last_clean":"2023-10-09T10:11:47.978498+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:11:47.978498+0000","last_undegraded":"2023-10-09T10:11:47.978498+0000","last_fullsized":"2023-10-09T10:11:47.978498+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"304'10800","reported_seq":10841,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:00.039897+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:12:00.039897+0000","last_peered":"2023-10-09T10:12:00.039897+0000","last_clean":"2023-10-09T10:12:00.039897+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:12:00.039897+0000","last_undegraded":"2023-10-09T10:12:00.039897+0000","last_fullsized":"2023-10-09T10:12:00.039897+0000","mapping_epoch":271,"log_start":"280'2200","ondisk_log_start":"280'2200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8600,"log_dups_size":0,"ondisk_log_size":8600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195952640,"num_objects":2990,"num_object_clones":0,"num_object_copies":8970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10800,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":324,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.990951+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:11:47.990951+0000","last_peered":"2023-10-09T10:11:47.990951+0000","last_clean":"2023-10-09T10:11:47.990951+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:11:47.990951+0000","last_undegraded":"2023-10-09T10:11:47.990951+0000","last_fullsized":"2023-10-09T10:11:47.990951+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":215,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.986902+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:11:47.986902+0000","last_peered":"2023-10-09T10:11:47.986902+0000","last_clean":"2023-10-09T10:11:47.986902+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:11:47.986902+0000","last_undegraded":"2023-10-09T10:11:47.986902+0000","last_fullsized":"2023-10-09T10:11:47.986902+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"304'10655","reported_seq":10698,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:01.714725+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:12:01.714725+0000","last_peered":"2023-10-09T10:12:01.714725+0000","last_clean":"2023-10-09T10:12:01.714725+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:12:01.714725+0000","last_undegraded":"2023-10-09T10:12:01.714725+0000","last_fullsized":"2023-10-09T10:12:01.714725+0000","mapping_epoch":271,"log_start":"277'1200","ondisk_log_start":"277'1200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9455,"log_dups_size":0,"ondisk_log_size":9455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190775296,"num_objects":2911,"num_object_clones":0,"num_object_copies":8733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10655,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":126,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.979888+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:11:47.979888+0000","last_peered":"2023-10-09T10:11:47.979888+0000","last_clean":"2023-10-09T10:11:47.979888+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:11:47.979888+0000","last_undegraded":"2023-10-09T10:11:47.979888+0000","last_fullsized":"2023-10-09T10:11:47.979888+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.996737+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:11:47.996737+0000","last_peered":"2023-10-09T10:11:47.996737+0000","last_clean":"2023-10-09T10:11:47.996737+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:11:47.996737+0000","last_undegraded":"2023-10-09T10:11:47.996737+0000","last_fullsized":"2023-10-09T10:11:47.996737+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"304'10391","reported_seq":10434,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:58.565470+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:11:58.565470+0000","last_peered":"2023-10-09T10:11:58.565470+0000","last_clean":"2023-10-09T10:11:58.565470+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:11:58.565470+0000","last_undegraded":"2023-10-09T10:11:58.565470+0000","last_fullsized":"2023-10-09T10:11:58.565470+0000","mapping_epoch":271,"log_start":"280'2600","ondisk_log_start":"280'2600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7791,"log_dups_size":0,"ondisk_log_size":7791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197853184,"num_objects":3019,"num_object_clones":0,"num_object_copies":9057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10391,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":137,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.979854+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:11:47.979854+0000","last_peered":"2023-10-09T10:11:47.979854+0000","last_clean":"2023-10-09T10:11:47.979854+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:11:47.979854+0000","last_undegraded":"2023-10-09T10:11:47.979854+0000","last_fullsized":"2023-10-09T10:11:47.979854+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":171,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.984062+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:11:47.984062+0000","last_peered":"2023-10-09T10:11:47.984062+0000","last_clean":"2023-10-09T10:11:47.984062+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:11:47.984062+0000","last_undegraded":"2023-10-09T10:11:47.984062+0000","last_fullsized":"2023-10-09T10:11:47.984062+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"304'10906","reported_seq":10949,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:01.488428+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:12:01.488428+0000","last_peered":"2023-10-09T10:12:01.488428+0000","last_clean":"2023-10-09T10:12:01.488428+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:12:01.488428+0000","last_undegraded":"2023-10-09T10:12:01.488428+0000","last_fullsized":"2023-10-09T10:12:01.488428+0000","mapping_epoch":271,"log_start":"280'2700","ondisk_log_start":"280'2700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8206,"log_dups_size":0,"ondisk_log_size":8206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192282624,"num_objects":2934,"num_object_clones":0,"num_object_copies":8802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10906,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":297,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977700+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:11:47.977700+0000","last_peered":"2023-10-09T10:11:47.977700+0000","last_clean":"2023-10-09T10:11:47.977700+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:11:47.977700+0000","last_undegraded":"2023-10-09T10:11:47.977700+0000","last_fullsized":"2023-10-09T10:11:47.977700+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":167,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.979737+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:11:47.979737+0000","last_peered":"2023-10-09T10:11:47.979737+0000","last_clean":"2023-10-09T10:11:47.979737+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:11:47.979737+0000","last_undegraded":"2023-10-09T10:11:47.979737+0000","last_fullsized":"2023-10-09T10:11:47.979737+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"304'10744","reported_seq":10787,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:58.539119+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:11:58.539119+0000","last_peered":"2023-10-09T10:11:58.539119+0000","last_clean":"2023-10-09T10:11:58.539119+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:11:58.539119+0000","last_undegraded":"2023-10-09T10:11:58.539119+0000","last_fullsized":"2023-10-09T10:11:58.539119+0000","mapping_epoch":271,"log_start":"280'3000","ondisk_log_start":"280'3000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193986560,"num_objects":2960,"num_object_clones":0,"num_object_copies":8880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10744,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":229,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987172+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:11:47.987172+0000","last_peered":"2023-10-09T10:11:47.987172+0000","last_clean":"2023-10-09T10:11:47.987172+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:11:47.987172+0000","last_undegraded":"2023-10-09T10:11:47.987172+0000","last_fullsized":"2023-10-09T10:11:47.987172+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":235,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987561+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:11:47.987561+0000","last_peered":"2023-10-09T10:11:47.987561+0000","last_clean":"2023-10-09T10:11:47.987561+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:11:47.987561+0000","last_undegraded":"2023-10-09T10:11:47.987561+0000","last_fullsized":"2023-10-09T10:11:47.987561+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"304'10562","reported_seq":10605,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:01.681947+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:12:01.681947+0000","last_peered":"2023-10-09T10:12:01.681947+0000","last_clean":"2023-10-09T10:12:01.681947+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:12:01.681947+0000","last_undegraded":"2023-10-09T10:12:01.681947+0000","last_fullsized":"2023-10-09T10:12:01.681947+0000","mapping_epoch":271,"log_start":"277'1100","ondisk_log_start":"277'1100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9462,"log_dups_size":0,"ondisk_log_size":9462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188088320,"num_objects":2870,"num_object_clones":0,"num_object_copies":8610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10562,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":198,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.983887+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:11:47.983887+0000","last_peered":"2023-10-09T10:11:47.983887+0000","last_clean":"2023-10-09T10:11:47.983887+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:11:47.983887+0000","last_undegraded":"2023-10-09T10:11:47.983887+0000","last_fullsized":"2023-10-09T10:11:47.983887+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":171,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.991218+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:11:47.991218+0000","last_peered":"2023-10-09T10:11:47.991218+0000","last_clean":"2023-10-09T10:11:47.991218+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:11:47.991218+0000","last_undegraded":"2023-10-09T10:11:47.991218+0000","last_fullsized":"2023-10-09T10:11:47.991218+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"304'10619","reported_seq":10661,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:58.564936+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:11:58.564936+0000","last_peered":"2023-10-09T10:11:58.564936+0000","last_clean":"2023-10-09T10:11:58.564936+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:11:58.564936+0000","last_undegraded":"2023-10-09T10:11:58.564936+0000","last_fullsized":"2023-10-09T10:11:58.564936+0000","mapping_epoch":271,"log_start":"280'2900","ondisk_log_start":"280'2900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7719,"log_dups_size":0,"ondisk_log_size":7719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195231744,"num_objects":2979,"num_object_clones":0,"num_object_copies":8937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10619,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":258,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.074349+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:11:57.074349+0000","last_peered":"2023-10-09T10:11:57.074349+0000","last_clean":"2023-10-09T10:11:57.074349+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:11:57.074349+0000","last_undegraded":"2023-10-09T10:11:57.074349+0000","last_fullsized":"2023-10-09T10:11:57.074349+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":307,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.990982+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:11:47.990982+0000","last_peered":"2023-10-09T10:11:47.990982+0000","last_clean":"2023-10-09T10:11:47.990982+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:11:47.990982+0000","last_undegraded":"2023-10-09T10:11:47.990982+0000","last_fullsized":"2023-10-09T10:11:47.990982+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"304'10622","reported_seq":10665,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:01.638766+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:12:01.638766+0000","last_peered":"2023-10-09T10:12:01.638766+0000","last_clean":"2023-10-09T10:12:01.638766+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:12:01.638766+0000","last_undegraded":"2023-10-09T10:12:01.638766+0000","last_fullsized":"2023-10-09T10:12:01.638766+0000","mapping_epoch":271,"log_start":"277'1200","ondisk_log_start":"277'1200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9422,"log_dups_size":0,"ondisk_log_size":9422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186777600,"num_objects":2850,"num_object_clones":0,"num_object_copies":8550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10622,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":307,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987228+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:11:47.987228+0000","last_peered":"2023-10-09T10:11:47.987228+0000","last_clean":"2023-10-09T10:11:47.987228+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:11:47.987228+0000","last_undegraded":"2023-10-09T10:11:47.987228+0000","last_fullsized":"2023-10-09T10:11:47.987228+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":146,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.979820+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:11:47.979820+0000","last_peered":"2023-10-09T10:11:47.979820+0000","last_clean":"2023-10-09T10:11:47.979820+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:11:47.979820+0000","last_undegraded":"2023-10-09T10:11:47.979820+0000","last_fullsized":"2023-10-09T10:11:47.979820+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":263,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987478+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:11:47.987478+0000","last_peered":"2023-10-09T10:11:47.987478+0000","last_clean":"2023-10-09T10:11:47.987478+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:11:47.987478+0000","last_undegraded":"2023-10-09T10:11:47.987478+0000","last_fullsized":"2023-10-09T10:11:47.987478+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"304'10778","reported_seq":10819,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:02.867653+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:12:02.867653+0000","last_peered":"2023-10-09T10:12:02.867653+0000","last_clean":"2023-10-09T10:12:02.867653+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:12:02.867653+0000","last_undegraded":"2023-10-09T10:12:02.867653+0000","last_fullsized":"2023-10-09T10:12:02.867653+0000","mapping_epoch":271,"log_start":"277'700","ondisk_log_start":"277'700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10078,"log_dups_size":0,"ondisk_log_size":10078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183238656,"num_objects":2796,"num_object_clones":0,"num_object_copies":8388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10778,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":122,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.978616+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:11:47.978616+0000","last_peered":"2023-10-09T10:11:47.978616+0000","last_clean":"2023-10-09T10:11:47.978616+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:11:47.978616+0000","last_undegraded":"2023-10-09T10:11:47.978616+0000","last_fullsized":"2023-10-09T10:11:47.978616+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":120,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.983912+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:11:47.983912+0000","last_peered":"2023-10-09T10:11:47.983912+0000","last_clean":"2023-10-09T10:11:47.983912+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:11:47.983912+0000","last_undegraded":"2023-10-09T10:11:47.983912+0000","last_fullsized":"2023-10-09T10:11:47.983912+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":330,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987159+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:11:47.987159+0000","last_peered":"2023-10-09T10:11:47.987159+0000","last_clean":"2023-10-09T10:11:47.987159+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:11:47.987159+0000","last_undegraded":"2023-10-09T10:11:47.987159+0000","last_fullsized":"2023-10-09T10:11:47.987159+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"304'10694","reported_seq":10737,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:01.419697+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:12:01.419697+0000","last_peered":"2023-10-09T10:12:01.419697+0000","last_clean":"2023-10-09T10:12:01.419697+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:12:01.419697+0000","last_undegraded":"2023-10-09T10:12:01.419697+0000","last_fullsized":"2023-10-09T10:12:01.419697+0000","mapping_epoch":271,"log_start":"280'2500","ondisk_log_start":"280'2500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186646528,"num_objects":2848,"num_object_clones":0,"num_object_copies":8544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10694,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":224,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:56.489059+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:11:56.489059+0000","last_peered":"2023-10-09T10:11:56.489059+0000","last_clean":"2023-10-09T10:11:56.489059+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:11:56.489059+0000","last_undegraded":"2023-10-09T10:11:56.489059+0000","last_fullsized":"2023-10-09T10:11:56.489059+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":299,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987387+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:11:47.987387+0000","last_peered":"2023-10-09T10:11:47.987387+0000","last_clean":"2023-10-09T10:11:47.987387+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:11:47.987387+0000","last_undegraded":"2023-10-09T10:11:47.987387+0000","last_fullsized":"2023-10-09T10:11:47.987387+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":320,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:53.572425+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:11:53.572425+0000","last_peered":"2023-10-09T10:11:53.572425+0000","last_clean":"2023-10-09T10:11:53.572425+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:11:53.572425+0000","last_undegraded":"2023-10-09T10:11:53.572425+0000","last_fullsized":"2023-10-09T10:11:53.572425+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"304'10642","reported_seq":10683,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:02.910908+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:12:02.910908+0000","last_peered":"2023-10-09T10:12:02.910908+0000","last_clean":"2023-10-09T10:12:02.910908+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:12:02.910908+0000","last_undegraded":"2023-10-09T10:12:02.910908+0000","last_fullsized":"2023-10-09T10:12:02.910908+0000","mapping_epoch":271,"log_start":"276'600","ondisk_log_start":"276'600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10042,"log_dups_size":0,"ondisk_log_size":10042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184680448,"num_objects":2818,"num_object_clones":0,"num_object_copies":8454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10642,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":131,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.978680+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:11:47.978680+0000","last_peered":"2023-10-09T10:11:47.978680+0000","last_clean":"2023-10-09T10:11:47.978680+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:11:47.978680+0000","last_undegraded":"2023-10-09T10:11:47.978680+0000","last_fullsized":"2023-10-09T10:11:47.978680+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":550,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.988073+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:11:47.988073+0000","last_peered":"2023-10-09T10:11:47.988073+0000","last_clean":"2023-10-09T10:11:47.988073+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:11:47.988073+0000","last_undegraded":"2023-10-09T10:11:47.988073+0000","last_fullsized":"2023-10-09T10:11:47.988073+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":208,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:51.459915+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:11:51.459915+0000","last_peered":"2023-10-09T10:11:51.459915+0000","last_clean":"2023-10-09T10:11:51.459915+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:11:51.459915+0000","last_undegraded":"2023-10-09T10:11:51.459915+0000","last_fullsized":"2023-10-09T10:11:51.459915+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":219,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.986226+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:11:47.986226+0000","last_peered":"2023-10-09T10:11:47.986226+0000","last_clean":"2023-10-09T10:11:47.986226+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:11:47.986226+0000","last_undegraded":"2023-10-09T10:11:47.986226+0000","last_fullsized":"2023-10-09T10:11:47.986226+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"304'10609","reported_seq":10652,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:58.525113+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:11:58.525113+0000","last_peered":"2023-10-09T10:11:58.525113+0000","last_clean":"2023-10-09T10:11:58.525113+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:11:58.525113+0000","last_undegraded":"2023-10-09T10:11:58.525113+0000","last_fullsized":"2023-10-09T10:11:58.525113+0000","mapping_epoch":271,"log_start":"280'2900","ondisk_log_start":"280'2900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7709,"log_dups_size":0,"ondisk_log_size":7709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197459968,"num_objects":3013,"num_object_clones":0,"num_object_copies":9039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10609,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":279,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987530+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:11:47.987530+0000","last_peered":"2023-10-09T10:11:47.987530+0000","last_clean":"2023-10-09T10:11:47.987530+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:11:47.987530+0000","last_undegraded":"2023-10-09T10:11:47.987530+0000","last_fullsized":"2023-10-09T10:11:47.987530+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":181,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.985927+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:11:47.985927+0000","last_peered":"2023-10-09T10:11:47.985927+0000","last_clean":"2023-10-09T10:11:47.985927+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:11:47.985927+0000","last_undegraded":"2023-10-09T10:11:47.985927+0000","last_fullsized":"2023-10-09T10:11:47.985927+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":209,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.984617+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:11:47.984617+0000","last_peered":"2023-10-09T10:11:47.984617+0000","last_clean":"2023-10-09T10:11:47.984617+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:11:47.984617+0000","last_undegraded":"2023-10-09T10:11:47.984617+0000","last_fullsized":"2023-10-09T10:11:47.984617+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"304'10716","reported_seq":17486,"reported_epoch":304,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:01.466990+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:12:01.466990+0000","last_peered":"2023-10-09T10:12:01.466990+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:12:01.466990+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"280'2600","ondisk_log_start":"280'2600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8116,"log_dups_size":0,"ondisk_log_size":8116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192806912,"num_objects":2942,"num_object_clones":0,"num_object_copies":8826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":555,"num_objects_misplaced":1369,"num_objects_unfound":0,"num_objects_dirty":2942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10716,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3360,"num_bytes_recovered":220200960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1018},{"shards":"1,3,4","objects":1369}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":323,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.986065+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:11:47.986065+0000","last_peered":"2023-10-09T10:11:47.986065+0000","last_clean":"2023-10-09T10:11:47.986065+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:11:47.986065+0000","last_undegraded":"2023-10-09T10:11:47.986065+0000","last_fullsized":"2023-10-09T10:11:47.986065+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":336,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987304+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:11:47.987304+0000","last_peered":"2023-10-09T10:11:47.987304+0000","last_clean":"2023-10-09T10:11:47.987304+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:11:47.987304+0000","last_undegraded":"2023-10-09T10:11:47.987304+0000","last_fullsized":"2023-10-09T10:11:47.987304+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":173,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.983960+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:11:47.983960+0000","last_peered":"2023-10-09T10:11:47.983960+0000","last_clean":"2023-10-09T10:11:47.983960+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:11:47.983960+0000","last_undegraded":"2023-10-09T10:11:47.983960+0000","last_fullsized":"2023-10-09T10:11:47.983960+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"304'10704","reported_seq":10747,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:01.693145+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:12:01.693145+0000","last_peered":"2023-10-09T10:12:01.693145+0000","last_clean":"2023-10-09T10:12:01.693145+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:12:01.693145+0000","last_undegraded":"2023-10-09T10:12:01.693145+0000","last_fullsized":"2023-10-09T10:12:01.693145+0000","mapping_epoch":271,"log_start":"277'1300","ondisk_log_start":"277'1300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9404,"log_dups_size":0,"ondisk_log_size":9404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187039744,"num_objects":2854,"num_object_clones":0,"num_object_copies":8562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10704,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":267,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.073135+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:11:57.073135+0000","last_peered":"2023-10-09T10:11:57.073135+0000","last_clean":"2023-10-09T10:11:57.073135+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:11:57.073135+0000","last_undegraded":"2023-10-09T10:11:57.073135+0000","last_fullsized":"2023-10-09T10:11:57.073135+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":281,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:53.572396+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:11:53.572396+0000","last_peered":"2023-10-09T10:11:53.572396+0000","last_clean":"2023-10-09T10:11:53.572396+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:11:53.572396+0000","last_undegraded":"2023-10-09T10:11:53.572396+0000","last_fullsized":"2023-10-09T10:11:53.572396+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":256,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977997+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:11:47.977997+0000","last_peered":"2023-10-09T10:11:47.977997+0000","last_clean":"2023-10-09T10:11:47.977997+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:11:47.977997+0000","last_undegraded":"2023-10-09T10:11:47.977997+0000","last_fullsized":"2023-10-09T10:11:47.977997+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"304'10613","reported_seq":10655,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:12:02.935095+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:12:02.935095+0000","last_peered":"2023-10-09T10:12:02.935095+0000","last_clean":"2023-10-09T10:12:02.935095+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:12:02.935095+0000","last_undegraded":"2023-10-09T10:12:02.935095+0000","last_fullsized":"2023-10-09T10:12:02.935095+0000","mapping_epoch":271,"log_start":"276'600","ondisk_log_start":"276'600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10013,"log_dups_size":0,"ondisk_log_size":10013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187170840,"num_objects":2857,"num_object_clones":0,"num_object_copies":8571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2857,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10613,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":212,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.996887+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:11:47.996887+0000","last_peered":"2023-10-09T10:11:47.996887+0000","last_clean":"2023-10-09T10:11:47.996887+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:11:47.996887+0000","last_undegraded":"2023-10-09T10:11:47.996887+0000","last_fullsized":"2023-10-09T10:11:47.996887+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":175,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.984537+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:11:47.984537+0000","last_peered":"2023-10-09T10:11:47.984537+0000","last_clean":"2023-10-09T10:11:47.984537+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:11:47.984537+0000","last_undegraded":"2023-10-09T10:11:47.984537+0000","last_fullsized":"2023-10-09T10:11:47.984537+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":157,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.978600+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:11:47.978600+0000","last_peered":"2023-10-09T10:11:47.978600+0000","last_clean":"2023-10-09T10:11:47.978600+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:11:47.978600+0000","last_undegraded":"2023-10-09T10:11:47.978600+0000","last_fullsized":"2023-10-09T10:11:47.978600+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":169,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.983851+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:11:47.983851+0000","last_peered":"2023-10-09T10:11:47.983851+0000","last_clean":"2023-10-09T10:11:47.983851+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:11:47.983851+0000","last_undegraded":"2023-10-09T10:11:47.983851+0000","last_fullsized":"2023-10-09T10:11:47.983851+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":242,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987386+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:11:47.987386+0000","last_peered":"2023-10-09T10:11:47.987386+0000","last_clean":"2023-10-09T10:11:47.987386+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:11:47.987386+0000","last_undegraded":"2023-10-09T10:11:47.987386+0000","last_fullsized":"2023-10-09T10:11:47.987386+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":141,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.979770+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:11:47.979770+0000","last_peered":"2023-10-09T10:11:47.979770+0000","last_clean":"2023-10-09T10:11:47.979770+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:11:47.979770+0000","last_undegraded":"2023-10-09T10:11:47.979770+0000","last_fullsized":"2023-10-09T10:11:47.979770+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":182,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977612+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:11:47.977612+0000","last_peered":"2023-10-09T10:11:47.977612+0000","last_clean":"2023-10-09T10:11:47.977612+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:11:47.977612+0000","last_undegraded":"2023-10-09T10:11:47.977612+0000","last_fullsized":"2023-10-09T10:11:47.977612+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":141,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.979724+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:11:47.979724+0000","last_peered":"2023-10-09T10:11:47.979724+0000","last_clean":"2023-10-09T10:11:47.979724+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:11:47.979724+0000","last_undegraded":"2023-10-09T10:11:47.979724+0000","last_fullsized":"2023-10-09T10:11:47.979724+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":191,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977413+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:11:47.977413+0000","last_peered":"2023-10-09T10:11:47.977413+0000","last_clean":"2023-10-09T10:11:47.977413+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:11:47.977413+0000","last_undegraded":"2023-10-09T10:11:47.977413+0000","last_fullsized":"2023-10-09T10:11:47.977413+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":321,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977681+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:11:47.977681+0000","last_peered":"2023-10-09T10:11:47.977681+0000","last_clean":"2023-10-09T10:11:47.977681+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:11:47.977681+0000","last_undegraded":"2023-10-09T10:11:47.977681+0000","last_fullsized":"2023-10-09T10:11:47.977681+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":341,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.977756+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:11:47.977756+0000","last_peered":"2023-10-09T10:11:47.977756+0000","last_clean":"2023-10-09T10:11:47.977756+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:11:47.977756+0000","last_undegraded":"2023-10-09T10:11:47.977756+0000","last_fullsized":"2023-10-09T10:11:47.977756+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":251,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.984388+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:11:47.984388+0000","last_peered":"2023-10-09T10:11:47.984388+0000","last_clean":"2023-10-09T10:11:47.984388+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:11:47.984388+0000","last_undegraded":"2023-10-09T10:11:47.984388+0000","last_fullsized":"2023-10-09T10:11:47.984388+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":171,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-10-09T10:11:47.987155+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:11:47.987155+0000","last_peered":"2023-10-09T10:11:47.987155+0000","last_clean":"2023-10-09T10:11:47.987155+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:11:47.987155+0000","last_undegraded":"2023-10-09T10:11:47.987155+0000","last_fullsized":"2023-10-09T10:11:47.987155+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3042377752,"num_objects":46424,"num_object_clones":0,"num_object_copies":139272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5928,"num_objects_misplaced":1369,"num_objects_unfound":0,"num_objects_dirty":46424,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170938,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3360,"num_bytes_recovered":220200960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9641734144,"data_stored":9641721928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142038,"ondisk_log_size":142038,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430508,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1260592,"kb_used_data":951080,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92484560,"statfs":{"total":95995035648,"available":94704189440,"internally_reserved":0,"allocated":973905920,"data_stored":973366653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:43 2023","interfaces":[{"interface":"back","average":{"1min":3.508,"5min":1.986,"15min":1.911},"min":{"1min":0.90100000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":9.9870000000000001,"5min":9.9870000000000001,"15min":50.591999999999999},"last":11.164},{"interface":"front","average":{"1min":3.661,"5min":2.133,"15min":1.905},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.974,"5min":15.974,"15min":50.747},"last":3.7709999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:11:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7029999999999998,"5min":2.117,"15min":1.728},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":14.311,"5min":14.311,"15min":25.338000000000001},"last":1.6419999999999999},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.2679999999999998,"15min":1.5509999999999999},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":16.526,"5min":18.27,"15min":18.27},"last":1.2929999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.752,"15min":2.2770000000000001},"min":{"1min":0.36099999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":12.621},{"interface":"front","average":{"1min":2.8029999999999999,"5min":1.845,"15min":2.1030000000000002},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.872,"5min":11.872,"15min":11.872},"last":3.5299999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4550000000000001,"5min":2.629,"15min":2.0430000000000001},"min":{"1min":0.436,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.6289999999999996,"5min":29.058,"15min":29.058},"last":3.8839999999999999},{"interface":"front","average":{"1min":3.6629999999999998,"5min":2.2559999999999998,"15min":2.0489999999999999},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":16.715}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.0469999999999999,"15min":0.95699999999999996},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.9820000000000002,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.97599999999999998,"15min":0.93100000000000005},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.6239999999999997,"5min":4.6239999999999997,"15min":4.6239999999999997},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.1519999999999999,"15min":1.006},"min":{"1min":0.30199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":5.7149999999999999,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.012,"5min":1.145,"15min":0.99299999999999999},"min":{"1min":0.38200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.5960000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.44}]},{"osd":6,"last update":"Mon Oct 9 10:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.93899999999999995,"15min":0.90900000000000003},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.6259999999999999,"5min":5.3129999999999997,"15min":5.4820000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.226,"5min":0.95599999999999996,"15min":0.87},"min":{"1min":0.36099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.419,"5min":9.1920000000000002,"15min":9.1920000000000002},"last":0.48499999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626736,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1603872,"kb_used_data":1294360,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92141280,"statfs":{"total":95995035648,"available":94352670720,"internally_reserved":0,"allocated":1325424640,"data_stored":1324902687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":1.919,"15min":1.6060000000000001},"min":{"1min":0.252,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":6.9630000000000001,"5min":15.435,"15min":15.435},"last":4.3150000000000004},{"interface":"front","average":{"1min":2.2469999999999999,"5min":1.921,"15min":1.591},"min":{"1min":0.33900000000000002,"5min":0.252,"15min":0.23999999999999999},"max":{"1min":8.2959999999999994,"5min":15.308999999999999,"15min":15.308999999999999},"last":13.385}]},{"osd":1,"last update":"Mon Oct 9 10:11:04 2023","interfaces":[{"interface":"back","average":{"1min":3.032,"5min":2.7160000000000002,"15min":1.8779999999999999},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":14.888999999999999,"5min":35.609999999999999,"15min":35.609999999999999},"last":6.4699999999999998},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.3580000000000001,"15min":1.8320000000000001},"min":{"1min":0.26700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.962,"5min":32.396000000000001,"15min":32.396000000000001},"last":6.8540000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":1.637,"15min":1.7929999999999999},"min":{"1min":0.35199999999999998,"5min":0.26900000000000002,"15min":0.26200000000000001},"max":{"1min":7.1280000000000001,"5min":12.898,"15min":12.898},"last":1.3200000000000001},{"interface":"front","average":{"1min":3.4860000000000002,"5min":2.1400000000000001,"15min":3.0379999999999998},"min":{"1min":0.44600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.542999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":2.2959999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":4.1079999999999997,"5min":2.3639999999999999,"15min":1.694},"min":{"1min":0.432,"5min":0.308,"15min":0.308},"max":{"1min":19.222000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":2.5329999999999999},{"interface":"front","average":{"1min":3.2490000000000001,"5min":2.4500000000000002,"15min":2.4039999999999999},"min":{"1min":0.39500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":8.2270000000000003,"5min":11.211,"15min":11.211},"last":3.8599999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.67100000000000004,"15min":0.51700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.8970000000000002,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":1.2869999999999999},{"interface":"front","average":{"1min":1.284,"5min":0.75800000000000001,"15min":0.58499999999999996},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":2.0659999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:11:37 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.81599999999999995},"min":{"1min":0.29399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.4580000000000002,"5min":7.9260000000000002,"15min":14.09},"last":1.976},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.042,"15min":0.872},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":6.3369999999999997,"5min":7.508,"15min":7.508},"last":2.105}]},{"osd":7,"last update":"Mon Oct 9 10:11:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":0.91100000000000003,"15min":0.82299999999999995},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.2709999999999999,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.538,"5min":0.95499999999999996,"15min":0.872},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.3159999999999998,"5min":7.5570000000000004,"15min":7.5570000000000004},"last":2.254}]}]},{"osd":1,"up_from":84,"seq":360777253049,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1768404,"kb_used_data":1356812,"kb_used_omap":9,"kb_used_meta":411574,"kb_avail":91976748,"statfs":{"total":95995035648,"available":94184189952,"internally_reserved":0,"allocated":1389375488,"data_stored":1388331357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":421452777},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.3980000000000001,"15min":2.1800000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":7.3609999999999998,"5min":25.59,"15min":25.59},"last":5.1280000000000001},{"interface":"front","average":{"1min":6.2140000000000004,"5min":3.206,"15min":2.3639999999999999},"min":{"1min":0.314,"5min":0.245,"15min":0.192},"max":{"1min":20.501999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":4.1760000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":3.0249999999999999,"15min":3.8069999999999999},"min":{"1min":0.70099999999999996,"5min":0.217,"15min":0.217},"max":{"1min":10.573,"5min":17.369,"15min":17.369},"last":0.91800000000000004},{"interface":"front","average":{"1min":3.3479999999999999,"5min":2.5449999999999999,"15min":3.7530000000000001},"min":{"1min":0.26400000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":13.414999999999999,"5min":13.414999999999999,"15min":16.023},"last":4.2229999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3170000000000002,"5min":3.6110000000000002,"15min":3.5590000000000002},"min":{"1min":0.41699999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.119,"5min":18.587,"15min":18.587},"last":0.72699999999999998},{"interface":"front","average":{"1min":2.7959999999999998,"5min":3.8029999999999999,"15min":4.5350000000000001},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.103999999999999,"5min":17.289000000000001,"15min":17.289000000000001},"last":5.2169999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.536,"15min":1.3480000000000001},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.976000000000001,"5min":11.976000000000001,"15min":11.976000000000001},"last":4.2690000000000001},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.177,"15min":1.6120000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.904,"5min":12.904,"15min":12.904},"last":4.1310000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":2.3519999999999999,"15min":1.9330000000000001},"min":{"1min":0.41999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.811,"5min":14.416,"15min":15.531000000000001},"last":0.80500000000000005},{"interface":"front","average":{"1min":2.742,"5min":2.1859999999999999,"15min":2.1080000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.9039999999999999,"5min":16.277999999999999,"15min":19.706},"last":0.95699999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.9450000000000001,"15min":1.825},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":7.3049999999999997,"5min":14.215,"15min":16.193999999999999},"last":5.3449999999999998},{"interface":"front","average":{"1min":4.1509999999999998,"5min":2.343,"15min":1.984},"min":{"1min":0.46200000000000002,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.868,"5min":14.471,"15min":15.481999999999999},"last":0.83699999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.306,"15min":2.1360000000000001},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.8620000000000001,"5min":16.331,"15min":16.331},"last":0.98599999999999999},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.0489999999999999,"15min":1.901},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3339999999999996,"5min":14.242000000000001,"15min":16.100999999999999},"last":5.3760000000000003}]}]},{"osd":0,"up_from":79,"seq":339302416576,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487348,"kb_used_data":1110700,"kb_used_omap":28,"kb_used_meta":376611,"kb_avail":92257804,"statfs":{"total":95995035648,"available":94471991296,"internally_reserved":0,"allocated":1137356800,"data_stored":1136341660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":385650512},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:11:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":2.0259999999999998,"15min":1.6559999999999999},"min":{"1min":0.216,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":12.975,"5min":12.975,"15min":13.084},"last":8.1489999999999991},{"interface":"front","average":{"1min":5.8680000000000003,"5min":3.653,"15min":2.5920000000000001},"min":{"1min":0.29099999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":22.934999999999999,"5min":22.934999999999999,"15min":22.934999999999999},"last":15.066000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0469999999999997,"5min":2.181,"15min":2.274},"min":{"1min":0.28899999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.003,"5min":18.003,"15min":18.003},"last":3.6419999999999999},{"interface":"front","average":{"1min":2.9660000000000002,"5min":2.0710000000000002,"15min":3.0190000000000001},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.432,"5min":15.529,"15min":15.529},"last":8.1210000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":4.6509999999999998,"15min":4.7930000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":23.151,"5min":25.734999999999999,"15min":25.734999999999999},"last":14.984999999999999},{"interface":"front","average":{"1min":5.3250000000000002,"5min":4.0739999999999998,"15min":4.298},"min":{"1min":0.33300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":18.279,"5min":21.923999999999999,"15min":21.923999999999999},"last":16.524999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3780000000000001,"5min":2.3079999999999998,"15min":1.6970000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.989000000000001,"5min":20.494,"15min":20.494},"last":7.9409999999999998},{"interface":"front","average":{"1min":3.6819999999999999,"5min":2.395,"15min":2.089},"min":{"1min":0.40200000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.907,"5min":15.907,"15min":15.907},"last":15.797000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5739999999999998,"15min":2.1869999999999998},"min":{"1min":0.36099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.868,"5min":17.096,"15min":17.096},"last":14.944000000000001},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.52,"15min":1.4299999999999999},"min":{"1min":0.371,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.371,"5min":10.371,"15min":12.478},"last":2.9950000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.734,"15min":1.5860000000000001},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":8.827,"5min":9.4719999999999995,"15min":11.699},"last":8.0449999999999999},{"interface":"front","average":{"1min":4.5800000000000001,"5min":2.9740000000000002,"15min":2.4660000000000002},"min":{"1min":0.52100000000000002,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":23.106999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":2.585}]},{"osd":7,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2679999999999998,"5min":2.6699999999999999,"15min":2.3540000000000001},"min":{"1min":0.437,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":16.460000000000001,"5min":16.460000000000001,"15min":26.978000000000002},"last":15.734},{"interface":"front","average":{"1min":2.677,"5min":1.7490000000000001,"15min":1.738},"min":{"1min":0.45100000000000001,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":8.8040000000000003,"5min":10.122,"15min":26.768000000000001},"last":8.0939999999999994}]}]},{"osd":2,"up_from":217,"seq":932007903309,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1220088,"kb_used_data":958960,"kb_used_omap":83,"kb_used_meta":261036,"kb_avail":92525064,"statfs":{"total":95995035648,"available":94745665536,"internally_reserved":0,"allocated":981975040,"data_stored":980956817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":267301359},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.0609999999999999,"15min":1.8660000000000001},"min":{"1min":0.24299999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.334,"5min":19.372,"15min":19.372},"last":4.1440000000000001},{"interface":"front","average":{"1min":5.5940000000000003,"5min":3.5150000000000001,"15min":3.133},"min":{"1min":0.32000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.047000000000001,"5min":16.754999999999999,"15min":16.754999999999999},"last":0.85999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":4.6630000000000003,"5min":2.653,"15min":2.6760000000000002},"min":{"1min":0.26500000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.276,"5min":14.276,"15min":14.276},"last":0.97999999999999998},{"interface":"front","average":{"1min":5.4710000000000001,"5min":3.0760000000000001,"15min":2.823},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.227,"5min":15.569000000000001,"15min":15.569000000000001},"last":0.82999999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.2989999999999999,"15min":2.1150000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.359999999999999,"5min":13.851000000000001,"15min":13.851000000000001},"last":4.1050000000000004},{"interface":"front","average":{"1min":4.1879999999999997,"5min":3.2360000000000002,"15min":3.6880000000000002},"min":{"1min":0.40799999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.51,"5min":15.811,"15min":15.811},"last":2.8849999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":1.3,"15min":0.90400000000000003},"min":{"1min":0.29399999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.413,"5min":12.413,"15min":12.413},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.597,"5min":1.329,"15min":0.996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.314},"max":{"1min":7.0810000000000004,"5min":12.385,"15min":12.385},"last":0.44500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.8120000000000001,"15min":1.5860000000000001},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1230000000000002,"5min":12.473000000000001,"15min":12.473000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.044,"5min":1.5980000000000001,"15min":1.706},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.923,"5min":13.587,"15min":13.587},"last":0.34300000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5529999999999999,"5min":1.9219999999999999,"15min":1.2689999999999999},"min":{"1min":0.374,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.484,"5min":15.292999999999999,"15min":15.292999999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.9670000000000001,"15min":1.8180000000000001},"min":{"1min":0.41099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.5679999999999996,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.50900000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.9359999999999999,"15min":1.294},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.457000000000001,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.94399999999999995},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.1539999999999999,"15min":1.758},"min":{"1min":0.38600000000000001,"5min":0.376,"15min":0.376},"max":{"1min":9.6039999999999992,"5min":12.433999999999999,"15min":12.433999999999999},"last":6.4059999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824053,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375936,"kb_used_data":1125688,"kb_used_omap":10,"kb_used_meta":250229,"kb_avail":92369216,"statfs":{"total":95995035648,"available":94586077184,"internally_reserved":0,"allocated":1152704512,"data_stored":1151502840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":256234563},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5139999999999998,"5min":4.0330000000000004,"15min":4.4800000000000004},"min":{"1min":0.27500000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.749000000000001,"5min":23.381,"15min":23.381},"last":17.373000000000001},{"interface":"front","average":{"1min":6.5919999999999996,"5min":4.5750000000000002,"15min":4.3109999999999999},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.782,"5min":20.239999999999998,"15min":20.239999999999998},"last":17.077000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2909999999999999,"5min":3.4820000000000002,"15min":3.7959999999999998},"min":{"1min":0.41399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.856999999999999,"5min":20.422999999999998,"15min":20.422999999999998},"last":40.618000000000002},{"interface":"front","average":{"1min":4.5620000000000003,"5min":3.726,"15min":3.5289999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.73,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.48799999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":3.1269999999999998,"15min":3.536},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.331,"5min":20.594999999999999,"15min":20.594999999999999},"last":29.466999999999999},{"interface":"front","average":{"1min":4.298,"5min":3.1680000000000001,"15min":2.9660000000000002},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":14.802,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.46800000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":2.048,"15min":1.413},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.696999999999999,"5min":15.676,"15min":15.676},"last":0.437},{"interface":"front","average":{"1min":4.7910000000000004,"5min":2.2360000000000002,"15min":1.752},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.478999999999999,"5min":16.478999999999999,"15min":16.478999999999999},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":2.9260000000000002,"15min":2.718},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.125999999999999,"5min":13.125999999999999,"15min":13.125999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":3.0699999999999998,"5min":2.6600000000000001,"15min":2.456},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":10.891999999999999,"5min":15.992000000000001,"15min":15.992000000000001},"last":17.242999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9649999999999999,"5min":2.9809999999999999,"15min":2.819},"min":{"1min":0.42099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.504999999999999,"5min":16.504999999999999,"15min":16.504999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.921,"5min":2.4239999999999999,"15min":2.544},"min":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.952,"5min":16.16,"15min":16.16},"last":17.286999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.1219999999999999,"15min":1.633},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":11.048999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":5.8399999999999999,"5min":3.073,"15min":2.8069999999999999},"min":{"1min":0.44600000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":16.367000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":0.44900000000000001}]}]},{"osd":4,"up_from":266,"seq":1142461300779,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1561316,"kb_used_data":1242588,"kb_used_omap":35,"kb_used_meta":318620,"kb_avail":92183836,"statfs":{"total":95995035648,"available":94396248064,"internally_reserved":0,"allocated":1272410112,"data_stored":1271882495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":326267118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.0109999999999999,"15min":0.84899999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.4829999999999997,"5min":10.881,"15min":10.881},"last":6.8419999999999996},{"interface":"front","average":{"1min":3.0150000000000001,"5min":1.5629999999999999,"15min":1.161},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":12.246,"5min":12.246,"15min":12.246},"last":2.4590000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2429999999999999,"5min":1.655,"15min":1.0960000000000001},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":12.161,"5min":12.161,"15min":12.161},"last":4.6299999999999999},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.7290000000000001,"15min":1.4259999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.504,"5min":11.141,"15min":11.141},"last":3.831}]},{"osd":2,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.4430000000000001,"15min":1.143},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.2469999999999999,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.905,"5min":1.242,"15min":1.018},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":6.2080000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":1.9490000000000001,"15min":1.4910000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.065,"5min":13.065,"15min":13.065},"last":2.2999999999999998},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.766,"15min":1.264},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.218,"5min":12.218,"15min":12.218},"last":4.4909999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":0.93200000000000005,"15min":0.79600000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.6869999999999994,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.56100000000000005,"15min":0.61899999999999999},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0609999999999999,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.36399999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.69799999999999995,"15min":0.58299999999999996},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.311},"max":{"1min":6.4119999999999999,"5min":6.4119999999999999,"15min":6.4119999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.7270000000000001,"5min":0.83899999999999997,"15min":0.66900000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.255000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.443}]},{"osd":7,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":0.89100000000000001,"15min":0.71099999999999997},"min":{"1min":0.38800000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":12.237,"5min":12.237,"15min":12.237},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.244,"5min":0.90600000000000003,"15min":0.85399999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.7590000000000003,"5min":9.7590000000000003,"15min":9.7590000000000003},"last":0.502}]}]},{"osd":5,"up_from":36,"seq":154618822964,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1758936,"kb_used_data":1453520,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91986216,"statfs":{"total":95995035648,"available":94193885184,"internally_reserved":0,"allocated":1488404480,"data_stored":1487884919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4710000000000001,"15min":1.5720000000000001},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.8179999999999996,"5min":7.8540000000000001,"15min":15.773999999999999},"last":1.8220000000000001},{"interface":"front","average":{"1min":2.7770000000000001,"5min":1.6879999999999999,"15min":1.6819999999999999},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.9950000000000001,"5min":10.244999999999999,"15min":15.699},"last":0.40200000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":1.748,"15min":1.5629999999999999},"min":{"1min":0.40300000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.4250000000000007,"5min":11.335000000000001,"15min":15.689},"last":2.0590000000000002},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.903,"15min":1.7110000000000001},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5979999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":2.222}]},{"osd":2,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.7130000000000001,"5min":1.905,"15min":2.456},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.259,"5min":10.259,"15min":15.640000000000001},"last":2.5720000000000001},{"interface":"front","average":{"1min":1.581,"5min":1.458,"15min":2.1739999999999999},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":4.1619999999999999,"5min":9.1329999999999991,"15min":10.656000000000001},"last":1.7050000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":2.1829999999999998,"15min":1.96},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2260000000000009,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":9.8699999999999992},{"interface":"front","average":{"1min":3.2370000000000001,"5min":2.4820000000000002,"15min":2.8399999999999999},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.9900000000000002,"5min":17.34,"15min":17.34},"last":5.1109999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.84899999999999998,"15min":0.65900000000000003},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3460000000000001,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.70499999999999996,"15min":0.73099999999999998},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.462,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.43099999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":0.97599999999999998,"15min":0.996},"min":{"1min":0.26200000000000001,"5min":0.25,"15min":0.25},"max":{"1min":8.5259999999999998,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.198,"5min":1.075,"15min":0.92600000000000005},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.9889999999999999,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.137,"15min":1.0389999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4710000000000001,"5min":7.891,"15min":10.334},"last":0.439},{"interface":"front","average":{"1min":1.173,"5min":0.80300000000000005,"15min":0.96899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":12.356},"last":0.45100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1125056512,"data_stored":1125056512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1377042432,"data_stored":1377042432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":970002432,"data_stored":969998360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1136525312,"data_stored":1136525312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1262747648,"data_stored":1262747648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1480200192,"data_stored":1480196120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1321402368,"data_stored":1321402368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":968757248,"data_stored":968753176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:12:05.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:12:05.207 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:05.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:05.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:05.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:05 smithi012 ceph-mon[152263]: pgmap v834: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 5928/139344 objects degraded (4.254%); 1369/139344 objects misplaced (0.982%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:12:05.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:05 smithi012 ceph-mon[147926]: pgmap v834: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 5928/139344 objects degraded (4.254%); 1369/139344 objects misplaced (0.982%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:12:05.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:05.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:05 smithi110 ceph-mon[136032]: pgmap v834: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 5928/139344 objects degraded (4.254%); 1369/139344 objects misplaced (0.982%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:12:05.454 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:05.565 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:05.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:05.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:05.611 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:12:05.672 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:12:05.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:05.710 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:12:05.717 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:05.774 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:05.780 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:12:05.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:05.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:05.827 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:12:05.863 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:12:05.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:05.905 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:12:05.965 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:12:05.981 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:05.998 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:12:05.998 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:12:05.998 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:12:05.998 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:12:05.998 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:12:05.998 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:12:05.998 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:12:05.998 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:12:06.217 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:12:06.257 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:06 smithi012 ceph-mon[147926]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:06.257 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3946538089' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:06.257 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:06 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:06.257 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4248371818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:06.258 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:06 smithi012 ceph-mon[152263]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:06.258 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3946538089' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:06.258 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:06 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:06.258 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4248371818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:06.549 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:06.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:06.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:06.685 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:06 smithi110 ceph-mon[136032]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3946538089' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:06 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4248371818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:06.768 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:06.991 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:06.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:06.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:07.100 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:07.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:07.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:07.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:07.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:07.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:07.324 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[147926]: pgmap v835: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 5928/136227 objects degraded (4.352%); 1369/136227 objects misplaced (1.005%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[147926]: failed to encode map e305 with expected crc 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[147926]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[147926]: failed to encode map e305 with expected crc 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[152263]: pgmap v835: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 5928/136227 objects degraded (4.352%); 1369/136227 objects misplaced (1.005%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:07.325 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[152263]: failed to encode map e305 with expected crc 2023-10-09T10:12:07.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:12:07.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[152263]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T10:12:07.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[152263]: failed to encode map e305 with expected crc 2023-10-09T10:12:07.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:12:07.357 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:12:07.357 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:12:07.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:12:07.357 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191677 6.19 6:9839b5ab:::benchmark_data_smithi012_182576_object86156:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e165)", 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.18473161300005, 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5138246070000001, 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.358 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191677 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167170+0000", 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167168+0000", 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167175+0000", 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4989999999999999e-06 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.360 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167184+0000", 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7329999999999998e-06 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167193+0000", 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1599999999999998e-06 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174376+0000", 2023-10-09T10:12:07.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071834530000000002 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174387+0000", 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1457999999999999e-05 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.658706+0000", 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4843186290000001 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.658834+0000", 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012832699999999999 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680928+0000", 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022093278000000001 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680994+0000", 2023-10-09T10:12:07.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6519000000000005e-05 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191684 6.19 6:9f873d67:::benchmark_data_smithi012_182576_object86163:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e165)", 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.18470253999999, 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5032064950000001, 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.364 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191684 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167199+0000", 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167198+0000", 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167201+0000", 2023-10-09T10:12:07.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3e-06 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167206+0000", 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8479999999999997e-06 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.167212+0000", 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1420000000000001e-06 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174497+0000", 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072854699999999996 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.174507+0000", 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0071000000000001e-05 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659145+0000", 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4846380749999999 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659343+0000", 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019807000000000001 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.659860+0000", 2023-10-09T10:12:07.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051712700000000004 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670116+0000", 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010255966 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670223+0000", 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010616 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670251+0000", 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8269999999999999e-05 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670302+0000", 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1674e-05 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670405+0000", 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102691 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191706 6.19 6:9a3f0857:::benchmark_data_smithi012_182576_object86185:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.16453143599995, 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.483309049, 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.372 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191706 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187370+0000", 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187368+0000", 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187373+0000", 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.087e-06 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187381+0000", 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6139999999999992e-06 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187388+0000", 2023-10-09T10:12:07.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9029999999999998e-06 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191191+0000", 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038029449999999998 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191201+0000", 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0463e-05 2023-10-09T10:12:07.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.662490+0000", 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.471288079 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.662641+0000", 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015126899999999999 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.663089+0000", 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00044850599999999999 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670456+0000", 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007366956 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670469+0000", 2023-10-09T10:12:07.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2931e-05 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670520+0000", 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1273000000000002e-05 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670579+0000", 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8789999999999998e-05 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670679+0000", 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9545999999999999e-05 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191710 6.19 6:9b680818:::benchmark_data_smithi012_182576_object86189:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.16450797200002, 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4835924069999999, 2023-10-09T10:12:07.380 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191710 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187393+0000", 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187395+0000", 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.238e-06 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187400+0000", 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.074e-06 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.187405+0000", 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5569999999999997e-06 2023-10-09T10:12:07.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191222+0000", 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038174189999999998 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.191229+0000", 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.665559+0000", 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4743292960000001 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.665709+0000", 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015078499999999999 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.666141+0000", 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043125999999999999 2023-10-09T10:12:07.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670478+0000", 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043378319999999998 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670789+0000", 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00031025900000000001 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670831+0000", 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2349999999999999e-05 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670880+0000", 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9397999999999997e-05 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.670986+0000", 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105147 2023-10-09T10:12:07.387 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191721 6.19 6:98cdd829:::benchmark_data_smithi012_182576_object86200:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.14647615800004, 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4866997529999999, 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.388 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191721 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205425+0000", 2023-10-09T10:12:07.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205423+0000", 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205432+0000", 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1829999999999994e-06 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205449+0000", 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7813e-05 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205460+0000", 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0966e-05 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205526+0000", 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5363000000000004e-05 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.205537+0000", 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.148e-05 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671019+0000", 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.465481982 2023-10-09T10:12:07.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671151+0000", 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013229600000000001 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.671734+0000", 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058208800000000003 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.680757+0000", 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090230250000000005 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681028+0000", 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027170299999999998 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.691971+0000", 2023-10-09T10:12:07.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010942708000000001 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692026+0000", 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5075999999999999e-05 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692125+0000", 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8728999999999996e-05 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.395 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191734 6.19 6:9ad5bbea:::benchmark_data_smithi012_182576_object86213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.13628827900004, 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.476698235, 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.396 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191734 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215613+0000", 2023-10-09T10:12:07.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215612+0000", 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215617+0000", 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1109999999999999e-06 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215630+0000", 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.347e-05 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215639+0000", 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2190000000000004e-06 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215696+0000", 2023-10-09T10:12:07.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7519000000000003e-05 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.215707+0000", 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1391e-05 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678298+0000", 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.462590391 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678424+0000", 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012647799999999999 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.678879+0000", 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045511999999999999 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689119+0000", 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010239244999999999 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692138+0000", 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030192180000000002 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692181+0000", 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2972999999999997e-05 2023-10-09T10:12:07.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692224+0000", 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3328999999999998e-05 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692311+0000", 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6976000000000002e-05 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.403 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191762 6.19 6:98c0cf1b:::benchmark_data_smithi012_182576_object86241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.10415233399999, 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.470776208, 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.404 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191762 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247749+0000", 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247747+0000", 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247756+0000", 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.003e-06 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247771+0000", 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4816e-05 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247779+0000", 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.551e-06 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247839+0000", 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9710999999999998e-05 2023-10-09T10:12:07.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.247851+0000", 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.221e-05 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681049+0000", 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4331976559999999 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681140+0000", 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1250999999999998e-05 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.681494+0000", 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035398299999999999 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689149+0000", 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076551509999999998 2023-10-09T10:12:07.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692143+0000", 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029938090000000001 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718366+0000", 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026223026999999999 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718421+0000", 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4262999999999999e-05 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718525+0000", 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000104549 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.411 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191773 6.19 6:9b9ae800:::benchmark_data_smithi012_182576_object86252:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.09255182100003, 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.45934027, 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191773 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259349+0000", 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259348+0000", 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259355+0000", 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8310000000000003e-06 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259368+0000", 2023-10-09T10:12:07.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3388e-05 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.259378+0000", 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4809999999999995e-06 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.262058+0000", 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026803130000000001 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.262070+0000", 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1612e-05 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.683593+0000", 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.421522669 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.683700+0000", 2023-10-09T10:12:07.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107054 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.684074+0000", 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037416800000000003 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689177+0000", 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051031940000000001 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692342+0000", 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003165026 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718562+0000", 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026219606999999999 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718600+0000", 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8834999999999997e-05 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718690+0000", 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9263000000000002e-05 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191842 6.19 6:9f5df5a7:::benchmark_data_smithi012_182576_object86321:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:12:07.419 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.00695736499995, 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.373904045, 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191842 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344944+0000", 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344942+0000", 2023-10-09T10:12:07.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344950+0000", 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2900000000000002e-06 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344964+0000", 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3373000000000001e-05 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.344973+0000", 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8200000000000003e-06 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.345026+0000", 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3804999999999997e-05 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.345038+0000", 2023-10-09T10:12:07.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1443999999999999e-05 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686361+0000", 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3413228020000001 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686486+0000", 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012562000000000001 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.686879+0000", 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039255099999999999 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689204+0000", 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002325503 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.692347+0000", 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031431340000000001 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718723+0000", 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026375604 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718760+0000", 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7398999999999999e-05 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718848+0000", 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7406999999999998e-05 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:191919 6.19 6:9f034d99:::benchmark_data_smithi012_182576_object86398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 570.90239651499996, 2023-10-09T10:12:07.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2695003030000001, 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 191919 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449505+0000", 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449503+0000", 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449510+0000", 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.08e-06 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449527+0000", 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6714999999999998e-05 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449537+0000", 2023-10-09T10:12:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5950000000000005e-06 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449603+0000", 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5975000000000004e-05 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:36.449617+0000", 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4163e-05 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689226+0000", 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.239609637 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689349+0000", 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012278500000000001 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.689741+0000", 2023-10-09T10:12:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039135500000000002 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.697237+0000", 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074966369999999996 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.697277+0000", 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9959000000000001e-05 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718881+0000", 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021603832999999999 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.718918+0000", 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7262999999999999e-05 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:37.719005+0000", 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6716000000000001e-05 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195708 6.a 6:5cc4cade:::benchmark_data_smithi012_182576_object90187:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 565.08050646200002, 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.548119284, 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.435 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195708 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271395+0000", 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271393+0000", 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271398+0000", 2023-10-09T10:12:07.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9450000000000001e-06 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271408+0000", 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0183000000000001e-05 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271416+0000", 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7559999999999999e-06 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271479+0000", 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2669000000000006e-05 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.271486+0000", 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5900000000000002e-06 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717349+0000", 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.44586224400000002 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717362+0000", 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3382e-05 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.803595+0000", 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0862334440000001 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.806135+0000", 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025396630000000002 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.806223+0000", 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7700000000000004e-05 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.813674+0000", 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074509019999999997 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819055+0000", 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053811429999999997 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819227+0000", 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017252200000000001 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819303+0000", 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5587000000000007e-05 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819381+0000", 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7582000000000004e-05 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819514+0000", 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013344400000000001 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195730 6.a 6:51d763b1:::benchmark_data_smithi012_182576_object90209:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 565.02253715699999, 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4903754600000001, 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.444 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195730 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329364+0000", 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329363+0000", 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329371+0000", 2023-10-09T10:12:07.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1990000000000006e-06 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329394+0000", 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2909999999999999e-05 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329404+0000", 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9809999999999998e-06 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329466+0000", 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2092000000000001e-05 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329479+0000", 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.359e-05 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717394+0000", 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38791446600000001 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717405+0000", 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1208e-05 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809765+0000", 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0923597 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809844+0000", 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8974999999999995e-05 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.809918+0000", 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4696000000000004e-05 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819132+0000", 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092136140000000002 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819148+0000", 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6092000000000001e-05 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819267+0000", 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119096 2023-10-09T10:12:07.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819562+0000", 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029480100000000003 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819615+0000", 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2502e-05 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819740+0000", 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012496799999999999 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195764 6.a 6:515d6e24:::benchmark_data_smithi012_182576_object90243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 564.95214846800002, 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.426156247, 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.453 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195764 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399753+0000", 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399751+0000", 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399758+0000", 2023-10-09T10:12:07.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3080000000000002e-06 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399771+0000", 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2680000000000001e-05 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399779+0000", 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1049999999999995e-06 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399843+0000", 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3441999999999993e-05 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.399856+0000", 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3008e-05 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717436+0000", 2023-10-09T10:12:07.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.317580268 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717445+0000", 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7660000000000004e-06 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814690+0000", 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0972454300000001 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814827+0000", 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013737299999999999 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.814985+0000", 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015749700000000001 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.819155+0000", 2023-10-09T10:12:07.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041702220000000003 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.823975+0000", 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048196610000000003 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825698+0000", 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017233140000000001 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825737+0000", 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8433000000000003e-05 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825794+0000", 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7871000000000002e-05 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.825909+0000", 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114583 2023-10-09T10:12:07.461 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154067 7.e 7:729cdabe:::benchmark_data_smithi012_221426_object46289:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 364.518755462, 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.711856684, 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.462 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154067 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:12:07.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833144+0000", 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833150+0000", 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0079999999999999e-06 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833167+0000", 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6198999999999999e-05 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833176+0000", 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4490000000000005e-06 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833926+0000", 2023-10-09T10:12:07.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00074946799999999999 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833934+0000", 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1589999999999993e-06 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841253+0000", 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073188610000000003 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841264+0000", 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1069e-05 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497295+0000", 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6560310870000001 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497435+0000", 2023-10-09T10:12:07.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013998400000000001 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.544884+0000", 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.047449005000000002 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.545002+0000", 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118678 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.468 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154074 7.e 7:7a8c6f84:::benchmark_data_smithi012_221426_object46296:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 364.51802178700001, 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7164543729999999, 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:07.469 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154074 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833884+0000", 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1059999999999999e-06 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833895+0000", 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1639e-05 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833906+0000", 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1104000000000001e-05 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833999+0000", 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2387999999999997e-05 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841286+0000", 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072812739999999999 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841293+0000", 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7290000000000003e-06 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500331+0000", 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.659038185 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500464+0000", 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013265799999999999 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550212+0000", 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049747711999999999 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550334+0000", 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012227200000000001 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154080 7.5 7:a9f132ab:::benchmark_data_smithi012_221426_object46302:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 364.51798532700002, 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5768529529999999, 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154080 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833915+0000", 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833918+0000", 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6740000000000001e-06 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833924+0000", 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2179999999999996e-06 2023-10-09T10:12:07.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833929+0000", 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2009999999999998e-06 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834017+0000", 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7529999999999997e-05 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834024+0000", 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2350000000000002e-06 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837176+0000", 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031517139999999999 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837194+0000", 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7634000000000001e-05 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395551+0000", 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5583572540000001 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395681+0000", 2023-10-09T10:12:07.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130059 2023-10-09T10:12:07.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:12:07.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395763+0000", 2023-10-09T10:12:07.483 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:07 smithi110 ceph-mon[136032]: pgmap v835: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 5928/136227 objects degraded (4.352%); 1369/136227 objects misplaced (1.005%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:12:07.483 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:07 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:07.483 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:07 smithi110 ceph-mon[136032]: failed to encode map e305 with expected crc 2023-10-09T10:12:07.483 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:12:07.483 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:07 smithi110 ceph-mon[136032]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T10:12:07.483 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:07 smithi110 ceph-mon[136032]: failed to encode map e305 with expected crc 2023-10-09T10:12:07.483 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:07 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1928999999999994e-05 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400135+0000", 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043726360000000001 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409675+0000", 2023-10-09T10:12:07.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095396509999999997 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409717+0000", 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2036000000000002e-05 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410592+0000", 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00087471800000000002 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410648+0000", 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6589000000000003e-05 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410769+0000", 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120383 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154083 7.e 7:757b33cc:::benchmark_data_smithi012_221426_object46305:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 364.51795738800001, 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7033852390000002, 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154083 2023-10-09T10:12:07.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833943+0000", 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833946+0000", 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0000000000000001e-06 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833952+0000", 2023-10-09T10:12:07.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9000000000000003e-06 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833957+0000", 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9629999999999997e-06 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834051+0000", 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4141999999999994e-05 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834060+0000", 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8079999999999994e-06 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841319+0000", 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072592009999999998 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841327+0000", 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.571e-06 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500500+0000", 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6591724889999999 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500595+0000", 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5142000000000005e-05 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.501295+0000", 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00069987000000000001 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510695+0000", 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094008289999999994 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520701+0000", 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010005623 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534431+0000", 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013730086000000001 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534503+0000", 2023-10-09T10:12:07.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2304000000000004e-05 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537329+0000", 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028256510000000002 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154085 7.5 7:a7e74dfa:::benchmark_data_smithi012_221426_object46307:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:12:07.495 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 364.51793948599999, 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5985369870000001, 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154085 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833961+0000", 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833964+0000", 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.294e-06 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833970+0000", 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8540000000000002e-06 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833975+0000", 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9710000000000003e-06 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834061+0000", 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6417999999999998e-05 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834067+0000", 2023-10-09T10:12:07.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.421e-06 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837215+0000", 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031475790000000002 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837222+0000", 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4370000000000001e-06 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400164+0000", 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5629411879999999 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400262+0000", 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8307000000000006e-05 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400392+0000", 2023-10-09T10:12:07.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013009100000000001 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409725+0000", 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093329940000000007 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409759+0000", 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4034e-05 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.425006+0000", 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015246994 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432361+0000", 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073547029999999998 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432413+0000", 2023-10-09T10:12:07.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2305999999999997e-05 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432499+0000", 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5711000000000007e-05 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154090 7.5 7:a5e0254e:::benchmark_data_smithi012_221426_object46312:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:12:07.504 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 364.51792023100001, 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.598732759, 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154090 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:07.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833983+0000", 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.26e-06 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833988+0000", 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4190000000000001e-06 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833993+0000", 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5419999999999999e-06 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834104+0000", 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011161499999999999 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834111+0000", 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0650000000000004e-06 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837242+0000", 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031316959999999998 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837250+0000", 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1799999999999996e-06 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.404888+0000", 2023-10-09T10:12:07.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5676373880000001 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.405039+0000", 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151543 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.405181+0000", 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014114600000000001 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409999+0000", 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048189210000000003 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410086+0000", 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6949999999999999e-05 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.425038+0000", 2023-10-09T10:12:07.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014951341999999999 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432561+0000", 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0075235340000000001 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432611+0000", 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9985000000000003e-05 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432714+0000", 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102497 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154097 7.e 7:7fa3f1e3:::benchmark_data_smithi012_221426_object46319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 364.51790314700003, 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.706036967, 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:07.513 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154097 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:07.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834000+0000", 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.384e-06 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2499999999999997e-06 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834010+0000", 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8359999999999999e-06 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834090+0000", 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9957999999999996e-05 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834096+0000", 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3350000000000003e-06 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841347+0000", 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072508479999999998 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841354+0000", 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7159999999999997e-06 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510724+0000", 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.669369753 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510903+0000", 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017934200000000001 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.511077+0000", 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017371899999999999 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520762+0000", 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0096847530000000008 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.525146+0000", 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043842450000000002 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537388+0000", 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012242494 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537446+0000", 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7565999999999997e-05 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.540035+0000", 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025890919999999999 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:07.521 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:07.522 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:12:07.522 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:07.598 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:07.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:07.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:07.884 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:08.181 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:12:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:12:07] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T10:12:08.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/136227 objects degraded (4.352%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:08.674 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/136227 objects degraded (4.352%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/136227 objects degraded (4.352%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:08.709 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:12:08.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:12:08.817 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:08.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:08.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:08.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:08.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:12:09.115 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:09.267 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:09.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:09.383 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:09.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:09 smithi012 ceph-mon[147926]: pgmap v837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 5928/132279 objects degraded (4.481%); 1187/132279 objects misplaced (0.897%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:09.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:09 smithi012 ceph-mon[152263]: pgmap v837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 5928/132279 objects degraded (4.481%); 1187/132279 objects misplaced (0.897%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:09.519 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:09.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:09.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:09.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:09 smithi110 ceph-mon[136032]: pgmap v837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 5928/132279 objects degraded (4.481%); 1187/132279 objects misplaced (0.897%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:09.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:09.777 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:09.843 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:09.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:09.951 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:09.988 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:10.035 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:10.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:10.139 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:10.187 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:10.229 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:10.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:10.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:10.257 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:10.261 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:10.333 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:10.347 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:10.347 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5928/127563 objects degraded (4.647%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1047,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":305,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":42521,"data_bytes":2785613752,"bytes_used":11606536192,"bytes_avail":756353748992,"bytes_total":767960285184,"degraded_objects":5928,"degraded_total":127563,"degraded_ratio":0.046471155429082103,"misplaced_objects":1187,"misplaced_total":127563,"misplaced_ratio":0.0093052060550473104,"recovering_objects_per_sec":37,"recovering_bytes_per_sec":2482217,"recovering_keys_per_sec":0,"num_objects_recovered":379,"num_bytes_recovered":24838144,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":626},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T10:09:33.293227+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T10:12:10.366 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:10.382 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:10.417 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:10.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:10.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:10.521 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:10.598 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:10.693 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:10.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:10.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:10.801 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:10.852 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:10.918 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:10.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:10.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:11.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:11.059 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:12:11.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:12:11.095 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:11.303 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:11.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:11.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:11.433 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:11.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:11 smithi110 ceph-mon[136032]: pgmap v838: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5928/127563 objects degraded (4.647%); 1187/127563 objects misplaced (0.931%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:11.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/170082736' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:11.481 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:11.496 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:11 smithi012 ceph-mon[147926]: pgmap v838: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5928/127563 objects degraded (4.647%); 1187/127563 objects misplaced (0.931%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:11.497 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/170082736' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:11.497 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:11 smithi012 ceph-mon[152263]: pgmap v838: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5928/127563 objects degraded (4.647%); 1187/127563 objects misplaced (0.931%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:11.497 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/170082736' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:11.519 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:11.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:11.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:11.706 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:11.845 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:11.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:11.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:12.037 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:12.180 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:12.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:12.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:12.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:12.396 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:12.569 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:12.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:12.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:12.736 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:12.787 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:12.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:12.892 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:12.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:12.990 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:12.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:13.095 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:13.262 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:12:13.262 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:178170 8.3 8:ccd33bd6:::benchmark_data_smithi012_271002_object69487:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.230521+0000", 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.015978908, 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016012489000000001, 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:13.263 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178170 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230521+0000", 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230521+0000", 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.264 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230519+0000", 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230530+0000", 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1545e-05 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.265 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230553+0000", 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2583999999999999e-05 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230564+0000", 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1194e-05 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230632+0000", 2023-10-09T10:12:13.266 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7595999999999995e-05 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230780+0000", 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014788900000000001 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230920+0000", 2023-10-09T10:12:13.267 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014065599999999999 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.236845+0000", 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059245419999999997 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.236857+0000", 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2318e-05 2023-10-09T10:12:13.268 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:178172 8.7 8:e71655c3:::benchmark_data_smithi012_271002_object69489:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.230583+0000", 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.015916531000000001, 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016059830000000001, 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:13.269 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178172 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230583+0000", 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.270 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230583+0000", 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230583+0000", 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.271 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.230586+0000", 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.14e-06 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.235298+0000", 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004711664 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.272 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.235306+0000", 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6119999999999999e-06 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.236391+0000", 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010847879999999999 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.273 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.236509+0000", 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011847500000000001 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.236594+0000", 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4115000000000004e-05 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.274 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.275 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.275 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.275 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.275 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:178178 8.1 8:84386a47:::benchmark_data_smithi012_271002_object69495:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-10-09T10:12:13.275 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.240923+0000", 2023-10-09T10:12:13.275 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0055769779999999998, 2023-10-09T10:12:13.275 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005809608, 2023-10-09T10:12:13.275 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178178 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240923+0000", 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240923+0000", 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240921+0000", 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.277 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240929+0000", 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0660000000000004e-06 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240946+0000", 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6379e-05 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240955+0000", 2023-10-09T10:12:13.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3279999999999995e-06 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.241582+0000", 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062684800000000005 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.241710+0000", 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127778 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.241833+0000", 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000123712 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:178180 8.1 8:85d0afbf:::benchmark_data_smithi012_271002_object69497:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.240971+0000", 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0055291389999999998, 2023-10-09T10:12:13.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058471080000000002, 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 178180 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240971+0000", 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240971+0000", 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240970+0000", 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240974+0000", 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5700000000000001e-06 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240980+0000", 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.003e-06 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.240985+0000", 2023-10-09T10:12:13.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8359999999999999e-06 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246033+0000", 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050485410000000001 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246163+0000", 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130316 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246289+0000", 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012578599999999999 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:178166 8.b e305/271 8:d5422dab:::benchmark_data_smithi012_271002_object69483:head v 305'11128, mlcod=305'11126)", 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.243719+0000", 2023-10-09T10:12:13.285 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0027811379999999998, 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031855120000000002, 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243719+0000", 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.286 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243719+0000", 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243717+0000", 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243727+0000", 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9699999999999994e-06 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243740+0000", 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2709e-05 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.288 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243748+0000", 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8210000000000002e-06 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243823+0000", 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5238e-05 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243848+0000", 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5111999999999999e-05 2023-10-09T10:12:13.289 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:178174 8.f e305/288)", 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.243770+0000", 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0027295900000000001, 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003205028, 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.290 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243770+0000", 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243770+0000", 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243770+0000", 2023-10-09T10:12:13.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243774+0000", 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5109999999999997e-06 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243781+0000", 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9929999999999998e-06 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.243787+0000", 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5629999999999999e-06 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45518.0:178170 8.3 e305/271)", 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.246438+0000", 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 6.1608999999999999e-05, 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00059111299999999999, 2023-10-09T10:12:13.293 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246438+0000", 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246438+0000", 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246437+0000", 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246443+0000", 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5829999999999997e-06 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246453+0000", 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0142e-05 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246460+0000", 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4839999999999998e-06 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45518.0:178172 8.7 e305/271)", 2023-10-09T10:12:13.296 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.246471+0000", 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.8530999999999998e-05, 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062108200000000001, 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246471+0000", 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246471+0000", 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246471+0000", 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246473+0000", 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1210000000000001e-06 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246478+0000", 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1270000000000002e-06 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.246483+0000", 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4610000000000001e-06 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.299 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:178168 8.0 e305/271)", 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.247067+0000", 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.999433, 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7859000000000001e-05, 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247067+0000", 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247067+0000", 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247066+0000", 2023-10-09T10:12:13.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247082+0000", 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5937e-05 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247093+0000", 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0672999999999999e-05 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247099+0000", 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4409999999999998e-06 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:178179 8.b e305/271)", 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.247113+0000", 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9993873, 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5308000000000004e-05, 2023-10-09T10:12:13.303 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247113+0000", 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247113+0000", 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.304 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247112+0000", 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247116+0000", 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9009999999999996e-06 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.305 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247121+0000", 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9869999999999999e-06 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247125+0000", 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7000000000000002e-06 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.45518.0:178172 8.7 e305/271)", 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:13.247129+0000", 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9993711, 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012448599999999999, 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247129+0000", 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247129+0000", 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247128+0000", 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247131+0000", 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2500000000000001e-06 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247135+0000", 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4530000000000004e-06 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:13.247139+0000", 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6569999999999999e-06 2023-10-09T10:12:13.309 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 11 2023-10-09T10:12:13.310 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:12:13.310 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:13.312 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:13.383 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:13.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:13.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:13.514 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:13.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:13 smithi012 ceph-mon[152263]: pgmap v839: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5928/127050 objects degraded (4.666%); 1187/127050 objects misplaced (0.934%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:13.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:13 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:13.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:13 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:13.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/127050 objects degraded (4.666%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:13.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:13 smithi012 ceph-mon[147926]: pgmap v839: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5928/127050 objects degraded (4.666%); 1187/127050 objects misplaced (0.934%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:13.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:13 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:13.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:13 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:13.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/127050 objects degraded (4.666%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:13.616 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:13.631 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:12:13.672 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:12:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:13 smithi110 ceph-mon[136032]: pgmap v839: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5928/127050 objects degraded (4.666%); 1187/127050 objects misplaced (0.934%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:12:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:13 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:13 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/127050 objects degraded (4.666%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:13.718 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:12:13.762 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:12:13.809 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:12:13.847 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:12:13.889 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:12:13.932 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:12:13.961 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:12:13.961 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:12:13.961 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:12:13.962 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:12:13.962 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:12:13.962 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:12:13.962 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:12:13.962 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:12:14.103 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:14.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:14.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:14.216 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:14.262 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:14.284 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1635120898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:14.284 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3883319964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:14.284 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:14.284 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:14.284 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:14.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1635120898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:14.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3883319964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:14.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:14.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:14.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:14.313 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:12:14.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:14.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:14.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:14.507 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:14.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1635120898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3883319964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:15.514 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:15.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:15.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:15.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[152263]: pgmap v840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 5928/121293 objects degraded (4.887%); 1005/121293 objects misplaced (0.829%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T10:12:15.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:15.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:15.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[152263]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T10:12:15.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[152263]: failed to encode map e306 with expected crc 2023-10-09T10:12:15.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[152263]: failed to encode map e306 with expected crc 2023-10-09T10:12:15.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[147926]: pgmap v840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 5928/121293 objects degraded (4.887%); 1005/121293 objects misplaced (0.829%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T10:12:15.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:15.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:15.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[147926]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T10:12:15.627 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[147926]: failed to encode map e306 with expected crc 2023-10-09T10:12:15.627 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:15 smithi012 ceph-mon[147926]: failed to encode map e306 with expected crc 2023-10-09T10:12:15.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:15.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:15 smithi110 ceph-mon[136032]: pgmap v840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 5928/121293 objects degraded (4.887%); 1005/121293 objects misplaced (0.829%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T10:12:15.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:15 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:15 smithi110 ceph-mon[136032]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T10:12:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:15 smithi110 ceph-mon[136032]: failed to encode map e306 with expected crc 2023-10-09T10:12:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:15 smithi110 ceph-mon[136032]: failed to encode map e306 with expected crc 2023-10-09T10:12:15.724 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:15.972 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:15.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:15.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:15.986 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:12:16.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:16.194 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:16.315 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:16 smithi012 ceph-mon[147926]: pgmap v842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 5928/118572 objects degraded (4.999%); 1005/118572 objects misplaced (0.848%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:16.506 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:16.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:16.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:16.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:16 smithi012 ceph-mon[152263]: pgmap v842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 5928/118572 objects degraded (4.999%); 1005/118572 objects misplaced (0.848%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:16.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:16.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:16 smithi110 ceph-mon[136032]: pgmap v842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 5928/118572 objects degraded (4.999%); 1005/118572 objects misplaced (0.848%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:16.708 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:16.991 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:16.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:16.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:17.100 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:17.236 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:17.327 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:17.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:17.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:17.431 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:17.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:17.551 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:17.702 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:17.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:17.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:17.887 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:17.926 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:12:17.952 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:17.971 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:12:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:12:17] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-09T10:12:18.176 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:12:18.188 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:18.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:18.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:18.300 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:18.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:12:18.455 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:18.558 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[147926]: pgmap v843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 5928/114540 objects degraded (5.175%); 787/114540 objects misplaced (0.687%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:18.558 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[147926]: from='client.35784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:18.558 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/114540 objects degraded (5.175%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:18.558 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:18.559 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:18.559 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[152263]: pgmap v843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 5928/114540 objects degraded (5.175%); 787/114540 objects misplaced (0.687%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:18.559 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[152263]: from='client.35784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:18.559 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/114540 objects degraded (5.175%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:18.559 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:18.559 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:18 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:18.846 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:18.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:18.943 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":843,"stamp":"2023-10-09T10:12:17.411397+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2501121976,"num_objects":38180,"num_object_clones":0,"num_object_copies":114540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5928,"num_objects_misplaced":787,"num_objects_unfound":0,"num_objects_dirty":38180,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":179466,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4011,"num_bytes_recovered":259533516,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142200,"ondisk_log_size":142200,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10548860,"kb_used_data":7999612,"kb_used_omap":167,"kb_used_meta":2548248,"kb_avail":739412356,"statfs":{"total":767960285184,"available":757158252544,"internally_reserved":0,"allocated":8191602688,"data_stored":8185197208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2609406553},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,4,2,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-387514368,"num_objects":-5913,"num_object_clones":0,"num_object_copies":-17739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-400,"num_objects_unfound":0,"num_objects_dirty":-5913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5913,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":400,"num_bytes_recovered":26214400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.010277"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":147,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.076835+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:11:57.076835+0000","last_peered":"2023-10-09T10:11:57.076835+0000","last_clean":"2023-10-09T10:11:57.076835+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:11:57.076835+0000","last_undegraded":"2023-10-09T10:11:57.076835+0000","last_fullsized":"2023-10-09T10:11:57.076835+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":263,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220634+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:12:06.220634+0000","last_peered":"2023-10-09T10:12:06.220634+0000","last_clean":"2023-10-09T10:12:06.220634+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:12:06.220634+0000","last_undegraded":"2023-10-09T10:12:06.220634+0000","last_fullsized":"2023-10-09T10:12:06.220634+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080964+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:11:57.080964+0000","last_peered":"2023-10-09T10:11:57.080964+0000","last_clean":"2023-10-09T10:11:57.080964+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:11:57.080964+0000","last_undegraded":"2023-10-09T10:11:57.080964+0000","last_fullsized":"2023-10-09T10:11:57.080964+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":172,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.211353+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:12:06.211353+0000","last_peered":"2023-10-09T10:12:06.211353+0000","last_clean":"2023-10-09T10:12:06.211353+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:12:06.211353+0000","last_undegraded":"2023-10-09T10:12:06.211353+0000","last_fullsized":"2023-10-09T10:12:06.211353+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":277,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.228886+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:12:06.228886+0000","last_peered":"2023-10-09T10:12:06.228886+0000","last_clean":"2023-10-09T10:12:06.228886+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:12:06.228886+0000","last_undegraded":"2023-10-09T10:12:06.228886+0000","last_fullsized":"2023-10-09T10:12:06.228886+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.081260+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:11:57.081260+0000","last_peered":"2023-10-09T10:11:57.081260+0000","last_clean":"2023-10-09T10:11:57.081260+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:11:57.081260+0000","last_undegraded":"2023-10-09T10:11:57.081260+0000","last_fullsized":"2023-10-09T10:11:57.081260+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":190,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:11.768365+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:12:11.768365+0000","last_peered":"2023-10-09T10:12:11.768365+0000","last_clean":"2023-10-09T10:12:11.768365+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:12:11.768365+0000","last_undegraded":"2023-10-09T10:12:11.768365+0000","last_fullsized":"2023-10-09T10:12:11.768365+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":258,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.228830+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:12:06.228830+0000","last_peered":"2023-10-09T10:12:06.228830+0000","last_clean":"2023-10-09T10:12:06.228830+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:12:06.228830+0000","last_undegraded":"2023-10-09T10:12:06.228830+0000","last_fullsized":"2023-10-09T10:12:06.228830+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":122,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.081100+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:11:57.081100+0000","last_peered":"2023-10-09T10:11:57.081100+0000","last_clean":"2023-10-09T10:11:57.081100+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:11:57.081100+0000","last_undegraded":"2023-10-09T10:11:57.081100+0000","last_fullsized":"2023-10-09T10:11:57.081100+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":219,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220434+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:12:06.220434+0000","last_peered":"2023-10-09T10:12:06.220434+0000","last_clean":"2023-10-09T10:12:06.220434+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:12:06.220434+0000","last_undegraded":"2023-10-09T10:12:06.220434+0000","last_fullsized":"2023-10-09T10:12:06.220434+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":344,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207740+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:12:06.207740+0000","last_peered":"2023-10-09T10:12:06.207740+0000","last_clean":"2023-10-09T10:12:06.207740+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:12:06.207740+0000","last_undegraded":"2023-10-09T10:12:06.207740+0000","last_fullsized":"2023-10-09T10:12:06.207740+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":300,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.076656+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:11:57.076656+0000","last_peered":"2023-10-09T10:11:57.076656+0000","last_clean":"2023-10-09T10:11:57.076656+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:11:57.076656+0000","last_undegraded":"2023-10-09T10:11:57.076656+0000","last_fullsized":"2023-10-09T10:11:57.076656+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":193,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.203940+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:12:06.203940+0000","last_peered":"2023-10-09T10:12:06.203940+0000","last_clean":"2023-10-09T10:12:06.203940+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:12:06.203940+0000","last_undegraded":"2023-10-09T10:12:06.203940+0000","last_fullsized":"2023-10-09T10:12:06.203940+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":172,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.081527+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:11:57.081527+0000","last_peered":"2023-10-09T10:11:57.081527+0000","last_clean":"2023-10-09T10:11:57.081527+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:11:57.081527+0000","last_undegraded":"2023-10-09T10:11:57.081527+0000","last_fullsized":"2023-10-09T10:11:57.081527+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":377,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.203767+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:12:06.203767+0000","last_peered":"2023-10-09T10:12:06.203767+0000","last_clean":"2023-10-09T10:12:06.203767+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:12:06.203767+0000","last_undegraded":"2023-10-09T10:12:06.203767+0000","last_fullsized":"2023-10-09T10:12:06.203767+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":245,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.221294+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:12:06.221294+0000","last_peered":"2023-10-09T10:12:06.221294+0000","last_clean":"2023-10-09T10:12:06.221294+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:12:06.221294+0000","last_undegraded":"2023-10-09T10:12:06.221294+0000","last_fullsized":"2023-10-09T10:12:06.221294+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":172,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.206407+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:12:06.206407+0000","last_peered":"2023-10-09T10:12:06.206407+0000","last_clean":"2023-10-09T10:12:06.206407+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:12:06.206407+0000","last_undegraded":"2023-10-09T10:12:06.206407+0000","last_fullsized":"2023-10-09T10:12:06.206407+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":138,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.081041+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:11:57.081041+0000","last_peered":"2023-10-09T10:11:57.081041+0000","last_clean":"2023-10-09T10:11:57.081041+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:11:57.081041+0000","last_undegraded":"2023-10-09T10:11:57.081041+0000","last_fullsized":"2023-10-09T10:11:57.081041+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":174,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.203621+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:12:06.203621+0000","last_peered":"2023-10-09T10:12:06.203621+0000","last_clean":"2023-10-09T10:12:06.203621+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:12:06.203621+0000","last_undegraded":"2023-10-09T10:12:06.203621+0000","last_fullsized":"2023-10-09T10:12:06.203621+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:11.768339+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:12:11.768339+0000","last_peered":"2023-10-09T10:12:11.768339+0000","last_clean":"2023-10-09T10:12:11.768339+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:12:11.768339+0000","last_undegraded":"2023-10-09T10:12:11.768339+0000","last_fullsized":"2023-10-09T10:12:11.768339+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":195,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.206042+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:12:06.206042+0000","last_peered":"2023-10-09T10:12:06.206042+0000","last_clean":"2023-10-09T10:12:06.206042+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:12:06.206042+0000","last_undegraded":"2023-10-09T10:12:06.206042+0000","last_fullsized":"2023-10-09T10:12:06.206042+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":258,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.224327+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:12:06.224327+0000","last_peered":"2023-10-09T10:12:06.224327+0000","last_clean":"2023-10-09T10:12:06.224327+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:12:06.224327+0000","last_undegraded":"2023-10-09T10:12:06.224327+0000","last_fullsized":"2023-10-09T10:12:06.224327+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"305'11251","reported_seq":11301,"reported_epoch":305,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:12.868805+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:12:12.868805+0000","last_peered":"2023-10-09T10:12:12.868805+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:12:12.868805+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"277'1200","ondisk_log_start":"277'1200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10051,"log_dups_size":0,"ondisk_log_size":10051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158269440,"num_objects":2415,"num_object_clones":0,"num_object_copies":7245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11251,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-2958},{"shards":"4,5,6","objects":955}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":338,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207844+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:12:06.207844+0000","last_peered":"2023-10-09T10:12:06.207844+0000","last_clean":"2023-10-09T10:12:06.207844+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:12:06.207844+0000","last_undegraded":"2023-10-09T10:12:06.207844+0000","last_fullsized":"2023-10-09T10:12:06.207844+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":123,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080706+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:11:57.080706+0000","last_peered":"2023-10-09T10:11:57.080706+0000","last_clean":"2023-10-09T10:11:57.080706+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:11:57.080706+0000","last_undegraded":"2023-10-09T10:11:57.080706+0000","last_fullsized":"2023-10-09T10:11:57.080706+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"306'11369","reported_seq":11411,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:15.077304+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:12:15.077304+0000","last_peered":"2023-10-09T10:12:15.077304+0000","last_clean":"2023-10-09T10:12:15.077304+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:12:15.077304+0000","last_undegraded":"2023-10-09T10:12:15.077304+0000","last_fullsized":"2023-10-09T10:12:15.077304+0000","mapping_epoch":271,"log_start":"280'2700","ondisk_log_start":"280'2700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8669,"log_dups_size":0,"ondisk_log_size":8669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11369,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":326,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207692+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:12:06.207692+0000","last_peered":"2023-10-09T10:12:06.207692+0000","last_clean":"2023-10-09T10:12:06.207692+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:12:06.207692+0000","last_undegraded":"2023-10-09T10:12:06.207692+0000","last_fullsized":"2023-10-09T10:12:06.207692+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":216,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080888+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:11:57.080888+0000","last_peered":"2023-10-09T10:11:57.080888+0000","last_clean":"2023-10-09T10:11:57.080888+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:11:57.080888+0000","last_undegraded":"2023-10-09T10:11:57.080888+0000","last_fullsized":"2023-10-09T10:11:57.080888+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"306'11225","reported_seq":11270,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:16.762360+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:12:16.762360+0000","last_peered":"2023-10-09T10:12:16.762360+0000","last_clean":"2023-10-09T10:12:16.762360+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:12:16.762360+0000","last_undegraded":"2023-10-09T10:12:16.762360+0000","last_fullsized":"2023-10-09T10:12:16.762360+0000","mapping_epoch":271,"log_start":"279'1800","ondisk_log_start":"279'1800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9425,"log_dups_size":0,"ondisk_log_size":9425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419776,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11225,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":127,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.081108+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:11:57.081108+0000","last_peered":"2023-10-09T10:11:57.081108+0000","last_clean":"2023-10-09T10:11:57.081108+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:11:57.081108+0000","last_undegraded":"2023-10-09T10:11:57.081108+0000","last_fullsized":"2023-10-09T10:11:57.081108+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":149,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209386+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:12:06.209386+0000","last_peered":"2023-10-09T10:12:06.209386+0000","last_clean":"2023-10-09T10:12:06.209386+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:12:06.209386+0000","last_undegraded":"2023-10-09T10:12:06.209386+0000","last_fullsized":"2023-10-09T10:12:06.209386+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"305'10989","reported_seq":11033,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:13.534694+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:12:13.534694+0000","last_peered":"2023-10-09T10:12:13.534694+0000","last_clean":"2023-10-09T10:12:13.534694+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:12:13.534694+0000","last_undegraded":"2023-10-09T10:12:13.534694+0000","last_fullsized":"2023-10-09T10:12:13.534694+0000","mapping_epoch":271,"log_start":"282'3200","ondisk_log_start":"282'3200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7789,"log_dups_size":0,"ondisk_log_size":7789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10989,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":138,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.084262+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:11:57.084262+0000","last_peered":"2023-10-09T10:11:57.084262+0000","last_clean":"2023-10-09T10:11:57.084262+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:11:57.084262+0000","last_undegraded":"2023-10-09T10:11:57.084262+0000","last_fullsized":"2023-10-09T10:11:57.084262+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":173,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207781+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:12:06.207781+0000","last_peered":"2023-10-09T10:12:06.207781+0000","last_clean":"2023-10-09T10:12:06.207781+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:12:06.207781+0000","last_undegraded":"2023-10-09T10:12:06.207781+0000","last_fullsized":"2023-10-09T10:12:06.207781+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"306'11485","reported_seq":11530,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:16.448673+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:12:16.448673+0000","last_peered":"2023-10-09T10:12:16.448673+0000","last_clean":"2023-10-09T10:12:16.448673+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:12:16.448673+0000","last_undegraded":"2023-10-09T10:12:16.448673+0000","last_fullsized":"2023-10-09T10:12:16.448673+0000","mapping_epoch":271,"log_start":"282'3300","ondisk_log_start":"282'3300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8185,"log_dups_size":0,"ondisk_log_size":8185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154337280,"num_objects":2355,"num_object_clones":0,"num_object_copies":7065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11485,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":300,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:11.509058+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:12:11.509058+0000","last_peered":"2023-10-09T10:12:11.509058+0000","last_clean":"2023-10-09T10:12:11.509058+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:12:11.509058+0000","last_undegraded":"2023-10-09T10:12:11.509058+0000","last_fullsized":"2023-10-09T10:12:11.509058+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:11.509079+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:12:11.509079+0000","last_peered":"2023-10-09T10:12:11.509079+0000","last_clean":"2023-10-09T10:12:11.509079+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:12:11.509079+0000","last_undegraded":"2023-10-09T10:12:11.509079+0000","last_fullsized":"2023-10-09T10:12:11.509079+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"305'11323","reported_seq":11367,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:13.584400+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:12:13.584400+0000","last_peered":"2023-10-09T10:12:13.584400+0000","last_clean":"2023-10-09T10:12:13.584400+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:12:13.584400+0000","last_undegraded":"2023-10-09T10:12:13.584400+0000","last_fullsized":"2023-10-09T10:12:13.584400+0000","mapping_epoch":271,"log_start":"283'3600","ondisk_log_start":"283'3600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7723,"log_dups_size":0,"ondisk_log_size":7723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156041216,"num_objects":2381,"num_object_clones":0,"num_object_copies":7143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11323,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":230,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.076587+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:11:57.076587+0000","last_peered":"2023-10-09T10:11:57.076587+0000","last_clean":"2023-10-09T10:11:57.076587+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:11:57.076587+0000","last_undegraded":"2023-10-09T10:11:57.076587+0000","last_fullsized":"2023-10-09T10:11:57.076587+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":236,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.077349+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:11:57.077349+0000","last_peered":"2023-10-09T10:11:57.077349+0000","last_clean":"2023-10-09T10:11:57.077349+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:11:57.077349+0000","last_undegraded":"2023-10-09T10:11:57.077349+0000","last_fullsized":"2023-10-09T10:11:57.077349+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"306'11105","reported_seq":11150,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:16.693737+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:12:16.693737+0000","last_peered":"2023-10-09T10:12:16.693737+0000","last_clean":"2023-10-09T10:12:16.693737+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:12:16.693737+0000","last_undegraded":"2023-10-09T10:12:16.693737+0000","last_fullsized":"2023-10-09T10:12:16.693737+0000","mapping_epoch":271,"log_start":"279'1700","ondisk_log_start":"279'1700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9405,"log_dups_size":0,"ondisk_log_size":9405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152502272,"num_objects":2327,"num_object_clones":0,"num_object_copies":6981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11105,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":200,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.203588+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:12:06.203588+0000","last_peered":"2023-10-09T10:12:06.203588+0000","last_clean":"2023-10-09T10:12:06.203588+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:12:06.203588+0000","last_undegraded":"2023-10-09T10:12:06.203588+0000","last_fullsized":"2023-10-09T10:12:06.203588+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":172,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.077481+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:11:57.077481+0000","last_peered":"2023-10-09T10:11:57.077481+0000","last_clean":"2023-10-09T10:11:57.077481+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:11:57.077481+0000","last_undegraded":"2023-10-09T10:11:57.077481+0000","last_fullsized":"2023-10-09T10:11:57.077481+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"305'11203","reported_seq":11247,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:13.589228+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:12:13.589228+0000","last_peered":"2023-10-09T10:12:13.589228+0000","last_clean":"2023-10-09T10:12:13.589228+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:12:13.589228+0000","last_undegraded":"2023-10-09T10:12:13.589228+0000","last_fullsized":"2023-10-09T10:12:13.589228+0000","mapping_epoch":271,"log_start":"282'3500","ondisk_log_start":"282'3500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7703,"log_dups_size":0,"ondisk_log_size":7703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156958720,"num_objects":2395,"num_object_clones":0,"num_object_copies":7185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11203,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":260,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.228783+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:12:06.228783+0000","last_peered":"2023-10-09T10:12:06.228783+0000","last_clean":"2023-10-09T10:12:06.228783+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:12:06.228783+0000","last_undegraded":"2023-10-09T10:12:06.228783+0000","last_fullsized":"2023-10-09T10:12:06.228783+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":308,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.084154+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:11:57.084154+0000","last_peered":"2023-10-09T10:11:57.084154+0000","last_clean":"2023-10-09T10:11:57.084154+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:11:57.084154+0000","last_undegraded":"2023-10-09T10:11:57.084154+0000","last_fullsized":"2023-10-09T10:11:57.084154+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"306'11158","reported_seq":11203,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:16.687046+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:12:16.687046+0000","last_peered":"2023-10-09T10:12:16.687046+0000","last_clean":"2023-10-09T10:12:16.687046+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:12:16.687046+0000","last_undegraded":"2023-10-09T10:12:16.687046+0000","last_fullsized":"2023-10-09T10:12:16.687046+0000","mapping_epoch":271,"log_start":"279'1700","ondisk_log_start":"279'1700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9458,"log_dups_size":0,"ondisk_log_size":9458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151650304,"num_objects":2314,"num_object_clones":0,"num_object_copies":6942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11158,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":308,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.077063+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:11:57.077063+0000","last_peered":"2023-10-09T10:11:57.077063+0000","last_clean":"2023-10-09T10:11:57.077063+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:11:57.077063+0000","last_undegraded":"2023-10-09T10:11:57.077063+0000","last_fullsized":"2023-10-09T10:11:57.077063+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":147,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.081020+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:11:57.081020+0000","last_peered":"2023-10-09T10:11:57.081020+0000","last_clean":"2023-10-09T10:11:57.081020+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:11:57.081020+0000","last_undegraded":"2023-10-09T10:11:57.081020+0000","last_fullsized":"2023-10-09T10:11:57.081020+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":264,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.076906+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:11:57.076906+0000","last_peered":"2023-10-09T10:11:57.076906+0000","last_clean":"2023-10-09T10:11:57.076906+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:11:57.076906+0000","last_undegraded":"2023-10-09T10:11:57.076906+0000","last_fullsized":"2023-10-09T10:11:57.076906+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"305'11140","reported_seq":11182,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:12.925863+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:12:12.925863+0000","last_peered":"2023-10-09T10:12:12.925863+0000","last_clean":"2023-10-09T10:12:12.925863+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:12:12.925863+0000","last_undegraded":"2023-10-09T10:12:12.925863+0000","last_fullsized":"2023-10-09T10:12:12.925863+0000","mapping_epoch":271,"log_start":"277'1100","ondisk_log_start":"277'1100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10040,"log_dups_size":0,"ondisk_log_size":10040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11140,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":123,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080784+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:11:57.080784+0000","last_peered":"2023-10-09T10:11:57.080784+0000","last_clean":"2023-10-09T10:11:57.080784+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:11:57.080784+0000","last_undegraded":"2023-10-09T10:11:57.080784+0000","last_fullsized":"2023-10-09T10:11:57.080784+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":122,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207562+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:12:06.207562+0000","last_peered":"2023-10-09T10:12:06.207562+0000","last_clean":"2023-10-09T10:12:06.207562+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:12:06.207562+0000","last_undegraded":"2023-10-09T10:12:06.207562+0000","last_fullsized":"2023-10-09T10:12:06.207562+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":331,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.076997+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:11:57.076997+0000","last_peered":"2023-10-09T10:11:57.076997+0000","last_clean":"2023-10-09T10:11:57.076997+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:11:57.076997+0000","last_undegraded":"2023-10-09T10:11:57.076997+0000","last_fullsized":"2023-10-09T10:11:57.076997+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"306'11278","reported_seq":11322,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:16.486917+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:12:16.486917+0000","last_peered":"2023-10-09T10:12:16.486917+0000","last_clean":"2023-10-09T10:12:16.486917+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:12:16.486917+0000","last_undegraded":"2023-10-09T10:12:16.486917+0000","last_fullsized":"2023-10-09T10:12:16.486917+0000","mapping_epoch":271,"log_start":"281'3100","ondisk_log_start":"281'3100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8178,"log_dups_size":0,"ondisk_log_size":8178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11278,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":226,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220297+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:12:06.220297+0000","last_peered":"2023-10-09T10:12:06.220297+0000","last_clean":"2023-10-09T10:12:06.220297+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:12:06.220297+0000","last_undegraded":"2023-10-09T10:12:06.220297+0000","last_fullsized":"2023-10-09T10:12:06.220297+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":300,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080440+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:11:57.080440+0000","last_peered":"2023-10-09T10:11:57.080440+0000","last_clean":"2023-10-09T10:11:57.080440+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:11:57.080440+0000","last_undegraded":"2023-10-09T10:11:57.080440+0000","last_fullsized":"2023-10-09T10:11:57.080440+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":321,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.084087+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:11:57.084087+0000","last_peered":"2023-10-09T10:11:57.084087+0000","last_clean":"2023-10-09T10:11:57.084087+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:11:57.084087+0000","last_undegraded":"2023-10-09T10:11:57.084087+0000","last_fullsized":"2023-10-09T10:11:57.084087+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"305'11012","reported_seq":11054,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:12.886393+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:12:12.886393+0000","last_peered":"2023-10-09T10:12:12.886393+0000","last_clean":"2023-10-09T10:12:12.886393+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:12:12.886393+0000","last_undegraded":"2023-10-09T10:12:12.886393+0000","last_fullsized":"2023-10-09T10:12:12.886393+0000","mapping_epoch":271,"log_start":"277'1000","ondisk_log_start":"277'1000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10012,"log_dups_size":0,"ondisk_log_size":10012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160432128,"num_objects":2448,"num_object_clones":0,"num_object_copies":7344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11012,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":132,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080820+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:11:57.080820+0000","last_peered":"2023-10-09T10:11:57.080820+0000","last_clean":"2023-10-09T10:11:57.080820+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:11:57.080820+0000","last_undegraded":"2023-10-09T10:11:57.080820+0000","last_fullsized":"2023-10-09T10:11:57.080820+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":552,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.203555+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:12:06.203555+0000","last_peered":"2023-10-09T10:12:06.203555+0000","last_clean":"2023-10-09T10:12:06.203555+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:12:06.203555+0000","last_undegraded":"2023-10-09T10:12:06.203555+0000","last_fullsized":"2023-10-09T10:12:06.203555+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":210,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220487+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:12:06.220487+0000","last_peered":"2023-10-09T10:12:06.220487+0000","last_clean":"2023-10-09T10:12:06.220487+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:12:06.220487+0000","last_undegraded":"2023-10-09T10:12:06.220487+0000","last_fullsized":"2023-10-09T10:12:06.220487+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":221,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207816+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:12:06.207816+0000","last_peered":"2023-10-09T10:12:06.207816+0000","last_clean":"2023-10-09T10:12:06.207816+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:12:06.207816+0000","last_undegraded":"2023-10-09T10:12:06.207816+0000","last_fullsized":"2023-10-09T10:12:06.207816+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"305'11193","reported_seq":11237,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:13.540442+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:12:13.540442+0000","last_peered":"2023-10-09T10:12:13.540442+0000","last_clean":"2023-10-09T10:12:13.540442+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:12:13.540442+0000","last_undegraded":"2023-10-09T10:12:13.540442+0000","last_fullsized":"2023-10-09T10:12:13.540442+0000","mapping_epoch":271,"log_start":"283'3500","ondisk_log_start":"283'3500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7693,"log_dups_size":0,"ondisk_log_size":7693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159186944,"num_objects":2429,"num_object_clones":0,"num_object_copies":7287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11193,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":280,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080596+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:11:57.080596+0000","last_peered":"2023-10-09T10:11:57.080596+0000","last_clean":"2023-10-09T10:11:57.080596+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:11:57.080596+0000","last_undegraded":"2023-10-09T10:11:57.080596+0000","last_fullsized":"2023-10-09T10:11:57.080596+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":183,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210713+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:12:06.210713+0000","last_peered":"2023-10-09T10:12:06.210713+0000","last_clean":"2023-10-09T10:12:06.210713+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:12:06.210713+0000","last_undegraded":"2023-10-09T10:12:06.210713+0000","last_fullsized":"2023-10-09T10:12:06.210713+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":211,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.211281+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:12:06.211281+0000","last_peered":"2023-10-09T10:12:06.211281+0000","last_clean":"2023-10-09T10:12:06.211281+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:12:06.211281+0000","last_undegraded":"2023-10-09T10:12:06.211281+0000","last_fullsized":"2023-10-09T10:12:06.211281+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"306'11269","reported_seq":19206,"reported_epoch":306,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:16.512210+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:12:16.512210+0000","last_peered":"2023-10-09T10:12:16.512210+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:12:16.512210+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"281'3100","ondisk_log_start":"281'3100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8169,"log_dups_size":0,"ondisk_log_size":8169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156565504,"num_objects":2389,"num_object_clones":0,"num_object_copies":7167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":555,"num_objects_misplaced":787,"num_objects_unfound":0,"num_objects_dirty":2389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11269,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3942,"num_bytes_recovered":258342912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1047},{"shards":"1,3,4","objects":787}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":325,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207643+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:12:06.207643+0000","last_peered":"2023-10-09T10:12:06.207643+0000","last_clean":"2023-10-09T10:12:06.207643+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:12:06.207643+0000","last_undegraded":"2023-10-09T10:12:06.207643+0000","last_fullsized":"2023-10-09T10:12:06.207643+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":337,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.077132+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:11:57.077132+0000","last_peered":"2023-10-09T10:11:57.077132+0000","last_clean":"2023-10-09T10:11:57.077132+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:11:57.077132+0000","last_undegraded":"2023-10-09T10:11:57.077132+0000","last_fullsized":"2023-10-09T10:11:57.077132+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":175,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207643+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:12:06.207643+0000","last_peered":"2023-10-09T10:12:06.207643+0000","last_clean":"2023-10-09T10:12:06.207643+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:12:06.207643+0000","last_undegraded":"2023-10-09T10:12:06.207643+0000","last_fullsized":"2023-10-09T10:12:06.207643+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"306'11208","reported_seq":11253,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:16.727301+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:12:16.727301+0000","last_peered":"2023-10-09T10:12:16.727301+0000","last_clean":"2023-10-09T10:12:16.727301+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:12:16.727301+0000","last_undegraded":"2023-10-09T10:12:16.727301+0000","last_fullsized":"2023-10-09T10:12:16.727301+0000","mapping_epoch":271,"log_start":"279'1800","ondisk_log_start":"279'1800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9408,"log_dups_size":0,"ondisk_log_size":9408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154009600,"num_objects":2350,"num_object_clones":0,"num_object_copies":7050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11208,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":269,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.224430+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:12:06.224430+0000","last_peered":"2023-10-09T10:12:06.224430+0000","last_clean":"2023-10-09T10:12:06.224430+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:12:06.224430+0000","last_undegraded":"2023-10-09T10:12:06.224430+0000","last_fullsized":"2023-10-09T10:12:06.224430+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":282,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.077547+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:11:57.077547+0000","last_peered":"2023-10-09T10:11:57.077547+0000","last_clean":"2023-10-09T10:11:57.077547+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:11:57.077547+0000","last_undegraded":"2023-10-09T10:11:57.077547+0000","last_fullsized":"2023-10-09T10:11:57.077547+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":258,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.228697+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:12:06.228697+0000","last_peered":"2023-10-09T10:12:06.228697+0000","last_clean":"2023-10-09T10:12:06.228697+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:12:06.228697+0000","last_undegraded":"2023-10-09T10:12:06.228697+0000","last_fullsized":"2023-10-09T10:12:06.228697+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"305'10998","reported_seq":11041,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:12.939093+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:12:12.939093+0000","last_peered":"2023-10-09T10:12:12.939093+0000","last_clean":"2023-10-09T10:12:12.939093+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:12:12.939093+0000","last_undegraded":"2023-10-09T10:12:12.939093+0000","last_fullsized":"2023-10-09T10:12:12.939093+0000","mapping_epoch":271,"log_start":"277'900","ondisk_log_start":"277'900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10098,"log_dups_size":0,"ondisk_log_size":10098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161939480,"num_objects":2472,"num_object_clones":0,"num_object_copies":7416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2472,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10998,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":214,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209780+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:12:06.209780+0000","last_peered":"2023-10-09T10:12:06.209780+0000","last_clean":"2023-10-09T10:12:06.209780+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:12:06.209780+0000","last_undegraded":"2023-10-09T10:12:06.209780+0000","last_fullsized":"2023-10-09T10:12:06.209780+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":177,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.211212+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:12:06.211212+0000","last_peered":"2023-10-09T10:12:06.211212+0000","last_clean":"2023-10-09T10:12:06.211212+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:12:06.211212+0000","last_undegraded":"2023-10-09T10:12:06.211212+0000","last_fullsized":"2023-10-09T10:12:06.211212+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":158,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080682+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:11:57.080682+0000","last_peered":"2023-10-09T10:11:57.080682+0000","last_clean":"2023-10-09T10:11:57.080682+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:11:57.080682+0000","last_undegraded":"2023-10-09T10:11:57.080682+0000","last_fullsized":"2023-10-09T10:11:57.080682+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":171,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207457+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:12:06.207457+0000","last_peered":"2023-10-09T10:12:06.207457+0000","last_clean":"2023-10-09T10:12:06.207457+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:12:06.207457+0000","last_undegraded":"2023-10-09T10:12:06.207457+0000","last_fullsized":"2023-10-09T10:12:06.207457+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":243,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.077199+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:11:57.077199+0000","last_peered":"2023-10-09T10:11:57.077199+0000","last_clean":"2023-10-09T10:11:57.077199+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:11:57.077199+0000","last_undegraded":"2023-10-09T10:11:57.077199+0000","last_fullsized":"2023-10-09T10:11:57.077199+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":142,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.084175+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:11:57.084175+0000","last_peered":"2023-10-09T10:11:57.084175+0000","last_clean":"2023-10-09T10:11:57.084175+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:11:57.084175+0000","last_undegraded":"2023-10-09T10:11:57.084175+0000","last_fullsized":"2023-10-09T10:11:57.084175+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":184,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220927+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:12:06.220927+0000","last_peered":"2023-10-09T10:12:06.220927+0000","last_clean":"2023-10-09T10:12:06.220927+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:12:06.220927+0000","last_undegraded":"2023-10-09T10:12:06.220927+0000","last_fullsized":"2023-10-09T10:12:06.220927+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":142,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080966+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:11:57.080966+0000","last_peered":"2023-10-09T10:11:57.080966+0000","last_clean":"2023-10-09T10:11:57.080966+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:11:57.080966+0000","last_undegraded":"2023-10-09T10:11:57.080966+0000","last_fullsized":"2023-10-09T10:11:57.080966+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":193,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220405+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:12:06.220405+0000","last_peered":"2023-10-09T10:12:06.220405+0000","last_clean":"2023-10-09T10:12:06.220405+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:12:06.220405+0000","last_undegraded":"2023-10-09T10:12:06.220405+0000","last_fullsized":"2023-10-09T10:12:06.220405+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":323,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220977+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:12:06.220977+0000","last_peered":"2023-10-09T10:12:06.220977+0000","last_clean":"2023-10-09T10:12:06.220977+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:12:06.220977+0000","last_undegraded":"2023-10-09T10:12:06.220977+0000","last_fullsized":"2023-10-09T10:12:06.220977+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":343,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220558+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:12:06.220558+0000","last_peered":"2023-10-09T10:12:06.220558+0000","last_clean":"2023-10-09T10:12:06.220558+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:12:06.220558+0000","last_undegraded":"2023-10-09T10:12:06.220558+0000","last_fullsized":"2023-10-09T10:12:06.220558+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":253,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.206019+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:12:06.206019+0000","last_peered":"2023-10-09T10:12:06.206019+0000","last_clean":"2023-10-09T10:12:06.206019+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:12:06.206019+0000","last_undegraded":"2023-10-09T10:12:06.206019+0000","last_fullsized":"2023-10-09T10:12:06.206019+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":172,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-10-09T10:11:57.080148+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:11:57.080148+0000","last_peered":"2023-10-09T10:11:57.080148+0000","last_clean":"2023-10-09T10:11:57.080148+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:11:57.080148+0000","last_undegraded":"2023-10-09T10:11:57.080148+0000","last_fullsized":"2023-10-09T10:11:57.080148+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2500526104,"num_objects":38156,"num_object_clones":0,"num_object_copies":114468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5928,"num_objects_misplaced":787,"num_objects_unfound":0,"num_objects_dirty":38156,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179206,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3942,"num_bytes_recovered":258342912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8114483200,"data_stored":8114470984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142006,"ondisk_log_size":142006,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430511,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1072000,"kb_used_data":770680,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92673152,"statfs":{"total":95995035648,"available":94897307648,"internally_reserved":0,"allocated":789176320,"data_stored":788636577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:43 2023","interfaces":[{"interface":"back","average":{"1min":3.508,"5min":1.986,"15min":1.911},"min":{"1min":0.90100000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":9.9870000000000001,"5min":9.9870000000000001,"15min":50.591999999999999},"last":2.1509999999999998},{"interface":"front","average":{"1min":3.661,"5min":2.133,"15min":1.905},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.974,"5min":15.974,"15min":50.747},"last":4.0410000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:12:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9940000000000002,"5min":2.3639999999999999,"15min":1.958},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":11.92,"5min":14.311,"15min":25.338000000000001},"last":0.374},{"interface":"front","average":{"1min":2.6320000000000001,"5min":2.218,"15min":1.6870000000000001},"min":{"1min":0.42999999999999999,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":10.663,"5min":18.27,"15min":18.27},"last":3.698}]},{"osd":2,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":3.7639999999999998,"5min":1.9970000000000001,"15min":2.359},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":13.775,"5min":13.775,"15min":13.775},"last":6.8810000000000002},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.008,"15min":2.157},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.101000000000001,"5min":12.101000000000001,"15min":12.101000000000001},"last":2.2400000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4550000000000001,"5min":2.629,"15min":2.0430000000000001},"min":{"1min":0.436,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.6289999999999996,"5min":29.058,"15min":29.058},"last":2.427},{"interface":"front","average":{"1min":3.6629999999999998,"5min":2.2559999999999998,"15min":2.0489999999999999},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":5.6050000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.0469999999999999,"15min":0.95699999999999996},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.9820000000000002,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":2.1160000000000001},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.97599999999999998,"15min":0.93100000000000005},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.6239999999999997,"5min":4.6239999999999997,"15min":4.6239999999999997},"last":1.8500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3260000000000001,"5min":1.403,"15min":1.1259999999999999},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":7.8159999999999998,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.2110000000000001,"15min":1.0580000000000001},"min":{"1min":0.38500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.806,"5min":25.754000000000001,"15min":25.754000000000001},"last":1.873}]},{"osd":6,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":1.0660000000000001,"15min":1.006},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":1.895},{"interface":"front","average":{"1min":1.327,"5min":0.86899999999999999,"15min":0.91900000000000004},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":9.1920000000000002},"last":0.33900000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626739,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1420464,"kb_used_data":1115048,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92324688,"statfs":{"total":95995035648,"available":94540480512,"internally_reserved":0,"allocated":1141809152,"data_stored":1141286723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":1.919,"15min":1.6060000000000001},"min":{"1min":0.252,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":6.9630000000000001,"5min":15.435,"15min":15.435},"last":2.702},{"interface":"front","average":{"1min":2.2469999999999999,"5min":1.921,"15min":1.591},"min":{"1min":0.33900000000000002,"5min":0.252,"15min":0.23999999999999999},"max":{"1min":8.2959999999999994,"5min":15.308999999999999,"15min":15.308999999999999},"last":1.5009999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:12:06 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":2.839,"15min":2.0579999999999998},"min":{"1min":0.30199999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.0519999999999996,"5min":35.609999999999999,"15min":35.609999999999999},"last":2.7919999999999998},{"interface":"front","average":{"1min":2.919,"5min":2.46,"15min":1.9910000000000001},"min":{"1min":0.36099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":8.2680000000000007,"5min":32.396000000000001,"15min":32.396000000000001},"last":1.468}]},{"osd":2,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":1.637,"15min":1.7929999999999999},"min":{"1min":0.35199999999999998,"5min":0.26900000000000002,"15min":0.26200000000000001},"max":{"1min":7.1280000000000001,"5min":12.898,"15min":12.898},"last":0.438},{"interface":"front","average":{"1min":3.4860000000000002,"5min":2.1400000000000001,"15min":3.0379999999999998},"min":{"1min":0.44600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.542999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":6.1769999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":4.1079999999999997,"5min":2.3639999999999999,"15min":1.694},"min":{"1min":0.432,"5min":0.308,"15min":0.308},"max":{"1min":19.222000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.2490000000000001,"5min":2.4500000000000002,"15min":2.4039999999999999},"min":{"1min":0.39500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":8.2270000000000003,"5min":11.211,"15min":11.211},"last":1.587}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.67100000000000004,"15min":0.51700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.8970000000000002,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.284,"5min":0.75800000000000001,"15min":0.58499999999999996},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":2.6339999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:11:37 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.81599999999999995},"min":{"1min":0.29399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.4580000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.499},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.042,"15min":0.872},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":6.3369999999999997,"5min":7.508,"15min":7.508},"last":2.6699999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.92600000000000005,"15min":0.89500000000000002},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.0700000000000003,"5min":6.2709999999999999,"15min":7.5350000000000001},"last":2.7240000000000002},{"interface":"front","average":{"1min":1.673,"5min":0.97199999999999998,"15min":0.94299999999999995},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.101,"5min":6.3159999999999998,"15min":7.5570000000000004},"last":2.7679999999999998}]}]},{"osd":1,"up_from":84,"seq":360777253052,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1506908,"kb_used_data":1098900,"kb_used_omap":9,"kb_used_meta":407478,"kb_avail":92238244,"statfs":{"total":95995035648,"available":94451961856,"internally_reserved":0,"allocated":1125273600,"data_stored":1124229231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":417258473},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.3980000000000001,"15min":2.1800000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":7.3609999999999998,"5min":25.59,"15min":25.59},"last":31.007000000000001},{"interface":"front","average":{"1min":6.2140000000000004,"5min":3.206,"15min":2.3639999999999999},"min":{"1min":0.314,"5min":0.245,"15min":0.192},"max":{"1min":20.501999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":1.23}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":3.0249999999999999,"15min":3.8069999999999999},"min":{"1min":0.70099999999999996,"5min":0.217,"15min":0.217},"max":{"1min":10.573,"5min":17.369,"15min":17.369},"last":7.702},{"interface":"front","average":{"1min":3.3479999999999999,"5min":2.5449999999999999,"15min":3.7530000000000001},"min":{"1min":0.26400000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":13.414999999999999,"5min":13.414999999999999,"15min":16.023},"last":21.645}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3170000000000002,"5min":3.6110000000000002,"15min":3.5590000000000002},"min":{"1min":0.41699999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.119,"5min":18.587,"15min":18.587},"last":0.64100000000000001},{"interface":"front","average":{"1min":2.7959999999999998,"5min":3.8029999999999999,"15min":4.5350000000000001},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.103999999999999,"5min":17.289000000000001,"15min":17.289000000000001},"last":21.547999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.536,"15min":1.3480000000000001},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.976000000000001,"5min":11.976000000000001,"15min":11.976000000000001},"last":21.620000000000001},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.177,"15min":1.6120000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.904,"5min":12.904,"15min":12.904},"last":0.76500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":2.3519999999999999,"15min":1.9330000000000001},"min":{"1min":0.41999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.811,"5min":14.416,"15min":15.531000000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":2.742,"5min":2.1859999999999999,"15min":2.1080000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.9039999999999999,"5min":16.277999999999999,"15min":19.706},"last":0.53900000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.9450000000000001,"15min":1.825},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":7.3049999999999997,"5min":14.215,"15min":16.193999999999999},"last":21.670999999999999},{"interface":"front","average":{"1min":4.1509999999999998,"5min":2.343,"15min":1.984},"min":{"1min":0.46200000000000002,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.868,"5min":14.471,"15min":15.481999999999999},"last":1.2629999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:11:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1080000000000001,"5min":2.306,"15min":2.1360000000000001},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.8620000000000001,"5min":16.331,"15min":16.331},"last":0.69399999999999995},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.0489999999999999,"15min":1.901},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":7.3339999999999996,"5min":14.242000000000001,"15min":16.100999999999999},"last":21.593}]}]},{"osd":0,"up_from":79,"seq":339302416579,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1291912,"kb_used_data":902976,"kb_used_omap":28,"kb_used_meta":388899,"kb_avail":92453240,"statfs":{"total":95995035648,"available":94672117760,"internally_reserved":0,"allocated":924647424,"data_stored":923628043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":398233424},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:11:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":2.0259999999999998,"15min":1.6559999999999999},"min":{"1min":0.216,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":12.975,"5min":12.975,"15min":13.084},"last":9.952},{"interface":"front","average":{"1min":5.8680000000000003,"5min":3.653,"15min":2.5920000000000001},"min":{"1min":0.29099999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":22.934999999999999,"5min":22.934999999999999,"15min":22.934999999999999},"last":0.47499999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0469999999999997,"5min":2.181,"15min":2.274},"min":{"1min":0.28899999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.003,"5min":18.003,"15min":18.003},"last":10.41},{"interface":"front","average":{"1min":2.9660000000000002,"5min":2.0710000000000002,"15min":3.0190000000000001},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.432,"5min":15.529,"15min":15.529},"last":10.374000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":4.6509999999999998,"15min":4.7930000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":23.151,"5min":25.734999999999999,"15min":25.734999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":5.3250000000000002,"5min":4.0739999999999998,"15min":4.298},"min":{"1min":0.33300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":18.279,"5min":21.923999999999999,"15min":21.923999999999999},"last":8.1509999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3780000000000001,"5min":2.3079999999999998,"15min":1.6970000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.989000000000001,"5min":20.494,"15min":20.494},"last":0.54200000000000004},{"interface":"front","average":{"1min":3.6819999999999999,"5min":2.395,"15min":2.089},"min":{"1min":0.40200000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.907,"5min":15.907,"15min":15.907},"last":6.149}]},{"osd":5,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5739999999999998,"15min":2.1869999999999998},"min":{"1min":0.36099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.868,"5min":17.096,"15min":17.096},"last":6.2249999999999996},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.52,"15min":1.4299999999999999},"min":{"1min":0.371,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.371,"5min":10.371,"15min":12.478},"last":10.284000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.734,"15min":1.5860000000000001},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":8.827,"5min":9.4719999999999995,"15min":11.699},"last":10.324},{"interface":"front","average":{"1min":4.5800000000000001,"5min":2.9740000000000002,"15min":2.4660000000000002},"min":{"1min":0.52100000000000002,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":23.106999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":0.56899999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2679999999999998,"5min":2.6699999999999999,"15min":2.3540000000000001},"min":{"1min":0.437,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":16.460000000000001,"5min":16.460000000000001,"15min":26.978000000000002},"last":6.2560000000000002},{"interface":"front","average":{"1min":2.677,"5min":1.7490000000000001,"15min":1.738},"min":{"1min":0.45100000000000001,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":8.8040000000000003,"5min":10.122,"15min":26.768000000000001},"last":10.35}]}]},{"osd":2,"up_from":217,"seq":932007903312,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1070212,"kb_used_data":813180,"kb_used_omap":83,"kb_used_meta":256940,"kb_avail":92674940,"statfs":{"total":95995035648,"available":94899138560,"internally_reserved":0,"allocated":832696320,"data_stored":831674094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":263107055},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.0609999999999999,"15min":1.8660000000000001},"min":{"1min":0.24299999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.334,"5min":19.372,"15min":19.372},"last":10.231999999999999},{"interface":"front","average":{"1min":5.5940000000000003,"5min":3.5150000000000001,"15min":3.133},"min":{"1min":0.32000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.047000000000001,"5min":16.754999999999999,"15min":16.754999999999999},"last":5.6840000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":4.6630000000000003,"5min":2.653,"15min":2.6760000000000002},"min":{"1min":0.26500000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.276,"5min":14.276,"15min":14.276},"last":10.282999999999999},{"interface":"front","average":{"1min":5.4710000000000001,"5min":3.0760000000000001,"15min":2.823},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.227,"5min":15.569000000000001,"15min":15.569000000000001},"last":5.6509999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.2989999999999999,"15min":2.1150000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.359999999999999,"5min":13.851000000000001,"15min":13.851000000000001},"last":10.342000000000001},{"interface":"front","average":{"1min":4.1879999999999997,"5min":3.2360000000000002,"15min":3.6880000000000002},"min":{"1min":0.40799999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.51,"5min":15.811,"15min":15.811},"last":5.5389999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":1.3,"15min":0.90400000000000003},"min":{"1min":0.29399999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.413,"5min":12.413,"15min":12.413},"last":5.5759999999999996},{"interface":"front","average":{"1min":1.597,"5min":1.329,"15min":0.996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.314},"max":{"1min":7.0810000000000004,"5min":12.385,"15min":12.385},"last":5.5190000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.8120000000000001,"15min":1.5860000000000001},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1230000000000002,"5min":12.473000000000001,"15min":12.473000000000001},"last":2.3319999999999999},{"interface":"front","average":{"1min":2.044,"5min":1.5980000000000001,"15min":1.706},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.923,"5min":13.587,"15min":13.587},"last":10.314}]},{"osd":6,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5529999999999999,"5min":1.9219999999999999,"15min":1.2689999999999999},"min":{"1min":0.374,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.484,"5min":15.292999999999999,"15min":15.292999999999999},"last":5.617},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.9670000000000001,"15min":1.8180000000000001},"min":{"1min":0.41099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.5679999999999996,"5min":12.502000000000001,"15min":12.502000000000001},"last":2.2909999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.9359999999999999,"15min":1.294},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.457000000000001,"5min":15.343999999999999,"15min":15.343999999999999},"last":5.7220000000000004},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.1539999999999999,"15min":1.758},"min":{"1min":0.38600000000000001,"5min":0.376,"15min":0.376},"max":{"1min":9.6039999999999992,"5min":12.433999999999999,"15min":12.433999999999999},"last":2.3660000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824056,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1169944,"kb_used_data":905744,"kb_used_omap":10,"kb_used_meta":264181,"kb_avail":92575208,"statfs":{"total":95995035648,"available":94797012992,"internally_reserved":0,"allocated":927481856,"data_stored":926272178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":270521411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5139999999999998,"5min":4.0330000000000004,"15min":4.4800000000000004},"min":{"1min":0.27500000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.749000000000001,"5min":23.381,"15min":23.381},"last":3.7040000000000002},{"interface":"front","average":{"1min":6.5919999999999996,"5min":4.5750000000000002,"15min":4.3109999999999999},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.782,"5min":20.239999999999998,"15min":20.239999999999998},"last":5.6520000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2909999999999999,"5min":3.4820000000000002,"15min":3.7959999999999998},"min":{"1min":0.41399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.856999999999999,"5min":20.422999999999998,"15min":20.422999999999998},"last":8.2539999999999996},{"interface":"front","average":{"1min":4.5620000000000003,"5min":3.726,"15min":3.5289999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.73,"5min":23.423999999999999,"15min":23.423999999999999},"last":2.5150000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":3.1269999999999998,"15min":3.536},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.331,"5min":20.594999999999999,"15min":20.594999999999999},"last":6.6609999999999996},{"interface":"front","average":{"1min":4.298,"5min":3.1680000000000001,"15min":2.9660000000000002},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":14.802,"5min":23.452000000000002,"15min":23.452000000000002},"last":5.4610000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":2.048,"15min":1.413},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.696999999999999,"5min":15.676,"15min":15.676},"last":2.5499999999999998},{"interface":"front","average":{"1min":4.7910000000000004,"5min":2.2360000000000002,"15min":1.752},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.478999999999999,"5min":16.478999999999999,"15min":16.478999999999999},"last":5.5670000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":2.9260000000000002,"15min":2.718},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.125999999999999,"5min":13.125999999999999,"15min":13.125999999999999},"last":8.1039999999999992},{"interface":"front","average":{"1min":3.0699999999999998,"5min":2.6600000000000001,"15min":2.456},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":10.891999999999999,"5min":15.992000000000001,"15min":15.992000000000001},"last":6.633}]},{"osd":6,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9649999999999999,"5min":2.9809999999999999,"15min":2.819},"min":{"1min":0.42099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.504999999999999,"5min":16.504999999999999,"15min":16.504999999999999},"last":7.4119999999999999},{"interface":"front","average":{"1min":1.921,"5min":2.4239999999999999,"15min":2.544},"min":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.952,"5min":16.16,"15min":16.16},"last":6.5899999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.1219999999999999,"15min":1.633},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":11.048999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":2.5779999999999998},{"interface":"front","average":{"1min":5.8399999999999999,"5min":3.073,"15min":2.8069999999999999},"min":{"1min":0.44600000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":16.367000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":5.6180000000000003}]}]},{"osd":4,"up_from":266,"seq":1142461300781,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1456492,"kb_used_data":1129636,"kb_used_omap":35,"kb_used_meta":326812,"kb_avail":92288660,"statfs":{"total":95995035648,"available":94503587840,"internally_reserved":0,"allocated":1156747264,"data_stored":1156219409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":334655726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.0109999999999999,"15min":0.84899999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.4829999999999997,"5min":10.881,"15min":10.881},"last":0.252},{"interface":"front","average":{"1min":3.0150000000000001,"5min":1.5629999999999999,"15min":1.161},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":12.246,"5min":12.246,"15min":12.246},"last":7.8280000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2429999999999999,"5min":1.655,"15min":1.0960000000000001},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":12.161,"5min":12.161,"15min":12.161},"last":7.1280000000000001},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.7290000000000001,"15min":1.4259999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.504,"5min":11.141,"15min":11.141},"last":0.44800000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.4430000000000001,"15min":1.143},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.2469999999999999,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":5.8170000000000002},{"interface":"front","average":{"1min":1.905,"5min":1.242,"15min":1.018},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":2.3940000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":1.9490000000000001,"15min":1.4910000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.065,"5min":13.065,"15min":13.065},"last":5.234},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.766,"15min":1.264},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.218,"5min":12.218,"15min":12.218},"last":6.7910000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":0.93200000000000005,"15min":0.79600000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.6869999999999994,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.436,"5min":0.56100000000000005,"15min":0.61899999999999999},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0609999999999999,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.312}]},{"osd":6,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.69799999999999995,"15min":0.58299999999999996},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.311},"max":{"1min":6.4119999999999999,"5min":6.4119999999999999,"15min":6.4119999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":0.83899999999999997,"15min":0.66900000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.255000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.35299999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":0.89100000000000001,"15min":0.71099999999999997},"min":{"1min":0.38800000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":12.237,"5min":12.237,"15min":12.237},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.244,"5min":0.90600000000000003,"15min":0.85399999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.7590000000000003,"5min":9.7590000000000003,"15min":9.7590000000000003},"last":0.46500000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822966,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1560928,"kb_used_data":1263448,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92184224,"statfs":{"total":95995035648,"available":94396645376,"internally_reserved":0,"allocated":1293770752,"data_stored":1293250953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4710000000000001,"15min":1.5720000000000001},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.8179999999999996,"5min":7.8540000000000001,"15min":15.773999999999999},"last":3.4849999999999999},{"interface":"front","average":{"1min":2.7770000000000001,"5min":1.6879999999999999,"15min":1.6819999999999999},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.9950000000000001,"5min":10.244999999999999,"15min":15.699},"last":1.431}]},{"osd":1,"last update":"Mon Oct 9 10:11:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":1.748,"15min":1.5629999999999999},"min":{"1min":0.40300000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.4250000000000007,"5min":11.335000000000001,"15min":15.689},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.903,"15min":1.7110000000000001},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5979999999999999,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.81499999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.7130000000000001,"5min":1.905,"15min":2.456},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.259,"5min":10.259,"15min":15.640000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.581,"5min":1.458,"15min":2.1739999999999999},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":4.1619999999999999,"5min":9.1329999999999991,"15min":10.656000000000001},"last":3.0379999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":2.1829999999999998,"15min":1.96},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2260000000000009,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":1.4970000000000001},{"interface":"front","average":{"1min":3.2370000000000001,"5min":2.4820000000000002,"15min":2.8399999999999999},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.9900000000000002,"5min":17.34,"15min":17.34},"last":17.193999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.84899999999999998,"15min":0.65900000000000003},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3460000000000001,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.70499999999999996,"15min":0.73099999999999998},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.462,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.41499999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":0.97599999999999998,"15min":0.996},"min":{"1min":0.26200000000000001,"5min":0.25,"15min":0.25},"max":{"1min":8.5259999999999998,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.198,"5min":1.075,"15min":0.92600000000000005},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.9889999999999999,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.35199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.137,"15min":1.0389999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4710000000000001,"5min":7.891,"15min":10.334},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.173,"5min":0.80300000000000005,"15min":0.96899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":12.356},"last":1.1200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":911736832,"data_stored":911736832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1113391104,"data_stored":1113391104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":821104640,"data_stored":821100568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":915472384,"data_stored":915472384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1147797504,"data_stored":1147797504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1285492736,"data_stored":1285488664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1134624768,"data_stored":1134624768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":784863232,"data_stored":784859160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:12:18.951 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:18 smithi110 ceph-mon[136032]: pgmap v843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 5928/114540 objects degraded (5.175%); 787/114540 objects misplaced (0.687%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:18.951 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:18 smithi110 ceph-mon[136032]: from='client.35784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:18.951 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/114540 objects degraded (5.175%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:18.951 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:18 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:18.951 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:18 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:18.953 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:18.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:19.047 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:19.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:19.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:19.241 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:19.327 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:19.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:19.445 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:19.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:19.533 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:19.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:19.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:19.542 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:19.632 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:12:19.633 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:12:19.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:19.749 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:19.833 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:19.848 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:19.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:19.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:20.022 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:20.216 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:20.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:20.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:20.417 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:20.487 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:20 smithi012 ceph-mon[147926]: pgmap v844: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 5928/110025 objects degraded (5.388%); 787/110025 objects misplaced (0.715%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:20.488 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:20 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:12:20.488 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:20 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:12:20.488 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:20 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:20.764 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:20.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:20.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:20 smithi012 ceph-mon[152263]: pgmap v844: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 5928/110025 objects degraded (5.388%); 787/110025 objects misplaced (0.715%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:20.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:20 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:12:20.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:20 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:12:20.776 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:20 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:20.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:12:20.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:20.908 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:20 smithi110 ceph-mon[136032]: pgmap v844: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 5928/110025 objects degraded (5.388%); 787/110025 objects misplaced (0.715%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:20.908 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:20 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:12:20.908 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:20 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:12:20.909 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:20 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:12:20.910 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:20.956 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:20.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:21.069 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:21.111 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:21.165 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:21.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:21.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:21.271 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:21.302 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:21.361 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:21.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:21.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:21.468 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:21.562 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:21.849 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:21.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:21.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:21.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:12:21.965 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:22.003 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:22.060 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:22.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:22.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:22.163 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:22.192 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:22.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:22.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:22.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:22.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:22.453 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:22.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:22 smithi012 ceph-mon[152263]: pgmap v845: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 5928/109464 objects degraded (5.415%); 787/109464 objects misplaced (0.719%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:22.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:22 smithi012 ceph-mon[147926]: pgmap v845: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 5928/109464 objects degraded (5.415%); 787/109464 objects misplaced (0.719%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:22.702 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:22.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:22.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:22.815 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:22.842 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:22 smithi110 ceph-mon[136032]: pgmap v845: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 5928/109464 objects degraded (5.415%); 787/109464 objects misplaced (0.719%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:12:22.844 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:22.881 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:22.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:22.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:22.987 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:23.055 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:23.157 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:23.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:23.264 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:23.294 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:23.316 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:23.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:23.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:23.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:23.440 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:23.449 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:23.473 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:23.524 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:23.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:23.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:23.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:23.639 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:12:23.664 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/109464 objects degraded (5.415%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:23.667 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:23.693 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:12:23.716 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:23.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:23.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:23.753 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:12:23.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/109464 objects degraded (5.415%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:23.775 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/109464 objects degraded (5.415%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:23.803 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:12:23.822 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:23.854 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:12:23.862 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:23.912 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:23.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:12:23.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:23.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:23.960 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:12:23.994 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:12:24.005 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:12:24.005 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:12:24.005 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:12:24.005 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:12:24.005 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:12:24.005 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:12:24.005 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:12:24.005 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:12:24.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:24.065 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:24.127 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:24.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:24.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:24.280 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:24.336 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:24.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:24.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:24.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:24.524 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:24.617 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:12:24.641 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:24.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:24.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:24 smithi110 ceph-mon[136032]: pgmap v846: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5928/103347 objects degraded (5.736%); 589/103347 objects misplaced (0.570%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T10:12:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3996099019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2136063804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:24.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:24.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:24 smithi012 ceph-mon[147926]: pgmap v846: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5928/103347 objects degraded (5.736%); 589/103347 objects misplaced (0.570%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T10:12:24.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3996099019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:24.702 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2136063804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:24.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:24.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:24 smithi012 ceph-mon[152263]: pgmap v846: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 5928/103347 objects degraded (5.736%); 589/103347 objects misplaced (0.570%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T10:12:24.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3996099019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:24.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2136063804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:24.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:24.752 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:24.785 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:24.850 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:24.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:24.956 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:25.026 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:25.371 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:25.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:25.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:25.478 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:25.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:25.571 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:25.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:25.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:25.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:25.745 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:12:25.745 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:12:25.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:12:25.746 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:12:25.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.746 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195715 6.14 6:2f73fb93:::benchmark_data_smithi012_182576_object90194:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:25.746 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:12:25.746 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.44500320999998, 2023-10-09T10:12:25.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.606497794, 2023-10-09T10:12:25.746 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.747 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.747 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.747 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:25.747 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:25.747 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195715 2023-10-09T10:12:25.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.747 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297211+0000", 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297210+0000", 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297215+0000", 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2869999999999998e-06 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297225+0000", 2023-10-09T10:12:25.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9709999999999999e-06 2023-10-09T10:12:25.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.297233+0000", 2023-10-09T10:12:25.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1200000000000002e-06 2023-10-09T10:12:25.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.301089+0000", 2023-10-09T10:12:25.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038566210000000002 2023-10-09T10:12:25.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.301096+0000", 2023-10-09T10:12:25.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0010000000000004e-06 2023-10-09T10:12:25.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.715860+0000", 2023-10-09T10:12:25.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.41476375300000001 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.715875+0000", 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5322e-05 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.886573+0000", 2023-10-09T10:12:25.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1706976149999999 2023-10-09T10:12:25.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.886704+0000", 2023-10-09T10:12:25.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000131271 2023-10-09T10:12:25.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:12:25.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.887173+0000", 2023-10-09T10:12:25.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00046823600000000002 2023-10-09T10:12:25.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.899644+0000", 2023-10-09T10:12:25.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012471365 2023-10-09T10:12:25.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.899817+0000", 2023-10-09T10:12:25.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000172919 2023-10-09T10:12:25.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900021+0000", 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000204556 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903542+0000", 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035206030000000002 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903591+0000", 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8568e-05 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.903709+0000", 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118261 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.759 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195727 6.4 6:27a88943:::benchmark_data_smithi012_182576_object90206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:25.759 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:12:25.759 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.413357959, 2023-10-09T10:12:25.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5877643209999999, 2023-10-09T10:12:25.759 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.760 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.760 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.760 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:25.760 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:25.760 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195727 2023-10-09T10:12:25.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.761 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:12:25.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328856+0000", 2023-10-09T10:12:25.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328855+0000", 2023-10-09T10:12:25.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:12:25.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328861+0000", 2023-10-09T10:12:25.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0430000000000004e-06 2023-10-09T10:12:25.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328874+0000", 2023-10-09T10:12:25.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3360000000000001e-05 2023-10-09T10:12:25.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328882+0000", 2023-10-09T10:12:25.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2023-10-09T10:12:25.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.328994+0000", 2023-10-09T10:12:25.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111865 2023-10-09T10:12:25.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.329006+0000", 2023-10-09T10:12:25.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.255e-05 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716358+0000", 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.38735175199999999 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716368+0000", 2023-10-09T10:12:25.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8479999999999996e-06 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900045+0000", 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1836774349999999 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900183+0000", 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013796500000000001 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900820+0000", 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00063694999999999995 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.907584+0000", 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067633459999999999 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916106+0000", 2023-10-09T10:12:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085223179999999992 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916306+0000", 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020026700000000001 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916459+0000", 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000153323 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916508+0000", 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8109999999999998e-05 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916621+0000", 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000113142 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195736 6.d 6:b34830fd:::benchmark_data_smithi012_182576_object90215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.40482194000003, 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6770504659999999, 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195736 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.772 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337392+0000", 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337391+0000", 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337398+0000", 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2910000000000002e-06 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337410+0000", 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2217e-05 2023-10-09T10:12:25.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337419+0000", 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6989999999999992e-06 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339092+0000", 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016734510000000001 2023-10-09T10:12:25.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339102+0000", 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7520000000000008e-06 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718181+0000", 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37907882999999998 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718193+0000", 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1955e-05 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.972506+0000", 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2543128530000001 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.975318+0000", 2023-10-09T10:12:25.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028124920000000002 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.975658+0000", 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000340136 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999408+0000", 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023749816 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999451+0000", 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3038999999999999e-05 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999490+0000", 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8520999999999999e-05 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014261+0000", 2023-10-09T10:12:25.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014770873 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014326+0000", 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5018999999999995e-05 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014443+0000", 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117192 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.780 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195737 6.d 6:b631f0af:::benchmark_data_smithi012_182576_object90216:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.40477009999995, 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6917381869999999, 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.781 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195737 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337444+0000", 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337447+0000", 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5310000000000002e-06 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337455+0000", 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8790000000000003e-06 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.337462+0000", 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0650000000000001e-06 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339139+0000", 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016767990000000001 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.339151+0000", 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1959e-05 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718219+0000", 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37906844499999998 2023-10-09T10:12:25.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718228+0000", 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6910000000000003e-06 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999515+0000", 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2812874459999999 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999630+0000", 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114138 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.999802+0000", 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000172535 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014252+0000", 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014449500000000001 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014454+0000", 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020259100000000001 2023-10-09T10:12:25.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014691+0000", 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023644399999999999 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029029+0000", 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014338379 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029086+0000", 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.694e-05 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029182+0000", 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6429999999999997e-05 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.790 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195738 6.d 6:b7beca9a:::benchmark_data_smithi012_182576_object90217:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.398924921, 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.686112541, 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:25.791 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195738 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343289+0000", 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343288+0000", 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343296+0000", 2023-10-09T10:12:25.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6650000000000003e-06 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343334+0000", 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8760999999999997e-05 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343345+0000", 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0872e-05 2023-10-09T10:12:25.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343426+0000", 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0876000000000003e-05 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343442+0000", 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6049000000000002e-05 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718250+0000", 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37480775999999999 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718258+0000", 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4360000000000002e-06 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.002824+0000", 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.284566216 2023-10-09T10:12:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.002952+0000", 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127133 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.003293+0000", 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00034169000000000002 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014460+0000", 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011166931999999999 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014643+0000", 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018320000000000001 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014731+0000", 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7837999999999995e-05 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029238+0000", 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014506865000000001 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029290+0000", 2023-10-09T10:12:25.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2261e-05 2023-10-09T10:12:25.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029402+0000", 2023-10-09T10:12:25.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111514 2023-10-09T10:12:25.800 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.801 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.801 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.801 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195740 6.d 6:ba67ed77:::benchmark_data_smithi012_182576_object90219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:25.801 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:12:25.801 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.39840041699995, 2023-10-09T10:12:25.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.685801323, 2023-10-09T10:12:25.802 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.802 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.802 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.802 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:25.802 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:25.802 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195740 2023-10-09T10:12:25.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.802 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:12:25.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343814+0000", 2023-10-09T10:12:25.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343813+0000", 2023-10-09T10:12:25.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:12:25.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343819+0000", 2023-10-09T10:12:25.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.049e-06 2023-10-09T10:12:25.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343834+0000", 2023-10-09T10:12:25.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5458999999999999e-05 2023-10-09T10:12:25.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343843+0000", 2023-10-09T10:12:25.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7250000000000003e-06 2023-10-09T10:12:25.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343908+0000", 2023-10-09T10:12:25.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4981999999999995e-05 2023-10-09T10:12:25.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.343924+0000", 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5843999999999999e-05 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718282+0000", 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.37435805300000002 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.718299+0000", 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6651999999999999e-05 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006544+0000", 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2882459150000001 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006671+0000", 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012675699999999999 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.006813+0000", 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141285 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014467+0000", 2023-10-09T10:12:25.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076547940000000004 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.014774+0000", 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00030705499999999998 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.028866+0000", 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014091942 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029464+0000", 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00059783999999999998 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029512+0000", 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7426000000000003e-05 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:44.029615+0000", 2023-10-09T10:12:25.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103649 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195753 6.14 6:2fd4c659:::benchmark_data_smithi012_182576_object90232:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.37774953899998, 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5530476879999999, 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.815 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195753 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364465+0000", 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364464+0000", 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364468+0000", 2023-10-09T10:12:25.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.433e-06 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364478+0000", 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0186e-05 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364484+0000", 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9349999999999999e-06 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364551+0000", 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7554999999999999e-05 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.364562+0000", 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0964999999999999e-05 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.717712+0000", 2023-10-09T10:12:25.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.35315011400000001 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.721170+0000", 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003457702 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900344+0000", 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.179173823 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900467+0000", 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000122846 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.900983+0000", 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051649900000000002 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.907573+0000", 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0065898750000000002 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.908808+0000", 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012348979999999999 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916252+0000", 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074441109999999998 2023-10-09T10:12:25.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916394+0000", 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014227099999999999 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916845+0000", 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00045041999999999999 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.917512+0000", 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00066757000000000003 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.34797.0:195755 6.4 6:21d202b2:::benchmark_data_smithi012_182576_object90234:head [delete] snapc 0=[] ondisk+write+known_if_redirected e168)", 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 583.37383893699996, 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5484353739999999, 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.824 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.825 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.825 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.34797", 2023-10-09T10:12:25.825 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/4059919325", 2023-10-09T10:12:25.825 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195755 2023-10-09T10:12:25.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.825 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:12:25.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:12:25.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368375+0000", 2023-10-09T10:12:25.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368378+0000", 2023-10-09T10:12:25.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7379999999999999e-06 2023-10-09T10:12:25.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368392+0000", 2023-10-09T10:12:25.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3913e-05 2023-10-09T10:12:25.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368399+0000", 2023-10-09T10:12:25.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6340000000000001e-06 2023-10-09T10:12:25.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368462+0000", 2023-10-09T10:12:25.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3113000000000004e-05 2023-10-09T10:12:25.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.368476+0000", 2023-10-09T10:12:25.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3366e-05 2023-10-09T10:12:25.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716390+0000", 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.347914734 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:42.716399+0000", 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6989999999999992e-06 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904175+0000", 2023-10-09T10:12:25.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.187776342 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904334+0000", 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015860900000000001 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.904813+0000", 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00047921500000000001 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916182+0000", 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011368645 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916200+0000", 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7609e-05 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916359+0000", 2023-10-09T10:12:25.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015949700000000001 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916659+0000", 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00029949099999999998 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916705+0000", 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.693e-05 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:02:43.916811+0000", 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105287 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:119305 8.1 8:8f490fc3:::benchmark_data_smithi012_271002_object10622:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:37.315208+0000", 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 108.427006783, 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082825955000000007, 2023-10-09T10:12:25.836 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 119305 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315208+0000", 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315208+0000", 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315207+0000", 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315210+0000", 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1470000000000001e-06 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315221+0000", 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0563e-05 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315226+0000", 2023-10-09T10:12:25.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5130000000000004e-06 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.324800+0000", 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095741039999999999 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.324936+0000", 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013576800000000001 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.325055+0000", 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119305 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.333879+0000", 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0088239340000000003 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.353803+0000", 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019923489999999999 2023-10-09T10:12:25.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.397960+0000", 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044156834999999998 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.397999+0000", 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9054999999999998e-05 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.398033+0000", 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4681000000000002e-05 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:119309 8.1 8:8c6804f7:::benchmark_data_smithi012_271002_object10626:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:37.315231+0000", 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 108.426983724, 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082907770000000006, 2023-10-09T10:12:25.843 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 119309 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315231+0000", 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315231+0000", 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315230+0000", 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315233+0000", 2023-10-09T10:12:25.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9859999999999999e-06 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315240+0000", 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7309999999999996e-06 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.315245+0000", 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1590000000000001e-06 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.329297+0000", 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014052113999999999 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.329466+0000", 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000168479 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.329621+0000", 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015526900000000001 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.334178+0000", 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045568859999999996 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.374494+0000", 2023-10-09T10:12:25.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.040315958999999998 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.398066+0000", 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023572536000000002 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.398104+0000", 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7644000000000001e-05 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:37.398138+0000", 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4306e-05 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:136590 8.7 8:ecc59e64:::benchmark_data_smithi012_271002_object27907:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:04.285941+0000", 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 81.456273261999996, 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.084553603000000005, 2023-10-09T10:12:25.850 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 136590 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.285941+0000", 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.285941+0000", 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.285940+0000", 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:25.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.285947+0000", 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5140000000000004e-06 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.285961+0000", 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3403e-05 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.285970+0000", 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5179999999999993e-06 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.288457+0000", 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024868310000000001 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.288582+0000", 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000125549 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.288706+0000", 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012342599999999999 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.299034+0000", 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010328641 2023-10-09T10:12:25.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.302380+0000", 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00334512 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.370408+0000", 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068027973000000005 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.370456+0000", 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.846e-05 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.370495+0000", 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8615999999999998e-05 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:174522 8.7 8:eb93958f:::benchmark_data_smithi012_271002_object65839:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:06.900398+0000", 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 18.84181646, 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.094634165000000006, 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.858 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 174522 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.900398+0000", 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.900398+0000", 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.900398+0000", 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.900400+0000", 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.243e-06 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.900405+0000", 2023-10-09T10:12:25.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9479999999999999e-06 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.900411+0000", 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.251e-06 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.912222+0000", 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011810955999999999 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.912360+0000", 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013791199999999999 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.912484+0000", 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000124244 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.917200+0000", 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0047158859999999999 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.927917+0000", 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010717304 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.994939+0000", 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067021832000000003 2023-10-09T10:12:25.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.994992+0000", 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3270000000000003e-05 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:06.995032+0000", 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9641999999999998e-05 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:175845 8.7 8:ea698d3f:::benchmark_data_smithi012_271002_object67162:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:09.012319+0000", 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 16.729895250999999, 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.098910872999999996, 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 175845 2023-10-09T10:12:25.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.012319+0000", 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.012319+0000", 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.012319+0000", 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.012322+0000", 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1719999999999999e-06 2023-10-09T10:12:25.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.012329+0000", 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9439999999999999e-06 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.012334+0000", 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6849999999999998e-06 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.028219+0000", 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015884598 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.028735+0000", 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051589300000000002 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.028880+0000", 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014500299999999999 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.035634+0000", 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067545640000000002 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.041430+0000", 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0057952730000000001 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.111130+0000", 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069700687999999997 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.111184+0000", 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3980999999999997e-05 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:09.111230+0000", 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5497999999999997e-05 2023-10-09T10:12:25.871 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:184526 8.7 8:ee385cb6:::benchmark_data_smithi012_271002_object75843:head [delete] snapc 0=[] ondisk+write+known_if_redirected e306)", 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:23.762932+0000", 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.97928207, 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053667291999999998, 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.872 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 184526 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.762932+0000", 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.762932+0000", 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.762932+0000", 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.762935+0000", 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1310000000000002e-06 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.762942+0000", 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0140000000000001e-06 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.762948+0000", 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9340000000000003e-06 2023-10-09T10:12:25.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.763348+0000", 2023-10-09T10:12:25.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000399806 2023-10-09T10:12:25.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.763456+0000", 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107996 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.763559+0000", 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103492 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.786414+0000", 2023-10-09T10:12:25.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022855039000000001 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.787258+0000", 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00084403500000000003 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.816514+0000", 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029255952000000002 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.816562+0000", 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8229000000000002e-05 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:23.816600+0000", 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7129999999999999e-05 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:184781 8.7 8:ec0b7349:::benchmark_data_smithi012_271002_object76098:head [delete] snapc 0=[] ondisk+write+known_if_redirected e306)", 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:24.183256+0000", 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.5589582449999999, 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053175544999999998, 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 184781 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.880 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183256+0000", 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183256+0000", 2023-10-09T10:12:25.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183254+0000", 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183261+0000", 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9809999999999997e-06 2023-10-09T10:12:25.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183274+0000", 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2271e-05 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183281+0000", 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6599999999999995e-06 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183338+0000", 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6428000000000003e-05 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183468+0000", 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013028000000000001 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.183583+0000", 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114479 2023-10-09T10:12:25.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.189045+0000", 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054624419999999996 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.189534+0000", 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000489333 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.236326+0000", 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.046791641000000002 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.236385+0000", 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8696999999999997e-05 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.236432+0000", 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6981999999999998e-05 2023-10-09T10:12:25.886 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:185028 8.1 8:8437f8d1:::benchmark_data_smithi012_271002_object76345:head [delete] snapc 0=[] ondisk+write+known_if_redirected e306)", 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:24.536174+0000", 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.206039981, 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053545035999999997, 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.887 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 185028 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.536174+0000", 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.536174+0000", 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.536174+0000", 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.536178+0000", 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6820000000000001e-06 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.536184+0000", 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9540000000000001e-06 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.536189+0000", 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3199999999999999e-06 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.543931+0000", 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007742156 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.544031+0000", 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9708000000000001e-05 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.544103+0000", 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1971999999999995e-05 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.558734+0000", 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014631606 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.568550+0000", 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0098153839999999999 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.589637+0000", 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021087661000000001 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.589679+0000", 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1328999999999997e-05 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.589719+0000", 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0599000000000002e-05 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:185106 8.7 8:ecf3f74a:::benchmark_data_smithi012_271002_object76423:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:24.672336+0000", 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.0698786899999999, 2023-10-09T10:12:25.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054562682000000001, 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 185106 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672336+0000", 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672336+0000", 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672334+0000", 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672341+0000", 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.64e-06 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672354+0000", 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3210000000000001e-05 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672363+0000", 2023-10-09T10:12:25.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3049999999999998e-06 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672422+0000", 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8445e-05 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672543+0000", 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120944 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.672661+0000", 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011864199999999999 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.677380+0000", 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0047183470000000003 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.686707+0000", 2023-10-09T10:12:25.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093273999999999996 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.726818+0000", 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.040110589000000002 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.726863+0000", 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5019999999999999e-05 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:24.726898+0000", 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5751999999999999e-05 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:185712 8.1 8:825db79a:::benchmark_data_smithi012_271002_object77029:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:25.596149+0000", 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.14606509200000001, 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057305109999999999, 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 185712 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.596149+0000", 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.596149+0000", 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.596149+0000", 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.596151+0000", 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6189999999999998e-06 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.596157+0000", 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7089999999999999e-06 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.596162+0000", 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.656e-06 2023-10-09T10:12:25.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601191+0000", 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050294470000000003 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601332+0000", 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014098600000000001 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:12:25.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601456+0000", 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012377800000000001 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.605882+0000", 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044255090000000002 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.647779+0000", 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041897118999999997 2023-10-09T10:12:25.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653357+0000", 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055780949999999999 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653404+0000", 2023-10-09T10:12:25.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6950000000000003e-05 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653454+0000", 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0639000000000001e-05 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.909 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:185713 8.1 8:829d7b09:::benchmark_data_smithi012_271002_object77030:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:25.601093+0000", 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.14112164099999999, 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052468709000000002, 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 185713 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.910 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601093+0000", 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601093+0000", 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.911 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601091+0000", 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601099+0000", 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3669999999999999e-06 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601113+0000", 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4071e-05 2023-10-09T10:12:25.912 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.601121+0000", 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5529999999999993e-06 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606106+0000", 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004985176 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.913 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606223+0000", 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011620000000000001 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606342+0000", 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119684 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.618185+0000", 2023-10-09T10:12:25.914 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011842302000000001 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.647820+0000", 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029634918 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653492+0000", 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056726179999999999 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.915 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653523+0000", 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0939000000000003e-05 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653561+0000", 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8238999999999998e-05 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.916 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:185717 8.1 8:8684c482:::benchmark_data_smithi012_271002_object77034:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:25.606746+0000", 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.13546781999999999, 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.046986554, 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 185717 2023-10-09T10:12:25.917 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606746+0000", 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606746+0000", 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.918 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:25.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606746+0000", 2023-10-09T10:12:25.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:25.919 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.919 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.919 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:25.919 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606749+0000", 2023-10-09T10:12:25.919 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5000000000000002e-06 2023-10-09T10:12:25.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.920 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.920 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:25.920 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606755+0000", 2023-10-09T10:12:25.920 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9460000000000003e-06 2023-10-09T10:12:25.920 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.921 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:25.921 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.606759+0000", 2023-10-09T10:12:25.921 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8439999999999997e-06 2023-10-09T10:12:25.921 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.921 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:25.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.614269+0000", 2023-10-09T10:12:25.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0075098739999999997 2023-10-09T10:12:25.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.922 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.922 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:25.922 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.614436+0000", 2023-10-09T10:12:25.922 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016705499999999999 2023-10-09T10:12:25.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:12:25.923 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.614528+0000", 2023-10-09T10:12:25.923 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2081000000000001e-05 2023-10-09T10:12:25.923 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.923 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.923 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:12:25.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.642263+0000", 2023-10-09T10:12:25.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027734992 2023-10-09T10:12:25.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.924 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.924 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.924 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.647910+0000", 2023-10-09T10:12:25.924 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056466679999999997 2023-10-09T10:12:25.924 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:25.925 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653673+0000", 2023-10-09T10:12:25.925 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0057627729999999997 2023-10-09T10:12:25.925 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.925 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.925 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:12:25.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653697+0000", 2023-10-09T10:12:25.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4431999999999999e-05 2023-10-09T10:12:25.926 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:25.926 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:25.926 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:12:25.926 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:25.653733+0000", 2023-10-09T10:12:25.926 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5775000000000002e-05 2023-10-09T10:12:25.927 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.927 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.927 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.927 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:25.927 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:25.927 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:12:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:25.928 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:25.928 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:25.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:25.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:25.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[147926]: failed to encode map e307 with expected crc 2023-10-09T10:12:25.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:12:25.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[147926]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T10:12:25.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[147926]: failed to encode map e307 with expected crc 2023-10-09T10:12:25.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[147926]: failed to encode map e307 with expected crc 2023-10-09T10:12:25.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:25.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[152263]: failed to encode map e307 with expected crc 2023-10-09T10:12:25.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:12:25.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[152263]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T10:12:25.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[152263]: failed to encode map e307 with expected crc 2023-10-09T10:12:25.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:25 smithi012 ceph-mon[152263]: failed to encode map e307 with expected crc 2023-10-09T10:12:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:25 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:25 smithi110 ceph-mon[136032]: failed to encode map e307 with expected crc 2023-10-09T10:12:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:12:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:25 smithi110 ceph-mon[136032]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T10:12:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:25 smithi110 ceph-mon[136032]: failed to encode map e307 with expected crc 2023-10-09T10:12:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:25 smithi110 ceph-mon[136032]: failed to encode map e307 with expected crc 2023-10-09T10:12:26.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:26.102 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:26.402 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:26.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:26.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:26.465 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:12:26.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:26.599 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:26.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:26 smithi012 ceph-mon[147926]: pgmap v848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 5928/100383 objects degraded (5.905%); 589/100383 objects misplaced (0.587%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T10:12:26.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:26 smithi012 ceph-mon[152263]: pgmap v848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 5928/100383 objects degraded (5.905%); 589/100383 objects misplaced (0.587%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T10:12:26.684 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:26.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:26.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:26.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:26.805 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:26.833 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:12:26.860 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:26.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:26 smithi110 ceph-mon[136032]: pgmap v848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 5928/100383 objects degraded (5.905%); 589/100383 objects misplaced (0.587%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T10:12:27.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:12:27.265 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:27.443 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:12:27.835 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:27 smithi110 ceph-mon[136032]: from='client.45779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:27.922 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:27 smithi012 ceph-mon[147926]: from='client.45779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:27.923 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:12:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:12:27] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-09T10:12:27.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:27 smithi012 ceph-mon[152263]: from='client.45779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:28.577 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:12:28.577 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:12:28.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.577 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:187632 8.0 e307/271 8:0ba88654:::benchmark_data_smithi012_271002_object78949:head v 307'11934, mlcod=307'11932)", 2023-10-09T10:12:28.577 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.567241+0000", 2023-10-09T10:12:28.577 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0080364119999999997, 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080664529999999995, 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.567241+0000", 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.567241+0000", 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.567240+0000", 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1329999999999999e-06 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.567248+0000", 2023-10-09T10:12:28.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5324000000000001e-05 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.567264+0000", 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4150000000000004e-06 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.567270+0000", 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7858999999999997e-05 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.567318+0000", 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2911000000000001e-05 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.567341+0000", 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9956000000000003e-05 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:187635 8.8 8:14c22b07:::benchmark_data_smithi012_271002_object78952:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.568538+0000", 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0067389559999999999, 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068490199999999999, 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 187635 2023-10-09T10:12:28.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568538+0000", 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568538+0000", 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568537+0000", 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3470000000000001e-06 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568544+0000", 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5705999999999998e-05 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568560+0000", 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0879999999999997e-06 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568567+0000", 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3161000000000003e-05 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568600+0000", 2023-10-09T10:12:28.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012495299999999999 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568725+0000", 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053594280000000003 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574084+0000", 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055459460000000004 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.586 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:187636 8.8 8.451a5848 (undecoded) ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.568577+0000", 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0067006100000000001, 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068913539999999997, 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:28.587 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 187636 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568577+0000", 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568577+0000", 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568576+0000", 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0860000000000002e-06 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568579+0000", 2023-10-09T10:12:28.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.614e-06 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568585+0000", 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7440000000000001e-06 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568589+0000", 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1963e-05 2023-10-09T10:12:28.590 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:187637 8.9 8:979270eb:::benchmark_data_smithi012_271002_object78954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.568593+0000", 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0066843600000000003, 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069774870000000001, 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:28.591 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 187637 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568593+0000", 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568593+0000", 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:28.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568593+0000", 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.452e-06 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568595+0000", 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.989e-06 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568600+0000", 2023-10-09T10:12:28.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5669999999999999e-06 2023-10-09T10:12:28.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568604+0000", 2023-10-09T10:12:28.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5138000000000001e-05 2023-10-09T10:12:28.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:28.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568639+0000", 2023-10-09T10:12:28.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3571000000000002e-05 2023-10-09T10:12:28.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:28.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568732+0000", 2023-10-09T10:12:28.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015156100000000001 2023-10-09T10:12:28.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:12:28.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568884+0000", 2023-10-09T10:12:28.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051929919999999996 2023-10-09T10:12:28.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:28.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574077+0000", 2023-10-09T10:12:28.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005483936 2023-10-09T10:12:28.598 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.598 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.598 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:187639 8.d 8:b3e3d575:::benchmark_data_smithi012_271002_object78956:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.568615+0000", 2023-10-09T10:12:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0066618509999999999, 2023-10-09T10:12:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070380829999999997, 2023-10-09T10:12:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:28.600 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:28.600 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:28.600 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:28.600 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 187639 2023-10-09T10:12:28.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.600 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568615+0000", 2023-10-09T10:12:28.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568615+0000", 2023-10-09T10:12:28.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:28.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568615+0000", 2023-10-09T10:12:28.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4399999999999999e-06 2023-10-09T10:12:28.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568617+0000", 2023-10-09T10:12:28.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0950000000000003e-06 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568623+0000", 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7560000000000001e-06 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568626+0000", 2023-10-09T10:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000272682 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.568899+0000", 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103788 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.569003+0000", 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119349 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.569122+0000", 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056588209999999996 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574781+0000", 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061654800000000001 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:187641 8.8 8.48e2edd8 (undecoded) ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.573368+0000", 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.001908943, 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023608829999999998, 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:28.607 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 187641 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.573368+0000", 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.573368+0000", 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.573367+0000", 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9089999999999997e-06 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.573373+0000", 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6209999999999999e-06 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.573383+0000", 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5230000000000003e-06 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.573388+0000", 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9885999999999999e-05 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.610 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.45518.0:187625 8.4 e307/288 ondisk, result = 0)", 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.574695+0000", 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00058264000000000005, 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010869199999999999, 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574695+0000", 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574695+0000", 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574694+0000", 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7350000000000001e-06 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574697+0000", 2023-10-09T10:12:28.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6920000000000002e-06 2023-10-09T10:12:28.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574705+0000", 2023-10-09T10:12:28.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4060000000000001e-06 2023-10-09T10:12:28.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574708+0000", 2023-10-09T10:12:28.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109498 2023-10-09T10:12:28.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:28.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574818+0000", 2023-10-09T10:12:28.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7850000000000003e-06 2023-10-09T10:12:28.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:28.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574828+0000", 2023-10-09T10:12:28.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132867 2023-10-09T10:12:28.616 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.616 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.616 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.617 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:187643 8.9 8:9c904dc0:::benchmark_data_smithi012_271002_object78960:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:28.617 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.575053+0000", 2023-10-09T10:12:28.617 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00022437600000000001, 2023-10-09T10:12:28.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00079587600000000005, 2023-10-09T10:12:28.618 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.618 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:28.618 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:28.618 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:28.619 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:28.619 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 187643 2023-10-09T10:12:28.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.619 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575053+0000", 2023-10-09T10:12:28.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575053+0000", 2023-10-09T10:12:28.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:28.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575052+0000", 2023-10-09T10:12:28.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6139999999999997e-06 2023-10-09T10:12:28.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575058+0000", 2023-10-09T10:12:28.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3529999999999999e-05 2023-10-09T10:12:28.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575072+0000", 2023-10-09T10:12:28.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3199999999999996e-06 2023-10-09T10:12:28.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575078+0000", 2023-10-09T10:12:28.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9950000000000001e-05 2023-10-09T10:12:28.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:28.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575128+0000", 2023-10-09T10:12:28.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000140779 2023-10-09T10:12:28.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:28.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575269+0000", 2023-10-09T10:12:28.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000149915 2023-10-09T10:12:28.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:12:28.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.575419+0000", 2023-10-09T10:12:28.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036574300000000003 2023-10-09T10:12:28.627 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.627 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.627 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.628 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:187623 8.8 8:18f83f8f:::benchmark_data_smithi012_271002_object78940:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:28.628 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.550747+0000", 2023-10-09T10:12:28.628 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.024530111, 2023-10-09T10:12:28.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025174643, 2023-10-09T10:12:28.629 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.629 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:28.629 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:28.629 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:28.629 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:28.630 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 187623 2023-10-09T10:12:28.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.630 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.550747+0000", 2023-10-09T10:12:28.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.550747+0000", 2023-10-09T10:12:28.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999957 2023-10-09T10:12:28.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.550743+0000", 2023-10-09T10:12:28.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5460000000000003e-06 2023-10-09T10:12:28.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.550750+0000", 2023-10-09T10:12:28.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8240000000000002e-06 2023-10-09T10:12:28.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.550757+0000", 2023-10-09T10:12:28.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6240000000000001e-06 2023-10-09T10:12:28.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.550762+0000", 2023-10-09T10:12:28.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4450999999999998e-05 2023-10-09T10:12:28.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:28.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.550806+0000", 2023-10-09T10:12:28.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011291899999999999 2023-10-09T10:12:28.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:28.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.550919+0000", 2023-10-09T10:12:28.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013744199999999999 2023-10-09T10:12:28.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:12:28.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.551057+0000", 2023-10-09T10:12:28.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056353640000000003 2023-10-09T10:12:28.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:28.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.556692+0000", 2023-10-09T10:12:28.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079511319999999996 2023-10-09T10:12:28.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:28.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.564643+0000", 2023-10-09T10:12:28.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013895981999999999 2023-10-09T10:12:28.639 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.639 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.639 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.640 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:187625 8.4 8:2538c1f5:::benchmark_data_smithi012_271002_object78942:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T10:12:28.640 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:28.556517+0000", 2023-10-09T10:12:28.640 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.018760446, 2023-10-09T10:12:28.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019488285000000001, 2023-10-09T10:12:28.640 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:28.640 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:28.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:28.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:28.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:28.641 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 187625 2023-10-09T10:12:28.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.641 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:28.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:28.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.556517+0000", 2023-10-09T10:12:28.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:28.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:28.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.556517+0000", 2023-10-09T10:12:28.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:28.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:28.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.556515+0000", 2023-10-09T10:12:28.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.259e-06 2023-10-09T10:12:28.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:28.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.556524+0000", 2023-10-09T10:12:28.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5750000000000007e-06 2023-10-09T10:12:28.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:28.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.556533+0000", 2023-10-09T10:12:28.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5060000000000003e-06 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.556539+0000", 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001961066 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.558500+0000", 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017968500000000001 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.558679+0000", 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018518500000000001 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.558865+0000", 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052580270000000002 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.564123+0000", 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025924989999999998 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.566715+0000", 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081223489999999992 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574837+0000", 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5408999999999997e-05 2023-10-09T10:12:28.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:28.574903+0000", 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018385997000000001 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 10 2023-10-09T10:12:28.648 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:12:28.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:28.649 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:28.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:28.650 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":848,"stamp":"2023-10-09T10:12:25.414288+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2191857592,"num_objects":33461,"num_object_clones":0,"num_object_copies":100383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5928,"num_objects_misplaced":589,"num_objects_unfound":0,"num_objects_dirty":33461,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":184185,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4209,"num_bytes_recovered":272509644,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142219,"ondisk_log_size":142219,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9607604,"kb_used_data":7038452,"kb_used_omap":167,"kb_used_meta":2568728,"kb_avail":740353612,"statfs":{"total":767960285184,"available":758122098688,"internally_reserved":0,"allocated":7207374848,"data_stored":7200960410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2630378073},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-397344768,"num_objects":-6063,"num_object_clones":0,"num_object_copies":-18189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-416,"num_objects_unfound":0,"num_objects_dirty":-6063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6063,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":416,"num_bytes_recovered":27262976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005038"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":148,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210629+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:12:06.210629+0000","last_peered":"2023-10-09T10:12:06.210629+0000","last_clean":"2023-10-09T10:12:06.210629+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:12:06.210629+0000","last_undegraded":"2023-10-09T10:12:06.210629+0000","last_fullsized":"2023-10-09T10:12:06.210629+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":263,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220634+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:12:06.220634+0000","last_peered":"2023-10-09T10:12:06.220634+0000","last_clean":"2023-10-09T10:12:06.220634+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:12:06.220634+0000","last_undegraded":"2023-10-09T10:12:06.220634+0000","last_fullsized":"2023-10-09T10:12:06.220634+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":309,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210647+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:12:06.210647+0000","last_peered":"2023-10-09T10:12:06.210647+0000","last_clean":"2023-10-09T10:12:06.210647+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:12:06.210647+0000","last_undegraded":"2023-10-09T10:12:06.210647+0000","last_fullsized":"2023-10-09T10:12:06.210647+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":172,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.211353+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:12:06.211353+0000","last_peered":"2023-10-09T10:12:06.211353+0000","last_clean":"2023-10-09T10:12:06.211353+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:12:06.211353+0000","last_undegraded":"2023-10-09T10:12:06.211353+0000","last_fullsized":"2023-10-09T10:12:06.211353+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":277,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.228886+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:12:06.228886+0000","last_peered":"2023-10-09T10:12:06.228886+0000","last_clean":"2023-10-09T10:12:06.228886+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:12:06.228886+0000","last_undegraded":"2023-10-09T10:12:06.228886+0000","last_fullsized":"2023-10-09T10:12:06.228886+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":124,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222402+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:12:06.222402+0000","last_peered":"2023-10-09T10:12:06.222402+0000","last_clean":"2023-10-09T10:12:06.222402+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:12:06.222402+0000","last_undegraded":"2023-10-09T10:12:06.222402+0000","last_fullsized":"2023-10-09T10:12:06.222402+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":190,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:11.768365+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:12:11.768365+0000","last_peered":"2023-10-09T10:12:11.768365+0000","last_clean":"2023-10-09T10:12:11.768365+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:12:11.768365+0000","last_undegraded":"2023-10-09T10:12:11.768365+0000","last_fullsized":"2023-10-09T10:12:11.768365+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":259,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.287113+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:12:14.287113+0000","last_peered":"2023-10-09T10:12:14.287113+0000","last_clean":"2023-10-09T10:12:14.287113+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:12:14.287113+0000","last_undegraded":"2023-10-09T10:12:14.287113+0000","last_fullsized":"2023-10-09T10:12:14.287113+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":123,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222228+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:12:06.222228+0000","last_peered":"2023-10-09T10:12:06.222228+0000","last_clean":"2023-10-09T10:12:06.222228+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:12:06.222228+0000","last_undegraded":"2023-10-09T10:12:06.222228+0000","last_fullsized":"2023-10-09T10:12:06.222228+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":220,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.289022+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:12:14.289022+0000","last_peered":"2023-10-09T10:12:14.289022+0000","last_clean":"2023-10-09T10:12:14.289022+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:12:14.289022+0000","last_undegraded":"2023-10-09T10:12:14.289022+0000","last_fullsized":"2023-10-09T10:12:14.289022+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":345,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.314048+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:12:14.314048+0000","last_peered":"2023-10-09T10:12:14.314048+0000","last_clean":"2023-10-09T10:12:14.314048+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:12:14.314048+0000","last_undegraded":"2023-10-09T10:12:14.314048+0000","last_fullsized":"2023-10-09T10:12:14.314048+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":301,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210236+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:12:06.210236+0000","last_peered":"2023-10-09T10:12:06.210236+0000","last_clean":"2023-10-09T10:12:06.210236+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:12:06.210236+0000","last_undegraded":"2023-10-09T10:12:06.210236+0000","last_fullsized":"2023-10-09T10:12:06.210236+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":194,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.311619+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:12:14.311619+0000","last_peered":"2023-10-09T10:12:14.311619+0000","last_clean":"2023-10-09T10:12:14.311619+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:12:14.311619+0000","last_undegraded":"2023-10-09T10:12:14.311619+0000","last_fullsized":"2023-10-09T10:12:14.311619+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":173,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210113+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:12:06.210113+0000","last_peered":"2023-10-09T10:12:06.210113+0000","last_clean":"2023-10-09T10:12:06.210113+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:12:06.210113+0000","last_undegraded":"2023-10-09T10:12:06.210113+0000","last_fullsized":"2023-10-09T10:12:06.210113+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":378,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.311353+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:12:14.311353+0000","last_peered":"2023-10-09T10:12:14.311353+0000","last_clean":"2023-10-09T10:12:14.311353+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:12:14.311353+0000","last_undegraded":"2023-10-09T10:12:14.311353+0000","last_fullsized":"2023-10-09T10:12:14.311353+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":246,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.287800+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:12:14.287800+0000","last_peered":"2023-10-09T10:12:14.287800+0000","last_clean":"2023-10-09T10:12:14.287800+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:12:14.287800+0000","last_undegraded":"2023-10-09T10:12:14.287800+0000","last_fullsized":"2023-10-09T10:12:14.287800+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":172,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.206407+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:12:06.206407+0000","last_peered":"2023-10-09T10:12:06.206407+0000","last_clean":"2023-10-09T10:12:06.206407+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:12:06.206407+0000","last_undegraded":"2023-10-09T10:12:06.206407+0000","last_fullsized":"2023-10-09T10:12:06.206407+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":139,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222473+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:12:06.222473+0000","last_peered":"2023-10-09T10:12:06.222473+0000","last_clean":"2023-10-09T10:12:06.222473+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:12:06.222473+0000","last_undegraded":"2023-10-09T10:12:06.222473+0000","last_fullsized":"2023-10-09T10:12:06.222473+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":175,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.311120+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:12:14.311120+0000","last_peered":"2023-10-09T10:12:14.311120+0000","last_clean":"2023-10-09T10:12:14.311120+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:12:14.311120+0000","last_undegraded":"2023-10-09T10:12:14.311120+0000","last_fullsized":"2023-10-09T10:12:14.311120+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:11.768339+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:12:11.768339+0000","last_peered":"2023-10-09T10:12:11.768339+0000","last_clean":"2023-10-09T10:12:11.768339+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:12:11.768339+0000","last_undegraded":"2023-10-09T10:12:11.768339+0000","last_fullsized":"2023-10-09T10:12:11.768339+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":195,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.206042+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:12:06.206042+0000","last_peered":"2023-10-09T10:12:06.206042+0000","last_clean":"2023-10-09T10:12:06.206042+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:12:06.206042+0000","last_undegraded":"2023-10-09T10:12:06.206042+0000","last_fullsized":"2023-10-09T10:12:06.206042+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":258,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.224327+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:12:06.224327+0000","last_peered":"2023-10-09T10:12:06.224327+0000","last_clean":"2023-10-09T10:12:06.224327+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:12:06.224327+0000","last_undegraded":"2023-10-09T10:12:06.224327+0000","last_fullsized":"2023-10-09T10:12:06.224327+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"306'11593","reported_seq":11644,"reported_epoch":306,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:22.973997+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:12:22.973997+0000","last_peered":"2023-10-09T10:12:22.973997+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:12:22.973997+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"278'1500","ondisk_log_start":"278'1500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10093,"log_dups_size":0,"ondisk_log_size":10093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135856128,"num_objects":2073,"num_object_clones":0,"num_object_copies":6219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11593,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-3300},{"shards":"4,5,6","objects":613}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":339,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.311976+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:12:14.311976+0000","last_peered":"2023-10-09T10:12:14.311976+0000","last_clean":"2023-10-09T10:12:14.311976+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:12:14.311976+0000","last_undegraded":"2023-10-09T10:12:14.311976+0000","last_fullsized":"2023-10-09T10:12:14.311976+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":124,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.221872+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:12:06.221872+0000","last_peered":"2023-10-09T10:12:06.221872+0000","last_clean":"2023-10-09T10:12:06.221872+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:12:06.221872+0000","last_undegraded":"2023-10-09T10:12:06.221872+0000","last_fullsized":"2023-10-09T10:12:06.221872+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"307'11737","reported_seq":11781,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:25.027693+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:12:25.027693+0000","last_peered":"2023-10-09T10:12:25.027693+0000","last_clean":"2023-10-09T10:12:25.027693+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:12:25.027693+0000","last_undegraded":"2023-10-09T10:12:25.027693+0000","last_fullsized":"2023-10-09T10:12:25.027693+0000","mapping_epoch":271,"log_start":"281'3100","ondisk_log_start":"281'3100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8637,"log_dups_size":0,"ondisk_log_size":8637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134545408,"num_objects":2053,"num_object_clones":0,"num_object_copies":6159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11737,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":327,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.311646+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:12:14.311646+0000","last_peered":"2023-10-09T10:12:14.311646+0000","last_clean":"2023-10-09T10:12:14.311646+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:12:14.311646+0000","last_undegraded":"2023-10-09T10:12:14.311646+0000","last_fullsized":"2023-10-09T10:12:14.311646+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":217,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210542+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:12:06.210542+0000","last_peered":"2023-10-09T10:12:06.210542+0000","last_clean":"2023-10-09T10:12:06.210542+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:12:06.210542+0000","last_undegraded":"2023-10-09T10:12:06.210542+0000","last_fullsized":"2023-10-09T10:12:06.210542+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"306'11416","reported_seq":11461,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:21.682910+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:12:21.682910+0000","last_peered":"2023-10-09T10:12:21.682910+0000","last_clean":"2023-10-09T10:12:21.682910+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:12:21.682910+0000","last_undegraded":"2023-10-09T10:12:21.682910+0000","last_fullsized":"2023-10-09T10:12:21.682910+0000","mapping_epoch":271,"log_start":"279'2000","ondisk_log_start":"279'2000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9416,"log_dups_size":0,"ondisk_log_size":9416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140902400,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11416,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":128,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222535+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:12:06.222535+0000","last_peered":"2023-10-09T10:12:06.222535+0000","last_clean":"2023-10-09T10:12:06.222535+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:12:06.222535+0000","last_undegraded":"2023-10-09T10:12:06.222535+0000","last_fullsized":"2023-10-09T10:12:06.222535+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":150,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.303726+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:12:14.303726+0000","last_peered":"2023-10-09T10:12:14.303726+0000","last_clean":"2023-10-09T10:12:14.303726+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:12:14.303726+0000","last_undegraded":"2023-10-09T10:12:14.303726+0000","last_fullsized":"2023-10-09T10:12:14.303726+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"306'11384","reported_seq":11429,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:23.480594+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:12:23.480594+0000","last_peered":"2023-10-09T10:12:23.480594+0000","last_clean":"2023-10-09T10:12:23.480594+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:12:23.480594+0000","last_undegraded":"2023-10-09T10:12:23.480594+0000","last_fullsized":"2023-10-09T10:12:23.480594+0000","mapping_epoch":271,"log_start":"283'3600","ondisk_log_start":"283'3600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7784,"log_dups_size":0,"ondisk_log_size":7784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132775936,"num_objects":2026,"num_object_clones":0,"num_object_copies":6078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11384,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":139,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222585+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:12:06.222585+0000","last_peered":"2023-10-09T10:12:06.222585+0000","last_clean":"2023-10-09T10:12:06.222585+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:12:06.222585+0000","last_undegraded":"2023-10-09T10:12:06.222585+0000","last_fullsized":"2023-10-09T10:12:06.222585+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":173,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207781+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:12:06.207781+0000","last_peered":"2023-10-09T10:12:06.207781+0000","last_clean":"2023-10-09T10:12:06.207781+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:12:06.207781+0000","last_undegraded":"2023-10-09T10:12:06.207781+0000","last_fullsized":"2023-10-09T10:12:06.207781+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"306'11657","reported_seq":11702,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:21.422069+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:12:21.422069+0000","last_peered":"2023-10-09T10:12:21.422069+0000","last_clean":"2023-10-09T10:12:21.422069+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:12:21.422069+0000","last_undegraded":"2023-10-09T10:12:21.422069+0000","last_fullsized":"2023-10-09T10:12:21.422069+0000","mapping_epoch":271,"log_start":"282'3500","ondisk_log_start":"282'3500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8157,"log_dups_size":0,"ondisk_log_size":8157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143065088,"num_objects":2183,"num_object_clones":0,"num_object_copies":6549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11657,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":300,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:11.509058+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:12:11.509058+0000","last_peered":"2023-10-09T10:12:11.509058+0000","last_clean":"2023-10-09T10:12:11.509058+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:12:11.509058+0000","last_undegraded":"2023-10-09T10:12:11.509058+0000","last_fullsized":"2023-10-09T10:12:11.509058+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:11.509079+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:12:11.509079+0000","last_peered":"2023-10-09T10:12:11.509079+0000","last_clean":"2023-10-09T10:12:11.509079+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:12:11.509079+0000","last_undegraded":"2023-10-09T10:12:11.509079+0000","last_fullsized":"2023-10-09T10:12:11.509079+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"306'11706","reported_seq":11751,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:23.570732+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:12:23.570732+0000","last_peered":"2023-10-09T10:12:23.570732+0000","last_clean":"2023-10-09T10:12:23.570732+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:12:23.570732+0000","last_undegraded":"2023-10-09T10:12:23.570732+0000","last_fullsized":"2023-10-09T10:12:23.570732+0000","mapping_epoch":271,"log_start":"283'4000","ondisk_log_start":"283'4000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7706,"log_dups_size":0,"ondisk_log_size":7706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130940928,"num_objects":1998,"num_object_clones":0,"num_object_copies":5994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11706,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":231,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210167+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:12:06.210167+0000","last_peered":"2023-10-09T10:12:06.210167+0000","last_clean":"2023-10-09T10:12:06.210167+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:12:06.210167+0000","last_undegraded":"2023-10-09T10:12:06.210167+0000","last_fullsized":"2023-10-09T10:12:06.210167+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":237,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209921+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:12:06.209921+0000","last_peered":"2023-10-09T10:12:06.209921+0000","last_clean":"2023-10-09T10:12:06.209921+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:12:06.209921+0000","last_undegraded":"2023-10-09T10:12:06.209921+0000","last_fullsized":"2023-10-09T10:12:06.209921+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"306'11278","reported_seq":11323,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:21.768161+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:12:21.768161+0000","last_peered":"2023-10-09T10:12:21.768161+0000","last_clean":"2023-10-09T10:12:21.768161+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:12:21.768161+0000","last_undegraded":"2023-10-09T10:12:21.768161+0000","last_fullsized":"2023-10-09T10:12:21.768161+0000","mapping_epoch":271,"log_start":"279'1900","ondisk_log_start":"279'1900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9378,"log_dups_size":0,"ondisk_log_size":9378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141164544,"num_objects":2154,"num_object_clones":0,"num_object_copies":6462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11278,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":201,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.311101+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:12:14.311101+0000","last_peered":"2023-10-09T10:12:14.311101+0000","last_clean":"2023-10-09T10:12:14.311101+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:12:14.311101+0000","last_undegraded":"2023-10-09T10:12:14.311101+0000","last_fullsized":"2023-10-09T10:12:14.311101+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":173,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209693+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:12:06.209693+0000","last_peered":"2023-10-09T10:12:06.209693+0000","last_clean":"2023-10-09T10:12:06.209693+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:12:06.209693+0000","last_undegraded":"2023-10-09T10:12:06.209693+0000","last_fullsized":"2023-10-09T10:12:06.209693+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"306'11603","reported_seq":11648,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:23.489514+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:12:23.489514+0000","last_peered":"2023-10-09T10:12:23.489514+0000","last_clean":"2023-10-09T10:12:23.489514+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:12:23.489514+0000","last_undegraded":"2023-10-09T10:12:23.489514+0000","last_fullsized":"2023-10-09T10:12:23.489514+0000","mapping_epoch":271,"log_start":"283'3900","ondisk_log_start":"283'3900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7703,"log_dups_size":0,"ondisk_log_size":7703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130744320,"num_objects":1995,"num_object_clones":0,"num_object_copies":5985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11603,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":260,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.228783+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:12:06.228783+0000","last_peered":"2023-10-09T10:12:06.228783+0000","last_clean":"2023-10-09T10:12:06.228783+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:12:06.228783+0000","last_undegraded":"2023-10-09T10:12:06.228783+0000","last_fullsized":"2023-10-09T10:12:06.228783+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":309,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.213503+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:12:06.213503+0000","last_peered":"2023-10-09T10:12:06.213503+0000","last_clean":"2023-10-09T10:12:06.213503+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:12:06.213503+0000","last_undegraded":"2023-10-09T10:12:06.213503+0000","last_fullsized":"2023-10-09T10:12:06.213503+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"306'11372","reported_seq":11417,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:21.721783+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:12:21.721783+0000","last_peered":"2023-10-09T10:12:21.721783+0000","last_clean":"2023-10-09T10:12:21.721783+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:12:21.721783+0000","last_undegraded":"2023-10-09T10:12:21.721783+0000","last_fullsized":"2023-10-09T10:12:21.721783+0000","mapping_epoch":271,"log_start":"279'1900","ondisk_log_start":"279'1900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9472,"log_dups_size":0,"ondisk_log_size":9472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137625600,"num_objects":2100,"num_object_clones":0,"num_object_copies":6300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11372,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":309,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209612+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:12:06.209612+0000","last_peered":"2023-10-09T10:12:06.209612+0000","last_clean":"2023-10-09T10:12:06.209612+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:12:06.209612+0000","last_undegraded":"2023-10-09T10:12:06.209612+0000","last_fullsized":"2023-10-09T10:12:06.209612+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":148,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222396+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:12:06.222396+0000","last_peered":"2023-10-09T10:12:06.222396+0000","last_clean":"2023-10-09T10:12:06.222396+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:12:06.222396+0000","last_undegraded":"2023-10-09T10:12:06.222396+0000","last_fullsized":"2023-10-09T10:12:06.222396+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":265,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210370+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:12:06.210370+0000","last_peered":"2023-10-09T10:12:06.210370+0000","last_clean":"2023-10-09T10:12:06.210370+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:12:06.210370+0000","last_undegraded":"2023-10-09T10:12:06.210370+0000","last_fullsized":"2023-10-09T10:12:06.210370+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"306'11511","reported_seq":11554,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:22.868404+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:12:22.868404+0000","last_peered":"2023-10-09T10:12:22.868404+0000","last_clean":"2023-10-09T10:12:22.868404+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:12:22.868404+0000","last_undegraded":"2023-10-09T10:12:22.868404+0000","last_fullsized":"2023-10-09T10:12:22.868404+0000","mapping_epoch":271,"log_start":"278'1500","ondisk_log_start":"278'1500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10011,"log_dups_size":0,"ondisk_log_size":10011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135200768,"num_objects":2063,"num_object_clones":0,"num_object_copies":6189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11511,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":124,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222122+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:12:06.222122+0000","last_peered":"2023-10-09T10:12:06.222122+0000","last_clean":"2023-10-09T10:12:06.222122+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:12:06.222122+0000","last_undegraded":"2023-10-09T10:12:06.222122+0000","last_fullsized":"2023-10-09T10:12:06.222122+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":122,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207562+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:12:06.207562+0000","last_peered":"2023-10-09T10:12:06.207562+0000","last_clean":"2023-10-09T10:12:06.207562+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:12:06.207562+0000","last_undegraded":"2023-10-09T10:12:06.207562+0000","last_fullsized":"2023-10-09T10:12:06.207562+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":332,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209538+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:12:06.209538+0000","last_peered":"2023-10-09T10:12:06.209538+0000","last_clean":"2023-10-09T10:12:06.209538+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:12:06.209538+0000","last_undegraded":"2023-10-09T10:12:06.209538+0000","last_fullsized":"2023-10-09T10:12:06.209538+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"306'11462","reported_seq":11507,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:21.515625+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:12:21.515625+0000","last_peered":"2023-10-09T10:12:21.515625+0000","last_clean":"2023-10-09T10:12:21.515625+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:12:21.515625+0000","last_undegraded":"2023-10-09T10:12:21.515625+0000","last_fullsized":"2023-10-09T10:12:21.515625+0000","mapping_epoch":271,"log_start":"282'3300","ondisk_log_start":"282'3300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8162,"log_dups_size":0,"ondisk_log_size":8162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136314880,"num_objects":2080,"num_object_clones":0,"num_object_copies":6240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11462,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":226,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220297+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:12:06.220297+0000","last_peered":"2023-10-09T10:12:06.220297+0000","last_clean":"2023-10-09T10:12:06.220297+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:12:06.220297+0000","last_undegraded":"2023-10-09T10:12:06.220297+0000","last_fullsized":"2023-10-09T10:12:06.220297+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":301,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209482+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:12:06.209482+0000","last_peered":"2023-10-09T10:12:06.209482+0000","last_clean":"2023-10-09T10:12:06.209482+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:12:06.209482+0000","last_undegraded":"2023-10-09T10:12:06.209482+0000","last_fullsized":"2023-10-09T10:12:06.209482+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":322,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.213431+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:12:06.213431+0000","last_peered":"2023-10-09T10:12:06.213431+0000","last_clean":"2023-10-09T10:12:06.213431+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:12:06.213431+0000","last_undegraded":"2023-10-09T10:12:06.213431+0000","last_fullsized":"2023-10-09T10:12:06.213431+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"306'11379","reported_seq":11422,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:22.922568+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:12:22.922568+0000","last_peered":"2023-10-09T10:12:22.922568+0000","last_clean":"2023-10-09T10:12:22.922568+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:12:22.922568+0000","last_undegraded":"2023-10-09T10:12:22.922568+0000","last_fullsized":"2023-10-09T10:12:22.922568+0000","mapping_epoch":271,"log_start":"277'1300","ondisk_log_start":"277'1300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10079,"log_dups_size":0,"ondisk_log_size":10079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136380416,"num_objects":2081,"num_object_clones":0,"num_object_copies":6243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11379,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":133,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222727+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:12:06.222727+0000","last_peered":"2023-10-09T10:12:06.222727+0000","last_clean":"2023-10-09T10:12:06.222727+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:12:06.222727+0000","last_undegraded":"2023-10-09T10:12:06.222727+0000","last_fullsized":"2023-10-09T10:12:06.222727+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":553,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.311302+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:12:14.311302+0000","last_peered":"2023-10-09T10:12:14.311302+0000","last_clean":"2023-10-09T10:12:14.311302+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:12:14.311302+0000","last_undegraded":"2023-10-09T10:12:14.311302+0000","last_fullsized":"2023-10-09T10:12:14.311302+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":210,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220487+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:12:06.220487+0000","last_peered":"2023-10-09T10:12:06.220487+0000","last_clean":"2023-10-09T10:12:06.220487+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:12:06.220487+0000","last_undegraded":"2023-10-09T10:12:06.220487+0000","last_fullsized":"2023-10-09T10:12:06.220487+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":222,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.314114+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:12:14.314114+0000","last_peered":"2023-10-09T10:12:14.314114+0000","last_clean":"2023-10-09T10:12:14.314114+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:12:14.314114+0000","last_undegraded":"2023-10-09T10:12:14.314114+0000","last_fullsized":"2023-10-09T10:12:14.314114+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"306'11583","reported_seq":11628,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:23.611949+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:12:23.611949+0000","last_peered":"2023-10-09T10:12:23.611949+0000","last_clean":"2023-10-09T10:12:23.611949+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:12:23.611949+0000","last_undegraded":"2023-10-09T10:12:23.611949+0000","last_fullsized":"2023-10-09T10:12:23.611949+0000","mapping_epoch":271,"log_start":"283'3800","ondisk_log_start":"283'3800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7783,"log_dups_size":0,"ondisk_log_size":7783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133627904,"num_objects":2039,"num_object_clones":0,"num_object_copies":6117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11583,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":281,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209609+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:12:06.209609+0000","last_peered":"2023-10-09T10:12:06.209609+0000","last_clean":"2023-10-09T10:12:06.209609+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:12:06.209609+0000","last_undegraded":"2023-10-09T10:12:06.209609+0000","last_fullsized":"2023-10-09T10:12:06.209609+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":183,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.210713+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:12:06.210713+0000","last_peered":"2023-10-09T10:12:06.210713+0000","last_clean":"2023-10-09T10:12:06.210713+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:12:06.210713+0000","last_undegraded":"2023-10-09T10:12:06.210713+0000","last_fullsized":"2023-10-09T10:12:06.210713+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":211,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.211281+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:12:06.211281+0000","last_peered":"2023-10-09T10:12:06.211281+0000","last_clean":"2023-10-09T10:12:06.211281+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:12:06.211281+0000","last_undegraded":"2023-10-09T10:12:06.211281+0000","last_fullsized":"2023-10-09T10:12:06.211281+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"306'11451","reported_seq":19784,"reported_epoch":306,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:21.510731+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:12:21.510731+0000","last_peered":"2023-10-09T10:12:21.510731+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:12:21.510731+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"282'3300","ondisk_log_start":"282'3300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8151,"log_dups_size":0,"ondisk_log_size":8151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144637952,"num_objects":2207,"num_object_clones":0,"num_object_copies":6621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":555,"num_objects_misplaced":589,"num_objects_unfound":0,"num_objects_dirty":2207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11451,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4140,"num_bytes_recovered":271319040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1063},{"shards":"1,3,4","objects":589}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":326,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.313925+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:12:14.313925+0000","last_peered":"2023-10-09T10:12:14.313925+0000","last_clean":"2023-10-09T10:12:14.313925+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:12:14.313925+0000","last_undegraded":"2023-10-09T10:12:14.313925+0000","last_fullsized":"2023-10-09T10:12:14.313925+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":338,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209691+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:12:06.209691+0000","last_peered":"2023-10-09T10:12:06.209691+0000","last_clean":"2023-10-09T10:12:06.209691+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:12:06.209691+0000","last_undegraded":"2023-10-09T10:12:06.209691+0000","last_fullsized":"2023-10-09T10:12:06.209691+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":175,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207643+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:12:06.207643+0000","last_peered":"2023-10-09T10:12:06.207643+0000","last_clean":"2023-10-09T10:12:06.207643+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:12:06.207643+0000","last_undegraded":"2023-10-09T10:12:06.207643+0000","last_fullsized":"2023-10-09T10:12:06.207643+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"306'11408","reported_seq":11453,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:21.766884+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:12:21.766884+0000","last_peered":"2023-10-09T10:12:21.766884+0000","last_clean":"2023-10-09T10:12:21.766884+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:12:21.766884+0000","last_undegraded":"2023-10-09T10:12:21.766884+0000","last_fullsized":"2023-10-09T10:12:21.766884+0000","mapping_epoch":271,"log_start":"279'2000","ondisk_log_start":"279'2000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9408,"log_dups_size":0,"ondisk_log_size":9408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140902400,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11408,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":269,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.224430+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:12:06.224430+0000","last_peered":"2023-10-09T10:12:06.224430+0000","last_clean":"2023-10-09T10:12:06.224430+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:12:06.224430+0000","last_undegraded":"2023-10-09T10:12:06.224430+0000","last_fullsized":"2023-10-09T10:12:06.224430+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":283,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209768+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:12:06.209768+0000","last_peered":"2023-10-09T10:12:06.209768+0000","last_clean":"2023-10-09T10:12:06.209768+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:12:06.209768+0000","last_undegraded":"2023-10-09T10:12:06.209768+0000","last_fullsized":"2023-10-09T10:12:06.209768+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":258,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.228697+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:12:06.228697+0000","last_peered":"2023-10-09T10:12:06.228697+0000","last_clean":"2023-10-09T10:12:06.228697+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:12:06.228697+0000","last_undegraded":"2023-10-09T10:12:06.228697+0000","last_fullsized":"2023-10-09T10:12:06.228697+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"306'11385","reported_seq":11429,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:22.945804+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:12:22.945804+0000","last_peered":"2023-10-09T10:12:22.945804+0000","last_clean":"2023-10-09T10:12:22.945804+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:12:22.945804+0000","last_undegraded":"2023-10-09T10:12:22.945804+0000","last_fullsized":"2023-10-09T10:12:22.945804+0000","mapping_epoch":271,"log_start":"277'1300","ondisk_log_start":"277'1300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10085,"log_dups_size":0,"ondisk_log_size":10085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136577048,"num_objects":2085,"num_object_clones":0,"num_object_copies":6255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2085,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11385,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":215,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:12:14.311362+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:12:14.311362+0000","last_peered":"2023-10-09T10:12:14.311362+0000","last_clean":"2023-10-09T10:12:14.311362+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:12:14.311362+0000","last_undegraded":"2023-10-09T10:12:14.311362+0000","last_fullsized":"2023-10-09T10:12:14.311362+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":177,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.211212+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:12:06.211212+0000","last_peered":"2023-10-09T10:12:06.211212+0000","last_clean":"2023-10-09T10:12:06.211212+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:12:06.211212+0000","last_undegraded":"2023-10-09T10:12:06.211212+0000","last_fullsized":"2023-10-09T10:12:06.211212+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":159,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222077+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:12:06.222077+0000","last_peered":"2023-10-09T10:12:06.222077+0000","last_clean":"2023-10-09T10:12:06.222077+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:12:06.222077+0000","last_undegraded":"2023-10-09T10:12:06.222077+0000","last_fullsized":"2023-10-09T10:12:06.222077+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":171,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.207457+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:12:06.207457+0000","last_peered":"2023-10-09T10:12:06.207457+0000","last_clean":"2023-10-09T10:12:06.207457+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:12:06.207457+0000","last_undegraded":"2023-10-09T10:12:06.207457+0000","last_fullsized":"2023-10-09T10:12:06.207457+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":244,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209769+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:12:06.209769+0000","last_peered":"2023-10-09T10:12:06.209769+0000","last_clean":"2023-10-09T10:12:06.209769+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:12:06.209769+0000","last_undegraded":"2023-10-09T10:12:06.209769+0000","last_fullsized":"2023-10-09T10:12:06.209769+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":143,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222493+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:12:06.222493+0000","last_peered":"2023-10-09T10:12:06.222493+0000","last_clean":"2023-10-09T10:12:06.222493+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:12:06.222493+0000","last_undegraded":"2023-10-09T10:12:06.222493+0000","last_fullsized":"2023-10-09T10:12:06.222493+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":184,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220927+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:12:06.220927+0000","last_peered":"2023-10-09T10:12:06.220927+0000","last_clean":"2023-10-09T10:12:06.220927+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:12:06.220927+0000","last_undegraded":"2023-10-09T10:12:06.220927+0000","last_fullsized":"2023-10-09T10:12:06.220927+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":143,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.222309+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:12:06.222309+0000","last_peered":"2023-10-09T10:12:06.222309+0000","last_clean":"2023-10-09T10:12:06.222309+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:12:06.222309+0000","last_undegraded":"2023-10-09T10:12:06.222309+0000","last_fullsized":"2023-10-09T10:12:06.222309+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":193,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220405+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:12:06.220405+0000","last_peered":"2023-10-09T10:12:06.220405+0000","last_clean":"2023-10-09T10:12:06.220405+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:12:06.220405+0000","last_undegraded":"2023-10-09T10:12:06.220405+0000","last_fullsized":"2023-10-09T10:12:06.220405+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":323,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220977+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:12:06.220977+0000","last_peered":"2023-10-09T10:12:06.220977+0000","last_clean":"2023-10-09T10:12:06.220977+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:12:06.220977+0000","last_undegraded":"2023-10-09T10:12:06.220977+0000","last_fullsized":"2023-10-09T10:12:06.220977+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":343,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.220558+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:12:06.220558+0000","last_peered":"2023-10-09T10:12:06.220558+0000","last_clean":"2023-10-09T10:12:06.220558+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:12:06.220558+0000","last_undegraded":"2023-10-09T10:12:06.220558+0000","last_fullsized":"2023-10-09T10:12:06.220558+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":253,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.206019+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:12:06.206019+0000","last_peered":"2023-10-09T10:12:06.206019+0000","last_clean":"2023-10-09T10:12:06.206019+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:12:06.206019+0000","last_undegraded":"2023-10-09T10:12:06.206019+0000","last_fullsized":"2023-10-09T10:12:06.206019+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T10:12:06.209118+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:12:06.209118+0000","last_peered":"2023-10-09T10:12:06.209118+0000","last_clean":"2023-10-09T10:12:06.209118+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:12:06.209118+0000","last_undegraded":"2023-10-09T10:12:06.209118+0000","last_fullsized":"2023-10-09T10:12:06.209118+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2191261720,"num_objects":33437,"num_object_clones":0,"num_object_copies":100311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5928,"num_objects_misplaced":589,"num_objects_unfound":0,"num_objects_dirty":33437,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183925,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4140,"num_bytes_recovered":271319040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7125151744,"data_stored":7125139528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142025,"ondisk_log_size":142025,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430513,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":961352,"kb_used_data":651712,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92783800,"statfs":{"total":95995035648,"available":95010611200,"internally_reserved":0,"allocated":667353088,"data_stored":666813107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:43 2023","interfaces":[{"interface":"back","average":{"1min":3.508,"5min":1.986,"15min":1.911},"min":{"1min":0.90100000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":9.9870000000000001,"5min":9.9870000000000001,"15min":50.591999999999999},"last":5.1689999999999996},{"interface":"front","average":{"1min":3.661,"5min":2.133,"15min":1.905},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.974,"5min":15.974,"15min":50.747},"last":1.913}]},{"osd":1,"last update":"Mon Oct 9 10:12:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9940000000000002,"5min":2.3639999999999999,"15min":1.958},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":11.92,"5min":14.311,"15min":25.338000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.6320000000000001,"5min":2.218,"15min":1.6870000000000001},"min":{"1min":0.42999999999999999,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":10.663,"5min":18.27,"15min":18.27},"last":1.855}]},{"osd":2,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":3.7639999999999998,"5min":1.9970000000000001,"15min":2.359},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":13.775,"5min":13.775,"15min":13.775},"last":7.1900000000000004},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.008,"15min":2.157},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.101000000000001,"5min":12.101000000000001,"15min":12.101000000000001},"last":5.4850000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4550000000000001,"5min":2.629,"15min":2.0430000000000001},"min":{"1min":0.436,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.6289999999999996,"5min":29.058,"15min":29.058},"last":5.5439999999999996},{"interface":"front","average":{"1min":3.6629999999999998,"5min":2.2559999999999998,"15min":2.0489999999999999},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.85499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.0469999999999999,"15min":0.95699999999999996},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.9820000000000002,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.97599999999999998,"15min":0.93100000000000005},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.6239999999999997,"5min":4.6239999999999997,"15min":4.6239999999999997},"last":0.36899999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3260000000000001,"5min":1.403,"15min":1.1259999999999999},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":7.8159999999999998,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.2110000000000001,"15min":1.0580000000000001},"min":{"1min":0.38500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.806,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.42899999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":1.0660000000000001,"15min":1.006},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.327,"5min":0.86899999999999999,"15min":0.91900000000000004},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":9.1920000000000002},"last":0.432}]}]},{"osd":6,"up_from":42,"seq":180388626741,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1302072,"kb_used_data":992560,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92443080,"statfs":{"total":95995035648,"available":94661713920,"internally_reserved":0,"allocated":1016381440,"data_stored":1015858773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":1.919,"15min":1.6060000000000001},"min":{"1min":0.252,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":6.9630000000000001,"5min":15.435,"15min":15.435},"last":7.3380000000000001},{"interface":"front","average":{"1min":2.2469999999999999,"5min":1.921,"15min":1.591},"min":{"1min":0.33900000000000002,"5min":0.252,"15min":0.23999999999999999},"max":{"1min":8.2959999999999994,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.94099999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:12:06 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":2.839,"15min":2.0579999999999998},"min":{"1min":0.30199999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.0519999999999996,"5min":35.609999999999999,"15min":35.609999999999999},"last":2.0409999999999999},{"interface":"front","average":{"1min":2.919,"5min":2.46,"15min":1.9910000000000001},"min":{"1min":0.36099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":8.2680000000000007,"5min":32.396000000000001,"15min":32.396000000000001},"last":5.8760000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":1.637,"15min":1.7929999999999999},"min":{"1min":0.35199999999999998,"5min":0.26900000000000002,"15min":0.26200000000000001},"max":{"1min":7.1280000000000001,"5min":12.898,"15min":12.898},"last":2.2549999999999999},{"interface":"front","average":{"1min":3.4860000000000002,"5min":2.1400000000000001,"15min":3.0379999999999998},"min":{"1min":0.44600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.542999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":0.76100000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":4.1079999999999997,"5min":2.3639999999999999,"15min":1.694},"min":{"1min":0.432,"5min":0.308,"15min":0.308},"max":{"1min":19.222000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":10.208},{"interface":"front","average":{"1min":3.2490000000000001,"5min":2.4500000000000002,"15min":2.4039999999999999},"min":{"1min":0.39500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":8.2270000000000003,"5min":11.211,"15min":11.211},"last":11.002000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.67100000000000004,"15min":0.51700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.8970000000000002,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.284,"5min":0.75800000000000001,"15min":0.58499999999999996},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.91200000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:11:37 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.81599999999999995},"min":{"1min":0.29399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.4580000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.042,"15min":0.872},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":6.3369999999999997,"5min":7.508,"15min":7.508},"last":0.85099999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.92600000000000005,"15min":0.89500000000000002},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.0700000000000003,"5min":6.2709999999999999,"15min":7.5350000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.673,"5min":0.97199999999999998,"15min":0.94299999999999995},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.101,"5min":6.3159999999999998,"15min":7.5570000000000004},"last":0.82199999999999995}]}]},{"osd":1,"up_from":84,"seq":360777253054,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1342248,"kb_used_data":926560,"kb_used_omap":9,"kb_used_meta":415670,"kb_avail":92402904,"statfs":{"total":95995035648,"available":94620573696,"internally_reserved":0,"allocated":948797440,"data_stored":947749068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":425647081},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":4.4160000000000004,"5min":2.5880000000000001,"15min":2.4500000000000002},"min":{"1min":0.217,"5min":0.183,"15min":0.183},"max":{"1min":31.007000000000001,"5min":31.007000000000001,"15min":31.007000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.8010000000000002,"5min":3.4079999999999999,"15min":2.5899999999999999},"min":{"1min":0.28599999999999998,"5min":0.245,"15min":0.192},"max":{"1min":10.577999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":1.9079999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":3.0249999999999999,"15min":3.8069999999999999},"min":{"1min":0.70099999999999996,"5min":0.217,"15min":0.217},"max":{"1min":10.573,"5min":17.369,"15min":17.369},"last":0.59699999999999998},{"interface":"front","average":{"1min":3.3479999999999999,"5min":2.5449999999999999,"15min":3.7530000000000001},"min":{"1min":0.26400000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":13.414999999999999,"5min":13.414999999999999,"15min":16.023},"last":0.31}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3170000000000002,"5min":3.6110000000000002,"15min":3.5590000000000002},"min":{"1min":0.41699999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.119,"5min":18.587,"15min":18.587},"last":21.808},{"interface":"front","average":{"1min":2.7959999999999998,"5min":3.8029999999999999,"15min":4.5350000000000001},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.103999999999999,"5min":17.289000000000001,"15min":17.289000000000001},"last":20.437000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.536,"15min":1.3480000000000001},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.976000000000001,"5min":11.976000000000001,"15min":11.976000000000001},"last":4.0460000000000003},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.177,"15min":1.6120000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.904,"5min":12.904,"15min":12.904},"last":3.5329999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.4430000000000001,"15min":2.0630000000000002},"min":{"1min":0.34100000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.8599999999999994,"5min":12.811,"15min":15.531000000000001},"last":3.758},{"interface":"front","average":{"1min":3.544,"5min":2.2949999999999999,"15min":2.2999999999999998},"min":{"1min":0.53900000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.855,"5min":16.277999999999999,"15min":19.706},"last":5.25}]},{"osd":6,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":2.0139999999999998,"15min":1.9810000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":21.670999999999999,"5min":21.670999999999999,"15min":21.670999999999999},"last":3.2789999999999999},{"interface":"front","average":{"1min":2.5219999999999998,"5min":2.4220000000000002,"15min":2.1070000000000002},"min":{"1min":0.36899999999999999,"5min":0.35499999999999998,"15min":0.32000000000000001},"max":{"1min":9.9440000000000008,"5min":12.868,"15min":15.481999999999999},"last":3.2650000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":2.3860000000000001,"15min":2.3330000000000002},"min":{"1min":0.68200000000000005,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.723000000000001,"5min":16.331,"15min":16.331},"last":2.9180000000000001},{"interface":"front","average":{"1min":3.101,"5min":2.0830000000000002,"15min":2.0619999999999998},"min":{"1min":0.42399999999999999,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":21.593,"5min":21.593,"15min":21.593},"last":3.141}]}]},{"osd":0,"up_from":79,"seq":339302416580,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1209352,"kb_used_data":832704,"kb_used_omap":28,"kb_used_meta":376611,"kb_avail":92535800,"statfs":{"total":95995035648,"available":94756659200,"internally_reserved":0,"allocated":852688896,"data_stored":851669515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":385650512},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":2.6800000000000002,"15min":2.0209999999999999},"min":{"1min":0.26400000000000001,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":16.797000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":4.117},{"interface":"front","average":{"1min":7.1150000000000002,"5min":4.2210000000000001,"15min":3.04},"min":{"1min":0.253,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":31.925999999999998,"5min":31.925999999999998,"15min":31.925999999999998},"last":0.63600000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0469999999999997,"5min":2.181,"15min":2.274},"min":{"1min":0.28899999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.003,"5min":18.003,"15min":18.003},"last":5.7809999999999997},{"interface":"front","average":{"1min":2.9660000000000002,"5min":2.0710000000000002,"15min":3.0190000000000001},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.432,"5min":15.529,"15min":15.529},"last":4.085}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":4.6509999999999998,"15min":4.7930000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":23.151,"5min":25.734999999999999,"15min":25.734999999999999},"last":5.5209999999999999},{"interface":"front","average":{"1min":5.3250000000000002,"5min":4.0739999999999998,"15min":4.298},"min":{"1min":0.33300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":18.279,"5min":21.923999999999999,"15min":21.923999999999999},"last":5.2489999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3780000000000001,"5min":2.3079999999999998,"15min":1.6970000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.989000000000001,"5min":20.494,"15min":20.494},"last":0.58299999999999996},{"interface":"front","average":{"1min":3.6819999999999999,"5min":2.395,"15min":2.089},"min":{"1min":0.40200000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.907,"5min":15.907,"15min":15.907},"last":5.1900000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5739999999999998,"15min":2.1869999999999998},"min":{"1min":0.36099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.868,"5min":17.096,"15min":17.096},"last":5.5419999999999998},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.52,"15min":1.4299999999999999},"min":{"1min":0.371,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.371,"5min":10.371,"15min":12.478},"last":5.8170000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.734,"15min":1.5860000000000001},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":8.827,"5min":9.4719999999999995,"15min":11.699},"last":5.8719999999999999},{"interface":"front","average":{"1min":4.5800000000000001,"5min":2.9740000000000002,"15min":2.4660000000000002},"min":{"1min":0.52100000000000002,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":23.106999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2679999999999998,"5min":2.6699999999999999,"15min":2.3540000000000001},"min":{"1min":0.437,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":16.460000000000001,"5min":16.460000000000001,"15min":26.978000000000002},"last":5.2240000000000002},{"interface":"front","average":{"1min":2.677,"5min":1.7490000000000001,"15min":1.738},"min":{"1min":0.45100000000000001,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":8.8040000000000003,"5min":10.122,"15min":26.768000000000001},"last":5.8440000000000003}]}]},{"osd":2,"up_from":217,"seq":932007903314,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":966288,"kb_used_data":717448,"kb_used_omap":83,"kb_used_meta":248748,"kb_avail":92778864,"statfs":{"total":95995035648,"available":95005556736,"internally_reserved":0,"allocated":734666752,"data_stored":733640523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":254718447},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.0609999999999999,"15min":1.8660000000000001},"min":{"1min":0.24299999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.334,"5min":19.372,"15min":19.372},"last":17.946000000000002},{"interface":"front","average":{"1min":5.5940000000000003,"5min":3.5150000000000001,"15min":3.133},"min":{"1min":0.32000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.047000000000001,"5min":16.754999999999999,"15min":16.754999999999999},"last":2.0800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":4.6630000000000003,"5min":2.653,"15min":2.6760000000000002},"min":{"1min":0.26500000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.276,"5min":14.276,"15min":14.276},"last":17.818000000000001},{"interface":"front","average":{"1min":5.4710000000000001,"5min":3.0760000000000001,"15min":2.823},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.227,"5min":15.569000000000001,"15min":15.569000000000001},"last":0.60199999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.2989999999999999,"15min":2.1150000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.359999999999999,"5min":13.851000000000001,"15min":13.851000000000001},"last":17.914999999999999},{"interface":"front","average":{"1min":4.1879999999999997,"5min":3.2360000000000002,"15min":3.6880000000000002},"min":{"1min":0.40799999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.51,"5min":15.811,"15min":15.811},"last":2.4849999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":1.3,"15min":0.90400000000000003},"min":{"1min":0.29399999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.413,"5min":12.413,"15min":12.413},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.597,"5min":1.329,"15min":0.996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.314},"max":{"1min":7.0810000000000004,"5min":12.385,"15min":12.385},"last":4.3849999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.8120000000000001,"15min":1.5860000000000001},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1230000000000002,"5min":12.473000000000001,"15min":12.473000000000001},"last":4.4260000000000002},{"interface":"front","average":{"1min":2.044,"5min":1.5980000000000001,"15min":1.706},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.923,"5min":13.587,"15min":13.587},"last":17.882999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5529999999999999,"5min":1.9219999999999999,"15min":1.2689999999999999},"min":{"1min":0.374,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.484,"5min":15.292999999999999,"15min":15.292999999999999},"last":3.0459999999999998},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.9670000000000001,"15min":1.8180000000000001},"min":{"1min":0.41099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.5679999999999996,"5min":12.502000000000001,"15min":12.502000000000001},"last":4.4489999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.9359999999999999,"15min":1.294},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.457000000000001,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.1539999999999999,"15min":1.758},"min":{"1min":0.38600000000000001,"5min":0.376,"15min":0.376},"max":{"1min":9.6039999999999992,"5min":12.433999999999999,"15min":12.433999999999999},"last":4.4770000000000003}]}]},{"osd":3,"up_from":250,"seq":1073741824057,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1103384,"kb_used_data":835024,"kb_used_omap":10,"kb_used_meta":268277,"kb_avail":92641768,"statfs":{"total":95995035648,"available":94865170432,"internally_reserved":0,"allocated":855064576,"data_stored":853854898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":274715715},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5139999999999998,"5min":4.0330000000000004,"15min":4.4800000000000004},"min":{"1min":0.27500000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.749000000000001,"5min":23.381,"15min":23.381},"last":2.4569999999999999},{"interface":"front","average":{"1min":6.5919999999999996,"5min":4.5750000000000002,"15min":4.3109999999999999},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.782,"5min":20.239999999999998,"15min":20.239999999999998},"last":18.141999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2909999999999999,"5min":3.4820000000000002,"15min":3.7959999999999998},"min":{"1min":0.41399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.856999999999999,"5min":20.422999999999998,"15min":20.422999999999998},"last":1.4390000000000001},{"interface":"front","average":{"1min":4.5620000000000003,"5min":3.726,"15min":3.5289999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.73,"5min":23.423999999999999,"15min":23.423999999999999},"last":6.0369999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":3.1269999999999998,"15min":3.536},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.331,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.377},{"interface":"front","average":{"1min":4.298,"5min":3.1680000000000001,"15min":2.9660000000000002},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":14.802,"5min":23.452000000000002,"15min":23.452000000000002},"last":5.7510000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":2.048,"15min":1.413},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.696999999999999,"5min":15.676,"15min":15.676},"last":2.3900000000000001},{"interface":"front","average":{"1min":4.7910000000000004,"5min":2.2360000000000002,"15min":1.752},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.478999999999999,"5min":16.478999999999999,"15min":16.478999999999999},"last":6.8479999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":2.9260000000000002,"15min":2.718},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.125999999999999,"5min":13.125999999999999,"15min":13.125999999999999},"last":8.4109999999999996},{"interface":"front","average":{"1min":3.0699999999999998,"5min":2.6600000000000001,"15min":2.456},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":10.891999999999999,"5min":15.992000000000001,"15min":15.992000000000001},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9649999999999999,"5min":2.9809999999999999,"15min":2.819},"min":{"1min":0.42099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.504999999999999,"5min":16.504999999999999,"15min":16.504999999999999},"last":8.3360000000000003},{"interface":"front","average":{"1min":1.921,"5min":2.4239999999999999,"15min":2.544},"min":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.952,"5min":16.16,"15min":16.16},"last":0.41099999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.1219999999999999,"15min":1.633},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":11.048999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":2.4209999999999998},{"interface":"front","average":{"1min":5.8399999999999999,"5min":3.073,"15min":2.8069999999999999},"min":{"1min":0.44600000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":16.367000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":8.3810000000000002}]}]},{"osd":4,"up_from":266,"seq":1142461300783,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1342004,"kb_used_data":1011052,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":92403148,"statfs":{"total":95995035648,"available":94620823552,"internally_reserved":0,"allocated":1035317248,"data_stored":1034789155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.0109999999999999,"15min":0.84899999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.4829999999999997,"5min":10.881,"15min":10.881},"last":1.536},{"interface":"front","average":{"1min":3.0150000000000001,"5min":1.5629999999999999,"15min":1.161},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":12.246,"5min":12.246,"15min":12.246},"last":0.61299999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2429999999999999,"5min":1.655,"15min":1.0960000000000001},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":12.161,"5min":12.161,"15min":12.161},"last":1.823},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.7290000000000001,"15min":1.4259999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.504,"5min":11.141,"15min":11.141},"last":1.7649999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.4430000000000001,"15min":1.143},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.2469999999999999,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":1.8140000000000001},{"interface":"front","average":{"1min":1.905,"5min":1.242,"15min":1.018},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":6.556}]},{"osd":3,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":1.9490000000000001,"15min":1.4910000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.065,"5min":13.065,"15min":13.065},"last":1.4830000000000001},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.766,"15min":1.264},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.218,"5min":12.218,"15min":12.218},"last":0.59299999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":0.93200000000000005,"15min":0.79600000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.6869999999999994,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":1.792},{"interface":"front","average":{"1min":0.436,"5min":0.56100000000000005,"15min":0.61899999999999999},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0609999999999999,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":2.3900000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.69799999999999995,"15min":0.58299999999999996},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.311},"max":{"1min":6.4119999999999999,"5min":6.4119999999999999,"15min":6.4119999999999999},"last":2.1659999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":0.83899999999999997,"15min":0.66900000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.255000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.56599999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":0.89100000000000001,"15min":0.71099999999999997},"min":{"1min":0.38800000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":12.237,"5min":12.237,"15min":12.237},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.244,"5min":0.90600000000000003,"15min":0.85399999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.7590000000000003,"5min":9.7590000000000003,"15min":9.7590000000000003},"last":2.004}]}]},{"osd":5,"up_from":36,"seq":154618822968,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1380904,"kb_used_data":1071392,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92364248,"statfs":{"total":95995035648,"available":94580989952,"internally_reserved":0,"allocated":1097105408,"data_stored":1096585371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4710000000000001,"15min":1.5720000000000001},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.8179999999999996,"5min":7.8540000000000001,"15min":15.773999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.7770000000000001,"5min":1.6879999999999999,"15min":1.6819999999999999},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.9950000000000001,"5min":10.244999999999999,"15min":15.699},"last":2.0110000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.1320000000000001,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.891,"5min":11.891,"15min":15.689},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.9350000000000001,"15min":1.823},"min":{"1min":0.41399999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.4489999999999998,"5min":19.402999999999999,"15min":19.402999999999999},"last":3.8879999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.7130000000000001,"5min":1.905,"15min":2.456},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.259,"5min":10.259,"15min":15.640000000000001},"last":4.0289999999999999},{"interface":"front","average":{"1min":1.581,"5min":1.458,"15min":2.1739999999999999},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":4.1619999999999999,"5min":9.1329999999999991,"15min":10.656000000000001},"last":2.2879999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":2.1829999999999998,"15min":1.96},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2260000000000009,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":7.8460000000000001},{"interface":"front","average":{"1min":3.2370000000000001,"5min":2.4820000000000002,"15min":2.8399999999999999},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.9900000000000002,"5min":17.34,"15min":17.34},"last":8.8849999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.84899999999999998,"15min":0.65900000000000003},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3460000000000001,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":3.0219999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.70499999999999996,"15min":0.73099999999999998},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.462,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":1.956}]},{"osd":6,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":0.97599999999999998,"15min":0.996},"min":{"1min":0.26200000000000001,"5min":0.25,"15min":0.25},"max":{"1min":8.5259999999999998,"5min":13.483000000000001,"15min":27.716000000000001},"last":3.0569999999999999},{"interface":"front","average":{"1min":1.198,"5min":1.075,"15min":0.92600000000000005},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.9889999999999999,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.374}]},{"osd":7,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.137,"15min":1.0389999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4710000000000001,"5min":7.891,"15min":10.334},"last":2.0630000000000002},{"interface":"front","average":{"1min":1.173,"5min":0.80300000000000005,"15min":0.96899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":12.356},"last":1.9850000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":836042752,"data_stored":836042752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":937164800,"data_stored":937164800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":723324928,"data_stored":723320856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":843186176,"data_stored":843186176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1026621440,"data_stored":1026621440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1089081344,"data_stored":1089077272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1009057792,"data_stored":1009057792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":660672512,"data_stored":660668440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:12:28.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:12:28.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:28.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:28 smithi012 ceph-mon[147926]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5928/96402 objects degraded (6.149%); 359/96402 objects misplaced (0.372%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:28.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/96402 objects degraded (6.149%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:28.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:28 smithi012 ceph-mon[152263]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5928/96402 objects degraded (6.149%); 359/96402 objects misplaced (0.372%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:28.900 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/96402 objects degraded (6.149%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:28.912 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:28 smithi110 ceph-mon[136032]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5928/96402 objects degraded (6.149%); 359/96402 objects misplaced (0.372%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/96402 objects degraded (6.149%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:28.952 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:12:29.008 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:29.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:29.081 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:29.113 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:29.139 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:29.152 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:29.239 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:29.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:29.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:29.432 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:29.572 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:29.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:29.679 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:29.680 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:29.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:29.783 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:29.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:29.948 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:29.955 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:29.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:30.063 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:30.130 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:30.191 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:30.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:30.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:30.297 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:30.345 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:30.482 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:30.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:30.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:30.588 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:30.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:30.721 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:30.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:30.826 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:30.832 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:30 smithi012 ceph-mon[152263]: pgmap v850: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5928/91557 objects degraded (6.475%); 359/91557 objects misplaced (0.392%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:30.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:30 smithi012 ceph-mon[147926]: pgmap v850: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5928/91557 objects degraded (6.475%); 359/91557 objects misplaced (0.392%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:30.897 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:30 smithi110 ceph-mon[136032]: pgmap v850: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5928/91557 objects degraded (6.475%); 359/91557 objects misplaced (0.392%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:30.902 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:30.964 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:30.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:30.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:31.081 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:31.126 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:12:31.126 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:12:31.126 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:12:31.126 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:12:31.126 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:12:31.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:31.203 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:31.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:31.300 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:12:31.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:31.346 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:12:31.385 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:12:31.423 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:31.424 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:12:31.468 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:12:31.504 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:12:31.509 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:31.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:31.553 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:12:31.555 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:12:31.598 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:12:31.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:31.631 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:12:31.631 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:12:31.631 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:12:31.631 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:12:31.631 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:12:31.631 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:12:31.632 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:12:31.632 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:12:31.704 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:31.800 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:31.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:31.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:31.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:31 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1831480038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:31.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:31 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1831480038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:31.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:31 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1831480038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:31.948 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:32.005 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:32.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:32.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:32.157 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:32.219 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:32.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:32.294 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:32.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:32.418 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:32.518 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:32.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:32.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:32.736 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:32.771 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:12:32.797 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:32.798 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5928/90924 objects degraded (6.520%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1069,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":308,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":30308,"data_bytes":1985222584,"bytes_used":9191870464,"bytes_avail":758768414720,"bytes_total":767960285184,"degraded_objects":5928,"degraded_total":90924,"degraded_ratio":0.065197307641546784,"misplaced_objects":359,"misplaced_total":90924,"misplaced_ratio":0.0039483524701948878,"recovering_objects_per_sec":42,"recovering_bytes_per_sec":2802239,"recovering_keys_per_sec":0,"num_objects_recovered":428,"num_bytes_recovered":28049408,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":617},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":25,"modified":"2023-10-09T10:12:13.408834+0000","services":{"osd":{"daemons":{"summary":"","1":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T10:12:32.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:32 smithi012 ceph-mon[152263]: pgmap v851: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5928/90924 objects degraded (6.520%); 359/90924 objects misplaced (0.395%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:32.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2630429348' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:32.817 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:32 smithi012 ceph-mon[147926]: pgmap v851: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5928/90924 objects degraded (6.520%); 359/90924 objects misplaced (0.395%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:32.817 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2630429348' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:32.838 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:32.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:32.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:32 smithi110 ceph-mon[136032]: pgmap v851: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5928/90924 objects degraded (6.520%); 359/90924 objects misplaced (0.395%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:12:32.892 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2630429348' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:32.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:33.050 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:33.415 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:33.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:33.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:33.456 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:12:33.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:12:33.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:33.680 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:33.730 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:33 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:12:33.730 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2630429348' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:33.730 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:33 smithi110 ceph-mon[136032]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T10:12:33.730 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:33 smithi110 ceph-mon[136032]: failed to encode map e308 with expected crc 2023-10-09T10:12:33.730 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:33 smithi110 ceph-mon[136032]: failed to encode map e308 with expected crc 2023-10-09T10:12:33.730 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1640750290' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:33.730 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5928/90924 objects degraded (6.520%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:33.732 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:12:33.732 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:12:33.776 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:33.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:33.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:33.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:33.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:12:33.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2630429348' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:33.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[152263]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T10:12:33.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[152263]: failed to encode map e308 with expected crc 2023-10-09T10:12:33.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[152263]: failed to encode map e308 with expected crc 2023-10-09T10:12:33.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1640750290' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:33.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5928/90924 objects degraded (6.520%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:33.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:12:33.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2630429348' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:33.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[147926]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T10:12:33.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[147926]: failed to encode map e308 with expected crc 2023-10-09T10:12:33.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[147926]: failed to encode map e308 with expected crc 2023-10-09T10:12:33.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1640750290' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:33.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5928/90924 objects degraded (6.520%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:33.989 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:34.068 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:34.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:34.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:34.173 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:34.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:12:34.219 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:34.278 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:34.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:34.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:34.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:12:34.379 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:34.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:34.530 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:34.978 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:35.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:34 smithi012 ceph-mon[152263]: failed to encode map e308 with expected crc 2023-10-09T10:12:35.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:34 smithi012 ceph-mon[152263]: failed to encode map e308 with expected crc 2023-10-09T10:12:35.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:34 smithi012 ceph-mon[152263]: pgmap v853: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5928/84612 objects degraded (7.006%); 98/84612 objects misplaced (0.116%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:12:35.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:34 smithi012 ceph-mon[147926]: failed to encode map e308 with expected crc 2023-10-09T10:12:35.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:34 smithi012 ceph-mon[147926]: failed to encode map e308 with expected crc 2023-10-09T10:12:35.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:34 smithi012 ceph-mon[147926]: pgmap v853: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5928/84612 objects degraded (7.006%); 98/84612 objects misplaced (0.116%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:12:35.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:34 smithi110 ceph-mon[136032]: failed to encode map e308 with expected crc 2023-10-09T10:12:35.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:34 smithi110 ceph-mon[136032]: failed to encode map e308 with expected crc 2023-10-09T10:12:35.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:34 smithi110 ceph-mon[136032]: pgmap v853: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5928/84612 objects degraded (7.006%); 98/84612 objects misplaced (0.116%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:12:35.484 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:12:35.600 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:35.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:35.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:35.705 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:35.738 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:35.893 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:35.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:35.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:36.001 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:36.053 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:36.106 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:36.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:36.216 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:36.264 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:36.370 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:36.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:36.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:36.465 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:12:36.465 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:12:36.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:36.558 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:36.765 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:36.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:36 smithi012 ceph-mon[147926]: pgmap v854: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 5928/81765 objects degraded (7.250%); 98/81765 objects misplaced (0.120%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T10:12:36.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:36 smithi012 ceph-mon[152263]: pgmap v854: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 5928/81765 objects degraded (7.250%); 98/81765 objects misplaced (0.120%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T10:12:36.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:36.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:36.807 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:36 smithi110 ceph-mon[136032]: pgmap v854: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 5928/81765 objects degraded (7.250%); 98/81765 objects misplaced (0.120%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T10:12:36.808 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:12:36.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:36.959 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:37.188 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:37.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:37.196 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:37.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:37.298 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:37.356 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:37.399 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:37.463 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:37.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:37.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:37.586 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:37.641 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:37.883 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:37.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:37.992 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:38.021 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:12:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:12:37] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-09T10:12:38.083 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:38.562 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:38.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:38.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:38.603 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:38.636 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:12:38.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:38.741 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:38.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:38 smithi012 ceph-mon[147926]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5742/77364 objects degraded (7.422%); 2.2 MiB/s, 54 objects/s recovering 2023-10-09T10:12:38.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5742/77364 objects degraded (7.422%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:38.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:38.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:38.906 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:38.917 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:12:38.960 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:12:38.960 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:38 smithi110 ceph-mon[136032]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5742/77364 objects degraded (7.422%); 2.2 MiB/s, 54 objects/s recovering 2023-10-09T10:12:38.960 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5742/77364 objects degraded (7.422%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:38.960 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:39.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:38 smithi012 ceph-mon[152263]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5742/77364 objects degraded (7.422%); 2.2 MiB/s, 54 objects/s recovering 2023-10-09T10:12:39.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5742/77364 objects degraded (7.422%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:39.254 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":855,"stamp":"2023-10-09T10:12:37.426986+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1688999864,"num_objects":25788,"num_object_clones":0,"num_object_copies":77364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5742,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25788,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":191858,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4984,"num_bytes_recovered":311175884,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142392,"ondisk_log_size":142392,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8295860,"kb_used_data":5742900,"kb_used_omap":167,"kb_used_meta":2552344,"kb_avail":741665356,"statfs":{"total":767960285184,"available":759465324544,"internally_reserved":0,"allocated":5880729600,"data_stored":5874289478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2613600857},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-415891456,"num_objects":-6346,"num_object_clones":0,"num_object_copies":-19038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-186,"num_objects_misplaced":-359,"num_objects_unfound":0,"num_objects_dirty":-6346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6346,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":545,"num_bytes_recovered":23592960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.012093"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":150,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573001+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:12:24.573001+0000","last_peered":"2023-10-09T10:12:24.573001+0000","last_clean":"2023-10-09T10:12:24.573001+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:12:24.573001+0000","last_undegraded":"2023-10-09T10:12:24.573001+0000","last_fullsized":"2023-10-09T10:12:24.573001+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":265,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.564834+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:12:24.564834+0000","last_peered":"2023-10-09T10:12:24.564834+0000","last_clean":"2023-10-09T10:12:24.564834+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:12:24.564834+0000","last_undegraded":"2023-10-09T10:12:24.564834+0000","last_fullsized":"2023-10-09T10:12:24.564834+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":311,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.572620+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:12:24.572620+0000","last_peered":"2023-10-09T10:12:24.572620+0000","last_clean":"2023-10-09T10:12:24.572620+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:12:24.572620+0000","last_undegraded":"2023-10-09T10:12:24.572620+0000","last_fullsized":"2023-10-09T10:12:24.572620+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":175,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.762198+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:12:32.762198+0000","last_peered":"2023-10-09T10:12:32.762198+0000","last_clean":"2023-10-09T10:12:32.762198+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:12:32.762198+0000","last_undegraded":"2023-10-09T10:12:32.762198+0000","last_fullsized":"2023-10-09T10:12:32.762198+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":280,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:31.544356+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:12:31.544356+0000","last_peered":"2023-10-09T10:12:31.544356+0000","last_clean":"2023-10-09T10:12:31.544356+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:12:31.544356+0000","last_undegraded":"2023-10-09T10:12:31.544356+0000","last_fullsized":"2023-10-09T10:12:31.544356+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":126,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.590492+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:12:24.590492+0000","last_peered":"2023-10-09T10:12:24.590492+0000","last_clean":"2023-10-09T10:12:24.590492+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:12:24.590492+0000","last_undegraded":"2023-10-09T10:12:24.590492+0000","last_fullsized":"2023-10-09T10:12:24.590492+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":192,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.577454+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:12:24.577454+0000","last_peered":"2023-10-09T10:12:24.577454+0000","last_clean":"2023-10-09T10:12:24.577454+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:12:24.577454+0000","last_undegraded":"2023-10-09T10:12:24.577454+0000","last_fullsized":"2023-10-09T10:12:24.577454+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":261,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.564866+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:12:24.564866+0000","last_peered":"2023-10-09T10:12:24.564866+0000","last_clean":"2023-10-09T10:12:24.564866+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:12:24.564866+0000","last_undegraded":"2023-10-09T10:12:24.564866+0000","last_fullsized":"2023-10-09T10:12:24.564866+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":125,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.590346+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:12:24.590346+0000","last_peered":"2023-10-09T10:12:24.590346+0000","last_clean":"2023-10-09T10:12:24.590346+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:12:24.590346+0000","last_undegraded":"2023-10-09T10:12:24.590346+0000","last_fullsized":"2023-10-09T10:12:24.590346+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":222,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.559017+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:12:24.559017+0000","last_peered":"2023-10-09T10:12:24.559017+0000","last_clean":"2023-10-09T10:12:24.559017+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:12:24.559017+0000","last_undegraded":"2023-10-09T10:12:24.559017+0000","last_fullsized":"2023-10-09T10:12:24.559017+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":346,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.566879+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:12:24.566879+0000","last_peered":"2023-10-09T10:12:24.566879+0000","last_clean":"2023-10-09T10:12:24.566879+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:12:24.566879+0000","last_undegraded":"2023-10-09T10:12:24.566879+0000","last_fullsized":"2023-10-09T10:12:24.566879+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":303,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.572870+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:12:24.572870+0000","last_peered":"2023-10-09T10:12:24.572870+0000","last_clean":"2023-10-09T10:12:24.572870+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:12:24.572870+0000","last_undegraded":"2023-10-09T10:12:24.572870+0000","last_fullsized":"2023-10-09T10:12:24.572870+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":195,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.567173+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:12:24.567173+0000","last_peered":"2023-10-09T10:12:24.567173+0000","last_clean":"2023-10-09T10:12:24.567173+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:12:24.567173+0000","last_undegraded":"2023-10-09T10:12:24.567173+0000","last_fullsized":"2023-10-09T10:12:24.567173+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":175,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573693+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:12:24.573693+0000","last_peered":"2023-10-09T10:12:24.573693+0000","last_clean":"2023-10-09T10:12:24.573693+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:12:24.573693+0000","last_undegraded":"2023-10-09T10:12:24.573693+0000","last_fullsized":"2023-10-09T10:12:24.573693+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":379,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.566751+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:12:24.566751+0000","last_peered":"2023-10-09T10:12:24.566751+0000","last_clean":"2023-10-09T10:12:24.566751+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:12:24.566751+0000","last_undegraded":"2023-10-09T10:12:24.566751+0000","last_fullsized":"2023-10-09T10:12:24.566751+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":248,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.559688+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:12:24.559688+0000","last_peered":"2023-10-09T10:12:24.559688+0000","last_clean":"2023-10-09T10:12:24.559688+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:12:24.559688+0000","last_undegraded":"2023-10-09T10:12:24.559688+0000","last_fullsized":"2023-10-09T10:12:24.559688+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":174,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.577098+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:12:24.577098+0000","last_peered":"2023-10-09T10:12:24.577098+0000","last_clean":"2023-10-09T10:12:24.577098+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:12:24.577098+0000","last_undegraded":"2023-10-09T10:12:24.577098+0000","last_fullsized":"2023-10-09T10:12:24.577098+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":141,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.590377+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:12:24.590377+0000","last_peered":"2023-10-09T10:12:24.590377+0000","last_clean":"2023-10-09T10:12:24.590377+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:12:24.590377+0000","last_undegraded":"2023-10-09T10:12:24.590377+0000","last_fullsized":"2023-10-09T10:12:24.590377+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":176,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.566591+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:12:24.566591+0000","last_peered":"2023-10-09T10:12:24.566591+0000","last_clean":"2023-10-09T10:12:24.566591+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:12:24.566591+0000","last_undegraded":"2023-10-09T10:12:24.566591+0000","last_fullsized":"2023-10-09T10:12:24.566591+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.579962+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:12:24.579962+0000","last_peered":"2023-10-09T10:12:24.579962+0000","last_clean":"2023-10-09T10:12:24.579962+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:12:24.579962+0000","last_undegraded":"2023-10-09T10:12:24.579962+0000","last_fullsized":"2023-10-09T10:12:24.579962+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":197,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.577500+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:12:24.577500+0000","last_peered":"2023-10-09T10:12:24.577500+0000","last_clean":"2023-10-09T10:12:24.577500+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:12:24.577500+0000","last_undegraded":"2023-10-09T10:12:24.577500+0000","last_fullsized":"2023-10-09T10:12:24.577500+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":261,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:31.544516+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:12:31.544516+0000","last_peered":"2023-10-09T10:12:31.544516+0000","last_clean":"2023-10-09T10:12:31.544516+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:12:31.544516+0000","last_undegraded":"2023-10-09T10:12:31.544516+0000","last_fullsized":"2023-10-09T10:12:31.544516+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"308'11987","reported_seq":12040,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:32.951655+0000","last_change":"2023-10-09T10:09:48.420521+0000","last_active":"2023-10-09T10:12:32.951655+0000","last_peered":"2023-10-09T10:12:32.951655+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:12:32.951655+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"279'1900","ondisk_log_start":"279'1900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10087,"log_dups_size":0,"ondisk_log_size":10087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110034944,"num_objects":1679,"num_object_clones":0,"num_object_copies":5037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11987,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-3694},{"shards":"4,5,6","objects":219}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":340,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.567707+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:12:24.567707+0000","last_peered":"2023-10-09T10:12:24.567707+0000","last_clean":"2023-10-09T10:12:24.567707+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:12:24.567707+0000","last_undegraded":"2023-10-09T10:12:24.567707+0000","last_fullsized":"2023-10-09T10:12:24.567707+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":126,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.589841+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:12:24.589841+0000","last_peered":"2023-10-09T10:12:24.589841+0000","last_clean":"2023-10-09T10:12:24.589841+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:12:24.589841+0000","last_undegraded":"2023-10-09T10:12:24.589841+0000","last_fullsized":"2023-10-09T10:12:24.589841+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"308'12139","reported_seq":12184,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:35.051403+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:12:35.051403+0000","last_peered":"2023-10-09T10:12:35.051403+0000","last_clean":"2023-10-09T10:12:35.051403+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:12:35.051403+0000","last_undegraded":"2023-10-09T10:12:35.051403+0000","last_fullsized":"2023-10-09T10:12:35.051403+0000","mapping_epoch":271,"log_start":"282'3500","ondisk_log_start":"282'3500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8639,"log_dups_size":0,"ondisk_log_size":8639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108199936,"num_objects":1651,"num_object_clones":0,"num_object_copies":4953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12139,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":328,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.566777+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:12:24.566777+0000","last_peered":"2023-10-09T10:12:24.566777+0000","last_clean":"2023-10-09T10:12:24.566777+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:12:24.566777+0000","last_undegraded":"2023-10-09T10:12:24.566777+0000","last_fullsized":"2023-10-09T10:12:24.566777+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":219,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.572531+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:12:24.572531+0000","last_peered":"2023-10-09T10:12:24.572531+0000","last_clean":"2023-10-09T10:12:24.572531+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:12:24.572531+0000","last_undegraded":"2023-10-09T10:12:24.572531+0000","last_fullsized":"2023-10-09T10:12:24.572531+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"308'12011","reported_seq":12058,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:36.773748+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:12:36.773748+0000","last_peered":"2023-10-09T10:12:36.773748+0000","last_clean":"2023-10-09T10:12:36.773748+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:12:36.773748+0000","last_undegraded":"2023-10-09T10:12:36.773748+0000","last_fullsized":"2023-10-09T10:12:36.773748+0000","mapping_epoch":271,"log_start":"280'2600","ondisk_log_start":"280'2600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9411,"log_dups_size":0,"ondisk_log_size":9411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101908480,"num_objects":1555,"num_object_clones":0,"num_object_copies":4665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12011,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":130,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.590392+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:12:24.590392+0000","last_peered":"2023-10-09T10:12:24.590392+0000","last_clean":"2023-10-09T10:12:24.590392+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:12:24.590392+0000","last_undegraded":"2023-10-09T10:12:24.590392+0000","last_fullsized":"2023-10-09T10:12:24.590392+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":151,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.568866+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:12:24.568866+0000","last_peered":"2023-10-09T10:12:24.568866+0000","last_clean":"2023-10-09T10:12:24.568866+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:12:24.568866+0000","last_undegraded":"2023-10-09T10:12:24.568866+0000","last_fullsized":"2023-10-09T10:12:24.568866+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"308'11787","reported_seq":11834,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:33.614904+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:12:33.614904+0000","last_peered":"2023-10-09T10:12:33.614904+0000","last_clean":"2023-10-09T10:12:33.614904+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:12:33.614904+0000","last_undegraded":"2023-10-09T10:12:33.614904+0000","last_fullsized":"2023-10-09T10:12:33.614904+0000","mapping_epoch":271,"log_start":"283'4000","ondisk_log_start":"283'4000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7787,"log_dups_size":0,"ondisk_log_size":7787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106364928,"num_objects":1623,"num_object_clones":0,"num_object_copies":4869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11787,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":141,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.594305+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:12:24.594305+0000","last_peered":"2023-10-09T10:12:24.594305+0000","last_clean":"2023-10-09T10:12:24.594305+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:12:24.594305+0000","last_undegraded":"2023-10-09T10:12:24.594305+0000","last_fullsized":"2023-10-09T10:12:24.594305+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":175,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.577122+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:12:24.577122+0000","last_peered":"2023-10-09T10:12:24.577122+0000","last_clean":"2023-10-09T10:12:24.577122+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:12:24.577122+0000","last_undegraded":"2023-10-09T10:12:24.577122+0000","last_fullsized":"2023-10-09T10:12:24.577122+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"308'12256","reported_seq":12303,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:36.488202+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:12:36.488202+0000","last_peered":"2023-10-09T10:12:36.488202+0000","last_clean":"2023-10-09T10:12:36.488202+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:12:36.488202+0000","last_undegraded":"2023-10-09T10:12:36.488202+0000","last_fullsized":"2023-10-09T10:12:36.488202+0000","mapping_epoch":271,"log_start":"283'4100","ondisk_log_start":"283'4100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103809024,"num_objects":1584,"num_object_clones":0,"num_object_copies":4752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12256,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":302,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.565377+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:12:24.565377+0000","last_peered":"2023-10-09T10:12:24.565377+0000","last_clean":"2023-10-09T10:12:24.565377+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:12:24.565377+0000","last_undegraded":"2023-10-09T10:12:24.565377+0000","last_fullsized":"2023-10-09T10:12:24.565377+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.559579+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:12:24.559579+0000","last_peered":"2023-10-09T10:12:24.559579+0000","last_clean":"2023-10-09T10:12:24.559579+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:12:24.559579+0000","last_undegraded":"2023-10-09T10:12:24.559579+0000","last_fullsized":"2023-10-09T10:12:24.559579+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"308'12095","reported_seq":12142,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:33.615250+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:12:33.615250+0000","last_peered":"2023-10-09T10:12:33.615250+0000","last_clean":"2023-10-09T10:12:33.615250+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:12:33.615250+0000","last_undegraded":"2023-10-09T10:12:33.615250+0000","last_fullsized":"2023-10-09T10:12:33.615250+0000","mapping_epoch":271,"log_start":"283'4400","ondisk_log_start":"283'4400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7695,"log_dups_size":0,"ondisk_log_size":7695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105447424,"num_objects":1609,"num_object_clones":0,"num_object_copies":4827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12095,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":233,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.572803+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:12:24.572803+0000","last_peered":"2023-10-09T10:12:24.572803+0000","last_clean":"2023-10-09T10:12:24.572803+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:12:24.572803+0000","last_undegraded":"2023-10-09T10:12:24.572803+0000","last_fullsized":"2023-10-09T10:12:24.572803+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.575104+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:12:24.575104+0000","last_peered":"2023-10-09T10:12:24.575104+0000","last_clean":"2023-10-09T10:12:24.575104+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:12:24.575104+0000","last_undegraded":"2023-10-09T10:12:24.575104+0000","last_fullsized":"2023-10-09T10:12:24.575104+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"308'11851","reported_seq":11898,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:36.773136+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:12:36.773136+0000","last_peered":"2023-10-09T10:12:36.773136+0000","last_clean":"2023-10-09T10:12:36.773136+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:12:36.773136+0000","last_undegraded":"2023-10-09T10:12:36.773136+0000","last_fullsized":"2023-10-09T10:12:36.773136+0000","mapping_epoch":271,"log_start":"280'2400","ondisk_log_start":"280'2400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9451,"log_dups_size":0,"ondisk_log_size":9451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103612416,"num_objects":1581,"num_object_clones":0,"num_object_copies":4743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11851,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":202,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.566499+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:12:24.566499+0000","last_peered":"2023-10-09T10:12:24.566499+0000","last_clean":"2023-10-09T10:12:24.566499+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:12:24.566499+0000","last_undegraded":"2023-10-09T10:12:24.566499+0000","last_fullsized":"2023-10-09T10:12:24.566499+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":175,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573412+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:12:24.573412+0000","last_peered":"2023-10-09T10:12:24.573412+0000","last_clean":"2023-10-09T10:12:24.573412+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:12:24.573412+0000","last_undegraded":"2023-10-09T10:12:24.573412+0000","last_fullsized":"2023-10-09T10:12:24.573412+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"308'11960","reported_seq":12007,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:33.637862+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:12:33.637862+0000","last_peered":"2023-10-09T10:12:33.637862+0000","last_clean":"2023-10-09T10:12:33.637862+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:12:33.637862+0000","last_undegraded":"2023-10-09T10:12:33.637862+0000","last_fullsized":"2023-10-09T10:12:33.637862+0000","mapping_epoch":271,"log_start":"283'4200","ondisk_log_start":"283'4200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7760,"log_dups_size":0,"ondisk_log_size":7760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107347968,"num_objects":1638,"num_object_clones":0,"num_object_copies":4914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11960,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":262,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.564773+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:12:24.564773+0000","last_peered":"2023-10-09T10:12:24.564773+0000","last_clean":"2023-10-09T10:12:24.564773+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:12:24.564773+0000","last_undegraded":"2023-10-09T10:12:24.564773+0000","last_fullsized":"2023-10-09T10:12:24.564773+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":312,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.760108+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:12:32.760108+0000","last_peered":"2023-10-09T10:12:32.760108+0000","last_clean":"2023-10-09T10:12:32.760108+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:12:32.760108+0000","last_undegraded":"2023-10-09T10:12:32.760108+0000","last_fullsized":"2023-10-09T10:12:32.760108+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"308'11973","reported_seq":12019,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:36.820126+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:12:36.820126+0000","last_peered":"2023-10-09T10:12:36.820126+0000","last_clean":"2023-10-09T10:12:36.820126+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:12:36.820126+0000","last_undegraded":"2023-10-09T10:12:36.820126+0000","last_fullsized":"2023-10-09T10:12:36.820126+0000","mapping_epoch":271,"log_start":"280'2500","ondisk_log_start":"280'2500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9473,"log_dups_size":0,"ondisk_log_size":9473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11973,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":311,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573854+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:12:24.573854+0000","last_peered":"2023-10-09T10:12:24.573854+0000","last_clean":"2023-10-09T10:12:24.573854+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:12:24.573854+0000","last_undegraded":"2023-10-09T10:12:24.573854+0000","last_fullsized":"2023-10-09T10:12:24.573854+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":150,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.590247+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:12:24.590247+0000","last_peered":"2023-10-09T10:12:24.590247+0000","last_clean":"2023-10-09T10:12:24.590247+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:12:24.590247+0000","last_undegraded":"2023-10-09T10:12:24.590247+0000","last_fullsized":"2023-10-09T10:12:24.590247+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":267,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573072+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:12:24.573072+0000","last_peered":"2023-10-09T10:12:24.573072+0000","last_clean":"2023-10-09T10:12:24.573072+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:12:24.573072+0000","last_undegraded":"2023-10-09T10:12:24.573072+0000","last_fullsized":"2023-10-09T10:12:24.573072+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"308'11915","reported_seq":11960,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.980391+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:12:32.980391+0000","last_peered":"2023-10-09T10:12:32.980391+0000","last_clean":"2023-10-09T10:12:32.980391+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:12:32.980391+0000","last_undegraded":"2023-10-09T10:12:32.980391+0000","last_fullsized":"2023-10-09T10:12:32.980391+0000","mapping_epoch":271,"log_start":"279'1900","ondisk_log_start":"279'1900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10015,"log_dups_size":0,"ondisk_log_size":10015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108724224,"num_objects":1659,"num_object_clones":0,"num_object_copies":4977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11915,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":126,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.590525+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:12:24.590525+0000","last_peered":"2023-10-09T10:12:24.590525+0000","last_clean":"2023-10-09T10:12:24.590525+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:12:24.590525+0000","last_undegraded":"2023-10-09T10:12:24.590525+0000","last_fullsized":"2023-10-09T10:12:24.590525+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:31.822802+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:12:31.822802+0000","last_peered":"2023-10-09T10:12:31.822802+0000","last_clean":"2023-10-09T10:12:31.822802+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:12:31.822802+0000","last_undegraded":"2023-10-09T10:12:31.822802+0000","last_fullsized":"2023-10-09T10:12:31.822802+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":334,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573781+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:12:24.573781+0000","last_peered":"2023-10-09T10:12:24.573781+0000","last_clean":"2023-10-09T10:12:24.573781+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:12:24.573781+0000","last_undegraded":"2023-10-09T10:12:24.573781+0000","last_fullsized":"2023-10-09T10:12:24.573781+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"308'11993","reported_seq":12040,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:36.522139+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:12:36.522139+0000","last_peered":"2023-10-09T10:12:36.522139+0000","last_clean":"2023-10-09T10:12:36.522139+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:12:36.522139+0000","last_undegraded":"2023-10-09T10:12:36.522139+0000","last_fullsized":"2023-10-09T10:12:36.522139+0000","mapping_epoch":271,"log_start":"283'3800","ondisk_log_start":"283'3800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8193,"log_dups_size":0,"ondisk_log_size":8193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101515264,"num_objects":1549,"num_object_clones":0,"num_object_copies":4647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11993,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":228,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.558886+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:12:24.558886+0000","last_peered":"2023-10-09T10:12:24.558886+0000","last_clean":"2023-10-09T10:12:24.558886+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:12:24.558886+0000","last_undegraded":"2023-10-09T10:12:24.558886+0000","last_fullsized":"2023-10-09T10:12:24.558886+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":303,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.574591+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:12:24.574591+0000","last_peered":"2023-10-09T10:12:24.574591+0000","last_clean":"2023-10-09T10:12:24.574591+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:12:24.574591+0000","last_undegraded":"2023-10-09T10:12:24.574591+0000","last_fullsized":"2023-10-09T10:12:24.574591+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":324,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573873+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:12:24.573873+0000","last_peered":"2023-10-09T10:12:24.573873+0000","last_clean":"2023-10-09T10:12:24.573873+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:12:24.573873+0000","last_undegraded":"2023-10-09T10:12:24.573873+0000","last_fullsized":"2023-10-09T10:12:24.573873+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"308'11780","reported_seq":11824,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.914228+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:12:32.914228+0000","last_peered":"2023-10-09T10:12:32.914228+0000","last_clean":"2023-10-09T10:12:32.914228+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:12:32.914228+0000","last_undegraded":"2023-10-09T10:12:32.914228+0000","last_fullsized":"2023-10-09T10:12:32.914228+0000","mapping_epoch":271,"log_start":"279'1700","ondisk_log_start":"279'1700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110100480,"num_objects":1680,"num_object_clones":0,"num_object_copies":5040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11780,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":135,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.590128+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:12:24.590128+0000","last_peered":"2023-10-09T10:12:24.590128+0000","last_clean":"2023-10-09T10:12:24.590128+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:12:24.590128+0000","last_undegraded":"2023-10-09T10:12:24.590128+0000","last_fullsized":"2023-10-09T10:12:24.590128+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":554,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.566505+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:12:24.566505+0000","last_peered":"2023-10-09T10:12:24.566505+0000","last_clean":"2023-10-09T10:12:24.566505+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:12:24.566505+0000","last_undegraded":"2023-10-09T10:12:24.566505+0000","last_fullsized":"2023-10-09T10:12:24.566505+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":212,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.559332+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:12:24.559332+0000","last_peered":"2023-10-09T10:12:24.559332+0000","last_clean":"2023-10-09T10:12:24.559332+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:12:24.559332+0000","last_undegraded":"2023-10-09T10:12:24.559332+0000","last_fullsized":"2023-10-09T10:12:24.559332+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":223,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.566923+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:12:24.566923+0000","last_peered":"2023-10-09T10:12:24.566923+0000","last_clean":"2023-10-09T10:12:24.566923+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:12:24.566923+0000","last_undegraded":"2023-10-09T10:12:24.566923+0000","last_fullsized":"2023-10-09T10:12:24.566923+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"308'11990","reported_seq":12036,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:33.587226+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:12:33.587226+0000","last_peered":"2023-10-09T10:12:33.587226+0000","last_clean":"2023-10-09T10:12:33.587226+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:12:33.587226+0000","last_undegraded":"2023-10-09T10:12:33.587226+0000","last_fullsized":"2023-10-09T10:12:33.587226+0000","mapping_epoch":271,"log_start":"283'4200","ondisk_log_start":"283'4200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7790,"log_dups_size":0,"ondisk_log_size":7790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106954752,"num_objects":1632,"num_object_clones":0,"num_object_copies":4896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11990,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":284,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:28.645728+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:12:28.645728+0000","last_peered":"2023-10-09T10:12:28.645728+0000","last_clean":"2023-10-09T10:12:28.645728+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:12:28.645728+0000","last_undegraded":"2023-10-09T10:12:28.645728+0000","last_fullsized":"2023-10-09T10:12:28.645728+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":185,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.577965+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:12:24.577965+0000","last_peered":"2023-10-09T10:12:24.577965+0000","last_clean":"2023-10-09T10:12:24.577965+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:12:24.577965+0000","last_undegraded":"2023-10-09T10:12:24.577965+0000","last_fullsized":"2023-10-09T10:12:24.577965+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":213,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.579207+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:12:24.579207+0000","last_peered":"2023-10-09T10:12:24.579207+0000","last_clean":"2023-10-09T10:12:24.579207+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:12:24.579207+0000","last_undegraded":"2023-10-09T10:12:24.579207+0000","last_fullsized":"2023-10-09T10:12:24.579207+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"308'12057","reported_seq":21942,"reported_epoch":308,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:36.544263+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:12:36.544263+0000","last_peered":"2023-10-09T10:12:36.544263+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:12:36.544263+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"283'3900","ondisk_log_start":"283'3900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8157,"log_dups_size":0,"ondisk_log_size":8157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104923136,"num_objects":1601,"num_object_clones":0,"num_object_copies":4803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":369,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12057,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4915,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1232}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":327,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.566791+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:12:24.566791+0000","last_peered":"2023-10-09T10:12:24.566791+0000","last_clean":"2023-10-09T10:12:24.566791+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:12:24.566791+0000","last_undegraded":"2023-10-09T10:12:24.566791+0000","last_fullsized":"2023-10-09T10:12:24.566791+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":340,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.574090+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:12:24.574090+0000","last_peered":"2023-10-09T10:12:24.574090+0000","last_clean":"2023-10-09T10:12:24.574090+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:12:24.574090+0000","last_undegraded":"2023-10-09T10:12:24.574090+0000","last_fullsized":"2023-10-09T10:12:24.574090+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":177,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.576962+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:12:24.576962+0000","last_peered":"2023-10-09T10:12:24.576962+0000","last_clean":"2023-10-09T10:12:24.576962+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:12:24.576962+0000","last_undegraded":"2023-10-09T10:12:24.576962+0000","last_fullsized":"2023-10-09T10:12:24.576962+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"308'12019","reported_seq":12066,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:36.773331+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:12:36.773331+0000","last_peered":"2023-10-09T10:12:36.773331+0000","last_clean":"2023-10-09T10:12:36.773331+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:12:36.773331+0000","last_undegraded":"2023-10-09T10:12:36.773331+0000","last_fullsized":"2023-10-09T10:12:36.773331+0000","mapping_epoch":271,"log_start":"280'2600","ondisk_log_start":"280'2600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9419,"log_dups_size":0,"ondisk_log_size":9419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100859904,"num_objects":1539,"num_object_clones":0,"num_object_copies":4617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12019,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":271,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.559104+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:12:24.559104+0000","last_peered":"2023-10-09T10:12:24.559104+0000","last_clean":"2023-10-09T10:12:24.559104+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:12:24.559104+0000","last_undegraded":"2023-10-09T10:12:24.559104+0000","last_fullsized":"2023-10-09T10:12:24.559104+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":285,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573482+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:12:24.573482+0000","last_peered":"2023-10-09T10:12:24.573482+0000","last_clean":"2023-10-09T10:12:24.573482+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:12:24.573482+0000","last_undegraded":"2023-10-09T10:12:24.573482+0000","last_fullsized":"2023-10-09T10:12:24.573482+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":260,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.564692+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:12:24.564692+0000","last_peered":"2023-10-09T10:12:24.564692+0000","last_clean":"2023-10-09T10:12:24.564692+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:12:24.564692+0000","last_undegraded":"2023-10-09T10:12:24.564692+0000","last_fullsized":"2023-10-09T10:12:24.564692+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"308'11785","reported_seq":11831,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.967791+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:12:32.967791+0000","last_peered":"2023-10-09T10:12:32.967791+0000","last_clean":"2023-10-09T10:12:32.967791+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:12:32.967791+0000","last_undegraded":"2023-10-09T10:12:32.967791+0000","last_fullsized":"2023-10-09T10:12:32.967791+0000","mapping_epoch":271,"log_start":"279'1700","ondisk_log_start":"279'1700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10085,"log_dups_size":0,"ondisk_log_size":10085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110362648,"num_objects":1685,"num_object_clones":0,"num_object_copies":5055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1685,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11785,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":216,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.569106+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:12:24.569106+0000","last_peered":"2023-10-09T10:12:24.569106+0000","last_clean":"2023-10-09T10:12:24.569106+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:12:24.569106+0000","last_undegraded":"2023-10-09T10:12:24.569106+0000","last_fullsized":"2023-10-09T10:12:24.569106+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":179,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.579130+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:12:24.579130+0000","last_peered":"2023-10-09T10:12:24.579130+0000","last_clean":"2023-10-09T10:12:24.579130+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:12:24.579130+0000","last_undegraded":"2023-10-09T10:12:24.579130+0000","last_fullsized":"2023-10-09T10:12:24.579130+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":161,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.589960+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:12:24.589960+0000","last_peered":"2023-10-09T10:12:24.589960+0000","last_clean":"2023-10-09T10:12:24.589960+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:12:24.589960+0000","last_undegraded":"2023-10-09T10:12:24.589960+0000","last_fullsized":"2023-10-09T10:12:24.589960+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":174,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:31.822780+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:12:31.822780+0000","last_peered":"2023-10-09T10:12:31.822780+0000","last_clean":"2023-10-09T10:12:31.822780+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:12:31.822780+0000","last_undegraded":"2023-10-09T10:12:31.822780+0000","last_fullsized":"2023-10-09T10:12:31.822780+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":246,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.573939+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:12:24.573939+0000","last_peered":"2023-10-09T10:12:24.573939+0000","last_clean":"2023-10-09T10:12:24.573939+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:12:24.573939+0000","last_undegraded":"2023-10-09T10:12:24.573939+0000","last_fullsized":"2023-10-09T10:12:24.573939+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":145,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.594263+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:12:24.594263+0000","last_peered":"2023-10-09T10:12:24.594263+0000","last_clean":"2023-10-09T10:12:24.594263+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:12:24.594263+0000","last_undegraded":"2023-10-09T10:12:24.594263+0000","last_fullsized":"2023-10-09T10:12:24.594263+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":186,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.565283+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:12:24.565283+0000","last_peered":"2023-10-09T10:12:24.565283+0000","last_clean":"2023-10-09T10:12:24.565283+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:12:24.565283+0000","last_undegraded":"2023-10-09T10:12:24.565283+0000","last_fullsized":"2023-10-09T10:12:24.565283+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":145,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.590248+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:12:24.590248+0000","last_peered":"2023-10-09T10:12:24.590248+0000","last_clean":"2023-10-09T10:12:24.590248+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:12:24.590248+0000","last_undegraded":"2023-10-09T10:12:24.590248+0000","last_fullsized":"2023-10-09T10:12:24.590248+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":195,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.559246+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:12:24.559246+0000","last_peered":"2023-10-09T10:12:24.559246+0000","last_clean":"2023-10-09T10:12:24.559246+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:12:24.559246+0000","last_undegraded":"2023-10-09T10:12:24.559246+0000","last_fullsized":"2023-10-09T10:12:24.559246+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":325,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.562539+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:12:24.562539+0000","last_peered":"2023-10-09T10:12:24.562539+0000","last_clean":"2023-10-09T10:12:24.562539+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:12:24.562539+0000","last_undegraded":"2023-10-09T10:12:24.562539+0000","last_fullsized":"2023-10-09T10:12:24.562539+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":346,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:31.544400+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:12:31.544400+0000","last_peered":"2023-10-09T10:12:31.544400+0000","last_clean":"2023-10-09T10:12:31.544400+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:12:31.544400+0000","last_undegraded":"2023-10-09T10:12:31.544400+0000","last_fullsized":"2023-10-09T10:12:31.544400+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":255,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.577227+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:12:24.577227+0000","last_peered":"2023-10-09T10:12:24.577227+0000","last_clean":"2023-10-09T10:12:24.577227+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:12:24.577227+0000","last_undegraded":"2023-10-09T10:12:24.577227+0000","last_fullsized":"2023-10-09T10:12:24.577227+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":175,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:12:24.574417+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:12:24.574417+0000","last_peered":"2023-10-09T10:12:24.574417+0000","last_clean":"2023-10-09T10:12:24.574417+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:12:24.574417+0000","last_undegraded":"2023-10-09T10:12:24.574417+0000","last_fullsized":"2023-10-09T10:12:24.574417+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1688403992,"num_objects":25764,"num_object_clones":0,"num_object_copies":77292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5742,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25764,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191598,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4915,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5790707712,"data_stored":5790695496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142198,"ondisk_log_size":142198,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430515,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":822352,"kb_used_data":521032,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92922800,"statfs":{"total":95995035648,"available":95152947200,"internally_reserved":0,"allocated":533536768,"data_stored":532996549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:43 2023","interfaces":[{"interface":"back","average":{"1min":3.508,"5min":1.986,"15min":1.911},"min":{"1min":0.90100000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":9.9870000000000001,"5min":9.9870000000000001,"15min":50.591999999999999},"last":2.3820000000000001},{"interface":"front","average":{"1min":3.661,"5min":2.133,"15min":1.905},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.974,"5min":15.974,"15min":50.747},"last":0.32600000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:12:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9940000000000002,"5min":2.3639999999999999,"15min":1.958},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":11.92,"5min":14.311,"15min":25.338000000000001},"last":2.4889999999999999},{"interface":"front","average":{"1min":2.6320000000000001,"5min":2.218,"15min":1.6870000000000001},"min":{"1min":0.42999999999999999,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":10.663,"5min":18.27,"15min":18.27},"last":0.27900000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":3.7639999999999998,"5min":1.9970000000000001,"15min":2.359},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":13.775,"5min":13.775,"15min":13.775},"last":2.6400000000000001},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.008,"15min":2.157},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.101000000000001,"5min":12.101000000000001,"15min":12.101000000000001},"last":0.35199999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4550000000000001,"5min":2.629,"15min":2.0430000000000001},"min":{"1min":0.436,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.6289999999999996,"5min":29.058,"15min":29.058},"last":0.40300000000000002},{"interface":"front","average":{"1min":3.6629999999999998,"5min":2.2559999999999998,"15min":2.0489999999999999},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.56299999999999994}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.0469999999999999,"15min":0.95699999999999996},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.9820000000000002,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":2.4199999999999999},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.97599999999999998,"15min":0.93100000000000005},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.6239999999999997,"5min":4.6239999999999997,"15min":4.6239999999999997},"last":0.377}]},{"osd":5,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3260000000000001,"5min":1.403,"15min":1.1259999999999999},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":7.8159999999999998,"5min":25.277999999999999,"15min":25.277999999999999},"last":5.0709999999999997},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.2110000000000001,"15min":1.0580000000000001},"min":{"1min":0.38500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.806,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.42899999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":1.0660000000000001,"15min":1.006},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":4.2789999999999999},{"interface":"front","average":{"1min":1.327,"5min":0.86899999999999999,"15min":0.91900000000000004},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":9.1920000000000002},"last":0.39000000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626743,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1168000,"kb_used_data":866680,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92577152,"statfs":{"total":95995035648,"available":94799003648,"internally_reserved":0,"allocated":887480320,"data_stored":886957415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":1.919,"15min":1.6060000000000001},"min":{"1min":0.252,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":6.9630000000000001,"5min":15.435,"15min":15.435},"last":2.8010000000000002},{"interface":"front","average":{"1min":2.2469999999999999,"5min":1.921,"15min":1.591},"min":{"1min":0.33900000000000002,"5min":0.252,"15min":0.23999999999999999},"max":{"1min":8.2959999999999994,"5min":15.308999999999999,"15min":15.308999999999999},"last":3.7989999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:12:06 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":2.839,"15min":2.0579999999999998},"min":{"1min":0.30199999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.0519999999999996,"5min":35.609999999999999,"15min":35.609999999999999},"last":9.8680000000000003},{"interface":"front","average":{"1min":2.919,"5min":2.46,"15min":1.9910000000000001},"min":{"1min":0.36099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":8.2680000000000007,"5min":32.396000000000001,"15min":32.396000000000001},"last":5.5199999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":1.637,"15min":1.7929999999999999},"min":{"1min":0.35199999999999998,"5min":0.26900000000000002,"15min":0.26200000000000001},"max":{"1min":7.1280000000000001,"5min":12.898,"15min":12.898},"last":3.8380000000000001},{"interface":"front","average":{"1min":3.4860000000000002,"5min":2.1400000000000001,"15min":3.0379999999999998},"min":{"1min":0.44600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.542999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":11.720000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":4.1079999999999997,"5min":2.3639999999999999,"15min":1.694},"min":{"1min":0.432,"5min":0.308,"15min":0.308},"max":{"1min":19.222000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":9.7509999999999994},{"interface":"front","average":{"1min":3.2490000000000001,"5min":2.4500000000000002,"15min":2.4039999999999999},"min":{"1min":0.39500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":8.2270000000000003,"5min":11.211,"15min":11.211},"last":4.6699999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.67100000000000004,"15min":0.51700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.8970000000000002,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":3.7730000000000001},{"interface":"front","average":{"1min":1.284,"5min":0.75800000000000001,"15min":0.58499999999999996},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:11:37 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.81599999999999995},"min":{"1min":0.29399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.4580000000000002,"5min":7.9260000000000002,"15min":14.09},"last":3.8149999999999999},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.042,"15min":0.872},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":6.3369999999999997,"5min":7.508,"15min":7.508},"last":2.8290000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.92600000000000005,"15min":0.89500000000000002},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.0700000000000003,"5min":6.2709999999999999,"15min":7.5350000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.673,"5min":0.97199999999999998,"15min":0.94299999999999995},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.101,"5min":6.3159999999999998,"15min":7.5570000000000004},"last":0.48599999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253056,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1170816,"kb_used_data":758904,"kb_used_omap":9,"kb_used_meta":411574,"kb_avail":92574336,"statfs":{"total":95995035648,"available":94796120064,"internally_reserved":0,"allocated":777117696,"data_stored":776061318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":421452777},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":4.4160000000000004,"5min":2.5880000000000001,"15min":2.4500000000000002},"min":{"1min":0.217,"5min":0.183,"15min":0.183},"max":{"1min":31.007000000000001,"5min":31.007000000000001,"15min":31.007000000000001},"last":0.495},{"interface":"front","average":{"1min":3.8010000000000002,"5min":3.4079999999999999,"15min":2.5899999999999999},"min":{"1min":0.28599999999999998,"5min":0.245,"15min":0.192},"max":{"1min":10.577999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":9.3710000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":3.0249999999999999,"15min":3.8069999999999999},"min":{"1min":0.70099999999999996,"5min":0.217,"15min":0.217},"max":{"1min":10.573,"5min":17.369,"15min":17.369},"last":9.2100000000000009},{"interface":"front","average":{"1min":3.3479999999999999,"5min":2.5449999999999999,"15min":3.7530000000000001},"min":{"1min":0.26400000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":13.414999999999999,"5min":13.414999999999999,"15min":16.023},"last":0.65700000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3170000000000002,"5min":3.6110000000000002,"15min":3.5590000000000002},"min":{"1min":0.41699999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.119,"5min":18.587,"15min":18.587},"last":9.3420000000000005},{"interface":"front","average":{"1min":2.7959999999999998,"5min":3.8029999999999999,"15min":4.5350000000000001},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.103999999999999,"5min":17.289000000000001,"15min":17.289000000000001},"last":0.85499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.536,"15min":1.3480000000000001},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.976000000000001,"5min":11.976000000000001,"15min":11.976000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.177,"15min":1.6120000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.904,"5min":12.904,"15min":12.904},"last":9.2569999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.4430000000000001,"15min":2.0630000000000002},"min":{"1min":0.34100000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.8599999999999994,"5min":12.811,"15min":15.531000000000001},"last":9.2870000000000008},{"interface":"front","average":{"1min":3.544,"5min":2.2949999999999999,"15min":2.2999999999999998},"min":{"1min":0.53900000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.855,"5min":16.277999999999999,"15min":19.706},"last":9.1300000000000008}]},{"osd":6,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":2.0139999999999998,"15min":1.9810000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":21.670999999999999,"5min":21.670999999999999,"15min":21.670999999999999},"last":4.0419999999999998},{"interface":"front","average":{"1min":2.5219999999999998,"5min":2.4220000000000002,"15min":2.1070000000000002},"min":{"1min":0.36899999999999999,"5min":0.35499999999999998,"15min":0.32000000000000001},"max":{"1min":9.9440000000000008,"5min":12.868,"15min":15.481999999999999},"last":9.3160000000000007}]},{"osd":7,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":2.3860000000000001,"15min":2.3330000000000002},"min":{"1min":0.68200000000000005,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.723000000000001,"5min":16.331,"15min":16.331},"last":9.1690000000000005},{"interface":"front","average":{"1min":3.101,"5min":2.0830000000000002,"15min":2.0619999999999998},"min":{"1min":0.42399999999999999,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":21.593,"5min":21.593,"15min":21.593},"last":4.6369999999999996}]}]},{"osd":0,"up_from":79,"seq":339302416583,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":990556,"kb_used_data":601620,"kb_used_omap":28,"kb_used_meta":388899,"kb_avail":92754596,"statfs":{"total":95995035648,"available":94980706304,"internally_reserved":0,"allocated":616058880,"data_stored":615035258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":398233424},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":2.6800000000000002,"15min":2.0209999999999999},"min":{"1min":0.26400000000000001,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":16.797000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":1.0680000000000001},{"interface":"front","average":{"1min":7.1150000000000002,"5min":4.2210000000000001,"15min":3.04},"min":{"1min":0.253,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":31.925999999999998,"5min":31.925999999999998,"15min":31.925999999999998},"last":5.7809999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0469999999999997,"5min":2.181,"15min":2.274},"min":{"1min":0.28899999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.003,"5min":18.003,"15min":18.003},"last":1.629},{"interface":"front","average":{"1min":2.9660000000000002,"5min":2.0710000000000002,"15min":3.0190000000000001},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.432,"5min":15.529,"15min":15.529},"last":1.6599999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":4.6509999999999998,"15min":4.7930000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":23.151,"5min":25.734999999999999,"15min":25.734999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":5.3250000000000002,"5min":4.0739999999999998,"15min":4.298},"min":{"1min":0.33300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":18.279,"5min":21.923999999999999,"15min":21.923999999999999},"last":1.171}]},{"osd":4,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3780000000000001,"5min":2.3079999999999998,"15min":1.6970000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.989000000000001,"5min":20.494,"15min":20.494},"last":0.40300000000000002},{"interface":"front","average":{"1min":3.6819999999999999,"5min":2.395,"15min":2.089},"min":{"1min":0.40200000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.907,"5min":15.907,"15min":15.907},"last":1.234}]},{"osd":5,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5739999999999998,"15min":2.1869999999999998},"min":{"1min":0.36099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.868,"5min":17.096,"15min":17.096},"last":1.2050000000000001},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.52,"15min":1.4299999999999999},"min":{"1min":0.371,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.371,"5min":10.371,"15min":12.478},"last":1.1040000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.734,"15min":1.5860000000000001},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":8.827,"5min":9.4719999999999995,"15min":11.699},"last":1.1319999999999999},{"interface":"front","average":{"1min":4.5800000000000001,"5min":2.9740000000000002,"15min":2.4660000000000002},"min":{"1min":0.52100000000000002,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":23.106999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":5.8700000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2679999999999998,"5min":2.6699999999999999,"15min":2.3540000000000001},"min":{"1min":0.437,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":16.460000000000001,"5min":16.460000000000001,"15min":26.978000000000002},"last":1.2629999999999999},{"interface":"front","average":{"1min":2.677,"5min":1.7490000000000001,"15min":1.738},"min":{"1min":0.45100000000000001,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":8.8040000000000003,"5min":10.122,"15min":26.768000000000001},"last":2.1520000000000001}]}]},{"osd":2,"up_from":217,"seq":932007903316,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":881764,"kb_used_data":624668,"kb_used_omap":83,"kb_used_meta":256940,"kb_avail":92863388,"statfs":{"total":95995035648,"available":95092109312,"internally_reserved":0,"allocated":639660032,"data_stored":638629562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":263107055},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.0609999999999999,"15min":1.8660000000000001},"min":{"1min":0.24299999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.334,"5min":19.372,"15min":19.372},"last":0.19400000000000001},{"interface":"front","average":{"1min":5.5940000000000003,"5min":3.5150000000000001,"15min":3.133},"min":{"1min":0.32000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.047000000000001,"5min":16.754999999999999,"15min":16.754999999999999},"last":4.4249999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":4.6630000000000003,"5min":2.653,"15min":2.6760000000000002},"min":{"1min":0.26500000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.276,"5min":14.276,"15min":14.276},"last":0.26700000000000002},{"interface":"front","average":{"1min":5.4710000000000001,"5min":3.0760000000000001,"15min":2.823},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.227,"5min":15.569000000000001,"15min":15.569000000000001},"last":0.254}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.2989999999999999,"15min":2.1150000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.359999999999999,"5min":13.851000000000001,"15min":13.851000000000001},"last":0.83999999999999997},{"interface":"front","average":{"1min":4.1879999999999997,"5min":3.2360000000000002,"15min":3.6880000000000002},"min":{"1min":0.40799999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.51,"5min":15.811,"15min":15.811},"last":0.88700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":1.3,"15min":0.90400000000000003},"min":{"1min":0.29399999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.413,"5min":12.413,"15min":12.413},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.597,"5min":1.329,"15min":0.996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.314},"max":{"1min":7.0810000000000004,"5min":12.385,"15min":12.385},"last":0.90400000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.8120000000000001,"15min":1.5860000000000001},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1230000000000002,"5min":12.473000000000001,"15min":12.473000000000001},"last":0.85299999999999998},{"interface":"front","average":{"1min":2.044,"5min":1.5980000000000001,"15min":1.706},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.923,"5min":13.587,"15min":13.587},"last":0.39900000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5529999999999999,"5min":1.9219999999999999,"15min":1.2689999999999999},"min":{"1min":0.374,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.484,"5min":15.292999999999999,"15min":15.292999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.9670000000000001,"15min":1.8180000000000001},"min":{"1min":0.41099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.5679999999999996,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.93300000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.9359999999999999,"15min":1.294},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.457000000000001,"5min":15.343999999999999,"15min":15.343999999999999},"last":1.079},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.1539999999999999,"15min":1.758},"min":{"1min":0.38600000000000001,"5min":0.376,"15min":0.376},"max":{"1min":9.6039999999999992,"5min":12.433999999999999,"15min":12.433999999999999},"last":0.95999999999999996}]}]},{"osd":3,"up_from":250,"seq":1073741824060,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":871600,"kb_used_data":607400,"kb_used_omap":10,"kb_used_meta":264181,"kb_avail":92873552,"statfs":{"total":95995035648,"available":95102517248,"internally_reserved":0,"allocated":621977600,"data_stored":620759916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":270521411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5139999999999998,"5min":4.0330000000000004,"15min":4.4800000000000004},"min":{"1min":0.27500000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.749000000000001,"5min":23.381,"15min":23.381},"last":3.3399999999999999},{"interface":"front","average":{"1min":6.5919999999999996,"5min":4.5750000000000002,"15min":4.3109999999999999},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.782,"5min":20.239999999999998,"15min":20.239999999999998},"last":2.9860000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2909999999999999,"5min":3.4820000000000002,"15min":3.7959999999999998},"min":{"1min":0.41399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.856999999999999,"5min":20.422999999999998,"15min":20.422999999999998},"last":13.457000000000001},{"interface":"front","average":{"1min":4.5620000000000003,"5min":3.726,"15min":3.5289999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.73,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.24299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":3.1269999999999998,"15min":3.536},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.331,"5min":20.594999999999999,"15min":20.594999999999999},"last":19.390999999999998},{"interface":"front","average":{"1min":4.298,"5min":3.1680000000000001,"15min":2.9660000000000002},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":14.802,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.29399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":2.048,"15min":1.413},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.696999999999999,"5min":15.676,"15min":15.676},"last":2.8420000000000001},{"interface":"front","average":{"1min":4.7910000000000004,"5min":2.2360000000000002,"15min":1.752},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.478999999999999,"5min":16.478999999999999,"15min":16.478999999999999},"last":2.9609999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":2.9260000000000002,"15min":2.718},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.125999999999999,"5min":13.125999999999999,"15min":13.125999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":3.0699999999999998,"5min":2.6600000000000001,"15min":2.456},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":10.891999999999999,"5min":15.992000000000001,"15min":15.992000000000001},"last":9.3629999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9649999999999999,"5min":2.9809999999999999,"15min":2.819},"min":{"1min":0.42099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.504999999999999,"5min":16.504999999999999,"15min":16.504999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.921,"5min":2.4239999999999999,"15min":2.544},"min":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.952,"5min":16.16,"15min":16.16},"last":9.3239999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.1219999999999999,"15min":1.633},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":11.048999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":2.7959999999999998},{"interface":"front","average":{"1min":5.8399999999999999,"5min":3.073,"15min":2.8069999999999999},"min":{"1min":0.44600000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":16.367000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":2.7669999999999999}]}]},{"osd":4,"up_from":266,"seq":1142461300785,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1215036,"kb_used_data":892276,"kb_used_omap":35,"kb_used_meta":322716,"kb_avail":92530116,"statfs":{"total":95995035648,"available":94750838784,"internally_reserved":0,"allocated":913690624,"data_stored":913162293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":330461422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.0109999999999999,"15min":0.84899999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.4829999999999997,"5min":10.881,"15min":10.881},"last":0.33900000000000002},{"interface":"front","average":{"1min":3.0150000000000001,"5min":1.5629999999999999,"15min":1.161},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":12.246,"5min":12.246,"15min":12.246},"last":6.1630000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2429999999999999,"5min":1.655,"15min":1.0960000000000001},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":12.161,"5min":12.161,"15min":12.161},"last":9.4700000000000006},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.7290000000000001,"15min":1.4259999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.504,"5min":11.141,"15min":11.141},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.4430000000000001,"15min":1.143},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.2469999999999999,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":0.316},{"interface":"front","average":{"1min":1.905,"5min":1.242,"15min":1.018},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":7.8920000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":1.9490000000000001,"15min":1.4910000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.065,"5min":13.065,"15min":13.065},"last":0.40400000000000003},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.766,"15min":1.264},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.218,"5min":12.218,"15min":12.218},"last":6.2770000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":0.93200000000000005,"15min":0.79600000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.6869999999999994,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.56100000000000005,"15min":0.61899999999999999},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0609999999999999,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.377}]},{"osd":6,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.69799999999999995,"15min":0.58299999999999996},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.311},"max":{"1min":6.4119999999999999,"5min":6.4119999999999999,"15min":6.4119999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":0.83899999999999997,"15min":0.66900000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.255000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":6.2060000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":0.89100000000000001,"15min":0.71099999999999997},"min":{"1min":0.38800000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":12.237,"5min":12.237,"15min":12.237},"last":8.1039999999999992},{"interface":"front","average":{"1min":1.244,"5min":0.90600000000000003,"15min":0.85399999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.7590000000000003,"5min":9.7590000000000003,"15min":9.7590000000000003},"last":0.65900000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822970,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1175736,"kb_used_data":870320,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92569416,"statfs":{"total":95995035648,"available":94791081984,"internally_reserved":0,"allocated":891207680,"data_stored":890687167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.4710000000000001,"15min":1.5720000000000001},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.8179999999999996,"5min":7.8540000000000001,"15min":15.773999999999999},"last":6.3849999999999998},{"interface":"front","average":{"1min":2.7770000000000001,"5min":1.6879999999999999,"15min":1.6819999999999999},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.9950000000000001,"5min":10.244999999999999,"15min":15.699},"last":0.32000000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.1320000000000001,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.891,"5min":11.891,"15min":15.689},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.9350000000000001,"15min":1.823},"min":{"1min":0.41399999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.4489999999999998,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.34799999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.7130000000000001,"5min":1.905,"15min":2.456},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.259,"5min":10.259,"15min":15.640000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.581,"5min":1.458,"15min":2.1739999999999999},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":4.1619999999999999,"5min":9.1329999999999991,"15min":10.656000000000001},"last":0.57899999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":2.1829999999999998,"15min":1.96},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2260000000000009,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.46200000000000002},{"interface":"front","average":{"1min":3.2370000000000001,"5min":2.4820000000000002,"15min":2.8399999999999999},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.9900000000000002,"5min":17.34,"15min":17.34},"last":0.42999999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.84899999999999998,"15min":0.65900000000000003},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3460000000000001,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.70499999999999996,"15min":0.73099999999999998},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.462,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.40400000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:11:34 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":0.97599999999999998,"15min":0.996},"min":{"1min":0.26200000000000001,"5min":0.25,"15min":0.25},"max":{"1min":8.5259999999999998,"5min":13.483000000000001,"15min":27.716000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.198,"5min":1.075,"15min":0.92600000000000005},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.9889999999999999,"5min":18.600999999999999,"15min":18.600999999999999},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.137,"15min":1.0389999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4710000000000001,"5min":7.891,"15min":10.334},"last":0.504},{"interface":"front","average":{"1min":1.173,"5min":0.80300000000000005,"15min":0.96899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":12.356},"last":0.53900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":601554944,"data_stored":601554944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":761004032,"data_stored":761004032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":622596096,"data_stored":622592024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":609550336,"data_stored":609550336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":901840896,"data_stored":901840896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":884871168,"data_stored":884867096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":880279552,"data_stored":880279552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":529010688,"data_stored":529006616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:12:39.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:39.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:39.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:39.365 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:12:39.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:39.448 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:39.597 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:39.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:39.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:39.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:39.759 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:12:39.783 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:39.822 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:12:39.862 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:12:39.875 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:39.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:39.901 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:12:39.943 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:12:39.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:39.999 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:12:40.039 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:12:40.072 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:40.080 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:39 smithi012 ceph-mon[147926]: from='client.55903 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:40.080 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4102134833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:40.080 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:39 smithi012 ceph-mon[152263]: from='client.55903 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:40.080 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4102134833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:40.081 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:12:40.091 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:40.095 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:12:40.095 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:12:40.095 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:12:40.095 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:12:40.095 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:12:40.095 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:12:40.095 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:12:40.095 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:12:40.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:39 smithi110 ceph-mon[136032]: from='client.55903 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:40.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4102134833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:40.241 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:40.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:40.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:40.414 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:40.483 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:40.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:40.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:40.589 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:40.653 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:40.807 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:40.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:40.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:40.871 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:12:40.915 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:40 smithi110 ceph-mon[136032]: pgmap v856: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 5742/72549 objects degraded (7.915%); 2.2 MiB/s, 54 objects/s recovering 2023-10-09T10:12:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2847065186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:40.967 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:41.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:40 smithi012 ceph-mon[152263]: pgmap v856: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 5742/72549 objects degraded (7.915%); 2.2 MiB/s, 54 objects/s recovering 2023-10-09T10:12:41.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2847065186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:41.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:41.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:40 smithi012 ceph-mon[147926]: pgmap v856: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 5742/72549 objects degraded (7.915%); 2.2 MiB/s, 54 objects/s recovering 2023-10-09T10:12:41.182 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2847065186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:41.183 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:12:41.742 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:41.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:41.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:41.854 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:41.888 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:41.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:41.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:41.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:42.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:42.168 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:42.169 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:12:42.169 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[147926]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T10:12:42.169 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[147926]: failed to encode map e309 with expected crc 2023-10-09T10:12:42.169 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[147926]: failed to encode map e309 with expected crc 2023-10-09T10:12:42.169 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:42.169 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:12:42.169 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[152263]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T10:12:42.170 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[152263]: failed to encode map e309 with expected crc 2023-10-09T10:12:42.170 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:41 smithi012 ceph-mon[152263]: failed to encode map e309 with expected crc 2023-10-09T10:12:42.178 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:41 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:41 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:12:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:41 smithi110 ceph-mon[136032]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T10:12:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:41 smithi110 ceph-mon[136032]: failed to encode map e309 with expected crc 2023-10-09T10:12:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:41 smithi110 ceph-mon[136032]: failed to encode map e309 with expected crc 2023-10-09T10:12:42.247 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:42.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:42.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:42.261 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:12:42.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:42.418 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:42.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:12:42.633 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:42.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:42.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:42.740 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:42.797 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:42.854 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:42.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:42.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:42.908 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:42 smithi012 ceph-mon[147926]: pgmap v858: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 5742/71994 objects degraded (7.976%); 730 KiB/s, 32 objects/s recovering 2023-10-09T10:12:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:42 smithi110 ceph-mon[136032]: pgmap v858: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 5742/71994 objects degraded (7.976%); 730 KiB/s, 32 objects/s recovering 2023-10-09T10:12:42.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:42.984 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:12:42.984 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:12:42.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:12:42.984 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:12:42.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.984 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190230 7.1 7:8724f90f:::benchmark_data_smithi012_221426_object82452:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:42.984 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.878911+0000", 2023-10-09T10:12:42.984 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10108480899999, 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.239926774, 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190230 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:42.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878911+0000", 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878911+0000", 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:42.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878910+0000", 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6420000000000007e-06 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878917+0000", 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3363999999999999e-05 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878931+0000", 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4010000000000002e-06 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878937+0000", 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4478999999999998e-05 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:42.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878991+0000", 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0682e-05 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879002+0000", 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080712920000000007 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887073+0000", 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4110000000000002e-06 2023-10-09T10:12:42.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887083+0000", 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2200019850000001 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.107085+0000", 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002097448 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.109182+0000", 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105568 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.109288+0000", 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0092099759999999999 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118498+0000", 2023-10-09T10:12:42.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8666999999999999e-05 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118557+0000", 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2558000000000001e-05 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118609+0000", 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022877999999999999 2023-10-09T10:12:42.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118838+0000", 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.239926774 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190272 7.1 7:8eae50ab:::benchmark_data_smithi012_221426_object82494:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.878946+0000", 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10104974699999, 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2401155699999999, 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:42.994 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190272 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878946+0000", 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878946+0000", 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878946+0000", 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6419999999999999e-06 2023-10-09T10:12:42.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878948+0000", 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7689999999999998e-06 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878952+0000", 2023-10-09T10:12:42.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4310000000000001e-06 2023-10-09T10:12:42.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:42.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878955+0000", 2023-10-09T10:12:42.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010689 2023-10-09T10:12:42.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879061+0000", 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2700000000000004e-06 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879067+0000", 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080352559999999993 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:42.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887102+0000", 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.835e-06 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887108+0000", 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2253334979999999 2023-10-09T10:12:43.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.112441+0000", 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010237499999999999 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.112544+0000", 2023-10-09T10:12:43.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6977000000000004e-05 2023-10-09T10:12:43.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.112631+0000", 2023-10-09T10:12:43.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062493150000000001 2023-10-09T10:12:43.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118880+0000", 2023-10-09T10:12:43.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1987999999999999e-05 2023-10-09T10:12:43.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118892+0000", 2023-10-09T10:12:43.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5519999999999999e-05 2023-10-09T10:12:43.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.118927+0000", 2023-10-09T10:12:43.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013421700000000001 2023-10-09T10:12:43.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119062+0000", 2023-10-09T10:12:43.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2401155699999999 2023-10-09T10:12:43.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.005 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.005 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190277 7.1 7:82c2f525:::benchmark_data_smithi012_221426_object82499:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.005 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.878979+0000", 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.101016507, 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2403365749999999, 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190277 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.006 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878979+0000", 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878979+0000", 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878979+0000", 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9800000000000001e-06 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878981+0000", 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5780000000000004e-06 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878988+0000", 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8729999999999999e-06 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.878991+0000", 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011209000000000001 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879103+0000", 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7549999999999994e-06 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879110+0000", 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080152250000000008 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887126+0000", 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6300000000000003e-06 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887131+0000", 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2282280270000001 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.115359+0000", 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000160617 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.115520+0000", 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014887599999999999 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.115669+0000", 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003501798 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119171+0000", 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5810000000000003e-06 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119180+0000", 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0796000000000001e-05 2023-10-09T10:12:43.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119211+0000", 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105062 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119316+0000", 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2403365749999999 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.014 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.015 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190284 7.1 7:8f7a2a8f:::benchmark_data_smithi012_221426_object82506:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879000+0000", 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10099548099998, 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2740284640000001, 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.016 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190284 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879000+0000", 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879000+0000", 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879000+0000", 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.689e-06 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879002+0000", 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3459999999999998e-06 2023-10-09T10:12:43.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879005+0000", 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1919999999999999e-06 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879007+0000", 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013018 2023-10-09T10:12:43.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879138+0000", 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8340000000000001e-06 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879141+0000", 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080076969999999994 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887149+0000", 2023-10-09T10:12:43.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.173e-06 2023-10-09T10:12:43.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887155+0000", 2023-10-09T10:12:43.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.23219162 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119347+0000", 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9079999999999995e-05 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119426+0000", 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0434999999999998e-05 2023-10-09T10:12:43.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.119506+0000", 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015469747000000001 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.134976+0000", 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017723145999999999 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.152699+0000", 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9720000000000002e-05 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.152779+0000", 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024988099999999997 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153029+0000", 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2740284640000001 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190290 7.1 7:8a45fb96:::benchmark_data_smithi012_221426_object82512:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.025 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879010+0000", 2023-10-09T10:12:43.026 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10098638900001, 2023-10-09T10:12:43.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.27435247, 2023-10-09T10:12:43.026 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.026 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.026 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.026 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190290 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879010+0000", 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879010+0000", 2023-10-09T10:12:43.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:43.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879009+0000", 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7489999999999999e-06 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879011+0000", 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8449999999999999e-06 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879014+0000", 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1009999999999999e-06 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879016+0000", 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015992600000000001 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879176+0000", 2023-10-09T10:12:43.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2989999999999999e-06 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879183+0000", 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079899610000000003 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887173+0000", 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.733e-06 2023-10-09T10:12:43.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887179+0000", 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.235754005 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.122933+0000", 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000144983 2023-10-09T10:12:43.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.123078+0000", 2023-10-09T10:12:43.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016969899999999999 2023-10-09T10:12:43.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.123248+0000", 2023-10-09T10:12:43.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011732963000000001 2023-10-09T10:12:43.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.134981+0000", 2023-10-09T10:12:43.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018158093 2023-10-09T10:12:43.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153139+0000", 2023-10-09T10:12:43.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9548e-05 2023-10-09T10:12:43.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153198+0000", 2023-10-09T10:12:43.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016380799999999999 2023-10-09T10:12:43.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153362+0000", 2023-10-09T10:12:43.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.27435247 2023-10-09T10:12:43.036 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.036 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.036 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.037 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190305 7.1 7:8f4b1bee:::benchmark_data_smithi012_221426_object82527:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.037 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879018+0000", 2023-10-09T10:12:43.037 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10097785099998, 2023-10-09T10:12:43.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2746395050000001, 2023-10-09T10:12:43.037 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.038 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.038 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190305 2023-10-09T10:12:43.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.038 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879018+0000", 2023-10-09T10:12:43.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879018+0000", 2023-10-09T10:12:43.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879018+0000", 2023-10-09T10:12:43.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5260000000000001e-06 2023-10-09T10:12:43.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879019+0000", 2023-10-09T10:12:43.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9170000000000002e-06 2023-10-09T10:12:43.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879022+0000", 2023-10-09T10:12:43.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9630000000000001e-06 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879024+0000", 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000183127 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879207+0000", 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.698e-06 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879211+0000", 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079855849999999999 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887197+0000", 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6919999999999999e-06 2023-10-09T10:12:43.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887202+0000", 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2399458830000001 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.127148+0000", 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8029000000000006e-05 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.127246+0000", 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101876 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.127348+0000", 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010995153000000001 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.138343+0000", 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015098489 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153442+0000", 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7228999999999997e-05 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153499+0000", 2023-10-09T10:12:43.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000158574 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153658+0000", 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2746395050000001 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190320 7.1 7:8f8740a7:::benchmark_data_smithi012_221426_object82542:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879026+0000", 2023-10-09T10:12:43.047 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10096958299999, 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2749258020000001, 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190320 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.048 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879026+0000", 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879026+0000", 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879026+0000", 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9659999999999999e-06 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879028+0000", 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8679999999999999e-06 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879031+0000", 2023-10-09T10:12:43.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1040000000000001e-06 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879033+0000", 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000216457 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879249+0000", 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8610000000000004e-06 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879258+0000", 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079627389999999999 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887221+0000", 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7089999999999999e-06 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887227+0000", 2023-10-09T10:12:43.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432745869999999 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.130501+0000", 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0001515 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.130653+0000", 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000165565 2023-10-09T10:12:43.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.130818+0000", 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018356846 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.149175+0000", 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045797640000000001 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153755+0000", 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7685000000000001e-05 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153813+0000", 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139376 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153952+0000", 2023-10-09T10:12:43.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2749258020000001 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190328 7.1 7:8450f8de:::benchmark_data_smithi012_221426_object82550:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879051+0000", 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10094453099998, 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2833133970000001, 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.056 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190328 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879051+0000", 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879051+0000", 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879051+0000", 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5710000000000001e-06 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.058 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879053+0000", 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0030000000000002e-06 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879058+0000", 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2369999999999999e-06 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879060+0000", 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023918499999999999 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879299+0000", 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.952e-06 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879308+0000", 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079363869999999996 2023-10-09T10:12:43.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887244+0000", 2023-10-09T10:12:43.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9599999999999997e-06 2023-10-09T10:12:43.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887250+0000", 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.247749566 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.135000+0000", 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3584000000000004e-05 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.135094+0000", 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103705 2023-10-09T10:12:43.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.135197+0000", 2023-10-09T10:12:43.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017849231 2023-10-09T10:12:43.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153046+0000", 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091115929999999994 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162158+0000", 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8216999999999997e-05 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162196+0000", 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016845900000000001 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162365+0000", 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2833133970000001 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.065 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190432 7.1 7:8d343cbd:::benchmark_data_smithi012_221426_object82654:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879062+0000", 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10093362999999, 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.283487313, 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.066 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.067 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190432 2023-10-09T10:12:43.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.067 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879062+0000", 2023-10-09T10:12:43.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879062+0000", 2023-10-09T10:12:43.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879062+0000", 2023-10-09T10:12:43.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0451999999999999e-05 2023-10-09T10:12:43.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879072+0000", 2023-10-09T10:12:43.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6150000000000001e-06 2023-10-09T10:12:43.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879076+0000", 2023-10-09T10:12:43.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.119e-06 2023-10-09T10:12:43.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879078+0000", 2023-10-09T10:12:43.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000267056 2023-10-09T10:12:43.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.072 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.072 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879345+0000", 2023-10-09T10:12:43.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5450000000000003e-06 2023-10-09T10:12:43.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879354+0000", 2023-10-09T10:12:43.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079143709999999999 2023-10-09T10:12:43.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887268+0000", 2023-10-09T10:12:43.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.75e-06 2023-10-09T10:12:43.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887274+0000", 2023-10-09T10:12:43.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2511190320000001 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.138393+0000", 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012326500000000001 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.138516+0000", 2023-10-09T10:12:43.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016769300000000001 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.138684+0000", 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014373889000000001 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153058+0000", 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0093553799999999999 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162413+0000", 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2394000000000002e-05 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162446+0000", 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103983 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162550+0000", 2023-10-09T10:12:43.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.283487313 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190433 7.1 7:8e64efc2:::benchmark_data_smithi012_221426_object82655:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879094+0000", 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10090195499998, 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2921292529999999, 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.078 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190433 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879094+0000", 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.079 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879094+0000", 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879094+0000", 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7120000000000001e-06 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879095+0000", 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9830000000000004e-06 2023-10-09T10:12:43.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879100+0000", 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0800000000000002e-06 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879103+0000", 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030411300000000001 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879408+0000", 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0961999999999999e-05 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879419+0000", 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078758939999999996 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887294+0000", 2023-10-09T10:12:43.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6760000000000001e-06 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887301+0000", 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2544338390000001 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.141735+0000", 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012397100000000001 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.141859+0000", 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014839000000000001 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.142007+0000", 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011061447 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153069+0000", 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017947520000000002 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.171016+0000", 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5809999999999998e-05 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.171052+0000", 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017113600000000001 2023-10-09T10:12:43.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.171223+0000", 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2921292529999999 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190435 7.1 7:8ff35f65:::benchmark_data_smithi012_221426_object82657:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879106+0000", 2023-10-09T10:12:43.086 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10088987300003, 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311420099, 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190435 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879106+0000", 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879106+0000", 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879106+0000", 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4610000000000001e-06 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879107+0000", 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9289999999999998e-06 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879110+0000", 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0190000000000001e-06 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879112+0000", 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034980699999999999 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879462+0000", 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3690000000000001e-06 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879469+0000", 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078489819999999991 2023-10-09T10:12:43.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887318+0000", 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9220000000000002e-06 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887324+0000", 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2580026360000001 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.145327+0000", 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000171764 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.145499+0000", 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017433100000000001 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.145673+0000", 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080011289999999992 2023-10-09T10:12:43.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153674+0000", 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036605540999999998 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190280+0000", 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2682000000000001e-05 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190332+0000", 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019375499999999999 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190526+0000", 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311420099 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.094 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190446 7.1 7:87cf5a1e:::benchmark_data_smithi012_221426_object82668:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879114+0000", 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10088163099999, 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311691205, 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.095 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190446 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879114+0000", 2023-10-09T10:12:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879114+0000", 2023-10-09T10:12:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879114+0000", 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3489999999999999e-06 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879115+0000", 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8270000000000002e-06 2023-10-09T10:12:43.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879118+0000", 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9860000000000001e-06 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879120+0000", 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000402023 2023-10-09T10:12:43.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879522+0000", 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0487e-05 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879533+0000", 2023-10-09T10:12:43.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078089259999999999 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887342+0000", 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5110000000000003e-06 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887347+0000", 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.261880992 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.149228+0000", 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014189699999999999 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.149370+0000", 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017402699999999999 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.149544+0000", 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041390330000000003 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153683+0000", 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036911868 2023-10-09T10:12:43.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190595+0000", 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4880000000000002e-05 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190660+0000", 2023-10-09T10:12:43.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014562900000000001 2023-10-09T10:12:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190805+0000", 2023-10-09T10:12:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311691205 2023-10-09T10:12:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.105 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190476 7.1 7:8e6b68f7:::benchmark_data_smithi012_221426_object82698:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879134+0000", 2023-10-09T10:12:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10086141300002, 2023-10-09T10:12:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311917888, 2023-10-09T10:12:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.106 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190476 2023-10-09T10:12:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879134+0000", 2023-10-09T10:12:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879134+0000", 2023-10-09T10:12:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879134+0000", 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.353e-06 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879136+0000", 2023-10-09T10:12:43.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2619999999999999e-06 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879139+0000", 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6290000000000002e-06 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879141+0000", 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044317599999999999 2023-10-09T10:12:43.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879585+0000", 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4009999999999997e-06 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879593+0000", 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077714439999999997 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887365+0000", 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6930000000000004e-06 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887370+0000", 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.266616481 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.153987+0000", 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011042100000000001 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.154097+0000", 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9776000000000002e-05 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.154187+0000", 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081925020000000008 2023-10-09T10:12:43.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162379+0000", 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02848268 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190862+0000", 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4193000000000001e-05 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.190916+0000", 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013610299999999999 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.191052+0000", 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.311917888 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.115 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190498 7.1 7:8b87c9fd:::benchmark_data_smithi012_221426_object82720:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879144+0000", 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10085228000003, 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3120772359999999, 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.116 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190498 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879144+0000", 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879144+0000", 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879143+0000", 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.46e-06 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879145+0000", 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2200000000000001e-06 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879148+0000", 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0999999999999998e-06 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879150+0000", 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047652399999999998 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879627+0000", 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.33e-06 2023-10-09T10:12:43.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879631+0000", 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077566129999999999 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887388+0000", 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8239999999999996e-06 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887393+0000", 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.270337568 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.157731+0000", 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1216000000000002e-05 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.157822+0000", 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100985 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.157923+0000", 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069121190000000004 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.164835+0000", 2023-10-09T10:12:43.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026256166000000001 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.191092+0000", 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2458e-05 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.191124+0000", 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6877000000000001e-05 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.191221+0000", 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3120772359999999 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190516 7.1 7:83f35799:::benchmark_data_smithi012_221426_object82738:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879152+0000", 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.10084358, 2023-10-09T10:12:43.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3136645630000001, 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190516 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879152+0000", 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879152+0000", 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879152+0000", 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.361e-06 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879153+0000", 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.909e-06 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879156+0000", 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3750000000000001e-06 2023-10-09T10:12:43.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879159+0000", 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050572799999999999 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879664+0000", 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8779999999999998e-06 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879672+0000", 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077386989999999999 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887411+0000", 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8189999999999997e-06 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887417+0000", 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.272863077 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.160280+0000", 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7027999999999994e-05 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.160367+0000", 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6990000000000006e-05 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.160454+0000", 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043883569999999998 2023-10-09T10:12:43.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.164842+0000", 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027848528000000001 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192691+0000", 2023-10-09T10:12:43.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1674000000000001e-05 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192733+0000", 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4364999999999996e-05 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192817+0000", 2023-10-09T10:12:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3136645630000001 2023-10-09T10:12:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190517 7.1 7:81260c21:::benchmark_data_smithi012_221426_object82739:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e251)", 2023-10-09T10:12:43.134 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:42.879167+0000", 2023-10-09T10:12:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 300.100829046, 2023-10-09T10:12:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.313835233, 2023-10-09T10:12:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:12:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:12:43.135 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190517 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879167+0000", 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879167+0000", 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879167+0000", 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4169999999999999e-06 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879168+0000", 2023-10-09T10:12:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2229999999999999e-06 2023-10-09T10:12:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879171+0000", 2023-10-09T10:12:43.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1289999999999999e-06 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879173+0000", 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00051873299999999995 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879692+0000", 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2479999999999998e-06 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.879696+0000", 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077381469999999999 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887435+0000", 2023-10-09T10:12:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7749999999999998e-06 2023-10-09T10:12:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:12:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:42.887440+0000", 2023-10-09T10:12:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2751414780000001 2023-10-09T10:12:43.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162582+0000", 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1100000000000006e-05 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162663+0000", 2023-10-09T10:12:43.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1507000000000004e-05 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3", 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.162744+0000", 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082317209999999991 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.170976+0000", 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021878416000000001 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192855+0000", 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4965999999999997e-05 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.192889+0000", 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112611 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:44.193002+0000", 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.313835233 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:157050 8.9 e301/271 8:96d45a28:::benchmark_data_smithi012_271002_object48367:head v 301'9756, mlcod=301'9754)", 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:11:38.118907+0000", 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 64.861088957000007, 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036639537999999999, 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.145 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.118907+0000", 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.118907+0000", 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.118906+0000", 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9590000000000007e-06 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.118916+0000", 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.643e-05 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.118932+0000", 2023-10-09T10:12:43.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2060000000000001e-06 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.118939+0000", 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4583000000000002e-05 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.119004+0000", 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4451e-05 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.119029+0000", 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036473516999999997 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.155502+0000", 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4452000000000001e-05 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:38.155546+0000", 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036639537999999999 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:187072 8.5 e307/271 8:ad9f3424:::benchmark_data_smithi012_271002_object78389:head v 307'11860, mlcod=307'11857)", 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:27.775042+0000", 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.204953977000001, 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027428726, 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.150 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.775042+0000", 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.775042+0000", 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:12:43.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.775041+0000", 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0726e-05 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.775052+0000", 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4975e-05 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.775067+0000", 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6900000000000003e-06 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.775073+0000", 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0087145220000000006 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.783788+0000", 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0403000000000001e-05 2023-10-09T10:12:43.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.783808+0000", 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018625787000000001 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.802434+0000", 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6454e-05 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:27.802471+0000", 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027428726 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:189026 8.6 e307/271 8:6402f4f6:::benchmark_data_smithi012_271002_object80343:head v 307'11758, mlcod=307'11756)", 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:30.873677+0000", 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 12.106318485999999, 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027951268000000001, 2023-10-09T10:12:43.155 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.873677+0000", 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.873677+0000", 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:12:43.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.873677+0000", 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.835e-06 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.873682+0000", 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4360000000000002e-06 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.873691+0000", 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.617e-06 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.873696+0000", 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6530999999999998e-05 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.873752+0000", 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0469999999999999e-05 2023-10-09T10:12:43.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.873773+0000", 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027817240999999999 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.901590+0000", 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8924000000000002e-05 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.901629+0000", 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027951268000000001 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:189037 8.6 e307/271 8:66b22552:::benchmark_data_smithi012_271002_object80354:head v 307'11762, mlcod=307'11756)", 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:30.894609+0000", 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 12.085387305999999, 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030567321000000001, 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:43.160 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:12:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.894609+0000", 2023-10-09T10:12:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.894609+0000", 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.894608+0000", 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6179999999999995e-06 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:43.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.894616+0000", 2023-10-09T10:12:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3162999999999999e-05 2023-10-09T10:12:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.894629+0000", 2023-10-09T10:12:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5609999999999998e-06 2023-10-09T10:12:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.894635+0000", 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01153972 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.906175+0000", 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9822000000000001e-05 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:43.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.906194+0000", 2023-10-09T10:12:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018936020000000001 2023-10-09T10:12:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:12:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.925130+0000", 2023-10-09T10:12:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5491000000000002e-05 2023-10-09T10:12:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:43.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:30.925176+0000", 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030567321000000001 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:43.166 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:12:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:43.168 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:43.168 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:43.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:43.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:42 smithi012 ceph-mon[152263]: pgmap v858: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 5742/71994 objects degraded (7.976%); 730 KiB/s, 32 objects/s recovering 2023-10-09T10:12:43.269 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:43.290 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:43.304 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:43.340 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:12:43.359 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:12:43.400 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:43.403 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:43.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:43.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:43.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:43.532 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:43.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:43.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:43.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:43.708 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:43.851 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:43.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:43.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:43.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:43 smithi012 ceph-mon[147926]: failed to encode map e310 with expected crc 2023-10-09T10:12:43.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:43 smithi012 ceph-mon[147926]: failed to encode map e310 with expected crc 2023-10-09T10:12:43.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:43 smithi012 ceph-mon[147926]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T10:12:43.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5742/71994 objects degraded (7.976%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:43.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:43 smithi012 ceph-mon[152263]: failed to encode map e310 with expected crc 2023-10-09T10:12:43.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:43 smithi012 ceph-mon[152263]: failed to encode map e310 with expected crc 2023-10-09T10:12:43.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:43 smithi012 ceph-mon[152263]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T10:12:43.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5742/71994 objects degraded (7.976%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:43.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:43.956 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:44.041 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:44.144 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:44.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:44.175 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:43 smithi110 ceph-mon[136032]: failed to encode map e310 with expected crc 2023-10-09T10:12:44.175 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:43 smithi110 ceph-mon[136032]: failed to encode map e310 with expected crc 2023-10-09T10:12:44.175 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:43 smithi110 ceph-mon[136032]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T10:12:44.175 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5742/71994 objects degraded (7.976%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:12:44.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:44.354 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:44.600 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:44.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:44.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:44.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:44.780 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:44.861 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:44.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:44.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:44.966 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:45.036 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:45.040 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:44 smithi110 ceph-mon[136032]: pgmap v860: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5401/65337 objects degraded (8.266%); 792 KiB/s, 78 objects/s recovering 2023-10-09T10:12:45.041 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:44 smithi110 ceph-mon[136032]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.041 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:44 smithi110 ceph-mon[136032]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.041 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:44 smithi110 ceph-mon[136032]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T10:12:45.041 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:45.041 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:44 smithi110 ceph-mon[136032]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.153 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:45.154 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:45.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:45.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[147926]: pgmap v860: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5401/65337 objects degraded (8.266%); 792 KiB/s, 78 objects/s recovering 2023-10-09T10:12:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[147926]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[147926]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[147926]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T10:12:45.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:45.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[147926]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[152263]: pgmap v860: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5401/65337 objects degraded (8.266%); 792 KiB/s, 78 objects/s recovering 2023-10-09T10:12:45.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[152263]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[152263]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[152263]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T10:12:45.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:45.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:44 smithi012 ceph-mon[152263]: failed to encode map e311 with expected crc 2023-10-09T10:12:45.214 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:12:45.235 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:197741 8.c e311/271 8:346f0baa:::benchmark_data_smithi012_271002_object89058:head v 311'12307, mlcod=311'12304)", 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:45.231279+0000", 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0040175499999999999, 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040480960000000002, 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.231279+0000", 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.231279+0000", 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.231277+0000", 2023-10-09T10:12:45.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.079e-05 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.231288+0000", 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4285e-05 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.231302+0000", 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8510000000000001e-06 2023-10-09T10:12:45.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.231309+0000", 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7212999999999997e-05 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.231357+0000", 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1053999999999999e-05 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:45.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.231378+0000", 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8913000000000007e-05 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:197744 8.5 8:a901a93e:::benchmark_data_smithi012_271002_object89061:head [delete] snapc 0=[] ondisk+write+known_if_redirected e311)", 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:45.233547+0000", 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0017495950000000001, 2023-10-09T10:12:45.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00185046, 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 197744 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233547+0000", 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233547+0000", 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233545+0000", 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4600000000000003e-06 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233553+0000", 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6705e-05 2023-10-09T10:12:45.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233570+0000", 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6939999999999998e-06 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233577+0000", 2023-10-09T10:12:45.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8891000000000002e-05 2023-10-09T10:12:45.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:12:45.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233636+0000", 2023-10-09T10:12:45.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014177800000000001 2023-10-09T10:12:45.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233777+0000", 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015241899999999999 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:45.233930+0000", 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000383118 2023-10-09T10:12:45.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:45.247 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:12:45.247 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:45.247 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:12:45.247 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:12:45.247 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 2 2023-10-09T10:12:45.247 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:12:45.256 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:12:45.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:45.321 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:12:45.347 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:45.386 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:12:45.435 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:12:45.474 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:12:45.496 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:45.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:45.522 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:12:45.563 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:12:45.588 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:12:45.588 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:12:45.588 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:12:45.588 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:12:45.588 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:12:45.588 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:12:45.588 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:12:45.588 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:12:45.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:45.660 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:12:45.708 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:45.825 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:45.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:45.931 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:45.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:45.952 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:12:46.031 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:46.124 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:46.134 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/241465268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:46.134 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/241465268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:46.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:46.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:46.198 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/241465268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:46.238 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:46.320 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:46.376 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:46.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:46.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:46.414 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:46.490 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:46.537 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:46.655 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:46.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:46.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:46.761 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:46.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:46.883 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:46.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:46.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:46.907 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":862,"stamp":"2023-10-09T10:12:45.432946+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1368594360,"num_objects":20899,"num_object_clones":0,"num_object_copies":62697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5401,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20899,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":196747,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5325,"num_bytes_recovered":311175884,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142181,"ondisk_log_size":142181,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7296392,"kb_used_data":4693384,"kb_used_omap":167,"kb_used_meta":2602520,"kb_avail":742664824,"statfs":{"total":767960285184,"available":760488779776,"internally_reserved":0,"allocated":4806025216,"data_stored":4799562266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2664981081},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,6,5,5,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-215220224,"num_objects":-3284,"num_object_clones":0,"num_object_copies":-9852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-341,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3284,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":341,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.005303"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":152,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877341+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:12:40.877341+0000","last_peered":"2023-10-09T10:12:40.877341+0000","last_clean":"2023-10-09T10:12:40.877341+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:12:40.877341+0000","last_undegraded":"2023-10-09T10:12:40.877341+0000","last_fullsized":"2023-10-09T10:12:40.877341+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":267,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.878889+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:12:40.878889+0000","last_peered":"2023-10-09T10:12:40.878889+0000","last_clean":"2023-10-09T10:12:40.878889+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:12:40.878889+0000","last_undegraded":"2023-10-09T10:12:40.878889+0000","last_fullsized":"2023-10-09T10:12:40.878889+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":313,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877924+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:12:40.877924+0000","last_peered":"2023-10-09T10:12:40.877924+0000","last_clean":"2023-10-09T10:12:40.877924+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:12:40.877924+0000","last_undegraded":"2023-10-09T10:12:40.877924+0000","last_fullsized":"2023-10-09T10:12:40.877924+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":176,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:36.832799+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:12:36.832799+0000","last_peered":"2023-10-09T10:12:36.832799+0000","last_clean":"2023-10-09T10:12:36.832799+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:12:36.832799+0000","last_undegraded":"2023-10-09T10:12:36.832799+0000","last_fullsized":"2023-10-09T10:12:36.832799+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":281,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.760304+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:12:32.760304+0000","last_peered":"2023-10-09T10:12:32.760304+0000","last_clean":"2023-10-09T10:12:32.760304+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:12:32.760304+0000","last_undegraded":"2023-10-09T10:12:32.760304+0000","last_fullsized":"2023-10-09T10:12:32.760304+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":128,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.878490+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:12:40.878490+0000","last_peered":"2023-10-09T10:12:40.878490+0000","last_clean":"2023-10-09T10:12:40.878490+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:12:40.878490+0000","last_undegraded":"2023-10-09T10:12:40.878490+0000","last_fullsized":"2023-10-09T10:12:40.878490+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.761666+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:12:32.761666+0000","last_peered":"2023-10-09T10:12:32.761666+0000","last_clean":"2023-10-09T10:12:32.761666+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:12:32.761666+0000","last_undegraded":"2023-10-09T10:12:32.761666+0000","last_fullsized":"2023-10-09T10:12:32.761666+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":262,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.760224+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:12:32.760224+0000","last_peered":"2023-10-09T10:12:32.760224+0000","last_clean":"2023-10-09T10:12:32.760224+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:12:32.760224+0000","last_undegraded":"2023-10-09T10:12:32.760224+0000","last_fullsized":"2023-10-09T10:12:32.760224+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":127,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.878398+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:12:40.878398+0000","last_peered":"2023-10-09T10:12:40.878398+0000","last_clean":"2023-10-09T10:12:40.878398+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:12:40.878398+0000","last_undegraded":"2023-10-09T10:12:40.878398+0000","last_fullsized":"2023-10-09T10:12:40.878398+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":223,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.751582+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:12:32.751582+0000","last_peered":"2023-10-09T10:12:32.751582+0000","last_clean":"2023-10-09T10:12:32.751582+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:12:32.751582+0000","last_undegraded":"2023-10-09T10:12:32.751582+0000","last_fullsized":"2023-10-09T10:12:32.751582+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":349,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.901433+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:12:42.901433+0000","last_peered":"2023-10-09T10:12:42.901433+0000","last_clean":"2023-10-09T10:12:42.901433+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:12:42.901433+0000","last_undegraded":"2023-10-09T10:12:42.901433+0000","last_fullsized":"2023-10-09T10:12:42.901433+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":305,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877198+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:12:40.877198+0000","last_peered":"2023-10-09T10:12:40.877198+0000","last_clean":"2023-10-09T10:12:40.877198+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:12:40.877198+0000","last_undegraded":"2023-10-09T10:12:40.877198+0000","last_fullsized":"2023-10-09T10:12:40.877198+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":198,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.901004+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:12:42.901004+0000","last_peered":"2023-10-09T10:12:42.901004+0000","last_clean":"2023-10-09T10:12:42.901004+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:12:42.901004+0000","last_undegraded":"2023-10-09T10:12:42.901004+0000","last_fullsized":"2023-10-09T10:12:42.901004+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":177,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877579+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:12:40.877579+0000","last_peered":"2023-10-09T10:12:40.877579+0000","last_clean":"2023-10-09T10:12:40.877579+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:12:40.877579+0000","last_undegraded":"2023-10-09T10:12:40.877579+0000","last_fullsized":"2023-10-09T10:12:40.877579+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":382,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.900284+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:12:42.900284+0000","last_peered":"2023-10-09T10:12:42.900284+0000","last_clean":"2023-10-09T10:12:42.900284+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:12:42.900284+0000","last_undegraded":"2023-10-09T10:12:42.900284+0000","last_fullsized":"2023-10-09T10:12:42.900284+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":249,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.752794+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:12:32.752794+0000","last_peered":"2023-10-09T10:12:32.752794+0000","last_clean":"2023-10-09T10:12:32.752794+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:12:32.752794+0000","last_undegraded":"2023-10-09T10:12:32.752794+0000","last_fullsized":"2023-10-09T10:12:32.752794+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":175,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.761690+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:12:32.761690+0000","last_peered":"2023-10-09T10:12:32.761690+0000","last_clean":"2023-10-09T10:12:32.761690+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:12:32.761690+0000","last_undegraded":"2023-10-09T10:12:32.761690+0000","last_fullsized":"2023-10-09T10:12:32.761690+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":143,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.878348+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:12:40.878348+0000","last_peered":"2023-10-09T10:12:40.878348+0000","last_clean":"2023-10-09T10:12:40.878348+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:12:40.878348+0000","last_undegraded":"2023-10-09T10:12:40.878348+0000","last_fullsized":"2023-10-09T10:12:40.878348+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":179,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.900024+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:12:42.900024+0000","last_peered":"2023-10-09T10:12:42.900024+0000","last_clean":"2023-10-09T10:12:42.900024+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:12:42.900024+0000","last_undegraded":"2023-10-09T10:12:42.900024+0000","last_fullsized":"2023-10-09T10:12:42.900024+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":175,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.762278+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:12:32.762278+0000","last_peered":"2023-10-09T10:12:32.762278+0000","last_clean":"2023-10-09T10:12:32.762278+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:12:32.762278+0000","last_undegraded":"2023-10-09T10:12:32.762278+0000","last_fullsized":"2023-10-09T10:12:32.762278+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":198,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.761289+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:12:32.761289+0000","last_peered":"2023-10-09T10:12:32.761289+0000","last_clean":"2023-10-09T10:12:32.761289+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:12:32.761289+0000","last_undegraded":"2023-10-09T10:12:32.761289+0000","last_fullsized":"2023-10-09T10:12:32.761289+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":262,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.751106+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:12:32.751106+0000","last_peered":"2023-10-09T10:12:32.751106+0000","last_clean":"2023-10-09T10:12:32.751106+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:12:32.751106+0000","last_undegraded":"2023-10-09T10:12:32.751106+0000","last_fullsized":"2023-10-09T10:12:32.751106+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"310'12426","reported_seq":12525,"reported_epoch":310,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:42.996865+0000","last_change":"2023-10-09T10:12:42.183681+0000","last_active":"2023-10-09T10:12:42.996865+0000","last_peered":"2023-10-09T10:12:42.996865+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:12:42.996865+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"280'2400","ondisk_log_start":"280'2400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10026,"log_dups_size":0,"ondisk_log_size":10026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81264640,"num_objects":1240,"num_object_clones":0,"num_object_copies":3720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5351,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12426,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-4111}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":343,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.900661+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:12:42.900661+0000","last_peered":"2023-10-09T10:12:42.900661+0000","last_clean":"2023-10-09T10:12:42.900661+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:12:42.900661+0000","last_undegraded":"2023-10-09T10:12:42.900661+0000","last_fullsized":"2023-10-09T10:12:42.900661+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.876623+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:12:40.876623+0000","last_peered":"2023-10-09T10:12:40.876623+0000","last_clean":"2023-10-09T10:12:40.876623+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:12:40.876623+0000","last_undegraded":"2023-10-09T10:12:40.876623+0000","last_fullsized":"2023-10-09T10:12:40.876623+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"311'12490","reported_seq":12538,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:45.065547+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:12:45.065547+0000","last_peered":"2023-10-09T10:12:45.065547+0000","last_clean":"2023-10-09T10:12:45.065547+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:12:45.065547+0000","last_undegraded":"2023-10-09T10:12:45.065547+0000","last_fullsized":"2023-10-09T10:12:45.065547+0000","mapping_epoch":271,"log_start":"283'3900","ondisk_log_start":"283'3900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8590,"log_dups_size":0,"ondisk_log_size":8590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85196800,"num_objects":1300,"num_object_clones":0,"num_object_copies":3900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12490,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":331,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.900650+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:12:42.900650+0000","last_peered":"2023-10-09T10:12:42.900650+0000","last_clean":"2023-10-09T10:12:42.900650+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:12:42.900650+0000","last_undegraded":"2023-10-09T10:12:42.900650+0000","last_fullsized":"2023-10-09T10:12:42.900650+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":222,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.907570+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:12:42.907570+0000","last_peered":"2023-10-09T10:12:42.907570+0000","last_clean":"2023-10-09T10:12:42.907570+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:12:42.907570+0000","last_undegraded":"2023-10-09T10:12:42.907570+0000","last_fullsized":"2023-10-09T10:12:42.907570+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"309'12202","reported_seq":12250,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:41.812540+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:12:41.812540+0000","last_peered":"2023-10-09T10:12:41.812540+0000","last_clean":"2023-10-09T10:12:41.812540+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:12:41.812540+0000","last_undegraded":"2023-10-09T10:12:41.812540+0000","last_fullsized":"2023-10-09T10:12:41.812540+0000","mapping_epoch":271,"log_start":"280'2800","ondisk_log_start":"280'2800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9402,"log_dups_size":0,"ondisk_log_size":9402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89391104,"num_objects":1364,"num_object_clones":0,"num_object_copies":4092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12202,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":132,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.878431+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:12:40.878431+0000","last_peered":"2023-10-09T10:12:40.878431+0000","last_clean":"2023-10-09T10:12:40.878431+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:12:40.878431+0000","last_undegraded":"2023-10-09T10:12:40.878431+0000","last_fullsized":"2023-10-09T10:12:40.878431+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":154,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.908923+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:12:42.908923+0000","last_peered":"2023-10-09T10:12:42.908923+0000","last_clean":"2023-10-09T10:12:42.908923+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:12:42.908923+0000","last_undegraded":"2023-10-09T10:12:42.908923+0000","last_fullsized":"2023-10-09T10:12:42.908923+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"310'12160","reported_seq":12209,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.212407+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:12:43.212407+0000","last_peered":"2023-10-09T10:12:43.212407+0000","last_clean":"2023-10-09T10:12:43.212407+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:12:43.212407+0000","last_undegraded":"2023-10-09T10:12:43.212407+0000","last_fullsized":"2023-10-09T10:12:43.212407+0000","mapping_epoch":271,"log_start":"284'4400","ondisk_log_start":"284'4400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7760,"log_dups_size":0,"ondisk_log_size":7760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81920000,"num_objects":1250,"num_object_clones":0,"num_object_copies":3750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12160,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":143,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.878530+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:12:40.878530+0000","last_peered":"2023-10-09T10:12:40.878530+0000","last_clean":"2023-10-09T10:12:40.878530+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:12:40.878530+0000","last_undegraded":"2023-10-09T10:12:40.878530+0000","last_fullsized":"2023-10-09T10:12:40.878530+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":176,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.761878+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:12:32.761878+0000","last_peered":"2023-10-09T10:12:32.761878+0000","last_clean":"2023-10-09T10:12:32.761878+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:12:32.761878+0000","last_undegraded":"2023-10-09T10:12:32.761878+0000","last_fullsized":"2023-10-09T10:12:32.761878+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"309'12466","reported_seq":12514,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:41.451666+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:12:41.451666+0000","last_peered":"2023-10-09T10:12:41.451666+0000","last_clean":"2023-10-09T10:12:41.451666+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:12:41.451666+0000","last_undegraded":"2023-10-09T10:12:41.451666+0000","last_fullsized":"2023-10-09T10:12:41.451666+0000","mapping_epoch":271,"log_start":"283'4300","ondisk_log_start":"283'4300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8166,"log_dups_size":0,"ondisk_log_size":8166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90046464,"num_objects":1374,"num_object_clones":0,"num_object_copies":4122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12466,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":303,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.750742+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:12:32.750742+0000","last_peered":"2023-10-09T10:12:32.750742+0000","last_clean":"2023-10-09T10:12:32.750742+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:12:32.750742+0000","last_undegraded":"2023-10-09T10:12:32.750742+0000","last_fullsized":"2023-10-09T10:12:32.750742+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":173,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.752604+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:12:32.752604+0000","last_peered":"2023-10-09T10:12:32.752604+0000","last_clean":"2023-10-09T10:12:32.752604+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:12:32.752604+0000","last_undegraded":"2023-10-09T10:12:32.752604+0000","last_fullsized":"2023-10-09T10:12:32.752604+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"310'12481","reported_seq":12530,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.252622+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:12:43.252622+0000","last_peered":"2023-10-09T10:12:43.252622+0000","last_clean":"2023-10-09T10:12:43.252622+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:12:43.252622+0000","last_undegraded":"2023-10-09T10:12:43.252622+0000","last_fullsized":"2023-10-09T10:12:43.252622+0000","mapping_epoch":271,"log_start":"285'4700","ondisk_log_start":"285'4700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7781,"log_dups_size":0,"ondisk_log_size":7781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80150528,"num_objects":1223,"num_object_clones":0,"num_object_copies":3669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12481,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":235,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877123+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:12:40.877123+0000","last_peered":"2023-10-09T10:12:40.877123+0000","last_clean":"2023-10-09T10:12:40.877123+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:12:40.877123+0000","last_undegraded":"2023-10-09T10:12:40.877123+0000","last_fullsized":"2023-10-09T10:12:40.877123+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":241,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877525+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:12:40.877525+0000","last_peered":"2023-10-09T10:12:40.877525+0000","last_clean":"2023-10-09T10:12:40.877525+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:12:40.877525+0000","last_undegraded":"2023-10-09T10:12:40.877525+0000","last_fullsized":"2023-10-09T10:12:40.877525+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"309'12073","reported_seq":12121,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:41.803529+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:12:41.803529+0000","last_peered":"2023-10-09T10:12:41.803529+0000","last_clean":"2023-10-09T10:12:41.803529+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:12:41.803529+0000","last_undegraded":"2023-10-09T10:12:41.803529+0000","last_fullsized":"2023-10-09T10:12:41.803529+0000","mapping_epoch":271,"log_start":"280'2600","ondisk_log_start":"280'2600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9473,"log_dups_size":0,"ondisk_log_size":9473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89063424,"num_objects":1359,"num_object_clones":0,"num_object_copies":4077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12073,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":205,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.899937+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:12:42.899937+0000","last_peered":"2023-10-09T10:12:42.899937+0000","last_clean":"2023-10-09T10:12:42.899937+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:12:42.899937+0000","last_undegraded":"2023-10-09T10:12:42.899937+0000","last_fullsized":"2023-10-09T10:12:42.899937+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":177,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877165+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:12:40.877165+0000","last_peered":"2023-10-09T10:12:40.877165+0000","last_clean":"2023-10-09T10:12:40.877165+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:12:40.877165+0000","last_undegraded":"2023-10-09T10:12:40.877165+0000","last_fullsized":"2023-10-09T10:12:40.877165+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"310'12335","reported_seq":12384,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.151479+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:12:43.151479+0000","last_peered":"2023-10-09T10:12:43.151479+0000","last_clean":"2023-10-09T10:12:43.151479+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:12:43.151479+0000","last_undegraded":"2023-10-09T10:12:43.151479+0000","last_fullsized":"2023-10-09T10:12:43.151479+0000","mapping_epoch":271,"log_start":"284'4600","ondisk_log_start":"284'4600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7735,"log_dups_size":0,"ondisk_log_size":7735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82771968,"num_objects":1263,"num_object_clones":0,"num_object_copies":3789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12335,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":263,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.760147+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:12:32.760147+0000","last_peered":"2023-10-09T10:12:32.760147+0000","last_clean":"2023-10-09T10:12:32.760147+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:12:32.760147+0000","last_undegraded":"2023-10-09T10:12:32.760147+0000","last_fullsized":"2023-10-09T10:12:32.760147+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":314,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877945+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:12:40.877945+0000","last_peered":"2023-10-09T10:12:40.877945+0000","last_clean":"2023-10-09T10:12:40.877945+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:12:40.877945+0000","last_undegraded":"2023-10-09T10:12:40.877945+0000","last_fullsized":"2023-10-09T10:12:40.877945+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"309'12209","reported_seq":12257,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:41.777332+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:12:41.777332+0000","last_peered":"2023-10-09T10:12:41.777332+0000","last_clean":"2023-10-09T10:12:41.777332+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:12:41.777332+0000","last_undegraded":"2023-10-09T10:12:41.777332+0000","last_fullsized":"2023-10-09T10:12:41.777332+0000","mapping_epoch":271,"log_start":"280'2800","ondisk_log_start":"280'2800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9409,"log_dups_size":0,"ondisk_log_size":9409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82771968,"num_objects":1263,"num_object_clones":0,"num_object_copies":3789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12209,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":313,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877456+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:12:40.877456+0000","last_peered":"2023-10-09T10:12:40.877456+0000","last_clean":"2023-10-09T10:12:40.877456+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:12:40.877456+0000","last_undegraded":"2023-10-09T10:12:40.877456+0000","last_fullsized":"2023-10-09T10:12:40.877456+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":152,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.878265+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:12:40.878265+0000","last_peered":"2023-10-09T10:12:40.878265+0000","last_clean":"2023-10-09T10:12:40.878265+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:12:40.878265+0000","last_undegraded":"2023-10-09T10:12:40.878265+0000","last_fullsized":"2023-10-09T10:12:40.878265+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":270,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.906823+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:12:42.906823+0000","last_peered":"2023-10-09T10:12:42.906823+0000","last_clean":"2023-10-09T10:12:42.906823+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:12:42.906823+0000","last_undegraded":"2023-10-09T10:12:42.906823+0000","last_fullsized":"2023-10-09T10:12:42.906823+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"310'12302","reported_seq":12349,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.949503+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:12:42.949503+0000","last_peered":"2023-10-09T10:12:42.949503+0000","last_clean":"2023-10-09T10:12:42.949503+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:12:42.949503+0000","last_undegraded":"2023-10-09T10:12:42.949503+0000","last_fullsized":"2023-10-09T10:12:42.949503+0000","mapping_epoch":271,"log_start":"280'2300","ondisk_log_start":"280'2300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83361792,"num_objects":1272,"num_object_clones":0,"num_object_copies":3816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12302,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":128,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.879904+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:12:40.879904+0000","last_peered":"2023-10-09T10:12:40.879904+0000","last_clean":"2023-10-09T10:12:40.879904+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:12:40.879904+0000","last_undegraded":"2023-10-09T10:12:40.879904+0000","last_fullsized":"2023-10-09T10:12:40.879904+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.761577+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:12:32.761577+0000","last_peered":"2023-10-09T10:12:32.761577+0000","last_clean":"2023-10-09T10:12:32.761577+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:12:32.761577+0000","last_undegraded":"2023-10-09T10:12:32.761577+0000","last_fullsized":"2023-10-09T10:12:32.761577+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":336,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877377+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:12:40.877377+0000","last_peered":"2023-10-09T10:12:40.877377+0000","last_clean":"2023-10-09T10:12:40.877377+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:12:40.877377+0000","last_undegraded":"2023-10-09T10:12:40.877377+0000","last_fullsized":"2023-10-09T10:12:40.877377+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"309'12172","reported_seq":12220,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:41.531932+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:12:41.531932+0000","last_peered":"2023-10-09T10:12:41.531932+0000","last_clean":"2023-10-09T10:12:41.531932+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:12:41.531932+0000","last_undegraded":"2023-10-09T10:12:41.531932+0000","last_fullsized":"2023-10-09T10:12:41.531932+0000","mapping_epoch":271,"log_start":"283'4000","ondisk_log_start":"283'4000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89784320,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12172,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":229,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.751418+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:12:32.751418+0000","last_peered":"2023-10-09T10:12:32.751418+0000","last_clean":"2023-10-09T10:12:32.751418+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:12:32.751418+0000","last_undegraded":"2023-10-09T10:12:32.751418+0000","last_fullsized":"2023-10-09T10:12:32.751418+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":305,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877093+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:12:40.877093+0000","last_peered":"2023-10-09T10:12:40.877093+0000","last_clean":"2023-10-09T10:12:40.877093+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:12:40.877093+0000","last_undegraded":"2023-10-09T10:12:40.877093+0000","last_fullsized":"2023-10-09T10:12:40.877093+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":326,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877850+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:12:40.877850+0000","last_peered":"2023-10-09T10:12:40.877850+0000","last_clean":"2023-10-09T10:12:40.877850+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:12:40.877850+0000","last_undegraded":"2023-10-09T10:12:40.877850+0000","last_fullsized":"2023-10-09T10:12:40.877850+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"310'12180","reported_seq":12227,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.996907+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:12:42.996907+0000","last_peered":"2023-10-09T10:12:42.996907+0000","last_clean":"2023-10-09T10:12:42.996907+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:12:42.996907+0000","last_undegraded":"2023-10-09T10:12:42.996907+0000","last_fullsized":"2023-10-09T10:12:42.996907+0000","mapping_epoch":271,"log_start":"280'2100","ondisk_log_start":"280'2100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83886080,"num_objects":1280,"num_object_clones":0,"num_object_copies":3840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12180,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":137,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.874276+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:12:40.874276+0000","last_peered":"2023-10-09T10:12:40.874276+0000","last_clean":"2023-10-09T10:12:40.874276+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:12:40.874276+0000","last_undegraded":"2023-10-09T10:12:40.874276+0000","last_fullsized":"2023-10-09T10:12:40.874276+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":557,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.900706+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:12:42.900706+0000","last_peered":"2023-10-09T10:12:42.900706+0000","last_clean":"2023-10-09T10:12:42.900706+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:12:42.900706+0000","last_undegraded":"2023-10-09T10:12:42.900706+0000","last_fullsized":"2023-10-09T10:12:42.900706+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":213,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.750369+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:12:32.750369+0000","last_peered":"2023-10-09T10:12:32.750369+0000","last_clean":"2023-10-09T10:12:32.750369+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:12:32.750369+0000","last_undegraded":"2023-10-09T10:12:32.750369+0000","last_fullsized":"2023-10-09T10:12:32.750369+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":226,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.901520+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:12:42.901520+0000","last_peered":"2023-10-09T10:12:42.901520+0000","last_clean":"2023-10-09T10:12:42.901520+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:12:42.901520+0000","last_undegraded":"2023-10-09T10:12:42.901520+0000","last_fullsized":"2023-10-09T10:12:42.901520+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"310'12378","reported_seq":12427,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.202438+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:12:43.202438+0000","last_peered":"2023-10-09T10:12:43.202438+0000","last_clean":"2023-10-09T10:12:43.202438+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:12:43.202438+0000","last_undegraded":"2023-10-09T10:12:43.202438+0000","last_fullsized":"2023-10-09T10:12:43.202438+0000","mapping_epoch":271,"log_start":"284'4600","ondisk_log_start":"284'4600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7778,"log_dups_size":0,"ondisk_log_size":7778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81526784,"num_objects":1244,"num_object_clones":0,"num_object_copies":3732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12378,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":286,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877229+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:12:40.877229+0000","last_peered":"2023-10-09T10:12:40.877229+0000","last_clean":"2023-10-09T10:12:40.877229+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:12:40.877229+0000","last_undegraded":"2023-10-09T10:12:40.877229+0000","last_fullsized":"2023-10-09T10:12:40.877229+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":186,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.762151+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:12:32.762151+0000","last_peered":"2023-10-09T10:12:32.762151+0000","last_clean":"2023-10-09T10:12:32.762151+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:12:32.762151+0000","last_undegraded":"2023-10-09T10:12:32.762151+0000","last_fullsized":"2023-10-09T10:12:32.762151+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":214,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.762118+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:12:32.762118+0000","last_peered":"2023-10-09T10:12:32.762118+0000","last_clean":"2023-10-09T10:12:32.762118+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:12:32.762118+0000","last_undegraded":"2023-10-09T10:12:32.762118+0000","last_fullsized":"2023-10-09T10:12:32.762118+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"309'12256","reported_seq":22780,"reported_epoch":309,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:41.536783+0000","last_change":"2023-10-09T10:09:48.467899+0000","last_active":"2023-10-09T10:12:41.536783+0000","last_peered":"2023-10-09T10:12:41.536783+0000","last_clean":"2023-10-09T10:09:45.909925+0000","last_became_active":"2023-10-09T10:09:48.399527+0000","last_became_peered":"2023-10-09T10:09:48.399527+0000","last_unstale":"2023-10-09T10:12:41.536783+0000","last_undegraded":"2023-10-09T10:09:48.027886+0000","last_fullsized":"2023-10-09T10:09:48.003678+0000","mapping_epoch":288,"log_start":"283'4100","ondisk_log_start":"283'4100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:06.333603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91881472,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":50,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12256,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5234,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":1352}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":330,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.901273+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:12:42.901273+0000","last_peered":"2023-10-09T10:12:42.901273+0000","last_clean":"2023-10-09T10:12:42.901273+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:12:42.901273+0000","last_undegraded":"2023-10-09T10:12:42.901273+0000","last_fullsized":"2023-10-09T10:12:42.901273+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":342,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877535+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:12:40.877535+0000","last_peered":"2023-10-09T10:12:40.877535+0000","last_clean":"2023-10-09T10:12:40.877535+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:12:40.877535+0000","last_undegraded":"2023-10-09T10:12:40.877535+0000","last_fullsized":"2023-10-09T10:12:40.877535+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":178,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.761692+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:12:32.761692+0000","last_peered":"2023-10-09T10:12:32.761692+0000","last_clean":"2023-10-09T10:12:32.761692+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:12:32.761692+0000","last_undegraded":"2023-10-09T10:12:32.761692+0000","last_fullsized":"2023-10-09T10:12:32.761692+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"309'12200","reported_seq":12248,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:41.828658+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:12:41.828658+0000","last_peered":"2023-10-09T10:12:41.828658+0000","last_clean":"2023-10-09T10:12:41.828658+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:12:41.828658+0000","last_undegraded":"2023-10-09T10:12:41.828658+0000","last_fullsized":"2023-10-09T10:12:41.828658+0000","mapping_epoch":271,"log_start":"280'2800","ondisk_log_start":"280'2800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9400,"log_dups_size":0,"ondisk_log_size":9400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88997888,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12200,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":272,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.751186+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:12:32.751186+0000","last_peered":"2023-10-09T10:12:32.751186+0000","last_clean":"2023-10-09T10:12:32.751186+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:12:32.751186+0000","last_undegraded":"2023-10-09T10:12:32.751186+0000","last_fullsized":"2023-10-09T10:12:32.751186+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":287,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877720+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:12:40.877720+0000","last_peered":"2023-10-09T10:12:40.877720+0000","last_clean":"2023-10-09T10:12:40.877720+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:12:40.877720+0000","last_undegraded":"2023-10-09T10:12:40.877720+0000","last_fullsized":"2023-10-09T10:12:40.877720+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":261,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.760061+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:12:32.760061+0000","last_peered":"2023-10-09T10:12:32.760061+0000","last_clean":"2023-10-09T10:12:32.760061+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:12:32.760061+0000","last_undegraded":"2023-10-09T10:12:32.760061+0000","last_fullsized":"2023-10-09T10:12:32.760061+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"310'12157","reported_seq":12205,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.892881+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:12:42.892881+0000","last_peered":"2023-10-09T10:12:42.892881+0000","last_clean":"2023-10-09T10:12:42.892881+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:12:42.892881+0000","last_undegraded":"2023-10-09T10:12:42.892881+0000","last_fullsized":"2023-10-09T10:12:42.892881+0000","mapping_epoch":271,"log_start":"280'2100","ondisk_log_start":"280'2100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10057,"log_dups_size":0,"ondisk_log_size":10057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85983256,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12157,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":219,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:12:42.909425+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:12:42.909425+0000","last_peered":"2023-10-09T10:12:42.909425+0000","last_clean":"2023-10-09T10:12:42.909425+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:12:42.909425+0000","last_undegraded":"2023-10-09T10:12:42.909425+0000","last_fullsized":"2023-10-09T10:12:42.909425+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":180,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.762000+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:12:32.762000+0000","last_peered":"2023-10-09T10:12:32.762000+0000","last_clean":"2023-10-09T10:12:32.762000+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:12:32.762000+0000","last_undegraded":"2023-10-09T10:12:32.762000+0000","last_fullsized":"2023-10-09T10:12:32.762000+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":163,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.874291+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:12:40.874291+0000","last_peered":"2023-10-09T10:12:40.874291+0000","last_clean":"2023-10-09T10:12:40.874291+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:12:40.874291+0000","last_undegraded":"2023-10-09T10:12:40.874291+0000","last_fullsized":"2023-10-09T10:12:40.874291+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":175,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.761431+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:12:32.761431+0000","last_peered":"2023-10-09T10:12:32.761431+0000","last_clean":"2023-10-09T10:12:32.761431+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:12:32.761431+0000","last_undegraded":"2023-10-09T10:12:32.761431+0000","last_fullsized":"2023-10-09T10:12:32.761431+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":248,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.877612+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:12:40.877612+0000","last_peered":"2023-10-09T10:12:40.877612+0000","last_clean":"2023-10-09T10:12:40.877612+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:12:40.877612+0000","last_undegraded":"2023-10-09T10:12:40.877612+0000","last_fullsized":"2023-10-09T10:12:40.877612+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":147,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.878437+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:12:40.878437+0000","last_peered":"2023-10-09T10:12:40.878437+0000","last_clean":"2023-10-09T10:12:40.878437+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:12:40.878437+0000","last_undegraded":"2023-10-09T10:12:40.878437+0000","last_fullsized":"2023-10-09T10:12:40.878437+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":188,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.885198+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:12:40.885198+0000","last_peered":"2023-10-09T10:12:40.885198+0000","last_clean":"2023-10-09T10:12:40.885198+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:12:40.885198+0000","last_undegraded":"2023-10-09T10:12:40.885198+0000","last_fullsized":"2023-10-09T10:12:40.885198+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":147,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.874470+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:12:40.874470+0000","last_peered":"2023-10-09T10:12:40.874470+0000","last_clean":"2023-10-09T10:12:40.874470+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:12:40.874470+0000","last_undegraded":"2023-10-09T10:12:40.874470+0000","last_fullsized":"2023-10-09T10:12:40.874470+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":197,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.884891+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:12:40.884891+0000","last_peered":"2023-10-09T10:12:40.884891+0000","last_clean":"2023-10-09T10:12:40.884891+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:12:40.884891+0000","last_undegraded":"2023-10-09T10:12:40.884891+0000","last_fullsized":"2023-10-09T10:12:40.884891+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":326,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.751310+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:12:32.751310+0000","last_peered":"2023-10-09T10:12:32.751310+0000","last_clean":"2023-10-09T10:12:32.751310+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:12:32.751310+0000","last_undegraded":"2023-10-09T10:12:32.751310+0000","last_fullsized":"2023-10-09T10:12:32.751310+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":347,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.751761+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:12:32.751761+0000","last_peered":"2023-10-09T10:12:32.751761+0000","last_clean":"2023-10-09T10:12:32.751761+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:12:32.751761+0000","last_undegraded":"2023-10-09T10:12:32.751761+0000","last_fullsized":"2023-10-09T10:12:32.751761+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":256,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:12:32.761425+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:12:32.761425+0000","last_peered":"2023-10-09T10:12:32.761425+0000","last_clean":"2023-10-09T10:12:32.761425+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:12:32.761425+0000","last_undegraded":"2023-10-09T10:12:32.761425+0000","last_fullsized":"2023-10-09T10:12:32.761425+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":177,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:12:40.876928+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:12:40.876928+0000","last_peered":"2023-10-09T10:12:40.876928+0000","last_clean":"2023-10-09T10:12:40.876928+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:12:40.876928+0000","last_undegraded":"2023-10-09T10:12:40.876928+0000","last_fullsized":"2023-10-09T10:12:40.876928+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1367998488,"num_objects":20875,"num_object_clones":0,"num_object_copies":62625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5401,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20875,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":196487,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5256,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4730662912,"data_stored":4730650696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141987,"ondisk_log_size":141987,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430517,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":716456,"kb_used_data":406944,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93028696,"statfs":{"total":95995035648,"available":95261384704,"internally_reserved":0,"allocated":416710656,"data_stored":416169665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.3130000000000002,"15min":2.0539999999999998},"min":{"1min":0.27400000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":11.164,"5min":11.164,"15min":50.591999999999999},"last":7.54},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.2770000000000001,"15min":2.0059999999999998},"min":{"1min":0.32600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.7880000000000003,"5min":15.974,"15min":50.747},"last":0.78300000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:12:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9940000000000002,"5min":2.3639999999999999,"15min":1.958},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":11.92,"5min":14.311,"15min":25.338000000000001},"last":0.26400000000000001},{"interface":"front","average":{"1min":2.6320000000000001,"5min":2.218,"15min":1.6870000000000001},"min":{"1min":0.42999999999999999,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":10.663,"5min":18.27,"15min":18.27},"last":0.51800000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":3.7639999999999998,"5min":1.9970000000000001,"15min":2.359},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":13.775,"5min":13.775,"15min":13.775},"last":0.29499999999999998},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.008,"15min":2.157},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.101000000000001,"5min":12.101000000000001,"15min":12.101000000000001},"last":5.5650000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4550000000000001,"5min":2.629,"15min":2.0430000000000001},"min":{"1min":0.436,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.6289999999999996,"5min":29.058,"15min":29.058},"last":3.206},{"interface":"front","average":{"1min":3.6629999999999998,"5min":2.2559999999999998,"15min":2.0489999999999999},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":1.5489999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.288,"15min":1.0369999999999999},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":7.7469999999999999,"5min":7.7469999999999999,"15min":7.7469999999999999},"last":2.2970000000000002},{"interface":"front","average":{"1min":1.724,"5min":1.139,"15min":0.98499999999999999},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":0.84199999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3260000000000001,"5min":1.403,"15min":1.1259999999999999},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":7.8159999999999998,"5min":25.277999999999999,"15min":25.277999999999999},"last":6.2489999999999997},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.2110000000000001,"15min":1.0580000000000001},"min":{"1min":0.38500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.806,"5min":25.754000000000001,"15min":25.754000000000001},"last":3.3460000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":1.0660000000000001,"15min":1.006},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.327,"5min":0.86899999999999999,"15min":0.91900000000000004},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":9.1920000000000002},"last":0.54700000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626745,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1060440,"kb_used_data":750928,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92684712,"statfs":{"total":95995035648,"available":94909145088,"internally_reserved":0,"allocated":768950272,"data_stored":768426595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":1.919,"15min":1.6060000000000001},"min":{"1min":0.252,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":6.9630000000000001,"5min":15.435,"15min":15.435},"last":2.8260000000000001},{"interface":"front","average":{"1min":2.2469999999999999,"5min":1.921,"15min":1.591},"min":{"1min":0.33900000000000002,"5min":0.252,"15min":0.23999999999999999},"max":{"1min":8.2959999999999994,"5min":15.308999999999999,"15min":15.308999999999999},"last":2.6269999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:12:06 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":2.839,"15min":2.0579999999999998},"min":{"1min":0.30199999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.0519999999999996,"5min":35.609999999999999,"15min":35.609999999999999},"last":4.5910000000000002},{"interface":"front","average":{"1min":2.919,"5min":2.46,"15min":1.9910000000000001},"min":{"1min":0.36099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":8.2680000000000007,"5min":32.396000000000001,"15min":32.396000000000001},"last":2.988}]},{"osd":2,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":1.637,"15min":1.7929999999999999},"min":{"1min":0.35199999999999998,"5min":0.26900000000000002,"15min":0.26200000000000001},"max":{"1min":7.1280000000000001,"5min":12.898,"15min":12.898},"last":5.0990000000000002},{"interface":"front","average":{"1min":3.4860000000000002,"5min":2.1400000000000001,"15min":3.0379999999999998},"min":{"1min":0.44600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.542999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":2.8599999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:11:50 2023","interfaces":[{"interface":"back","average":{"1min":4.1079999999999997,"5min":2.3639999999999999,"15min":1.694},"min":{"1min":0.432,"5min":0.308,"15min":0.308},"max":{"1min":19.222000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":2.7349999999999999},{"interface":"front","average":{"1min":3.2490000000000001,"5min":2.4500000000000002,"15min":2.4039999999999999},"min":{"1min":0.39500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":8.2270000000000003,"5min":11.211,"15min":11.211},"last":3.504}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.67100000000000004,"15min":0.51700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.8970000000000002,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.372},{"interface":"front","average":{"1min":1.284,"5min":0.75800000000000001,"15min":0.58499999999999996},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.50800000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:12:37 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.0660000000000001,"15min":0.91800000000000004},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.23699999999999999},"max":{"1min":6.3170000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.335,"5min":1.222,"15min":0.99399999999999999},"min":{"1min":0.34699999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":14.465999999999999,"5min":14.465999999999999,"15min":14.465999999999999},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.92600000000000005,"15min":0.89500000000000002},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.0700000000000003,"5min":6.2709999999999999,"15min":7.5350000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.673,"5min":0.97199999999999998,"15min":0.94299999999999995},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.101,"5min":6.3159999999999998,"15min":7.5570000000000004},"last":0.59699999999999998}]}]},{"osd":1,"up_from":84,"seq":360777253058,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1012436,"kb_used_data":576908,"kb_used_omap":9,"kb_used_meta":435510,"kb_avail":92732716,"statfs":{"total":95995035648,"available":94958301184,"internally_reserved":0,"allocated":590753792,"data_stored":589693144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":445963241},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":4.4160000000000004,"5min":2.5880000000000001,"15min":2.4500000000000002},"min":{"1min":0.217,"5min":0.183,"15min":0.183},"max":{"1min":31.007000000000001,"5min":31.007000000000001,"15min":31.007000000000001},"last":2.4940000000000002},{"interface":"front","average":{"1min":3.8010000000000002,"5min":3.4079999999999999,"15min":2.5899999999999999},"min":{"1min":0.28599999999999998,"5min":0.245,"15min":0.192},"max":{"1min":10.577999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":5.1680000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":3.0249999999999999,"15min":3.8069999999999999},"min":{"1min":0.70099999999999996,"5min":0.217,"15min":0.217},"max":{"1min":10.573,"5min":17.369,"15min":17.369},"last":2.2890000000000001},{"interface":"front","average":{"1min":3.3479999999999999,"5min":2.5449999999999999,"15min":3.7530000000000001},"min":{"1min":0.26400000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":13.414999999999999,"5min":13.414999999999999,"15min":16.023},"last":7.8230000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3170000000000002,"5min":3.6110000000000002,"15min":3.5590000000000002},"min":{"1min":0.41699999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.119,"5min":18.587,"15min":18.587},"last":0.88},{"interface":"front","average":{"1min":2.7959999999999998,"5min":3.8029999999999999,"15min":4.5350000000000001},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.103999999999999,"5min":17.289000000000001,"15min":17.289000000000001},"last":1.3899999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.8029999999999999,"15min":1.4370000000000001},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":21.620000000000001,"5min":21.620000000000001,"15min":21.620000000000001},"last":0.505},{"interface":"front","average":{"1min":2.4980000000000002,"5min":2.411,"15min":1.6899999999999999},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":9.2569999999999997,"5min":12.904,"15min":12.904},"last":0.93000000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.4430000000000001,"15min":2.0630000000000002},"min":{"1min":0.34100000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.8599999999999994,"5min":12.811,"15min":15.531000000000001},"last":0.90800000000000003},{"interface":"front","average":{"1min":3.544,"5min":2.2949999999999999,"15min":2.2999999999999998},"min":{"1min":0.53900000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.855,"5min":16.277999999999999,"15min":19.706},"last":19.513000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":2.0139999999999998,"15min":1.9810000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":21.670999999999999,"5min":21.670999999999999,"15min":21.670999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.5219999999999998,"5min":2.4220000000000002,"15min":2.1070000000000002},"min":{"1min":0.36899999999999999,"5min":0.35499999999999998,"15min":0.32000000000000001},"max":{"1min":9.9440000000000008,"5min":12.868,"15min":15.481999999999999},"last":0.95899999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":2.3860000000000001,"15min":2.3330000000000002},"min":{"1min":0.68200000000000005,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.723000000000001,"5min":16.331,"15min":16.331},"last":2.3319999999999999},{"interface":"front","average":{"1min":3.101,"5min":2.0830000000000002,"15min":2.0619999999999998},"min":{"1min":0.42399999999999999,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":21.593,"5min":21.593,"15min":21.593},"last":0.46000000000000002}]}]},{"osd":0,"up_from":79,"seq":339302416584,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900584,"kb_used_data":523872,"kb_used_omap":28,"kb_used_meta":376611,"kb_avail":92844568,"statfs":{"total":95995035648,"available":95072837632,"internally_reserved":0,"allocated":536444928,"data_stored":535417303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":385650512},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":2.6800000000000002,"15min":2.0209999999999999},"min":{"1min":0.26400000000000001,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":16.797000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":7.6070000000000002},{"interface":"front","average":{"1min":7.1150000000000002,"5min":4.2210000000000001,"15min":3.04},"min":{"1min":0.253,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":31.925999999999998,"5min":31.925999999999998,"15min":31.925999999999998},"last":4.3280000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0469999999999997,"5min":2.181,"15min":2.274},"min":{"1min":0.28899999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.003,"5min":18.003,"15min":18.003},"last":7.5880000000000001},{"interface":"front","average":{"1min":2.9660000000000002,"5min":2.0710000000000002,"15min":3.0190000000000001},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.432,"5min":15.529,"15min":15.529},"last":7.4850000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":4.6509999999999998,"15min":4.7930000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":23.151,"5min":25.734999999999999,"15min":25.734999999999999},"last":4.3789999999999996},{"interface":"front","average":{"1min":5.3250000000000002,"5min":4.0739999999999998,"15min":4.298},"min":{"1min":0.33300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":18.279,"5min":21.923999999999999,"15min":21.923999999999999},"last":4.444}]},{"osd":4,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3780000000000001,"5min":2.3079999999999998,"15min":1.6970000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.989000000000001,"5min":20.494,"15min":20.494},"last":7.6390000000000002},{"interface":"front","average":{"1min":3.6819999999999999,"5min":2.395,"15min":2.089},"min":{"1min":0.40200000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.907,"5min":15.907,"15min":15.907},"last":5.5540000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":2.5739999999999998,"15min":2.1869999999999998},"min":{"1min":0.36099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.868,"5min":17.096,"15min":17.096},"last":4.4029999999999996},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.52,"15min":1.4299999999999999},"min":{"1min":0.371,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.371,"5min":10.371,"15min":12.478},"last":7.5549999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.734,"15min":1.5860000000000001},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.31900000000000001},"max":{"1min":8.827,"5min":9.4719999999999995,"15min":11.699},"last":7.5300000000000002},{"interface":"front","average":{"1min":4.5800000000000001,"5min":2.9740000000000002,"15min":2.4660000000000002},"min":{"1min":0.52100000000000002,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":23.106999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":4.5279999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:11:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2679999999999998,"5min":2.6699999999999999,"15min":2.3540000000000001},"min":{"1min":0.437,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":16.460000000000001,"5min":16.460000000000001,"15min":26.978000000000002},"last":4.484},{"interface":"front","average":{"1min":2.677,"5min":1.7490000000000001,"15min":1.738},"min":{"1min":0.45100000000000001,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":8.8040000000000003,"5min":10.122,"15min":26.768000000000001},"last":7.5720000000000001}]}]},{"osd":2,"up_from":217,"seq":932007903318,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":753604,"kb_used_data":482492,"kb_used_omap":83,"kb_used_meta":270892,"kb_avail":92991548,"statfs":{"total":95995035648,"available":95223345152,"internally_reserved":0,"allocated":494071808,"data_stored":493033044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":277393903},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.0609999999999999,"15min":1.8660000000000001},"min":{"1min":0.24299999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.334,"5min":19.372,"15min":19.372},"last":0.66300000000000003},{"interface":"front","average":{"1min":5.5940000000000003,"5min":3.5150000000000001,"15min":3.133},"min":{"1min":0.32000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.047000000000001,"5min":16.754999999999999,"15min":16.754999999999999},"last":0.73199999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":4.6630000000000003,"5min":2.653,"15min":2.6760000000000002},"min":{"1min":0.26500000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.276,"5min":14.276,"15min":14.276},"last":1.155},{"interface":"front","average":{"1min":5.4710000000000001,"5min":3.0760000000000001,"15min":2.823},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.227,"5min":15.569000000000001,"15min":15.569000000000001},"last":1.361}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.2989999999999999,"15min":2.1150000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.359999999999999,"5min":13.851000000000001,"15min":13.851000000000001},"last":2.7370000000000001},{"interface":"front","average":{"1min":4.1879999999999997,"5min":3.2360000000000002,"15min":3.6880000000000002},"min":{"1min":0.40799999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.51,"5min":15.811,"15min":15.811},"last":2.6800000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":1.6200000000000001,"15min":1.0109999999999999},"min":{"1min":0.32000000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.725,"5min":12.725,"15min":12.725},"last":0.70299999999999996},{"interface":"front","average":{"1min":3.0270000000000001,"5min":1.7689999999999999,"15min":1.143},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":15.276,"5min":15.276,"15min":15.276},"last":5.2640000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.8120000000000001,"15min":1.5860000000000001},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1230000000000002,"5min":12.473000000000001,"15min":12.473000000000001},"last":5.2149999999999999},{"interface":"front","average":{"1min":2.044,"5min":1.5980000000000001,"15min":1.706},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.923,"5min":13.587,"15min":13.587},"last":1.7629999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5529999999999999,"5min":1.9219999999999999,"15min":1.2689999999999999},"min":{"1min":0.374,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.484,"5min":15.292999999999999,"15min":15.292999999999999},"last":5.8949999999999996},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.9670000000000001,"15min":1.8180000000000001},"min":{"1min":0.41099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.5679999999999996,"5min":12.502000000000001,"15min":12.502000000000001},"last":5.298}]},{"osd":7,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.9359999999999999,"15min":1.294},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.457000000000001,"5min":15.343999999999999,"15min":15.343999999999999},"last":3.7200000000000002},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.1539999999999999,"15min":1.758},"min":{"1min":0.38600000000000001,"5min":0.376,"15min":0.376},"max":{"1min":9.6039999999999992,"5min":12.433999999999999,"15min":12.433999999999999},"last":5.3239999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824061,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":801276,"kb_used_data":532916,"kb_used_omap":10,"kb_used_meta":268277,"kb_avail":92943876,"statfs":{"total":95995035648,"available":95174529024,"internally_reserved":0,"allocated":545705984,"data_stored":544484297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":274715715},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5139999999999998,"5min":4.0330000000000004,"15min":4.4800000000000004},"min":{"1min":0.27500000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.749000000000001,"5min":23.381,"15min":23.381},"last":9.5960000000000001},{"interface":"front","average":{"1min":6.5919999999999996,"5min":4.5750000000000002,"15min":4.3109999999999999},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.782,"5min":20.239999999999998,"15min":20.239999999999998},"last":1.5609999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2909999999999999,"5min":3.4820000000000002,"15min":3.7959999999999998},"min":{"1min":0.41399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.856999999999999,"5min":20.422999999999998,"15min":20.422999999999998},"last":1.8400000000000001},{"interface":"front","average":{"1min":4.5620000000000003,"5min":3.726,"15min":3.5289999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.73,"5min":23.423999999999999,"15min":23.423999999999999},"last":5.5129999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":3.1269999999999998,"15min":3.536},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.331,"5min":20.594999999999999,"15min":20.594999999999999},"last":14.332000000000001},{"interface":"front","average":{"1min":4.298,"5min":3.1680000000000001,"15min":2.9660000000000002},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":14.802,"5min":23.452000000000002,"15min":23.452000000000002},"last":5.5449999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":2.048,"15min":1.413},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.696999999999999,"5min":15.676,"15min":15.676},"last":5.5640000000000001},{"interface":"front","average":{"1min":4.7910000000000004,"5min":2.2360000000000002,"15min":1.752},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.478999999999999,"5min":16.478999999999999,"15min":16.478999999999999},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":2.9260000000000002,"15min":2.718},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.125999999999999,"5min":13.125999999999999,"15min":13.125999999999999},"last":6.8209999999999997},{"interface":"front","average":{"1min":3.0699999999999998,"5min":2.6600000000000001,"15min":2.456},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":10.891999999999999,"5min":15.992000000000001,"15min":15.992000000000001},"last":1.778}]},{"osd":6,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9649999999999999,"5min":2.9809999999999999,"15min":2.819},"min":{"1min":0.42099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.504999999999999,"5min":16.504999999999999,"15min":16.504999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.921,"5min":2.4239999999999999,"15min":2.544},"min":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.952,"5min":16.16,"15min":16.16},"last":1.8100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.1219999999999999,"15min":1.633},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":11.048999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":5.5830000000000002},{"interface":"front","average":{"1min":5.8399999999999999,"5min":3.073,"15min":2.8069999999999999},"min":{"1min":0.44600000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":16.367000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":6.7809999999999997}]}]},{"osd":4,"up_from":266,"seq":1142461300787,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1087628,"kb_used_data":756676,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":92657524,"statfs":{"total":95995035648,"available":94881304576,"internally_reserved":0,"allocated":774836224,"data_stored":774307417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.0109999999999999,"15min":0.84899999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.4829999999999997,"5min":10.881,"15min":10.881},"last":0.23200000000000001},{"interface":"front","average":{"1min":3.0150000000000001,"5min":1.5629999999999999,"15min":1.161},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":12.246,"5min":12.246,"15min":12.246},"last":2.2440000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2429999999999999,"5min":1.655,"15min":1.0960000000000001},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":12.161,"5min":12.161,"15min":12.161},"last":1.284},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.7290000000000001,"15min":1.4259999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.504,"5min":11.141,"15min":11.141},"last":0.32100000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.4430000000000001,"15min":1.143},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.2469999999999999,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.905,"5min":1.242,"15min":1.018},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":1.2509999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":1.9490000000000001,"15min":1.4910000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.065,"5min":13.065,"15min":13.065},"last":0.93400000000000005},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.766,"15min":1.264},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.218,"5min":12.218,"15min":12.218},"last":1.3200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":0.93200000000000005,"15min":0.79600000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.6869999999999994,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.56100000000000005,"15min":0.61899999999999999},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0609999999999999,"5min":2.6970000000000001,"15min":2.6970000000000001},"last":0.51300000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.69799999999999995,"15min":0.58299999999999996},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.311},"max":{"1min":6.4119999999999999,"5min":6.4119999999999999,"15min":6.4119999999999999},"last":2.7440000000000002},{"interface":"front","average":{"1min":1.7270000000000001,"5min":0.83899999999999997,"15min":0.66900000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.255000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":0.89100000000000001,"15min":0.71099999999999997},"min":{"1min":0.38800000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":12.237,"5min":12.237,"15min":12.237},"last":3.391},{"interface":"front","average":{"1min":1.244,"5min":0.90600000000000003,"15min":0.85399999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.7590000000000003,"5min":9.7590000000000003,"15min":9.7590000000000003},"last":2.8479999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822972,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":963968,"kb_used_data":662648,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92781184,"statfs":{"total":95995035648,"available":95007932416,"internally_reserved":0,"allocated":678551552,"data_stored":678030801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5950000000000002,"5min":1.7989999999999999,"15min":1.7090000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2639999999999993,"5min":8.2639999999999993,"15min":15.773999999999999},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.823,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.4939999999999998,"5min":10.244999999999999,"15min":15.699},"last":1.603}]},{"osd":1,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.1320000000000001,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.891,"5min":11.891,"15min":15.689},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.9350000000000001,"15min":1.823},"min":{"1min":0.41399999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.4489999999999998,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.78300000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.7130000000000001,"5min":1.905,"15min":2.456},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.259,"5min":10.259,"15min":15.640000000000001},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.581,"5min":1.458,"15min":2.1739999999999999},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":4.1619999999999999,"5min":9.1329999999999991,"15min":10.656000000000001},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":2.1829999999999998,"15min":1.96},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2260000000000009,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":4.0190000000000001},{"interface":"front","average":{"1min":3.2370000000000001,"5min":2.4820000000000002,"15min":2.8399999999999999},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.9900000000000002,"5min":17.34,"15min":17.34},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.0089999999999999,"15min":0.71199999999999997},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3419999999999996,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.754,"15min":0.747},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5559999999999992,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":0.50600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.0629999999999999,"15min":1.0820000000000001},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":27.716000000000001},"last":0.625},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.135,"15min":1.004},"min":{"1min":0.32700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2910000000000004,"5min":8.2910000000000004,"15min":18.600999999999999},"last":0.44800000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.137,"15min":1.0389999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4710000000000001,"5min":7.891,"15min":10.334},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.173,"5min":0.80300000000000005,"15min":0.96899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":12.356},"last":0.55000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":521207808,"data_stored":521207808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":585170944,"data_stored":585170944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":478416896,"data_stored":478412824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":534052864,"data_stored":534052864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":766443520,"data_stored":766443520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":672862208,"data_stored":672858136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":760872960,"data_stored":760872960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":411635712,"data_stored":411631640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:12:46.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:12:46.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:46 smithi110 ceph-mon[136032]: pgmap v862: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 5401/62697 objects degraded (8.614%); 0 B/s, 56 objects/s recovering 2023-10-09T10:12:46.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:46 smithi110 ceph-mon[136032]: from='client.55924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:46.988 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:47.026 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:46 smithi012 ceph-mon[152263]: pgmap v862: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 5401/62697 objects degraded (8.614%); 0 B/s, 56 objects/s recovering 2023-10-09T10:12:47.026 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:46 smithi012 ceph-mon[152263]: from='client.55924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:47.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:46 smithi012 ceph-mon[147926]: pgmap v862: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 5401/62697 objects degraded (8.614%); 0 B/s, 56 objects/s recovering 2023-10-09T10:12:47.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:46 smithi012 ceph-mon[147926]: from='client.55924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:47.034 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:47.144 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:47.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:47.252 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:47.306 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:47.420 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:47.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:47.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:47.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:47.617 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:47.674 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:47.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:47.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:47.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:47.786 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:12:47.787 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:12:47.850 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:47.855 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:47.927 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:47.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:47.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:47.986 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:12:48.036 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:48.101 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3323759770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:48.101 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:47 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:48.102 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:12:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:12:47] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T10:12:48.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3323759770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:48.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:47 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:48.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3323759770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:48.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:47 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:12:48.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:12:48.223 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:48.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:48.323 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:12:48.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:48.422 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:48.503 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:48.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:48.613 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:48.704 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:48.759 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:48.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:48.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:48.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:48.922 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:49.024 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:49.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:49.103 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:12:49.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[147926]: pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5351/59058 objects degraded (9.061%); 0 B/s, 59 objects/s recovering 2023-10-09T10:12:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[147926]: failed to encode map e312 with expected crc 2023-10-09T10:12:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[147926]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T10:12:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[147926]: failed to encode map e312 with expected crc 2023-10-09T10:12:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5351/59058 objects degraded (9.061%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[152263]: pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5351/59058 objects degraded (9.061%); 0 B/s, 59 objects/s recovering 2023-10-09T10:12:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[152263]: failed to encode map e312 with expected crc 2023-10-09T10:12:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[152263]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T10:12:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[152263]: failed to encode map e312 with expected crc 2023-10-09T10:12:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5351/59058 objects degraded (9.061%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:49.221 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:49.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:48 smithi110 ceph-mon[136032]: pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5351/59058 objects degraded (9.061%); 0 B/s, 59 objects/s recovering 2023-10-09T10:12:49.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:48 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:49.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:48 smithi110 ceph-mon[136032]: failed to encode map e312 with expected crc 2023-10-09T10:12:49.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:48 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:12:49.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:48 smithi110 ceph-mon[136032]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T10:12:49.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:48 smithi110 ceph-mon[136032]: failed to encode map e312 with expected crc 2023-10-09T10:12:49.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5351/59058 objects degraded (9.061%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:50.491 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:50.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:50.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:50.570 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:12:50.570 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:12:50.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:12:50.690 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:50.770 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:12:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:50.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:12:50.790 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:50.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:50.989 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:51.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:12:51.073 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:50 smithi012 ceph-mon[147926]: pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5167/54759 objects degraded (9.436%); 0 B/s, 39 objects/s recovering 2023-10-09T10:12:51.073 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:50 smithi012 ceph-mon[152263]: pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5167/54759 objects degraded (9.436%); 0 B/s, 39 objects/s recovering 2023-10-09T10:12:51.157 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:51.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:51.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:51.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:51.381 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:50 smithi110 ceph-mon[136032]: pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5167/54759 objects degraded (9.436%); 0 B/s, 39 objects/s recovering 2023-10-09T10:12:51.463 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:51.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:51.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:51.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:51.670 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:51.929 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:51.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:52.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:12:52.084 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:52.135 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:12:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:52.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:12:52.184 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:52.184 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5167/54135 objects degraded (9.545%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1089,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":312,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":18045,"data_bytes":1181554616,"bytes_used":6887444480,"bytes_avail":761072840704,"bytes_total":767960285184,"degraded_objects":5167,"degraded_total":54135,"degraded_ratio":0.095446568763277007,"recovering_objects_per_sec":29,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":234,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":466},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T10:12:45.433085+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 1s)","progress":0.98876404762268066,"add_to_ceph_s":true}}} 2023-10-09T10:12:52.237 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:52.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:52.370 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:52.492 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:52.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:52.600 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:52.611 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:12:52.663 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:52.673 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:12:52.715 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:12:52.717 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:52.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:52.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:52.759 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:12:52.802 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:12:52.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:52.845 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:12:52.887 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:12:52.897 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:12:52.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:12:52.948 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:52.948 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:12:52.991 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:12:53.036 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:53.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:53.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:53.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:12:53.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:12:53.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:12:53.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:12:53.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:12:53.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:12:53.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:12:53.059 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:12:53.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:53.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:52 smithi012 ceph-mon[147926]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 5167/54135 objects degraded (9.545%); 0 B/s, 29 objects/s recovering 2023-10-09T10:12:53.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3737002756' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:53.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3802573260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:53.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:53 smithi012 ceph-mon[152263]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 5167/54135 objects degraded (9.545%); 0 B/s, 29 objects/s recovering 2023-10-09T10:12:53.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3737002756' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:53.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3802573260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:53.273 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:53.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:52 smithi110 ceph-mon[136032]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 5167/54135 objects degraded (9.545%); 0 B/s, 29 objects/s recovering 2023-10-09T10:12:53.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3737002756' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:12:53.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3802573260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:12:53.839 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:53.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:53.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:53.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:54.039 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:54.075 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:54.081 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:54.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:54 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5167/54135 objects degraded (9.545%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:54.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:54 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5167/54135 objects degraded (9.545%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:54.277 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:54.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:54.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:54.387 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:54.428 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:54 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5167/54135 objects degraded (9.545%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:54.438 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:54.480 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:54.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:54.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:54.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:54.615 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:12:54.615 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:203738 8.4 e312/288)", 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:54.610045+0000", 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.002792138, 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028308109999999999, 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610045+0000", 2023-10-09T10:12:54.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610045+0000", 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610044+0000", 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610052+0000", 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2309999999999996e-06 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610064+0000", 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1581e-05 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610072+0000", 2023-10-09T10:12:54.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.055e-06 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:203737 8.f e312/310 8:f6cca675:::benchmark_data_smithi012_271002_object95054:head v 312'12799, mlcod=312'12795)", 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:54.610846+0000", 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0019911130000000001, 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020894400000000001, 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:54.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610846+0000", 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610846+0000", 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610845+0000", 2023-10-09T10:12:54.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610852+0000", 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1949999999999997e-06 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610862+0000", 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0138e-05 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610869+0000", 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9410000000000001e-06 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610930+0000", 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1014000000000002e-05 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.610951+0000", 2023-10-09T10:12:54.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0760000000000001e-05 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:203736 8.4 e312/288 8:2df7acfa:::benchmark_data_smithi012_271002_object95053:head v 312'12849, mlcod=293'7235)", 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:54.604959+0000", 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0078775930000000004, 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0080392680000000005, 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:12:54.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.604959+0000", 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.604959+0000", 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.604959+0000", 2023-10-09T10:12:54.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.604966+0000", 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.52e-06 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.604977+0000", 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0404000000000001e-05 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.604983+0000", 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5270000000000004e-06 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.606807+0000", 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001823994 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.606838+0000", 2023-10-09T10:12:54.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0670000000000003e-05 2023-10-09T10:12:54.627 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:54.627 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:12:54.627 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:54.627 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:12:54.627 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:12:54.627 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 3 2023-10-09T10:12:54.627 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:12:54.724 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:54.799 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:54.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:54.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:54.909 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:54.979 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:55.039 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:55.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:55.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:55.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:12:55.245 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:55.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:12:55.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:55 smithi012 ceph-mon[147926]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 986 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 5023/47367 objects degraded (10.604%); 0 B/s, 39 objects/s recovering 2023-10-09T10:12:55.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:55 smithi012 ceph-mon[152263]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 986 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 5023/47367 objects degraded (10.604%); 0 B/s, 39 objects/s recovering 2023-10-09T10:12:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:55 smithi110 ceph-mon[136032]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 986 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 5023/47367 objects degraded (10.604%); 0 B/s, 39 objects/s recovering 2023-10-09T10:12:55.582 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:12:56.036 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:12:56.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:56.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:12:56.144 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:12:56.198 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:56.256 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:12:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:56.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:12:56.369 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:56.409 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:56.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:56.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:56.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:56.642 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:56.738 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:56.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:56.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:56.849 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:56.892 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:56.899 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:56.952 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:12:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:56.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:12:57.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:57.160 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:12:57.175 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:57.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:57 smithi110 ceph-mon[136032]: pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 924 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 5023/44385 objects degraded (11.317%); 0 B/s, 37 objects/s recovering 2023-10-09T10:12:57.283 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:57.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:57.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:57.379 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:57 smithi012 ceph-mon[147926]: pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 924 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 5023/44385 objects degraded (11.317%); 0 B/s, 37 objects/s recovering 2023-10-09T10:12:57.380 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:57 smithi012 ceph-mon[152263]: pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 924 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 5023/44385 objects degraded (11.317%); 0 B/s, 37 objects/s recovering 2023-10-09T10:12:57.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:57.453 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:12:57.484 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:57.485 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:12:57.567 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:57.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:57.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:57.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:57.730 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:57.916 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:12:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:12:57] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T10:12:58.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2090484694' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:58.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:58.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5023/44385 objects degraded (11.317%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:58.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2090484694' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:58.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:58.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5023/44385 objects degraded (11.317%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:58.179 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:58.182 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:12:58.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:58.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:58.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:58.296 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":868,"stamp":"2023-10-09T10:12:55.441182+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":968562616,"num_objects":14795,"num_object_clones":0,"num_object_copies":44385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5023,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14795,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":202851,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5703,"num_bytes_recovered":311175884,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142285,"ondisk_log_size":142285,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6090668,"kb_used_data":3500204,"kb_used_omap":167,"kb_used_meta":2590232,"kb_avail":743870548,"statfs":{"total":767960285184,"available":761723441152,"internally_reserved":0,"allocated":3584208896,"data_stored":3577715105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2652398169},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-400031744,"num_objects":-6104,"num_object_clones":0,"num_object_copies":-18312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-378,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6104,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":378,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008235"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":154,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912184+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:12:43.912184+0000","last_peered":"2023-10-09T10:12:43.912184+0000","last_clean":"2023-10-09T10:12:43.912184+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:12:43.912184+0000","last_undegraded":"2023-10-09T10:12:43.912184+0000","last_fullsized":"2023-10-09T10:12:43.912184+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":270,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909912+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:12:43.909912+0000","last_peered":"2023-10-09T10:12:43.909912+0000","last_clean":"2023-10-09T10:12:43.909912+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:12:43.909912+0000","last_undegraded":"2023-10-09T10:12:43.909912+0000","last_fullsized":"2023-10-09T10:12:43.909912+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":315,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911846+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:12:43.911846+0000","last_peered":"2023-10-09T10:12:43.911846+0000","last_clean":"2023-10-09T10:12:43.911846+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:12:43.911846+0000","last_undegraded":"2023-10-09T10:12:43.911846+0000","last_fullsized":"2023-10-09T10:12:43.911846+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":179,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.951344+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:12:43.951344+0000","last_peered":"2023-10-09T10:12:43.951344+0000","last_clean":"2023-10-09T10:12:43.951344+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:12:43.951344+0000","last_undegraded":"2023-10-09T10:12:43.951344+0000","last_fullsized":"2023-10-09T10:12:43.951344+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":284,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.908873+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:12:43.908873+0000","last_peered":"2023-10-09T10:12:43.908873+0000","last_clean":"2023-10-09T10:12:43.908873+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:12:43.908873+0000","last_undegraded":"2023-10-09T10:12:43.908873+0000","last_fullsized":"2023-10-09T10:12:43.908873+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":130,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911576+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:12:43.911576+0000","last_peered":"2023-10-09T10:12:43.911576+0000","last_clean":"2023-10-09T10:12:43.911576+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:12:43.911576+0000","last_undegraded":"2023-10-09T10:12:43.911576+0000","last_fullsized":"2023-10-09T10:12:43.911576+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":196,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.951318+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:12:43.951318+0000","last_peered":"2023-10-09T10:12:43.951318+0000","last_clean":"2023-10-09T10:12:43.951318+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:12:43.951318+0000","last_undegraded":"2023-10-09T10:12:43.951318+0000","last_fullsized":"2023-10-09T10:12:43.951318+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":265,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.908810+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:12:43.908810+0000","last_peered":"2023-10-09T10:12:43.908810+0000","last_clean":"2023-10-09T10:12:43.908810+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:12:43.908810+0000","last_undegraded":"2023-10-09T10:12:43.908810+0000","last_fullsized":"2023-10-09T10:12:43.908810+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":129,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911702+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:12:43.911702+0000","last_peered":"2023-10-09T10:12:43.911702+0000","last_clean":"2023-10-09T10:12:43.911702+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:12:43.911702+0000","last_undegraded":"2023-10-09T10:12:43.911702+0000","last_fullsized":"2023-10-09T10:12:43.911702+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":226,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.908845+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:12:43.908845+0000","last_peered":"2023-10-09T10:12:43.908845+0000","last_clean":"2023-10-09T10:12:43.908845+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:12:43.908845+0000","last_undegraded":"2023-10-09T10:12:43.908845+0000","last_fullsized":"2023-10-09T10:12:43.908845+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":350,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.947848+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:12:43.947848+0000","last_peered":"2023-10-09T10:12:43.947848+0000","last_clean":"2023-10-09T10:12:43.947848+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:12:43.947848+0000","last_undegraded":"2023-10-09T10:12:43.947848+0000","last_fullsized":"2023-10-09T10:12:43.947848+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":307,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912031+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:12:43.912031+0000","last_peered":"2023-10-09T10:12:43.912031+0000","last_clean":"2023-10-09T10:12:43.912031+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:12:43.912031+0000","last_undegraded":"2023-10-09T10:12:43.912031+0000","last_fullsized":"2023-10-09T10:12:43.912031+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":199,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.945039+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:12:43.945039+0000","last_peered":"2023-10-09T10:12:43.945039+0000","last_clean":"2023-10-09T10:12:43.945039+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:12:43.945039+0000","last_undegraded":"2023-10-09T10:12:43.945039+0000","last_fullsized":"2023-10-09T10:12:43.945039+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":179,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.913592+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:12:43.913592+0000","last_peered":"2023-10-09T10:12:43.913592+0000","last_clean":"2023-10-09T10:12:43.913592+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:12:43.913592+0000","last_undegraded":"2023-10-09T10:12:43.913592+0000","last_fullsized":"2023-10-09T10:12:43.913592+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":383,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.944816+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:12:43.944816+0000","last_peered":"2023-10-09T10:12:43.944816+0000","last_clean":"2023-10-09T10:12:43.944816+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:12:43.944816+0000","last_undegraded":"2023-10-09T10:12:43.944816+0000","last_fullsized":"2023-10-09T10:12:43.944816+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":252,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909249+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:12:43.909249+0000","last_peered":"2023-10-09T10:12:43.909249+0000","last_clean":"2023-10-09T10:12:43.909249+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:12:43.909249+0000","last_undegraded":"2023-10-09T10:12:43.909249+0000","last_fullsized":"2023-10-09T10:12:43.909249+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":178,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.950812+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:12:43.950812+0000","last_peered":"2023-10-09T10:12:43.950812+0000","last_clean":"2023-10-09T10:12:43.950812+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:12:43.950812+0000","last_undegraded":"2023-10-09T10:12:43.950812+0000","last_fullsized":"2023-10-09T10:12:43.950812+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":145,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912066+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:12:43.912066+0000","last_peered":"2023-10-09T10:12:43.912066+0000","last_clean":"2023-10-09T10:12:43.912066+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:12:43.912066+0000","last_undegraded":"2023-10-09T10:12:43.912066+0000","last_fullsized":"2023-10-09T10:12:43.912066+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":180,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.944789+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:12:43.944789+0000","last_peered":"2023-10-09T10:12:43.944789+0000","last_clean":"2023-10-09T10:12:43.944789+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:12:43.944789+0000","last_undegraded":"2023-10-09T10:12:43.944789+0000","last_fullsized":"2023-10-09T10:12:43.944789+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":178,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.951420+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:12:43.951420+0000","last_peered":"2023-10-09T10:12:43.951420+0000","last_clean":"2023-10-09T10:12:43.951420+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:12:43.951420+0000","last_undegraded":"2023-10-09T10:12:43.951420+0000","last_fullsized":"2023-10-09T10:12:43.951420+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":201,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.950746+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:12:43.950746+0000","last_peered":"2023-10-09T10:12:43.950746+0000","last_clean":"2023-10-09T10:12:43.950746+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:12:43.950746+0000","last_undegraded":"2023-10-09T10:12:43.950746+0000","last_fullsized":"2023-10-09T10:12:43.950746+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":265,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909141+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:12:43.909141+0000","last_peered":"2023-10-09T10:12:43.909141+0000","last_clean":"2023-10-09T10:12:43.909141+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:12:43.909141+0000","last_undegraded":"2023-10-09T10:12:43.909141+0000","last_fullsized":"2023-10-09T10:12:43.909141+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"312'12793","reported_seq":13551,"reported_epoch":312,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:12:52.992169+0000","last_change":"2023-10-09T10:12:42.183681+0000","last_active":"2023-10-09T10:12:52.992169+0000","last_peered":"2023-10-09T10:12:52.992169+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:12:52.992169+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"280'2700","ondisk_log_start":"280'2700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10093,"log_dups_size":0,"ondisk_log_size":10093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57212928,"num_objects":873,"num_object_clones":0,"num_object_copies":2619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5023,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12793,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":350,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-4150}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":344,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.945066+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:12:43.945066+0000","last_peered":"2023-10-09T10:12:43.945066+0000","last_clean":"2023-10-09T10:12:43.945066+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:12:43.945066+0000","last_undegraded":"2023-10-09T10:12:43.945066+0000","last_fullsized":"2023-10-09T10:12:43.945066+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":130,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911426+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:12:43.911426+0000","last_peered":"2023-10-09T10:12:43.911426+0000","last_clean":"2023-10-09T10:12:43.911426+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:12:43.911426+0000","last_undegraded":"2023-10-09T10:12:43.911426+0000","last_fullsized":"2023-10-09T10:12:43.911426+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"312'12937","reported_seq":12986,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:55.050190+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:12:55.050190+0000","last_peered":"2023-10-09T10:12:55.050190+0000","last_clean":"2023-10-09T10:12:55.050190+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:12:55.050190+0000","last_undegraded":"2023-10-09T10:12:55.050190+0000","last_fullsized":"2023-10-09T10:12:55.050190+0000","mapping_epoch":271,"log_start":"283'4300","ondisk_log_start":"283'4300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8637,"log_dups_size":0,"ondisk_log_size":8637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55902208,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12937,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":332,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.945181+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:12:43.945181+0000","last_peered":"2023-10-09T10:12:43.945181+0000","last_clean":"2023-10-09T10:12:43.945181+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:12:43.945181+0000","last_undegraded":"2023-10-09T10:12:43.945181+0000","last_fullsized":"2023-10-09T10:12:43.945181+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":224,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911779+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:12:43.911779+0000","last_peered":"2023-10-09T10:12:43.911779+0000","last_clean":"2023-10-09T10:12:43.911779+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:12:43.911779+0000","last_undegraded":"2023-10-09T10:12:43.911779+0000","last_fullsized":"2023-10-09T10:12:43.911779+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"312'12604","reported_seq":12655,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:51.764535+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:12:51.764535+0000","last_peered":"2023-10-09T10:12:51.764535+0000","last_clean":"2023-10-09T10:12:51.764535+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:12:51.764535+0000","last_undegraded":"2023-10-09T10:12:51.764535+0000","last_fullsized":"2023-10-09T10:12:51.764535+0000","mapping_epoch":271,"log_start":"282'3200","ondisk_log_start":"282'3200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9404,"log_dups_size":0,"ondisk_log_size":9404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63045632,"num_objects":962,"num_object_clones":0,"num_object_copies":2886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12604,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":134,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912111+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:12:43.912111+0000","last_peered":"2023-10-09T10:12:43.912111+0000","last_clean":"2023-10-09T10:12:43.912111+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:12:43.912111+0000","last_undegraded":"2023-10-09T10:12:43.912111+0000","last_fullsized":"2023-10-09T10:12:43.912111+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":155,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.961396+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:12:43.961396+0000","last_peered":"2023-10-09T10:12:43.961396+0000","last_clean":"2023-10-09T10:12:43.961396+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:12:43.961396+0000","last_undegraded":"2023-10-09T10:12:43.961396+0000","last_fullsized":"2023-10-09T10:12:43.961396+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"312'12592","reported_seq":12643,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:53.661872+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:12:53.661872+0000","last_peered":"2023-10-09T10:12:53.661872+0000","last_clean":"2023-10-09T10:12:53.661872+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:12:53.661872+0000","last_undegraded":"2023-10-09T10:12:53.661872+0000","last_fullsized":"2023-10-09T10:12:53.661872+0000","mapping_epoch":271,"log_start":"285'4800","ondisk_log_start":"285'4800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7792,"log_dups_size":0,"ondisk_log_size":7792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53608448,"num_objects":818,"num_object_clones":0,"num_object_copies":2454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12592,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":145,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912267+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:12:43.912267+0000","last_peered":"2023-10-09T10:12:43.912267+0000","last_clean":"2023-10-09T10:12:43.912267+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:12:43.912267+0000","last_undegraded":"2023-10-09T10:12:43.912267+0000","last_fullsized":"2023-10-09T10:12:43.912267+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":179,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.950960+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:12:43.950960+0000","last_peered":"2023-10-09T10:12:43.950960+0000","last_clean":"2023-10-09T10:12:43.950960+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:12:43.950960+0000","last_undegraded":"2023-10-09T10:12:43.950960+0000","last_fullsized":"2023-10-09T10:12:43.950960+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"312'12839","reported_seq":12889,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:51.428918+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:12:51.428918+0000","last_peered":"2023-10-09T10:12:51.428918+0000","last_clean":"2023-10-09T10:12:51.428918+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:12:51.428918+0000","last_undegraded":"2023-10-09T10:12:51.428918+0000","last_fullsized":"2023-10-09T10:12:51.428918+0000","mapping_epoch":271,"log_start":"285'4700","ondisk_log_start":"285'4700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8139,"log_dups_size":0,"ondisk_log_size":8139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65601536,"num_objects":1001,"num_object_clones":0,"num_object_copies":3003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12839,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":306,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909443+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:12:43.909443+0000","last_peered":"2023-10-09T10:12:43.909443+0000","last_clean":"2023-10-09T10:12:43.909443+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:12:43.909443+0000","last_undegraded":"2023-10-09T10:12:43.909443+0000","last_fullsized":"2023-10-09T10:12:43.909443+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":176,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909125+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:12:43.909125+0000","last_peered":"2023-10-09T10:12:43.909125+0000","last_clean":"2023-10-09T10:12:43.909125+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:12:43.909125+0000","last_undegraded":"2023-10-09T10:12:43.909125+0000","last_fullsized":"2023-10-09T10:12:43.909125+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"312'12837","reported_seq":12889,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:53.630537+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:12:53.630537+0000","last_peered":"2023-10-09T10:12:53.630537+0000","last_clean":"2023-10-09T10:12:53.630537+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:12:53.630537+0000","last_undegraded":"2023-10-09T10:12:53.630537+0000","last_fullsized":"2023-10-09T10:12:53.630537+0000","mapping_epoch":271,"log_start":"285'5100","ondisk_log_start":"285'5100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7737,"log_dups_size":0,"ondisk_log_size":7737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56819712,"num_objects":867,"num_object_clones":0,"num_object_copies":2601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12837,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":238,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.968935+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:12:47.968935+0000","last_peered":"2023-10-09T10:12:47.968935+0000","last_clean":"2023-10-09T10:12:47.968935+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:12:47.968935+0000","last_undegraded":"2023-10-09T10:12:47.968935+0000","last_fullsized":"2023-10-09T10:12:47.968935+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":243,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911965+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:12:43.911965+0000","last_peered":"2023-10-09T10:12:43.911965+0000","last_clean":"2023-10-09T10:12:43.911965+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:12:43.911965+0000","last_undegraded":"2023-10-09T10:12:43.911965+0000","last_fullsized":"2023-10-09T10:12:43.911965+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"312'12456","reported_seq":12508,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:51.764423+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:12:51.764423+0000","last_peered":"2023-10-09T10:12:51.764423+0000","last_clean":"2023-10-09T10:12:51.764423+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:12:51.764423+0000","last_undegraded":"2023-10-09T10:12:51.764423+0000","last_fullsized":"2023-10-09T10:12:51.764423+0000","mapping_epoch":271,"log_start":"281'3000","ondisk_log_start":"281'3000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9456,"log_dups_size":0,"ondisk_log_size":9456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63963136,"num_objects":976,"num_object_clones":0,"num_object_copies":2928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12456,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":206,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.944709+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:12:43.944709+0000","last_peered":"2023-10-09T10:12:43.944709+0000","last_clean":"2023-10-09T10:12:43.944709+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:12:43.944709+0000","last_undegraded":"2023-10-09T10:12:43.944709+0000","last_fullsized":"2023-10-09T10:12:43.944709+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":179,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912005+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:12:43.912005+0000","last_peered":"2023-10-09T10:12:43.912005+0000","last_clean":"2023-10-09T10:12:43.912005+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:12:43.912005+0000","last_undegraded":"2023-10-09T10:12:43.912005+0000","last_fullsized":"2023-10-09T10:12:43.912005+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"312'12686","reported_seq":12736,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:53.643091+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:12:53.643091+0000","last_peered":"2023-10-09T10:12:53.643091+0000","last_clean":"2023-10-09T10:12:53.643091+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:12:53.643091+0000","last_undegraded":"2023-10-09T10:12:53.643091+0000","last_fullsized":"2023-10-09T10:12:53.643091+0000","mapping_epoch":271,"log_start":"285'4900","ondisk_log_start":"285'4900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7786,"log_dups_size":0,"ondisk_log_size":7786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59768832,"num_objects":912,"num_object_clones":0,"num_object_copies":2736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12686,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":266,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.908782+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:12:43.908782+0000","last_peered":"2023-10-09T10:12:43.908782+0000","last_clean":"2023-10-09T10:12:43.908782+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:12:43.908782+0000","last_undegraded":"2023-10-09T10:12:43.908782+0000","last_fullsized":"2023-10-09T10:12:43.908782+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":316,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912297+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:12:43.912297+0000","last_peered":"2023-10-09T10:12:43.912297+0000","last_clean":"2023-10-09T10:12:43.912297+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:12:43.912297+0000","last_undegraded":"2023-10-09T10:12:43.912297+0000","last_fullsized":"2023-10-09T10:12:43.912297+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"312'12565","reported_seq":12616,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:51.808338+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:12:51.808338+0000","last_peered":"2023-10-09T10:12:51.808338+0000","last_clean":"2023-10-09T10:12:51.808338+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:12:51.808338+0000","last_undegraded":"2023-10-09T10:12:51.808338+0000","last_fullsized":"2023-10-09T10:12:51.808338+0000","mapping_epoch":271,"log_start":"282'3100","ondisk_log_start":"282'3100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9465,"log_dups_size":0,"ondisk_log_size":9465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59441152,"num_objects":907,"num_object_clones":0,"num_object_copies":2721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12565,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":315,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911959+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:12:43.911959+0000","last_peered":"2023-10-09T10:12:43.911959+0000","last_clean":"2023-10-09T10:12:43.911959+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:12:43.911959+0000","last_undegraded":"2023-10-09T10:12:43.911959+0000","last_fullsized":"2023-10-09T10:12:43.911959+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":154,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912026+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:12:43.912026+0000","last_peered":"2023-10-09T10:12:43.912026+0000","last_clean":"2023-10-09T10:12:43.912026+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:12:43.912026+0000","last_undegraded":"2023-10-09T10:12:43.912026+0000","last_fullsized":"2023-10-09T10:12:43.912026+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":272,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912237+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:12:43.912237+0000","last_peered":"2023-10-09T10:12:43.912237+0000","last_clean":"2023-10-09T10:12:43.912237+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:12:43.912237+0000","last_undegraded":"2023-10-09T10:12:43.912237+0000","last_fullsized":"2023-10-09T10:12:43.912237+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"312'12640","reported_seq":12688,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:52.957792+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:12:52.957792+0000","last_peered":"2023-10-09T10:12:52.957792+0000","last_clean":"2023-10-09T10:12:52.957792+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:12:52.957792+0000","last_undegraded":"2023-10-09T10:12:52.957792+0000","last_fullsized":"2023-10-09T10:12:52.957792+0000","mapping_epoch":271,"log_start":"280'2600","ondisk_log_start":"280'2600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10040,"log_dups_size":0,"ondisk_log_size":10040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12640,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":130,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911473+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:12:43.911473+0000","last_peered":"2023-10-09T10:12:43.911473+0000","last_clean":"2023-10-09T10:12:43.911473+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:12:43.911473+0000","last_undegraded":"2023-10-09T10:12:43.911473+0000","last_fullsized":"2023-10-09T10:12:43.911473+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":129,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.950763+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:12:43.950763+0000","last_peered":"2023-10-09T10:12:43.950763+0000","last_clean":"2023-10-09T10:12:43.950763+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:12:43.950763+0000","last_undegraded":"2023-10-09T10:12:43.950763+0000","last_fullsized":"2023-10-09T10:12:43.950763+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":338,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911900+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:12:43.911900+0000","last_peered":"2023-10-09T10:12:43.911900+0000","last_clean":"2023-10-09T10:12:43.911900+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:12:43.911900+0000","last_undegraded":"2023-10-09T10:12:43.911900+0000","last_fullsized":"2023-10-09T10:12:43.911900+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"312'12529","reported_seq":12580,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:51.474530+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:12:51.474530+0000","last_peered":"2023-10-09T10:12:51.474530+0000","last_clean":"2023-10-09T10:12:51.474530+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:12:51.474530+0000","last_undegraded":"2023-10-09T10:12:51.474530+0000","last_fullsized":"2023-10-09T10:12:51.474530+0000","mapping_epoch":271,"log_start":"283'4400","ondisk_log_start":"283'4400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8129,"log_dups_size":0,"ondisk_log_size":8129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66387968,"num_objects":1013,"num_object_clones":0,"num_object_copies":3039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12529,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":232,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.908763+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:12:43.908763+0000","last_peered":"2023-10-09T10:12:43.908763+0000","last_clean":"2023-10-09T10:12:43.908763+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:12:43.908763+0000","last_undegraded":"2023-10-09T10:12:43.908763+0000","last_fullsized":"2023-10-09T10:12:43.908763+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":308,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.971479+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:12:47.971479+0000","last_peered":"2023-10-09T10:12:47.971479+0000","last_clean":"2023-10-09T10:12:47.971479+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:12:47.971479+0000","last_undegraded":"2023-10-09T10:12:47.971479+0000","last_fullsized":"2023-10-09T10:12:47.971479+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":328,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912206+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:12:43.912206+0000","last_peered":"2023-10-09T10:12:43.912206+0000","last_clean":"2023-10-09T10:12:43.912206+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:12:43.912206+0000","last_undegraded":"2023-10-09T10:12:43.912206+0000","last_fullsized":"2023-10-09T10:12:43.912206+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"312'12550","reported_seq":12599,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:52.989916+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:12:52.989916+0000","last_peered":"2023-10-09T10:12:52.989916+0000","last_clean":"2023-10-09T10:12:52.989916+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:12:52.989916+0000","last_undegraded":"2023-10-09T10:12:52.989916+0000","last_fullsized":"2023-10-09T10:12:52.989916+0000","mapping_epoch":271,"log_start":"280'2500","ondisk_log_start":"280'2500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10050,"log_dups_size":0,"ondisk_log_size":10050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59637760,"num_objects":910,"num_object_clones":0,"num_object_copies":2730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12550,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":139,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911427+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:12:43.911427+0000","last_peered":"2023-10-09T10:12:43.911427+0000","last_clean":"2023-10-09T10:12:43.911427+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:12:43.911427+0000","last_undegraded":"2023-10-09T10:12:43.911427+0000","last_fullsized":"2023-10-09T10:12:43.911427+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":558,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.944709+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:12:43.944709+0000","last_peered":"2023-10-09T10:12:43.944709+0000","last_clean":"2023-10-09T10:12:43.944709+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:12:43.944709+0000","last_undegraded":"2023-10-09T10:12:43.944709+0000","last_fullsized":"2023-10-09T10:12:43.944709+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":216,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909424+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:12:43.909424+0000","last_peered":"2023-10-09T10:12:43.909424+0000","last_clean":"2023-10-09T10:12:43.909424+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:12:43.909424+0000","last_undegraded":"2023-10-09T10:12:43.909424+0000","last_fullsized":"2023-10-09T10:12:43.909424+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":227,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.947873+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:12:43.947873+0000","last_peered":"2023-10-09T10:12:43.947873+0000","last_clean":"2023-10-09T10:12:43.947873+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:12:43.947873+0000","last_undegraded":"2023-10-09T10:12:43.947873+0000","last_fullsized":"2023-10-09T10:12:43.947873+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"312'12754","reported_seq":12805,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:53.665658+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:12:53.665658+0000","last_peered":"2023-10-09T10:12:53.665658+0000","last_clean":"2023-10-09T10:12:53.665658+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:12:53.665658+0000","last_undegraded":"2023-10-09T10:12:53.665658+0000","last_fullsized":"2023-10-09T10:12:53.665658+0000","mapping_epoch":271,"log_start":"285'5000","ondisk_log_start":"285'5000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7754,"log_dups_size":0,"ondisk_log_size":7754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56885248,"num_objects":868,"num_object_clones":0,"num_object_copies":2604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12754,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":288,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912158+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:12:43.912158+0000","last_peered":"2023-10-09T10:12:43.912158+0000","last_clean":"2023-10-09T10:12:43.912158+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:12:43.912158+0000","last_undegraded":"2023-10-09T10:12:43.912158+0000","last_fullsized":"2023-10-09T10:12:43.912158+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":189,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.951621+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:12:43.951621+0000","last_peered":"2023-10-09T10:12:43.951621+0000","last_clean":"2023-10-09T10:12:43.951621+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:12:43.951621+0000","last_undegraded":"2023-10-09T10:12:43.951621+0000","last_fullsized":"2023-10-09T10:12:43.951621+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":217,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.951260+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:12:43.951260+0000","last_peered":"2023-10-09T10:12:43.951260+0000","last_clean":"2023-10-09T10:12:43.951260+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:12:43.951260+0000","last_undegraded":"2023-10-09T10:12:43.951260+0000","last_fullsized":"2023-10-09T10:12:43.951260+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"312'12681","reported_seq":23316,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:51.540081+0000","last_change":"2023-10-09T10:12:43.942710+0000","last_active":"2023-10-09T10:12:51.540081+0000","last_peered":"2023-10-09T10:12:51.540081+0000","last_clean":"2023-10-09T10:12:51.540081+0000","last_became_active":"2023-10-09T10:12:43.912846+0000","last_became_peered":"2023-10-09T10:12:43.912846+0000","last_unstale":"2023-10-09T10:12:51.540081+0000","last_undegraded":"2023-10-09T10:12:51.540081+0000","last_fullsized":"2023-10-09T10:12:51.540081+0000","mapping_epoch":310,"log_start":"284'4500","ondisk_log_start":"284'4500","created":271,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8181,"log_dups_size":0,"ondisk_log_size":8181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:08.472095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64028672,"num_objects":977,"num_object_clones":0,"num_object_copies":2931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12681,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5284,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":331,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.945397+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:12:43.945397+0000","last_peered":"2023-10-09T10:12:43.945397+0000","last_clean":"2023-10-09T10:12:43.945397+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:12:43.945397+0000","last_undegraded":"2023-10-09T10:12:43.945397+0000","last_fullsized":"2023-10-09T10:12:43.945397+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":344,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912001+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:12:43.912001+0000","last_peered":"2023-10-09T10:12:43.912001+0000","last_clean":"2023-10-09T10:12:43.912001+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:12:43.912001+0000","last_undegraded":"2023-10-09T10:12:43.912001+0000","last_fullsized":"2023-10-09T10:12:43.912001+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":181,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.950835+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:12:43.950835+0000","last_peered":"2023-10-09T10:12:43.950835+0000","last_clean":"2023-10-09T10:12:43.950835+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:12:43.950835+0000","last_undegraded":"2023-10-09T10:12:43.950835+0000","last_fullsized":"2023-10-09T10:12:43.950835+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"312'12603","reported_seq":12654,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:51.833215+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:12:51.833215+0000","last_peered":"2023-10-09T10:12:51.833215+0000","last_clean":"2023-10-09T10:12:51.833215+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:12:51.833215+0000","last_undegraded":"2023-10-09T10:12:51.833215+0000","last_fullsized":"2023-10-09T10:12:51.833215+0000","mapping_epoch":271,"log_start":"282'3200","ondisk_log_start":"282'3200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9403,"log_dups_size":0,"ondisk_log_size":9403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62586880,"num_objects":955,"num_object_clones":0,"num_object_copies":2865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12603,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":275,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909190+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:12:43.909190+0000","last_peered":"2023-10-09T10:12:43.909190+0000","last_clean":"2023-10-09T10:12:43.909190+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:12:43.909190+0000","last_undegraded":"2023-10-09T10:12:43.909190+0000","last_fullsized":"2023-10-09T10:12:43.909190+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":289,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912089+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:12:43.912089+0000","last_peered":"2023-10-09T10:12:43.912089+0000","last_clean":"2023-10-09T10:12:43.912089+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:12:43.912089+0000","last_undegraded":"2023-10-09T10:12:43.912089+0000","last_fullsized":"2023-10-09T10:12:43.912089+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":264,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.908726+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:12:43.908726+0000","last_peered":"2023-10-09T10:12:43.908726+0000","last_clean":"2023-10-09T10:12:43.908726+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:12:43.908726+0000","last_undegraded":"2023-10-09T10:12:43.908726+0000","last_fullsized":"2023-10-09T10:12:43.908726+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"312'12525","reported_seq":12575,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:52.933806+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:12:52.933806+0000","last_peered":"2023-10-09T10:12:52.933806+0000","last_clean":"2023-10-09T10:12:52.933806+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:12:52.933806+0000","last_undegraded":"2023-10-09T10:12:52.933806+0000","last_fullsized":"2023-10-09T10:12:52.933806+0000","mapping_epoch":271,"log_start":"280'2500","ondisk_log_start":"280'2500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10025,"log_dups_size":0,"ondisk_log_size":10025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61866008,"num_objects":945,"num_object_clones":0,"num_object_copies":2835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":945,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12525,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":220,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.961043+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:12:43.961043+0000","last_peered":"2023-10-09T10:12:43.961043+0000","last_clean":"2023-10-09T10:12:43.961043+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:12:43.961043+0000","last_undegraded":"2023-10-09T10:12:43.961043+0000","last_fullsized":"2023-10-09T10:12:43.961043+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":183,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.951173+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:12:43.951173+0000","last_peered":"2023-10-09T10:12:43.951173+0000","last_clean":"2023-10-09T10:12:43.951173+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:12:43.951173+0000","last_undegraded":"2023-10-09T10:12:43.951173+0000","last_fullsized":"2023-10-09T10:12:43.951173+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":165,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911377+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:12:43.911377+0000","last_peered":"2023-10-09T10:12:43.911377+0000","last_clean":"2023-10-09T10:12:43.911377+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:12:43.911377+0000","last_undegraded":"2023-10-09T10:12:43.911377+0000","last_fullsized":"2023-10-09T10:12:43.911377+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":178,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.950675+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:12:43.950675+0000","last_peered":"2023-10-09T10:12:43.950675+0000","last_clean":"2023-10-09T10:12:43.950675+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:12:43.950675+0000","last_undegraded":"2023-10-09T10:12:43.950675+0000","last_fullsized":"2023-10-09T10:12:43.950675+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":250,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912063+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:12:43.912063+0000","last_peered":"2023-10-09T10:12:43.912063+0000","last_clean":"2023-10-09T10:12:43.912063+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:12:43.912063+0000","last_undegraded":"2023-10-09T10:12:43.912063+0000","last_fullsized":"2023-10-09T10:12:43.912063+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":149,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.912225+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:12:43.912225+0000","last_peered":"2023-10-09T10:12:43.912225+0000","last_clean":"2023-10-09T10:12:43.912225+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:12:43.912225+0000","last_undegraded":"2023-10-09T10:12:43.912225+0000","last_fullsized":"2023-10-09T10:12:43.912225+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":191,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909357+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:12:43.909357+0000","last_peered":"2023-10-09T10:12:43.909357+0000","last_clean":"2023-10-09T10:12:43.909357+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:12:43.909357+0000","last_undegraded":"2023-10-09T10:12:43.909357+0000","last_fullsized":"2023-10-09T10:12:43.909357+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":149,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911983+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:12:43.911983+0000","last_peered":"2023-10-09T10:12:43.911983+0000","last_clean":"2023-10-09T10:12:43.911983+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:12:43.911983+0000","last_undegraded":"2023-10-09T10:12:43.911983+0000","last_fullsized":"2023-10-09T10:12:43.911983+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":200,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909008+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:12:43.909008+0000","last_peered":"2023-10-09T10:12:43.909008+0000","last_clean":"2023-10-09T10:12:43.909008+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:12:43.909008+0000","last_undegraded":"2023-10-09T10:12:43.909008+0000","last_fullsized":"2023-10-09T10:12:43.909008+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":329,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909088+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:12:43.909088+0000","last_peered":"2023-10-09T10:12:43.909088+0000","last_clean":"2023-10-09T10:12:43.909088+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:12:43.909088+0000","last_undegraded":"2023-10-09T10:12:43.909088+0000","last_fullsized":"2023-10-09T10:12:43.909088+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":350,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.909053+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:12:43.909053+0000","last_peered":"2023-10-09T10:12:43.909053+0000","last_clean":"2023-10-09T10:12:43.909053+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:12:43.909053+0000","last_undegraded":"2023-10-09T10:12:43.909053+0000","last_fullsized":"2023-10-09T10:12:43.909053+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":259,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.951304+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:12:43.951304+0000","last_peered":"2023-10-09T10:12:43.951304+0000","last_clean":"2023-10-09T10:12:43.951304+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:12:43.951304+0000","last_undegraded":"2023-10-09T10:12:43.951304+0000","last_fullsized":"2023-10-09T10:12:43.951304+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:12:43.911781+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:12:43.911781+0000","last_peered":"2023-10-09T10:12:43.911781+0000","last_clean":"2023-10-09T10:12:43.911781+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:12:43.911781+0000","last_undegraded":"2023-10-09T10:12:43.911781+0000","last_fullsized":"2023-10-09T10:12:43.911781+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":967966744,"num_objects":14771,"num_object_clones":0,"num_object_copies":44313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5023,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14771,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202591,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5634,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3499175936,"data_stored":3499163720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142091,"ondisk_log_size":142091,"up":48,"acting":47,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430519,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":578480,"kb_used_data":277160,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93166672,"statfs":{"total":95995035648,"available":95402672128,"internally_reserved":0,"allocated":283811840,"data_stored":283270582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.3130000000000002,"15min":2.0539999999999998},"min":{"1min":0.27400000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":11.164,"5min":11.164,"15min":50.591999999999999},"last":3.0720000000000001},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.2770000000000001,"15min":2.0059999999999998},"min":{"1min":0.32600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.7880000000000003,"5min":15.974,"15min":50.747},"last":0.35399999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:12:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9940000000000002,"5min":2.3639999999999999,"15min":1.958},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":11.92,"5min":14.311,"15min":25.338000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.6320000000000001,"5min":2.218,"15min":1.6870000000000001},"min":{"1min":0.42999999999999999,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":10.663,"5min":18.27,"15min":18.27},"last":6.4779999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":3.7639999999999998,"5min":1.9970000000000001,"15min":2.359},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":13.775,"5min":13.775,"15min":13.775},"last":4.0839999999999996},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.008,"15min":2.157},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.101000000000001,"5min":12.101000000000001,"15min":12.101000000000001},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4550000000000001,"5min":2.629,"15min":2.0430000000000001},"min":{"1min":0.436,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.6289999999999996,"5min":29.058,"15min":29.058},"last":0.54200000000000004},{"interface":"front","average":{"1min":3.6629999999999998,"5min":2.2559999999999998,"15min":2.0489999999999999},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":2.403}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.288,"15min":1.0369999999999999},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":7.7469999999999999,"5min":7.7469999999999999,"15min":7.7469999999999999},"last":0.313},{"interface":"front","average":{"1min":1.724,"5min":1.139,"15min":0.98499999999999999},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":6.516}]},{"osd":5,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3260000000000001,"5min":1.403,"15min":1.1259999999999999},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":7.8159999999999998,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.2110000000000001,"15min":1.0580000000000001},"min":{"1min":0.38500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.806,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.51300000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":1.0660000000000001,"15min":1.006},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.327,"5min":0.86899999999999999,"15min":0.91900000000000004},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":9.1920000000000002},"last":1.325}]}]},{"osd":6,"up_from":42,"seq":180388626747,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919712,"kb_used_data":618392,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92825440,"statfs":{"total":95995035648,"available":95053250560,"internally_reserved":0,"allocated":633233408,"data_stored":632709464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8180000000000001,"5min":2.2330000000000001,"15min":1.756},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.6940000000000008,"5min":15.435,"15min":15.435},"last":2.677},{"interface":"front","average":{"1min":2.5129999999999999,"5min":2.1890000000000001,"15min":1.7250000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":13.385,"5min":15.308999999999999,"15min":15.308999999999999},"last":2.484}]},{"osd":1,"last update":"Mon Oct 9 10:12:06 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":2.839,"15min":2.0579999999999998},"min":{"1min":0.30199999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.0519999999999996,"5min":35.609999999999999,"15min":35.609999999999999},"last":1.3500000000000001},{"interface":"front","average":{"1min":2.919,"5min":2.46,"15min":1.9910000000000001},"min":{"1min":0.36099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":8.2680000000000007,"5min":32.396000000000001,"15min":32.396000000000001},"last":2.8069999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.8740000000000001,"15min":1.802},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":7.7359999999999998,"5min":12.898,"15min":12.898},"last":1.9490000000000001},{"interface":"front","average":{"1min":3.4609999999999999,"5min":2.6499999999999999,"15min":3.036},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.720000000000001,"5min":14.542999999999999,"15min":14.542999999999999},"last":3.8690000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.7839999999999998,"15min":1.8340000000000001},"min":{"1min":0.437,"5min":0.308,"15min":0.308},"max":{"1min":11.098000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":0.83099999999999996},{"interface":"front","average":{"1min":3.161,"5min":2.6059999999999999,"15min":2.456},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.002000000000001,"5min":11.211,"15min":11.211},"last":2.3700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":0.94699999999999995,"15min":0.60899999999999999},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.113,"5min":10.113,"15min":10.113},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.89500000000000002,"15min":0.63100000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.948,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:12:37 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.0660000000000001,"15min":0.91800000000000004},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.23699999999999999},"max":{"1min":6.3170000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.335,"5min":1.222,"15min":0.99399999999999999},"min":{"1min":0.34699999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":14.465999999999999,"5min":14.465999999999999,"15min":14.465999999999999},"last":0.41499999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.92600000000000005,"15min":0.89500000000000002},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.0700000000000003,"5min":6.2709999999999999,"15min":7.5350000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.673,"5min":0.97199999999999998,"15min":0.94299999999999995},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.101,"5min":6.3159999999999998,"15min":7.5570000000000004},"last":0.46100000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253060,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":853228,"kb_used_data":409508,"kb_used_omap":9,"kb_used_meta":443702,"kb_avail":92891924,"statfs":{"total":95995035648,"available":95121330176,"internally_reserved":0,"allocated":419336192,"data_stored":418267488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":454351849},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":4.4160000000000004,"5min":2.5880000000000001,"15min":2.4500000000000002},"min":{"1min":0.217,"5min":0.183,"15min":0.183},"max":{"1min":31.007000000000001,"5min":31.007000000000001,"15min":31.007000000000001},"last":5.7270000000000003},{"interface":"front","average":{"1min":3.8010000000000002,"5min":3.4079999999999999,"15min":2.5899999999999999},"min":{"1min":0.28599999999999998,"5min":0.245,"15min":0.192},"max":{"1min":10.577999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.997}]},{"osd":2,"last update":"Mon Oct 9 10:12:52 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":3.1619999999999999,"15min":3.7450000000000001},"min":{"1min":0.22800000000000001,"5min":0.217,"15min":0.217},"max":{"1min":10.262,"5min":12.487,"15min":17.369},"last":0.38900000000000001},{"interface":"front","average":{"1min":4.2009999999999996,"5min":3.1099999999999999,"15min":3.7429999999999999},"min":{"1min":0.31,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":21.645,"5min":21.645,"15min":21.645},"last":5.9480000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":3.6669999999999998,"15min":3.5779999999999998},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":21.808,"5min":21.808,"15min":21.808},"last":5.8209999999999997},{"interface":"front","average":{"1min":5.1740000000000004,"5min":3.8580000000000001,"15min":4.5540000000000003},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":21.547999999999998,"5min":21.547999999999998,"15min":21.547999999999998},"last":14.300000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.8029999999999999,"15min":1.4370000000000001},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":21.620000000000001,"5min":21.620000000000001,"15min":21.620000000000001},"last":5.8799999999999999},{"interface":"front","average":{"1min":2.4980000000000002,"5min":2.411,"15min":1.6899999999999999},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":9.2569999999999997,"5min":12.904,"15min":12.904},"last":1.1419999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.4430000000000001,"15min":2.0630000000000002},"min":{"1min":0.34100000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.8599999999999994,"5min":12.811,"15min":15.531000000000001},"last":1.0660000000000001},{"interface":"front","average":{"1min":3.544,"5min":2.2949999999999999,"15min":2.2999999999999998},"min":{"1min":0.53900000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.855,"5min":16.277999999999999,"15min":19.706},"last":0.42599999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":2.0139999999999998,"15min":1.9810000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":21.670999999999999,"5min":21.670999999999999,"15min":21.670999999999999},"last":5.7960000000000003},{"interface":"front","average":{"1min":2.5219999999999998,"5min":2.4220000000000002,"15min":2.1070000000000002},"min":{"1min":0.36899999999999999,"5min":0.35499999999999998,"15min":0.32000000000000001},"max":{"1min":9.9440000000000008,"5min":12.868,"15min":15.481999999999999},"last":1.169}]},{"osd":7,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":2.3860000000000001,"15min":2.3330000000000002},"min":{"1min":0.68200000000000005,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.723000000000001,"5min":16.331,"15min":16.331},"last":0.45500000000000002},{"interface":"front","average":{"1min":3.101,"5min":2.0830000000000002,"15min":2.0619999999999998},"min":{"1min":0.42399999999999999,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":21.593,"5min":21.593,"15min":21.593},"last":5.8360000000000003}]}]},{"osd":0,"up_from":79,"seq":339302416586,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":760648,"kb_used_data":375808,"kb_used_omap":28,"kb_used_meta":384803,"kb_avail":92984504,"statfs":{"total":95995035648,"available":95216132096,"internally_reserved":0,"allocated":384827392,"data_stored":383791444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":394039120},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":2.6800000000000002,"15min":2.0209999999999999},"min":{"1min":0.26400000000000001,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":16.797000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":4.3369999999999997},{"interface":"front","average":{"1min":7.1150000000000002,"5min":4.2210000000000001,"15min":3.04},"min":{"1min":0.253,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":31.925999999999998,"5min":31.925999999999998,"15min":31.925999999999998},"last":2.012}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.0469999999999997,"5min":2.181,"15min":2.274},"min":{"1min":0.28899999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.003,"5min":18.003,"15min":18.003},"last":4.5279999999999996},{"interface":"front","average":{"1min":2.9660000000000002,"5min":2.0710000000000002,"15min":3.0190000000000001},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.432,"5min":15.529,"15min":15.529},"last":4.5700000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":4.6509999999999998,"15min":4.7930000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":23.151,"5min":25.734999999999999,"15min":25.734999999999999},"last":4.3890000000000002},{"interface":"front","average":{"1min":5.3250000000000002,"5min":4.0739999999999998,"15min":4.298},"min":{"1min":0.33300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":18.279,"5min":21.923999999999999,"15min":21.923999999999999},"last":1.9199999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9109999999999996,"5min":3.012,"15min":1.9319999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":2.0350000000000001},{"interface":"front","average":{"1min":4.2830000000000004,"5min":2.8639999999999999,"15min":2.246},"min":{"1min":0.34599999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.797000000000001,"5min":15.907,"15min":15.907},"last":1.9810000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.5119999999999996,"5min":3.145,"15min":2.444},"min":{"1min":0.38500000000000001,"5min":0.34599999999999997,"15min":0.28000000000000003},"max":{"1min":14.944000000000001,"5min":17.096,"15min":17.096},"last":4.2240000000000002},{"interface":"front","average":{"1min":4.1159999999999997,"5min":2.0710000000000002,"15min":1.6599999999999999},"min":{"1min":0.34799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.547000000000001,"5min":13.547000000000001,"15min":13.547000000000001},"last":4.6319999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1500000000000004,"5min":2.2210000000000001,"15min":1.8169999999999999},"min":{"1min":0.41199999999999998,"5min":0.33700000000000002,"15min":0.31900000000000001},"max":{"1min":13.51,"5min":13.51,"15min":13.51},"last":4.6050000000000004},{"interface":"front","average":{"1min":5.2859999999999996,"5min":3.726,"15min":2.7719999999999998},"min":{"1min":0.42299999999999999,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":20.446999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":2.0640000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.2040000000000002,"15min":2.601},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":15.734,"5min":16.460000000000001,"15min":26.978000000000002},"last":1.9550000000000001},{"interface":"front","average":{"1min":4.5389999999999997,"5min":2.3650000000000002,"15min":1.9950000000000001},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":13.787000000000001,"5min":13.787000000000001,"15min":26.768000000000001},"last":4.6559999999999997}]}]},{"osd":2,"up_from":217,"seq":932007903320,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":602188,"kb_used_data":335300,"kb_used_omap":83,"kb_used_meta":266796,"kb_avail":93142964,"statfs":{"total":95995035648,"available":95378395136,"internally_reserved":0,"allocated":343347200,"data_stored":342308169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":273199599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.0609999999999999,"15min":1.8660000000000001},"min":{"1min":0.24299999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.334,"5min":19.372,"15min":19.372},"last":12.353},{"interface":"front","average":{"1min":5.5940000000000003,"5min":3.5150000000000001,"15min":3.133},"min":{"1min":0.32000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.047000000000001,"5min":16.754999999999999,"15min":16.754999999999999},"last":0.29799999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":4.6630000000000003,"5min":2.653,"15min":2.6760000000000002},"min":{"1min":0.26500000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.276,"5min":14.276,"15min":14.276},"last":19.283999999999999},{"interface":"front","average":{"1min":5.4710000000000001,"5min":3.0760000000000001,"15min":2.823},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.227,"5min":15.569000000000001,"15min":15.569000000000001},"last":0.32800000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.2989999999999999,"15min":2.1150000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.359999999999999,"5min":13.851000000000001,"15min":13.851000000000001},"last":19.370999999999999},{"interface":"front","average":{"1min":4.1879999999999997,"5min":3.2360000000000002,"15min":3.6880000000000002},"min":{"1min":0.40799999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.51,"5min":15.811,"15min":15.811},"last":6.2469999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":1.6200000000000001,"15min":1.0109999999999999},"min":{"1min":0.32000000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.725,"5min":12.725,"15min":12.725},"last":0.39100000000000001},{"interface":"front","average":{"1min":3.0270000000000001,"5min":1.7689999999999999,"15min":1.143},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":15.276,"5min":15.276,"15min":15.276},"last":11.993}]},{"osd":5,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.8120000000000001,"15min":1.5860000000000001},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1230000000000002,"5min":12.473000000000001,"15min":12.473000000000001},"last":11.93},{"interface":"front","average":{"1min":2.044,"5min":1.5980000000000001,"15min":1.706},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.923,"5min":13.587,"15min":13.587},"last":11.904999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5529999999999999,"5min":1.9219999999999999,"15min":1.2689999999999999},"min":{"1min":0.374,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.484,"5min":15.292999999999999,"15min":15.292999999999999},"last":0.91900000000000004},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.9670000000000001,"15min":1.8180000000000001},"min":{"1min":0.41099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.5679999999999996,"5min":12.502000000000001,"15min":12.502000000000001},"last":12.052}]},{"osd":7,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4969999999999999,"5min":1.9359999999999999,"15min":1.294},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.457000000000001,"5min":15.343999999999999,"15min":15.343999999999999},"last":6.2839999999999998},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.1539999999999999,"15min":1.758},"min":{"1min":0.38600000000000001,"5min":0.376,"15min":0.376},"max":{"1min":9.6039999999999992,"5min":12.433999999999999,"15min":12.433999999999999},"last":12.023999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824063,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":645408,"kb_used_data":385304,"kb_used_omap":10,"kb_used_meta":260085,"kb_avail":93099744,"statfs":{"total":95995035648,"available":95334137856,"internally_reserved":0,"allocated":394551296,"data_stored":393317550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":266327107},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5139999999999998,"5min":4.0330000000000004,"15min":4.4800000000000004},"min":{"1min":0.27500000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.749000000000001,"5min":23.381,"15min":23.381},"last":0.49299999999999999},{"interface":"front","average":{"1min":6.5919999999999996,"5min":4.5750000000000002,"15min":4.3109999999999999},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.782,"5min":20.239999999999998,"15min":20.239999999999998},"last":0.32400000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2909999999999999,"5min":3.4820000000000002,"15min":3.7959999999999998},"min":{"1min":0.41399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.856999999999999,"5min":20.422999999999998,"15min":20.422999999999998},"last":1.593},{"interface":"front","average":{"1min":4.5620000000000003,"5min":3.726,"15min":3.5289999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.73,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":3.1269999999999998,"15min":3.536},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.331,"5min":20.594999999999999,"15min":20.594999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":4.298,"5min":3.1680000000000001,"15min":2.9660000000000002},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":14.802,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.55400000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1000000000000001,"5min":2.2490000000000001,"15min":1.48},"min":{"1min":0.33300000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.8700000000000001,"5min":15.676,"15min":15.676},"last":9.6669999999999998},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.6600000000000001,"15min":1.8939999999999999},"min":{"1min":0.40600000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.9890000000000008,"5min":16.478999999999999,"15min":16.478999999999999},"last":0.379}]},{"osd":5,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":2.9260000000000002,"15min":2.718},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.125999999999999,"5min":13.125999999999999,"15min":13.125999999999999},"last":6.6849999999999996},{"interface":"front","average":{"1min":3.0699999999999998,"5min":2.6600000000000001,"15min":2.456},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":10.891999999999999,"5min":15.992000000000001,"15min":15.992000000000001},"last":2.0819999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9649999999999999,"5min":2.9809999999999999,"15min":2.819},"min":{"1min":0.42099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.504999999999999,"5min":16.504999999999999,"15min":16.504999999999999},"last":0.71699999999999997},{"interface":"front","average":{"1min":1.921,"5min":2.4239999999999999,"15min":2.544},"min":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.952,"5min":16.16,"15min":16.16},"last":7.3419999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:11:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.1219999999999999,"15min":1.633},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":11.048999999999999,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":5.8399999999999999,"5min":3.073,"15min":2.8069999999999999},"min":{"1min":0.44600000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":16.367000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":1.865}]}]},{"osd":4,"up_from":266,"seq":1142461300789,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":943780,"kb_used_data":621020,"kb_used_omap":35,"kb_used_meta":322716,"kb_avail":92801372,"statfs":{"total":95995035648,"available":95028604928,"internally_reserved":0,"allocated":635924480,"data_stored":635394872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":330461422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":1.256,"15min":0.93100000000000005},"min":{"1min":0.222,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.1319999999999997,"5min":10.881,"15min":10.881},"last":2.1280000000000001},{"interface":"front","average":{"1min":3.04,"5min":1.9790000000000001,"15min":1.2989999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":8.4740000000000002,"5min":12.246,"15min":12.246},"last":0.98199999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":3.8079999999999998,"5min":2.2530000000000001,"15min":1.2949999999999999},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":9.4700000000000006,"5min":12.161,"15min":12.161},"last":1.772},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.986,"15min":1.512},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":13.231999999999999,"5min":13.231999999999999,"15min":13.231999999999999},"last":1.8380000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":3.1160000000000001,"5min":1.867,"15min":1.2849999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.6550000000000002,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":2.7730000000000001},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.728,"15min":1.1799999999999999},"min":{"1min":0.311,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.8920000000000003,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":2.6720000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":2.1930000000000001,"15min":1.5720000000000001},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.8630000000000004,"5min":13.065,"15min":13.065},"last":1.9570000000000001},{"interface":"front","average":{"1min":4.4089999999999998,"5min":2.4449999999999998,"15min":1.4910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":14.577,"5min":14.577,"15min":14.577},"last":1.1679999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.196,"15min":0.88400000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.0589999999999993,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.66800000000000004,"15min":0.65500000000000003},"min":{"1min":0.28799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7549999999999999,"5min":5.7549999999999999,"15min":5.7549999999999999},"last":0.58499999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":0.95599999999999996,"15min":0.66900000000000004},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.851,"5min":6.851,"15min":6.851},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.0840000000000001,"15min":0.75},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.3860000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.372}]},{"osd":7,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4569999999999999,"5min":1.258,"15min":0.83299999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":8.1039999999999992,"5min":12.237,"15min":12.237},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.3410000000000002,"5min":1.208,"15min":0.95499999999999996},"min":{"1min":0.314,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":0.48999999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822974,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":787224,"kb_used_data":477712,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92957928,"statfs":{"total":95995035648,"available":95188918272,"internally_reserved":0,"allocated":489177088,"data_stored":488655536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5950000000000002,"5min":1.7989999999999999,"15min":1.7090000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2639999999999993,"5min":8.2639999999999993,"15min":15.773999999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.823,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.4939999999999998,"5min":10.244999999999999,"15min":15.699},"last":0.374}]},{"osd":1,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.1320000000000001,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.891,"5min":11.891,"15min":15.689},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.9350000000000001,"15min":1.823},"min":{"1min":0.41399999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.4489999999999998,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.996}]},{"osd":2,"last update":"Mon Oct 9 10:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.7130000000000001,"5min":1.905,"15min":2.456},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.259,"5min":10.259,"15min":15.640000000000001},"last":1.111},{"interface":"front","average":{"1min":1.581,"5min":1.458,"15min":2.1739999999999999},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":4.1619999999999999,"5min":9.1329999999999991,"15min":10.656000000000001},"last":0.497}]},{"osd":3,"last update":"Mon Oct 9 10:12:52 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":2.3879999999999999,"15min":2.028},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.8699999999999992,"5min":9.8699999999999992,"15min":9.8699999999999992},"last":0.35899999999999999},{"interface":"front","average":{"1min":4.1420000000000003,"5min":2.706,"15min":2.9140000000000001},"min":{"1min":0.42999999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":17.193999999999999,"5min":17.34,"15min":17.34},"last":3.1139999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.0089999999999999,"15min":0.71199999999999997},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3419999999999996,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.754,"15min":0.747},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5559999999999992,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":0.27800000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.0629999999999999,"15min":1.0820000000000001},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":27.716000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.135,"15min":1.004},"min":{"1min":0.32700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2910000000000004,"5min":8.2910000000000004,"15min":18.600999999999999},"last":0.34000000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.137,"15min":1.0389999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4710000000000001,"5min":7.891,"15min":10.334},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.173,"5min":0.80300000000000005,"15min":0.96899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":12.356},"last":0.34999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":372047872,"data_stored":372047872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":406192128,"data_stored":406192128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":325980160,"data_stored":325976088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":384172032,"data_stored":384172032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":625803264,"data_stored":625803264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":482217984,"data_stored":482213912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":626065408,"data_stored":626065408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":276697088,"data_stored":276693016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:12:58.383 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2090484694' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:12:58.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5023/44385 objects degraded (11.317%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:12:58.660 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:58.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:58.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:12:58.853 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:59.047 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:12:59.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:59.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:12:59.152 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:12:59.169 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:59.169 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 840 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5023/40359 objects degraded (12.446%); 0 B/s, 32 objects/s recovering 2023-10-09T10:12:59.169 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:59.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:12:59.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T10:12:59.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.170 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:59.170 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:59.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 840 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5023/40359 objects degraded (12.446%); 0 B/s, 32 objects/s recovering 2023-10-09T10:12:59.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:59.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:12:59.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T10:12:59.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:12:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:59.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:59.233 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:12:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:59.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:12:59.322 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:12:59.322 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:12:59.323 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:12:59.323 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:12:59.323 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:12:59.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:12:59.424 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:12:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:12:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 840 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5023/40359 objects degraded (12.446%); 0 B/s, 32 objects/s recovering 2023-10-09T10:12:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:12:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:12:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T10:12:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: failed to encode map e313 with expected crc 2023-10-09T10:12:59.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:12:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:12:59.846 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:12:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:12:59.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:12:59.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:12:59.953 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:12:59.992 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:00.015 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:13:00.045 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:00.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:00.074 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:13:00.112 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:13:00.144 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:13:00.152 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:00.199 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:13:00.208 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:00.253 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:00.256 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2817640292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:00.256 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:13:00.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:00.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:00.302 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:13:00.345 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:13:00.347 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:13:00.347 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:13:00.347 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:13:00.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:13:00.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:13:00.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:13:00.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:13:00.348 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:13:00.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:00.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:00.454 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:00.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:00.558 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:00.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2817640292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:00.627 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2817640292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:00.631 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:00.876 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:00.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:13:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:00.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:00.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:01.125 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:01.165 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:01.226 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:01.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:01.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:01.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:13:01.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:01.410 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:01.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:01 smithi110 ceph-mon[136032]: pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 745 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4868/35787 objects degraded (13.603%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:01.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:01 smithi012 ceph-mon[147926]: pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 745 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4868/35787 objects degraded (13.603%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:01 smithi012 ceph-mon[152263]: pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 745 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4868/35787 objects degraded (13.603%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:02.008 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:02.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:02.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:02.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:02.197 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:02.277 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:02.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:02.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:02.385 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:02.434 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:02.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:02.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:02.607 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:02.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:02.705 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:02.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:02.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:02.895 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:02.965 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:02.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:02.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:13:03.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:03.168 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:03.350 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:03.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:03.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:03.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:03 smithi012 ceph-mon[147926]: pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 732 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4868/35184 objects degraded (13.836%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:03.432 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4868/35184 objects degraded (13.836%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:03.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:03 smithi012 ceph-mon[152263]: pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 732 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4868/35184 objects degraded (13.836%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:03.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4868/35184 objects degraded (13.836%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:03.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:03.509 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:03 smithi110 ceph-mon[136032]: pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 732 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4868/35184 objects degraded (13.836%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:03.510 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4868/35184 objects degraded (13.836%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:03.526 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:03.583 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:03.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:03.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:03.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:03.772 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:03.846 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:03.869 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:03.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:03.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:03.902 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:03.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:04.006 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:04.069 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:04.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:04.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:04.166 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:04.173 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:04.274 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:04.570 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:04.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:04.603 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:13:04.603 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:13:04.688 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:04.754 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:04.806 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:04.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:04.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:04.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:05.008 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:05.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:05 smithi012 ceph-mon[152263]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 589 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 4705/28311 objects degraded (16.619%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:05.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:05 smithi012 ceph-mon[147926]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 589 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 4705/28311 objects degraded (16.619%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:05.580 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:05.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:05.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:05.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:13:05.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:05.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:05 smithi110 ceph-mon[136032]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 589 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 4705/28311 objects degraded (16.619%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:05.787 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:06.049 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:06.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:06.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:06.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:06.243 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:06.594 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:06.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:06.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:06.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:06.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:06.848 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:06.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:06.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:06.955 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:06.993 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:07.042 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:07.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:07.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:07.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:07.178 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:07.228 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:07.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:07.331 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:07.364 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:07 smithi110 ceph-mon[136032]: pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 526 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4705/25302 objects degraded (18.595%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:07.367 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:07.410 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:07.412 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:07.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:07.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:07.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:07 smithi012 ceph-mon[152263]: pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 526 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4705/25302 objects degraded (18.595%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:07.439 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:07 smithi012 ceph-mon[147926]: pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 526 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4705/25302 objects degraded (18.595%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:07.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:07.585 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:13:07.599 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:07.603 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:13:07.628 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:13:07.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:13:07.694 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:13:07.726 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:13:07.745 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:07.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:07.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:07.777 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:13:07.835 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:13:07.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:07.879 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:13:07.890 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:07.923 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:13:07.929 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:13:07.929 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:13:07.930 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:13:07.930 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:13:07.930 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:13:07.930 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:13:07.930 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:13:07.930 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:13:07.952 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:07.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:08.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:08.136 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:13:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:13:07] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T10:13:08.159 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:08.322 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:13:08.336 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":875,"stamp":"2023-10-09T10:13:07.450663+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":454039480,"num_objects":6944,"num_object_clones":0,"num_object_copies":20832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4705,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6944,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":210702,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6021,"num_bytes_recovered":311175884,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142236,"ondisk_log_size":142236,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4695928,"kb_used_data":2092984,"kb_used_omap":167,"kb_used_meta":2602520,"kb_avail":745265288,"statfs":{"total":767960285184,"available":763151654912,"internally_reserved":0,"allocated":2143215616,"data_stored":2136708394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2664981081},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-426573824,"num_objects":-6509,"num_object_clones":0,"num_object_copies":-19527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-318,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6509,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":318,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004907"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":155,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969136+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:12:47.969136+0000","last_peered":"2023-10-09T10:12:47.969136+0000","last_clean":"2023-10-09T10:12:47.969136+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:12:47.969136+0000","last_undegraded":"2023-10-09T10:12:47.969136+0000","last_fullsized":"2023-10-09T10:12:47.969136+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":271,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.963350+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:12:47.963350+0000","last_peered":"2023-10-09T10:12:47.963350+0000","last_clean":"2023-10-09T10:12:47.963350+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:12:47.963350+0000","last_undegraded":"2023-10-09T10:12:47.963350+0000","last_fullsized":"2023-10-09T10:12:47.963350+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":316,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.968944+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:12:47.968944+0000","last_peered":"2023-10-09T10:12:47.968944+0000","last_clean":"2023-10-09T10:12:47.968944+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:12:47.968944+0000","last_undegraded":"2023-10-09T10:12:47.968944+0000","last_fullsized":"2023-10-09T10:12:47.968944+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":180,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966948+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:12:47.966948+0000","last_peered":"2023-10-09T10:12:47.966948+0000","last_clean":"2023-10-09T10:12:47.966948+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:12:47.966948+0000","last_undegraded":"2023-10-09T10:12:47.966948+0000","last_fullsized":"2023-10-09T10:12:47.966948+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":285,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966622+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:12:47.966622+0000","last_peered":"2023-10-09T10:12:47.966622+0000","last_clean":"2023-10-09T10:12:47.966622+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:12:47.966622+0000","last_undegraded":"2023-10-09T10:12:47.966622+0000","last_fullsized":"2023-10-09T10:12:47.966622+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":131,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.970861+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:12:47.970861+0000","last_peered":"2023-10-09T10:12:47.970861+0000","last_clean":"2023-10-09T10:12:47.970861+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:12:47.970861+0000","last_undegraded":"2023-10-09T10:12:47.970861+0000","last_fullsized":"2023-10-09T10:12:47.970861+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":197,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.962645+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:12:47.962645+0000","last_peered":"2023-10-09T10:12:47.962645+0000","last_clean":"2023-10-09T10:12:47.962645+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:12:47.962645+0000","last_undegraded":"2023-10-09T10:12:47.962645+0000","last_fullsized":"2023-10-09T10:12:47.962645+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":266,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966547+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:12:47.966547+0000","last_peered":"2023-10-09T10:12:47.966547+0000","last_clean":"2023-10-09T10:12:47.966547+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:12:47.966547+0000","last_undegraded":"2023-10-09T10:12:47.966547+0000","last_fullsized":"2023-10-09T10:12:47.966547+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":130,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.970630+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:12:47.970630+0000","last_peered":"2023-10-09T10:12:47.970630+0000","last_clean":"2023-10-09T10:12:47.970630+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:12:47.970630+0000","last_undegraded":"2023-10-09T10:12:47.970630+0000","last_fullsized":"2023-10-09T10:12:47.970630+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":227,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.963133+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:12:47.963133+0000","last_peered":"2023-10-09T10:12:47.963133+0000","last_clean":"2023-10-09T10:12:47.963133+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:12:47.963133+0000","last_undegraded":"2023-10-09T10:12:47.963133+0000","last_fullsized":"2023-10-09T10:12:47.963133+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":351,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969502+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:12:47.969502+0000","last_peered":"2023-10-09T10:12:47.969502+0000","last_clean":"2023-10-09T10:12:47.969502+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:12:47.969502+0000","last_undegraded":"2023-10-09T10:12:47.969502+0000","last_fullsized":"2023-10-09T10:12:47.969502+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":309,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.166917+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:12:58.166917+0000","last_peered":"2023-10-09T10:12:58.166917+0000","last_clean":"2023-10-09T10:12:58.166917+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:12:58.166917+0000","last_undegraded":"2023-10-09T10:12:58.166917+0000","last_fullsized":"2023-10-09T10:12:58.166917+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":200,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969096+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:12:47.969096+0000","last_peered":"2023-10-09T10:12:47.969096+0000","last_clean":"2023-10-09T10:12:47.969096+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:12:47.969096+0000","last_undegraded":"2023-10-09T10:12:47.969096+0000","last_fullsized":"2023-10-09T10:12:47.969096+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":180,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969458+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:12:47.969458+0000","last_peered":"2023-10-09T10:12:47.969458+0000","last_clean":"2023-10-09T10:12:47.969458+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:12:47.969458+0000","last_undegraded":"2023-10-09T10:12:47.969458+0000","last_fullsized":"2023-10-09T10:12:47.969458+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":384,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.968721+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:12:47.968721+0000","last_peered":"2023-10-09T10:12:47.968721+0000","last_clean":"2023-10-09T10:12:47.968721+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:12:47.968721+0000","last_undegraded":"2023-10-09T10:12:47.968721+0000","last_fullsized":"2023-10-09T10:12:47.968721+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":253,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.963630+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:12:47.963630+0000","last_peered":"2023-10-09T10:12:47.963630+0000","last_clean":"2023-10-09T10:12:47.963630+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:12:47.963630+0000","last_undegraded":"2023-10-09T10:12:47.963630+0000","last_fullsized":"2023-10-09T10:12:47.963630+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":179,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966626+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:12:47.966626+0000","last_peered":"2023-10-09T10:12:47.966626+0000","last_clean":"2023-10-09T10:12:47.966626+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:12:47.966626+0000","last_undegraded":"2023-10-09T10:12:47.966626+0000","last_fullsized":"2023-10-09T10:12:47.966626+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":146,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.970981+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:12:47.970981+0000","last_peered":"2023-10-09T10:12:47.970981+0000","last_clean":"2023-10-09T10:12:47.970981+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:12:47.970981+0000","last_undegraded":"2023-10-09T10:12:47.970981+0000","last_fullsized":"2023-10-09T10:12:47.970981+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":181,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.972273+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:12:47.972273+0000","last_peered":"2023-10-09T10:12:47.972273+0000","last_clean":"2023-10-09T10:12:47.972273+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:12:47.972273+0000","last_undegraded":"2023-10-09T10:12:47.972273+0000","last_fullsized":"2023-10-09T10:12:47.972273+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":179,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.967018+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:12:47.967018+0000","last_peered":"2023-10-09T10:12:47.967018+0000","last_clean":"2023-10-09T10:12:47.967018+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:12:47.967018+0000","last_undegraded":"2023-10-09T10:12:47.967018+0000","last_fullsized":"2023-10-09T10:12:47.967018+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":202,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.961960+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:12:47.961960+0000","last_peered":"2023-10-09T10:12:47.961960+0000","last_clean":"2023-10-09T10:12:47.961960+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:12:47.961960+0000","last_undegraded":"2023-10-09T10:12:47.961960+0000","last_fullsized":"2023-10-09T10:12:47.961960+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":266,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.965038+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:12:47.965038+0000","last_peered":"2023-10-09T10:12:47.965038+0000","last_clean":"2023-10-09T10:12:47.965038+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:12:47.965038+0000","last_undegraded":"2023-10-09T10:12:47.965038+0000","last_fullsized":"2023-10-09T10:12:47.965038+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"313'13170","reported_seq":14565,"reported_epoch":313,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:13:02.999231+0000","last_change":"2023-10-09T10:12:42.183681+0000","last_active":"2023-10-09T10:13:02.999231+0000","last_peered":"2023-10-09T10:13:02.999231+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:13:02.999231+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"281'3100","ondisk_log_start":"281'3100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32505856,"num_objects":496,"num_object_clones":0,"num_object_copies":1488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4705,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13170,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":668,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-4209}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":345,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969117+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:12:47.969117+0000","last_peered":"2023-10-09T10:12:47.969117+0000","last_clean":"2023-10-09T10:12:47.969117+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:12:47.969117+0000","last_undegraded":"2023-10-09T10:12:47.969117+0000","last_fullsized":"2023-10-09T10:12:47.969117+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.961505+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:12:47.961505+0000","last_peered":"2023-10-09T10:12:47.961505+0000","last_clean":"2023-10-09T10:12:47.961505+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:12:47.961505+0000","last_undegraded":"2023-10-09T10:12:47.961505+0000","last_fullsized":"2023-10-09T10:12:47.961505+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"313'13341","reported_seq":13391,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:04.994895+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:13:04.994895+0000","last_peered":"2023-10-09T10:13:04.994895+0000","last_clean":"2023-10-09T10:13:04.994895+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:13:04.994895+0000","last_undegraded":"2023-10-09T10:13:04.994895+0000","last_fullsized":"2023-10-09T10:13:04.994895+0000","mapping_epoch":271,"log_start":"285'4700","ondisk_log_start":"285'4700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8641,"log_dups_size":0,"ondisk_log_size":8641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29425664,"num_objects":449,"num_object_clones":0,"num_object_copies":1347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13341,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":333,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969237+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:12:47.969237+0000","last_peered":"2023-10-09T10:12:47.969237+0000","last_clean":"2023-10-09T10:12:47.969237+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:12:47.969237+0000","last_undegraded":"2023-10-09T10:12:47.969237+0000","last_fullsized":"2023-10-09T10:12:47.969237+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":225,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.968864+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:12:47.968864+0000","last_peered":"2023-10-09T10:12:47.968864+0000","last_clean":"2023-10-09T10:12:47.968864+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:12:47.968864+0000","last_undegraded":"2023-10-09T10:12:47.968864+0000","last_fullsized":"2023-10-09T10:12:47.968864+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"313'13256","reported_seq":13309,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:06.859349+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:13:06.859349+0000","last_peered":"2023-10-09T10:13:06.859349+0000","last_clean":"2023-10-09T10:13:06.859349+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:13:06.859349+0000","last_undegraded":"2023-10-09T10:13:06.859349+0000","last_fullsized":"2023-10-09T10:13:06.859349+0000","mapping_epoch":271,"log_start":"283'3800","ondisk_log_start":"283'3800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9456,"log_dups_size":0,"ondisk_log_size":9456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20316160,"num_objects":310,"num_object_clones":0,"num_object_copies":930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13256,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":135,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.971057+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:12:47.971057+0000","last_peered":"2023-10-09T10:12:47.971057+0000","last_clean":"2023-10-09T10:12:47.971057+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:12:47.971057+0000","last_undegraded":"2023-10-09T10:12:47.971057+0000","last_fullsized":"2023-10-09T10:12:47.971057+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":156,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.965692+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:12:47.965692+0000","last_peered":"2023-10-09T10:12:47.965692+0000","last_clean":"2023-10-09T10:12:47.965692+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:12:47.965692+0000","last_undegraded":"2023-10-09T10:12:47.965692+0000","last_fullsized":"2023-10-09T10:12:47.965692+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"313'12948","reported_seq":13000,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:03.657740+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:13:03.657740+0000","last_peered":"2023-10-09T10:13:03.657740+0000","last_clean":"2023-10-09T10:13:03.657740+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:13:03.657740+0000","last_undegraded":"2023-10-09T10:13:03.657740+0000","last_fullsized":"2023-10-09T10:13:03.657740+0000","mapping_epoch":271,"log_start":"285'5200","ondisk_log_start":"285'5200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7748,"log_dups_size":0,"ondisk_log_size":7748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30277632,"num_objects":462,"num_object_clones":0,"num_object_copies":1386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12948,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":146,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.967709+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:12:47.967709+0000","last_peered":"2023-10-09T10:12:47.967709+0000","last_clean":"2023-10-09T10:12:47.967709+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:12:47.967709+0000","last_undegraded":"2023-10-09T10:12:47.967709+0000","last_fullsized":"2023-10-09T10:12:47.967709+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":180,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966980+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:12:47.966980+0000","last_peered":"2023-10-09T10:12:47.966980+0000","last_clean":"2023-10-09T10:12:47.966980+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:12:47.966980+0000","last_undegraded":"2023-10-09T10:12:47.966980+0000","last_fullsized":"2023-10-09T10:12:47.966980+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"313'13456","reported_seq":13509,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:06.544734+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:13:06.544734+0000","last_peered":"2023-10-09T10:13:06.544734+0000","last_clean":"2023-10-09T10:13:06.544734+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:13:06.544734+0000","last_undegraded":"2023-10-09T10:13:06.544734+0000","last_fullsized":"2023-10-09T10:13:06.544734+0000","mapping_epoch":271,"log_start":"285'5300","ondisk_log_start":"285'5300","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25165824,"num_objects":384,"num_object_clones":0,"num_object_copies":1152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13456,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":307,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.964584+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:12:47.964584+0000","last_peered":"2023-10-09T10:12:47.964584+0000","last_clean":"2023-10-09T10:12:47.964584+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:12:47.964584+0000","last_undegraded":"2023-10-09T10:12:47.964584+0000","last_fullsized":"2023-10-09T10:12:47.964584+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":177,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.963561+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:12:47.963561+0000","last_peered":"2023-10-09T10:12:47.963561+0000","last_clean":"2023-10-09T10:12:47.963561+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:12:47.963561+0000","last_undegraded":"2023-10-09T10:12:47.963561+0000","last_fullsized":"2023-10-09T10:12:47.963561+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"313'13227","reported_seq":13280,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:03.629872+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:13:03.629872+0000","last_peered":"2023-10-09T10:13:03.629872+0000","last_clean":"2023-10-09T10:13:03.629872+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:13:03.629872+0000","last_undegraded":"2023-10-09T10:13:03.629872+0000","last_fullsized":"2023-10-09T10:13:03.629872+0000","mapping_epoch":271,"log_start":"289'5500","ondisk_log_start":"289'5500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7727,"log_dups_size":0,"ondisk_log_size":7727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31260672,"num_objects":477,"num_object_clones":0,"num_object_copies":1431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13227,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":239,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:48.666970+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:12:48.666970+0000","last_peered":"2023-10-09T10:12:48.666970+0000","last_clean":"2023-10-09T10:12:48.666970+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:12:48.666970+0000","last_undegraded":"2023-10-09T10:12:48.666970+0000","last_fullsized":"2023-10-09T10:12:48.666970+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":244,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969840+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:12:47.969840+0000","last_peered":"2023-10-09T10:12:47.969840+0000","last_clean":"2023-10-09T10:12:47.969840+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:12:47.969840+0000","last_undegraded":"2023-10-09T10:12:47.969840+0000","last_fullsized":"2023-10-09T10:12:47.969840+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"313'13064","reported_seq":13117,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:06.855353+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:13:06.855353+0000","last_peered":"2023-10-09T10:13:06.855353+0000","last_clean":"2023-10-09T10:13:06.855353+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:13:06.855353+0000","last_undegraded":"2023-10-09T10:13:06.855353+0000","last_fullsized":"2023-10-09T10:13:06.855353+0000","mapping_epoch":271,"log_start":"283'3600","ondisk_log_start":"283'3600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9464,"log_dups_size":0,"ondisk_log_size":9464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24117248,"num_objects":368,"num_object_clones":0,"num_object_copies":1104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13064,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":207,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.972187+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:12:47.972187+0000","last_peered":"2023-10-09T10:12:47.972187+0000","last_clean":"2023-10-09T10:12:47.972187+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:12:47.972187+0000","last_undegraded":"2023-10-09T10:12:47.972187+0000","last_fullsized":"2023-10-09T10:12:47.972187+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":181,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.172254+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:12:58.172254+0000","last_peered":"2023-10-09T10:12:58.172254+0000","last_clean":"2023-10-09T10:12:58.172254+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:12:58.172254+0000","last_undegraded":"2023-10-09T10:12:58.172254+0000","last_fullsized":"2023-10-09T10:12:58.172254+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"313'13107","reported_seq":13160,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:03.667748+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:13:03.667748+0000","last_peered":"2023-10-09T10:13:03.667748+0000","last_clean":"2023-10-09T10:13:03.667748+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:13:03.667748+0000","last_undegraded":"2023-10-09T10:13:03.667748+0000","last_fullsized":"2023-10-09T10:13:03.667748+0000","mapping_epoch":271,"log_start":"289'5400","ondisk_log_start":"289'5400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7707,"log_dups_size":0,"ondisk_log_size":7707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32178176,"num_objects":491,"num_object_clones":0,"num_object_copies":1473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13107,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":267,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966476+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:12:47.966476+0000","last_peered":"2023-10-09T10:12:47.966476+0000","last_clean":"2023-10-09T10:12:47.966476+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:12:47.966476+0000","last_undegraded":"2023-10-09T10:12:47.966476+0000","last_fullsized":"2023-10-09T10:12:47.966476+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":317,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.972565+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:12:47.972565+0000","last_peered":"2023-10-09T10:12:47.972565+0000","last_clean":"2023-10-09T10:12:47.972565+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:12:47.972565+0000","last_undegraded":"2023-10-09T10:12:47.972565+0000","last_fullsized":"2023-10-09T10:12:47.972565+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"313'13164","reported_seq":13216,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:06.777599+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:13:06.777599+0000","last_peered":"2023-10-09T10:13:06.777599+0000","last_clean":"2023-10-09T10:13:06.777599+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:13:06.777599+0000","last_undegraded":"2023-10-09T10:13:06.777599+0000","last_fullsized":"2023-10-09T10:13:06.777599+0000","mapping_epoch":271,"log_start":"283'3700","ondisk_log_start":"283'3700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9464,"log_dups_size":0,"ondisk_log_size":9464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20185088,"num_objects":308,"num_object_clones":0,"num_object_copies":924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13164,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":316,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969558+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:12:47.969558+0000","last_peered":"2023-10-09T10:12:47.969558+0000","last_clean":"2023-10-09T10:12:47.969558+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:12:47.969558+0000","last_undegraded":"2023-10-09T10:12:47.969558+0000","last_fullsized":"2023-10-09T10:12:47.969558+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":155,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.970370+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:12:47.970370+0000","last_peered":"2023-10-09T10:12:47.970370+0000","last_clean":"2023-10-09T10:12:47.970370+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:12:47.970370+0000","last_undegraded":"2023-10-09T10:12:47.970370+0000","last_fullsized":"2023-10-09T10:12:47.970370+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":273,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969213+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:12:47.969213+0000","last_peered":"2023-10-09T10:12:47.969213+0000","last_clean":"2023-10-09T10:12:47.969213+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:12:47.969213+0000","last_undegraded":"2023-10-09T10:12:47.969213+0000","last_fullsized":"2023-10-09T10:12:47.969213+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"313'13062","reported_seq":13112,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:02.961795+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:13:02.961795+0000","last_peered":"2023-10-09T10:13:02.961795+0000","last_clean":"2023-10-09T10:13:02.961795+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:13:02.961795+0000","last_undegraded":"2023-10-09T10:13:02.961795+0000","last_fullsized":"2023-10-09T10:13:02.961795+0000","mapping_epoch":271,"log_start":"280'3000","ondisk_log_start":"280'3000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33554432,"num_objects":512,"num_object_clones":0,"num_object_copies":1536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13062,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":131,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.961924+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:12:47.961924+0000","last_peered":"2023-10-09T10:12:47.961924+0000","last_clean":"2023-10-09T10:12:47.961924+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:12:47.961924+0000","last_undegraded":"2023-10-09T10:12:47.961924+0000","last_fullsized":"2023-10-09T10:12:47.961924+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":130,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966818+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:12:47.966818+0000","last_peered":"2023-10-09T10:12:47.966818+0000","last_clean":"2023-10-09T10:12:47.966818+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:12:47.966818+0000","last_undegraded":"2023-10-09T10:12:47.966818+0000","last_fullsized":"2023-10-09T10:12:47.966818+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":339,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969489+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:12:47.969489+0000","last_peered":"2023-10-09T10:12:47.969489+0000","last_clean":"2023-10-09T10:12:47.969489+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:12:47.969489+0000","last_undegraded":"2023-10-09T10:12:47.969489+0000","last_fullsized":"2023-10-09T10:12:47.969489+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"313'13144","reported_seq":13196,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:06.539346+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:13:06.539346+0000","last_peered":"2023-10-09T10:13:06.539346+0000","last_clean":"2023-10-09T10:13:06.539346+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:13:06.539346+0000","last_undegraded":"2023-10-09T10:13:06.539346+0000","last_fullsized":"2023-10-09T10:13:06.539346+0000","mapping_epoch":271,"log_start":"285'5000","ondisk_log_start":"285'5000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26083328,"num_objects":398,"num_object_clones":0,"num_object_copies":1194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13144,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":233,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.963001+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:12:47.963001+0000","last_peered":"2023-10-09T10:12:47.963001+0000","last_clean":"2023-10-09T10:12:47.963001+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:12:47.963001+0000","last_undegraded":"2023-10-09T10:12:47.963001+0000","last_fullsized":"2023-10-09T10:12:47.963001+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":309,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:48.666895+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:12:48.666895+0000","last_peered":"2023-10-09T10:12:48.666895+0000","last_clean":"2023-10-09T10:12:48.666895+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:12:48.666895+0000","last_undegraded":"2023-10-09T10:12:48.666895+0000","last_fullsized":"2023-10-09T10:12:48.666895+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":329,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.972496+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:12:47.972496+0000","last_peered":"2023-10-09T10:12:47.972496+0000","last_clean":"2023-10-09T10:12:47.972496+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:12:47.972496+0000","last_undegraded":"2023-10-09T10:12:47.972496+0000","last_fullsized":"2023-10-09T10:12:47.972496+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"313'12961","reported_seq":13011,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:02.967640+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:13:02.967640+0000","last_peered":"2023-10-09T10:13:02.967640+0000","last_clean":"2023-10-09T10:13:02.967640+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:13:02.967640+0000","last_undegraded":"2023-10-09T10:13:02.967640+0000","last_fullsized":"2023-10-09T10:13:02.967640+0000","mapping_epoch":271,"log_start":"280'2900","ondisk_log_start":"280'2900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10061,"log_dups_size":0,"ondisk_log_size":10061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32702464,"num_objects":499,"num_object_clones":0,"num_object_copies":1497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12961,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":140,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.961775+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:12:47.961775+0000","last_peered":"2023-10-09T10:12:47.961775+0000","last_clean":"2023-10-09T10:12:47.961775+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:12:47.961775+0000","last_undegraded":"2023-10-09T10:12:47.961775+0000","last_fullsized":"2023-10-09T10:12:47.961775+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":559,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.968572+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:12:47.968572+0000","last_peered":"2023-10-09T10:12:47.968572+0000","last_clean":"2023-10-09T10:12:47.968572+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:12:47.968572+0000","last_undegraded":"2023-10-09T10:12:47.968572+0000","last_fullsized":"2023-10-09T10:12:47.968572+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":217,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.963157+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:12:47.963157+0000","last_peered":"2023-10-09T10:12:47.963157+0000","last_clean":"2023-10-09T10:12:47.963157+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:12:47.963157+0000","last_undegraded":"2023-10-09T10:12:47.963157+0000","last_fullsized":"2023-10-09T10:12:47.963157+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":228,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969554+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:12:47.969554+0000","last_peered":"2023-10-09T10:12:47.969554+0000","last_clean":"2023-10-09T10:12:47.969554+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:12:47.969554+0000","last_undegraded":"2023-10-09T10:12:47.969554+0000","last_fullsized":"2023-10-09T10:12:47.969554+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"313'13132","reported_seq":13184,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:03.586827+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:13:03.586827+0000","last_peered":"2023-10-09T10:13:03.586827+0000","last_clean":"2023-10-09T10:13:03.586827+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:13:03.586827+0000","last_undegraded":"2023-10-09T10:13:03.586827+0000","last_fullsized":"2023-10-09T10:13:03.586827+0000","mapping_epoch":271,"log_start":"289'5400","ondisk_log_start":"289'5400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7732,"log_dups_size":0,"ondisk_log_size":7732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32112640,"num_objects":490,"num_object_clones":0,"num_object_copies":1470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13132,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":289,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.971593+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:12:47.971593+0000","last_peered":"2023-10-09T10:12:47.971593+0000","last_clean":"2023-10-09T10:12:47.971593+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:12:47.971593+0000","last_undegraded":"2023-10-09T10:12:47.971593+0000","last_fullsized":"2023-10-09T10:12:47.971593+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":191,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:56.865278+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:12:56.865278+0000","last_peered":"2023-10-09T10:12:56.865278+0000","last_clean":"2023-10-09T10:12:56.865278+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:12:56.865278+0000","last_undegraded":"2023-10-09T10:12:56.865278+0000","last_fullsized":"2023-10-09T10:12:56.865278+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":218,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.967004+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:12:47.967004+0000","last_peered":"2023-10-09T10:12:47.967004+0000","last_clean":"2023-10-09T10:12:47.967004+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:12:47.967004+0000","last_undegraded":"2023-10-09T10:12:47.967004+0000","last_fullsized":"2023-10-09T10:12:47.967004+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"313'13281","reported_seq":23917,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:06.544634+0000","last_change":"2023-10-09T10:12:43.942710+0000","last_active":"2023-10-09T10:13:06.544634+0000","last_peered":"2023-10-09T10:13:06.544634+0000","last_clean":"2023-10-09T10:13:06.544634+0000","last_became_active":"2023-10-09T10:12:43.912846+0000","last_became_peered":"2023-10-09T10:12:43.912846+0000","last_unstale":"2023-10-09T10:13:06.544634+0000","last_undegraded":"2023-10-09T10:13:06.544634+0000","last_fullsized":"2023-10-09T10:13:06.544634+0000","mapping_epoch":310,"log_start":"285'5100","ondisk_log_start":"285'5100","created":271,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8181,"log_dups_size":0,"ondisk_log_size":8181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:08.472095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24707072,"num_objects":377,"num_object_clones":0,"num_object_copies":1131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13281,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5284,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":332,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969425+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:12:47.969425+0000","last_peered":"2023-10-09T10:12:47.969425+0000","last_clean":"2023-10-09T10:12:47.969425+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:12:47.969425+0000","last_undegraded":"2023-10-09T10:12:47.969425+0000","last_fullsized":"2023-10-09T10:12:47.969425+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":345,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969630+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:12:47.969630+0000","last_peered":"2023-10-09T10:12:47.969630+0000","last_clean":"2023-10-09T10:12:47.969630+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:12:47.969630+0000","last_undegraded":"2023-10-09T10:12:47.969630+0000","last_fullsized":"2023-10-09T10:12:47.969630+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":183,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:56.865351+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:12:56.865351+0000","last_peered":"2023-10-09T10:12:56.865351+0000","last_clean":"2023-10-09T10:12:56.865351+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:12:56.865351+0000","last_undegraded":"2023-10-09T10:12:56.865351+0000","last_fullsized":"2023-10-09T10:12:56.865351+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"313'13208","reported_seq":13259,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:06.861406+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:13:06.861406+0000","last_peered":"2023-10-09T10:13:06.861406+0000","last_clean":"2023-10-09T10:13:06.861406+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:13:06.861406+0000","last_undegraded":"2023-10-09T10:13:06.861406+0000","last_fullsized":"2023-10-09T10:13:06.861406+0000","mapping_epoch":271,"log_start":"283'3800","ondisk_log_start":"283'3800","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9408,"log_dups_size":0,"ondisk_log_size":9408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22937600,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13208,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":276,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.965088+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:12:47.965088+0000","last_peered":"2023-10-09T10:12:47.965088+0000","last_clean":"2023-10-09T10:12:47.965088+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:12:47.965088+0000","last_undegraded":"2023-10-09T10:12:47.965088+0000","last_fullsized":"2023-10-09T10:12:47.965088+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":290,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.972306+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:12:47.972306+0000","last_peered":"2023-10-09T10:12:47.972306+0000","last_clean":"2023-10-09T10:12:47.972306+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:12:47.972306+0000","last_undegraded":"2023-10-09T10:12:47.972306+0000","last_fullsized":"2023-10-09T10:12:47.972306+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":265,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966390+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:12:47.966390+0000","last_peered":"2023-10-09T10:12:47.966390+0000","last_clean":"2023-10-09T10:12:47.966390+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:12:47.966390+0000","last_undegraded":"2023-10-09T10:12:47.966390+0000","last_fullsized":"2023-10-09T10:12:47.966390+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"313'12921","reported_seq":12972,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:02.966196+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:13:02.966196+0000","last_peered":"2023-10-09T10:13:02.966196+0000","last_clean":"2023-10-09T10:13:02.966196+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:13:02.966196+0000","last_undegraded":"2023-10-09T10:13:02.966196+0000","last_fullsized":"2023-10-09T10:13:02.966196+0000","mapping_epoch":271,"log_start":"280'2900","ondisk_log_start":"280'2900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10021,"log_dups_size":0,"ondisk_log_size":10021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35913752,"num_objects":549,"num_object_clones":0,"num_object_copies":1647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":549,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12921,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":221,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966534+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:12:47.966534+0000","last_peered":"2023-10-09T10:12:47.966534+0000","last_clean":"2023-10-09T10:12:47.966534+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:12:47.966534+0000","last_undegraded":"2023-10-09T10:12:47.966534+0000","last_fullsized":"2023-10-09T10:12:47.966534+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":184,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966836+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:12:47.966836+0000","last_peered":"2023-10-09T10:12:47.966836+0000","last_clean":"2023-10-09T10:12:47.966836+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:12:47.966836+0000","last_undegraded":"2023-10-09T10:12:47.966836+0000","last_fullsized":"2023-10-09T10:12:47.966836+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":166,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.961879+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:12:47.961879+0000","last_peered":"2023-10-09T10:12:47.961879+0000","last_clean":"2023-10-09T10:12:47.961879+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:12:47.961879+0000","last_undegraded":"2023-10-09T10:12:47.961879+0000","last_fullsized":"2023-10-09T10:12:47.961879+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":179,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.966760+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:12:47.966760+0000","last_peered":"2023-10-09T10:12:47.966760+0000","last_clean":"2023-10-09T10:12:47.966760+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:12:47.966760+0000","last_undegraded":"2023-10-09T10:12:47.966760+0000","last_fullsized":"2023-10-09T10:12:47.966760+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":251,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.969696+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:12:47.969696+0000","last_peered":"2023-10-09T10:12:47.969696+0000","last_clean":"2023-10-09T10:12:47.969696+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:12:47.969696+0000","last_undegraded":"2023-10-09T10:12:47.969696+0000","last_fullsized":"2023-10-09T10:12:47.969696+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":150,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.967642+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:12:47.967642+0000","last_peered":"2023-10-09T10:12:47.967642+0000","last_clean":"2023-10-09T10:12:47.967642+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:12:47.967642+0000","last_undegraded":"2023-10-09T10:12:47.967642+0000","last_fullsized":"2023-10-09T10:12:47.967642+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":192,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.964502+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:12:47.964502+0000","last_peered":"2023-10-09T10:12:47.964502+0000","last_clean":"2023-10-09T10:12:47.964502+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:12:47.964502+0000","last_undegraded":"2023-10-09T10:12:47.964502+0000","last_fullsized":"2023-10-09T10:12:47.964502+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":150,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.970038+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:12:47.970038+0000","last_peered":"2023-10-09T10:12:47.970038+0000","last_clean":"2023-10-09T10:12:47.970038+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:12:47.970038+0000","last_undegraded":"2023-10-09T10:12:47.970038+0000","last_fullsized":"2023-10-09T10:12:47.970038+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":201,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.963073+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:12:47.963073+0000","last_peered":"2023-10-09T10:12:47.963073+0000","last_clean":"2023-10-09T10:12:47.963073+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:12:47.963073+0000","last_undegraded":"2023-10-09T10:12:47.963073+0000","last_fullsized":"2023-10-09T10:12:47.963073+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":330,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.967219+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:12:47.967219+0000","last_peered":"2023-10-09T10:12:47.967219+0000","last_clean":"2023-10-09T10:12:47.967219+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:12:47.967219+0000","last_undegraded":"2023-10-09T10:12:47.967219+0000","last_fullsized":"2023-10-09T10:12:47.967219+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":351,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.963262+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:12:47.963262+0000","last_peered":"2023-10-09T10:12:47.963262+0000","last_clean":"2023-10-09T10:12:47.963262+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:12:47.963262+0000","last_undegraded":"2023-10-09T10:12:47.963262+0000","last_fullsized":"2023-10-09T10:12:47.963262+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":261,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:56.876909+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:12:56.876909+0000","last_peered":"2023-10-09T10:12:56.876909+0000","last_clean":"2023-10-09T10:12:56.876909+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:12:56.876909+0000","last_undegraded":"2023-10-09T10:12:56.876909+0000","last_fullsized":"2023-10-09T10:12:56.876909+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":180,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:12:47.971293+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:12:47.971293+0000","last_peered":"2023-10-09T10:12:47.971293+0000","last_clean":"2023-10-09T10:12:47.971293+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:12:47.971293+0000","last_undegraded":"2023-10-09T10:12:47.971293+0000","last_fullsized":"2023-10-09T10:12:47.971293+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":453443608,"num_objects":6920,"num_object_clones":0,"num_object_copies":20760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4705,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6920,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":210442,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5952,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2057449472,"data_stored":2057437256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142042,"ondisk_log_size":142042,"up":48,"acting":47,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430521,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":451896,"kb_used_data":142384,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93293256,"statfs":{"total":95995035648,"available":95532294144,"internally_reserved":0,"allocated":145801216,"data_stored":145259691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.3130000000000002,"15min":2.0539999999999998},"min":{"1min":0.27400000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":11.164,"5min":11.164,"15min":50.591999999999999},"last":0.251},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.2770000000000001,"15min":2.0059999999999998},"min":{"1min":0.32600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.7880000000000003,"5min":15.974,"15min":50.747},"last":4.2869999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:12:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9940000000000002,"5min":2.3639999999999999,"15min":1.958},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":11.92,"5min":14.311,"15min":25.338000000000001},"last":2.75},{"interface":"front","average":{"1min":2.6320000000000001,"5min":2.218,"15min":1.6870000000000001},"min":{"1min":0.42999999999999999,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":10.663,"5min":18.27,"15min":18.27},"last":4.6769999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":3.8029999999999999,"5min":2.6030000000000002,"15min":2.4430000000000001},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":10.682,"5min":13.775,"15min":13.775},"last":3.4329999999999998},{"interface":"front","average":{"1min":3.7200000000000002,"5min":2.4350000000000001,"15min":2.2559999999999998},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.795999999999999,"5min":12.101000000000001,"15min":12.101000000000001},"last":4.8739999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":3.6110000000000002,"5min":3.0009999999999999,"15min":2.1669999999999998},"min":{"1min":0.40300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":10.827,"5min":29.058,"15min":29.058},"last":3.306},{"interface":"front","average":{"1min":4.0060000000000002,"5min":2.6680000000000001,"15min":2.1859999999999999},"min":{"1min":0.36699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":16.715,"5min":16.715,"15min":16.715},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.288,"15min":1.0369999999999999},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":7.7469999999999999,"5min":7.7469999999999999,"15min":7.7469999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.724,"5min":1.139,"15min":0.98499999999999999},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":0.40500000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8180000000000001,"15min":1.256},"min":{"1min":0.34000000000000002,"5min":0.219,"15min":0.219},"max":{"1min":9.2560000000000002,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.9089999999999998,"5min":1.7130000000000001,"15min":1.2150000000000001},"min":{"1min":0.38600000000000001,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":18.547999999999998,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.54100000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1000000000000001,"5min":1.3620000000000001,"15min":1.1080000000000001},"min":{"1min":0.40400000000000003,"5min":0.30199999999999999,"15min":0.26100000000000001},"max":{"1min":8.8569999999999993,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.1779999999999999,"5min":1.2090000000000001,"15min":1.026},"min":{"1min":0.33900000000000002,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":5.9660000000000002,"5min":6.4020000000000001,"15min":9.1920000000000002},"last":0.379}]}]},{"osd":6,"up_from":42,"seq":180388626749,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":797032,"kb_used_data":487456,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92948120,"statfs":{"total":95995035648,"available":95178874880,"internally_reserved":0,"allocated":499154944,"data_stored":498630733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8180000000000001,"5min":2.2330000000000001,"15min":1.756},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.6940000000000008,"5min":15.435,"15min":15.435},"last":1.339},{"interface":"front","average":{"1min":2.5129999999999999,"5min":2.1890000000000001,"15min":1.7250000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":13.385,"5min":15.308999999999999,"15min":15.308999999999999},"last":1.2589999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:12:06 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":2.839,"15min":2.0579999999999998},"min":{"1min":0.30199999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.0519999999999996,"5min":35.609999999999999,"15min":35.609999999999999},"last":1.5529999999999999},{"interface":"front","average":{"1min":2.919,"5min":2.46,"15min":1.9910000000000001},"min":{"1min":0.36099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":8.2680000000000007,"5min":32.396000000000001,"15min":32.396000000000001},"last":2.3900000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.8740000000000001,"15min":1.802},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":7.7359999999999998,"5min":12.898,"15min":12.898},"last":4.3209999999999997},{"interface":"front","average":{"1min":3.4609999999999999,"5min":2.6499999999999999,"15min":3.036},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.720000000000001,"5min":14.542999999999999,"15min":14.542999999999999},"last":7.3410000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.7839999999999998,"15min":1.8340000000000001},"min":{"1min":0.437,"5min":0.308,"15min":0.308},"max":{"1min":11.098000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":1.4430000000000001},{"interface":"front","average":{"1min":3.161,"5min":2.6059999999999999,"15min":2.456},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.002000000000001,"5min":11.211,"15min":11.211},"last":9.3599999999999994}]},{"osd":4,"last update":"Mon Oct 9 10:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":0.94699999999999995,"15min":0.60899999999999999},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.113,"5min":10.113,"15min":10.113},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.89500000000000002,"15min":0.63100000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.948,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.42399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:12:37 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.0660000000000001,"15min":0.91800000000000004},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.23699999999999999},"max":{"1min":6.3170000000000002,"5min":7.9260000000000002,"15min":14.09},"last":1.2250000000000001},{"interface":"front","average":{"1min":2.335,"5min":1.222,"15min":0.99399999999999999},"min":{"1min":0.34699999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":14.465999999999999,"5min":14.465999999999999,"15min":14.465999999999999},"last":0.51300000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.92600000000000005,"15min":0.89500000000000002},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.252},"max":{"1min":6.0700000000000003,"5min":6.2709999999999999,"15min":7.5350000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.673,"5min":0.97199999999999998,"15min":0.94299999999999995},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.101,"5min":6.3159999999999998,"15min":7.5570000000000004},"last":0.49099999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253062,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":669688,"kb_used_data":234160,"kb_used_omap":9,"kb_used_meta":435510,"kb_avail":93075464,"statfs":{"total":95995035648,"available":95309275136,"internally_reserved":0,"allocated":239779840,"data_stored":238707104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":445963241},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":4.4160000000000004,"5min":2.5880000000000001,"15min":2.4500000000000002},"min":{"1min":0.217,"5min":0.183,"15min":0.183},"max":{"1min":31.007000000000001,"5min":31.007000000000001,"15min":31.007000000000001},"last":4.9800000000000004},{"interface":"front","average":{"1min":3.8010000000000002,"5min":3.4079999999999999,"15min":2.5899999999999999},"min":{"1min":0.28599999999999998,"5min":0.245,"15min":0.192},"max":{"1min":10.577999999999999,"5min":20.501999999999999,"15min":20.501999999999999},"last":5.0730000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:12:52 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":3.1619999999999999,"15min":3.7450000000000001},"min":{"1min":0.22800000000000001,"5min":0.217,"15min":0.217},"max":{"1min":10.262,"5min":12.487,"15min":17.369},"last":10.711},{"interface":"front","average":{"1min":4.2009999999999996,"5min":3.1099999999999999,"15min":3.7429999999999999},"min":{"1min":0.31,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":21.645,"5min":21.645,"15min":21.645},"last":5.4180000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":3.6669999999999998,"15min":3.5779999999999998},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":21.808,"5min":21.808,"15min":21.808},"last":5.0250000000000004},{"interface":"front","average":{"1min":5.1740000000000004,"5min":3.8580000000000001,"15min":4.5540000000000003},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":21.547999999999998,"5min":21.547999999999998,"15min":21.547999999999998},"last":4.9279999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.8029999999999999,"15min":1.4370000000000001},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":21.620000000000001,"5min":21.620000000000001,"15min":21.620000000000001},"last":4.8140000000000001},{"interface":"front","average":{"1min":2.4980000000000002,"5min":2.411,"15min":1.6899999999999999},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":9.2569999999999997,"5min":12.904,"15min":12.904},"last":5.0499999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.4430000000000001,"15min":2.0630000000000002},"min":{"1min":0.34100000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.8599999999999994,"5min":12.811,"15min":15.531000000000001},"last":5},{"interface":"front","average":{"1min":3.544,"5min":2.2949999999999999,"15min":2.2999999999999998},"min":{"1min":0.53900000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.855,"5min":16.277999999999999,"15min":19.706},"last":11.628}]},{"osd":6,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":2.0139999999999998,"15min":1.9810000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":21.670999999999999,"5min":21.670999999999999,"15min":21.670999999999999},"last":4.8959999999999999},{"interface":"front","average":{"1min":2.5219999999999998,"5min":2.4220000000000002,"15min":2.1070000000000002},"min":{"1min":0.36899999999999999,"5min":0.35499999999999998,"15min":0.32000000000000001},"max":{"1min":9.9440000000000008,"5min":12.868,"15min":15.481999999999999},"last":4.9539999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":2.3860000000000001,"15min":2.3330000000000002},"min":{"1min":0.68200000000000005,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.723000000000001,"5min":16.331,"15min":16.331},"last":6.5910000000000002},{"interface":"front","average":{"1min":3.101,"5min":2.0830000000000002,"15min":2.0619999999999998},"min":{"1min":0.42399999999999999,"5min":0.378,"15min":0.34799999999999998},"max":{"1min":21.593,"5min":21.593,"15min":21.593},"last":4.8529999999999998}]}]},{"osd":0,"up_from":79,"seq":339302416589,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525140,"kb_used_data":144332,"kb_used_omap":28,"kb_used_meta":380707,"kb_avail":93220012,"statfs":{"total":95995035648,"available":95457292288,"internally_reserved":0,"allocated":147795968,"data_stored":146755988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":389844816},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":2.6800000000000002,"15min":2.0209999999999999},"min":{"1min":0.26400000000000001,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":16.797000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":0.629},{"interface":"front","average":{"1min":7.1150000000000002,"5min":4.2210000000000001,"15min":3.04},"min":{"1min":0.253,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":31.925999999999998,"5min":31.925999999999998,"15min":31.925999999999998},"last":4.7530000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":4.1130000000000004,"5min":2.7850000000000001,"15min":2.3929999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":13.483000000000001,"5min":18.003,"15min":18.003},"last":0.41599999999999998},{"interface":"front","average":{"1min":4.282,"5min":2.7949999999999999,"15min":3.0710000000000002},"min":{"1min":0.30099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":13.811,"5min":15.529,"15min":15.529},"last":0.38500000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":6.9710000000000001,"5min":5.0720000000000001,"15min":4.9340000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.013999999999999,"5min":25.734999999999999,"15min":25.734999999999999},"last":8.6799999999999997},{"interface":"front","average":{"1min":4.5730000000000004,"5min":4.1059999999999999,"15min":4.3090000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":18.928999999999998,"5min":21.923999999999999,"15min":21.923999999999999},"last":3.9460000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9109999999999996,"5min":3.012,"15min":1.9319999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":4.8070000000000004},{"interface":"front","average":{"1min":4.2830000000000004,"5min":2.8639999999999999,"15min":2.246},"min":{"1min":0.34599999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.797000000000001,"5min":15.907,"15min":15.907},"last":3.9809999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.5119999999999996,"5min":3.145,"15min":2.444},"min":{"1min":0.38500000000000001,"5min":0.34599999999999997,"15min":0.28000000000000003},"max":{"1min":14.944000000000001,"5min":17.096,"15min":17.096},"last":18.620000000000001},{"interface":"front","average":{"1min":4.1159999999999997,"5min":2.0710000000000002,"15min":1.6599999999999999},"min":{"1min":0.34799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.547000000000001,"5min":13.547000000000001,"15min":13.547000000000001},"last":19.798999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1500000000000004,"5min":2.2210000000000001,"15min":1.8169999999999999},"min":{"1min":0.41199999999999998,"5min":0.33700000000000002,"15min":0.31900000000000001},"max":{"1min":13.51,"5min":13.51,"15min":13.51},"last":19.875},{"interface":"front","average":{"1min":5.2859999999999996,"5min":3.726,"15min":2.7719999999999998},"min":{"1min":0.42299999999999999,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":20.446999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":4.8310000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.2040000000000002,"15min":2.601},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":15.734,"5min":16.460000000000001,"15min":26.978000000000002},"last":18.904},{"interface":"front","average":{"1min":4.5389999999999997,"5min":2.3650000000000002,"15min":1.9950000000000001},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":13.787000000000001,"5min":13.787000000000001,"15min":26.768000000000001},"last":19.843}]}]},{"osd":2,"up_from":217,"seq":932007903322,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":461268,"kb_used_data":186124,"kb_used_omap":83,"kb_used_meta":274988,"kb_avail":93283884,"statfs":{"total":95995035648,"available":95522697216,"internally_reserved":0,"allocated":190590976,"data_stored":189551678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":281588207},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3250000000000002,"5min":2.762,"15min":2.036},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17899999999999999},"max":{"1min":17.946000000000002,"5min":19.372,"15min":19.372},"last":0.68899999999999995},{"interface":"front","average":{"1min":4.7469999999999999,"5min":4.1379999999999999,"15min":3.254},"min":{"1min":0.29799999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.143000000000001,"5min":20.143000000000001,"15min":20.143000000000001},"last":1.018}]},{"osd":1,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":6.234,"5min":3.7509999999999999,"15min":2.9119999999999999},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.22800000000000001},"max":{"1min":19.283999999999999,"5min":19.283999999999999,"15min":19.283999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.5550000000000002,"5min":3.6520000000000001,"15min":2.8799999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":19.027000000000001,"5min":19.027000000000001,"15min":19.027000000000001},"last":0.47199999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":5.7430000000000003,"5min":3.0430000000000001,"15min":2.363},"min":{"1min":0.45000000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":19.370999999999999,"5min":19.370999999999999,"15min":19.370999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":4.585,"5min":3.3700000000000001,"15min":3.7330000000000001},"min":{"1min":0.36799999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":19.053000000000001,"5min":19.053000000000001,"15min":19.053000000000001},"last":0.50600000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":1.6200000000000001,"15min":1.0109999999999999},"min":{"1min":0.32000000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.725,"5min":12.725,"15min":12.725},"last":4.4189999999999996},{"interface":"front","average":{"1min":3.0270000000000001,"5min":1.7689999999999999,"15min":1.143},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":15.276,"5min":15.276,"15min":15.276},"last":1.385}]},{"osd":5,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.1820000000000004,"5min":2.4409999999999998,"15min":1.7669999999999999},"min":{"1min":0.438,"5min":0.32900000000000001,"15min":0.27700000000000002},"max":{"1min":15.32,"5min":15.32,"15min":15.32},"last":1.885},{"interface":"front","average":{"1min":4.452,"5min":2.3119999999999998,"15min":1.8859999999999999},"min":{"1min":0.34300000000000003,"5min":0.315,"15min":0.315},"max":{"1min":17.882999999999999,"5min":17.882999999999999,"15min":17.882999999999999},"last":2.8119999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":3.798,"5min":2.5310000000000001,"15min":1.4590000000000001},"min":{"1min":0.435,"5min":0.374,"15min":0.34899999999999998},"max":{"1min":18.943999999999999,"5min":18.943999999999999,"15min":18.943999999999999},"last":2.5680000000000001},{"interface":"front","average":{"1min":4.6109999999999998,"5min":2.6709999999999998,"15min":2.0089999999999999},"min":{"1min":0.46999999999999997,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":15.378,"5min":15.378,"15min":15.378},"last":3.1240000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.452,"5min":2.7170000000000001,"15min":1.526},"min":{"1min":0.40699999999999997,"5min":0.39900000000000002,"15min":0.35899999999999999},"max":{"1min":18.972999999999999,"5min":18.972999999999999,"15min":18.972999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":5.6260000000000003,"5min":3.0609999999999999,"15min":2.0289999999999999},"min":{"1min":0.499,"5min":0.38600000000000001,"15min":0.376},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":4.5910000000000002}]}]},{"osd":3,"up_from":250,"seq":1073741824066,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410092,"kb_used_data":154084,"kb_used_omap":10,"kb_used_meta":255989,"kb_avail":93335060,"statfs":{"total":95995035648,"available":95575101440,"internally_reserved":0,"allocated":157782016,"data_stored":156544238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":262132803},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.8439999999999999,"5min":3.8610000000000002,"15min":4.423},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.373000000000001,"5min":23.381,"15min":23.381},"last":0.28299999999999997},{"interface":"front","average":{"1min":5.9340000000000002,"5min":4.9260000000000002,"15min":4.4279999999999999},"min":{"1min":0.245,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":18.141999999999999,"5min":20.239999999999998,"15min":20.239999999999998},"last":0.27100000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":6.0229999999999997,"5min":3.8959999999999999,"15min":3.9340000000000002},"min":{"1min":0.27400000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":40.618000000000002,"5min":40.618000000000002,"15min":40.618000000000002},"last":4.1779999999999999},{"interface":"front","average":{"1min":2.2490000000000001,"5min":3.4889999999999999,"15min":3.4500000000000002},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.218},"max":{"1min":6.0369999999999999,"5min":23.423999999999999,"15min":23.423999999999999},"last":5.7960000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":6.8339999999999996,"5min":3.746,"15min":3.742},"min":{"1min":0.314,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":29.466999999999999,"5min":29.466999999999999,"15min":29.466999999999999},"last":3.883},{"interface":"front","average":{"1min":2.238,"5min":3.0430000000000001,"15min":2.9239999999999999},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.7510000000000003,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.46500000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1000000000000001,"5min":2.2490000000000001,"15min":1.48},"min":{"1min":0.33300000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.8700000000000001,"5min":15.676,"15min":15.676},"last":0.52400000000000002},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.6600000000000001,"15min":1.8939999999999999},"min":{"1min":0.40600000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.9890000000000008,"5min":16.478999999999999,"15min":16.478999999999999},"last":5.9850000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":4.1909999999999998,"5min":3.2410000000000001,"15min":2.823},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.307},"max":{"1min":9.0350000000000001,"5min":13.125999999999999,"15min":13.125999999999999},"last":6.024},{"interface":"front","average":{"1min":5.1120000000000001,"5min":3.2120000000000002,"15min":2.6400000000000001},"min":{"1min":0.373,"5min":0.309,"15min":0.309},"max":{"1min":17.242999999999999,"5min":17.242999999999999,"15min":17.242999999999999},"last":5.96}]},{"osd":6,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":3.0840000000000001,"15min":2.8540000000000001},"min":{"1min":0.38900000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.0660000000000007,"5min":16.504999999999999,"15min":16.504999999999999},"last":5.9089999999999998},{"interface":"front","average":{"1min":4.8070000000000004,"5min":2.8650000000000002,"15min":2.6909999999999998},"min":{"1min":0.41099999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":17.286999999999999,"5min":17.286999999999999,"15min":17.286999999999999},"last":3.9249999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7850000000000001,"5min":2.4009999999999998,"15min":1.726},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.6869999999999994,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":3.8820000000000001,"5min":3.3140000000000001,"15min":2.8879999999999999},"min":{"1min":0.44900000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":10.241,"5min":16.367000000000001,"15min":16.367000000000001},"last":0.60899999999999999}]}]},{"osd":4,"up_from":266,"seq":1142461300791,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":804204,"kb_used_data":473252,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":92940948,"statfs":{"total":95995035648,"available":95171530752,"internally_reserved":0,"allocated":484610048,"data_stored":484080173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":1.256,"15min":0.93100000000000005},"min":{"1min":0.222,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.1319999999999997,"5min":10.881,"15min":10.881},"last":0.32400000000000001},{"interface":"front","average":{"1min":3.04,"5min":1.9790000000000001,"15min":1.2989999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":8.4740000000000002,"5min":12.246,"15min":12.246},"last":1.397}]},{"osd":1,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":3.8079999999999998,"5min":2.2530000000000001,"15min":1.2949999999999999},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":9.4700000000000006,"5min":12.161,"15min":12.161},"last":1.976},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.986,"15min":1.512},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":13.231999999999999,"5min":13.231999999999999,"15min":13.231999999999999},"last":2.5249999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":3.1160000000000001,"5min":1.867,"15min":1.2849999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.6550000000000002,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.728,"15min":1.1799999999999999},"min":{"1min":0.311,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.8920000000000003,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":2.1930000000000001,"15min":1.5720000000000001},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.8630000000000004,"5min":13.065,"15min":13.065},"last":1.9990000000000001},{"interface":"front","average":{"1min":4.4089999999999998,"5min":2.4449999999999998,"15min":1.4910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":14.577,"5min":14.577,"15min":14.577},"last":2.9430000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.196,"15min":0.88400000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.0589999999999993,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.66800000000000004,"15min":0.65500000000000003},"min":{"1min":0.28799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7549999999999999,"5min":5.7549999999999999,"15min":5.7549999999999999},"last":0.38800000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":0.95599999999999996,"15min":0.66900000000000004},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.851,"5min":6.851,"15min":6.851},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.0840000000000001,"15min":0.75},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.3860000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.53100000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4569999999999999,"5min":1.258,"15min":0.83299999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":8.1039999999999992,"5min":12.237,"15min":12.237},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.3410000000000002,"5min":1.208,"15min":0.95499999999999996},"min":{"1min":0.314,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":0.40200000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822976,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":576608,"kb_used_data":271192,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93168544,"statfs":{"total":95995035648,"available":95404589056,"internally_reserved":0,"allocated":277700608,"data_stored":277178789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5950000000000002,"5min":1.7989999999999999,"15min":1.7090000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2639999999999993,"5min":8.2639999999999993,"15min":15.773999999999999},"last":2.7549999999999999},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.823,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.4939999999999998,"5min":10.244999999999999,"15min":15.699},"last":0.495}]},{"osd":1,"last update":"Mon Oct 9 10:12:16 2023","interfaces":[{"interface":"back","average":{"1min":3.1320000000000001,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.891,"5min":11.891,"15min":15.689},"last":3.1930000000000001},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.9350000000000001,"15min":1.823},"min":{"1min":0.41399999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.4489999999999998,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.65600000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":2.1739999999999999,"15min":2.4609999999999999},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.1620000000000008,"5min":10.259,"15min":15.640000000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.593,"15min":2.1219999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.313},"max":{"1min":6.5529999999999999,"5min":6.5529999999999999,"15min":10.656000000000001},"last":1.77}]},{"osd":3,"last update":"Mon Oct 9 10:12:52 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":2.3879999999999999,"15min":2.028},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.8699999999999992,"5min":9.8699999999999992,"15min":9.8699999999999992},"last":2.54},{"interface":"front","average":{"1min":4.1420000000000003,"5min":2.706,"15min":2.9140000000000001},"min":{"1min":0.42999999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":17.193999999999999,"5min":17.34,"15min":17.34},"last":1.7130000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.0089999999999999,"15min":0.71199999999999997},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3419999999999996,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":2.6019999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.754,"15min":0.747},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5559999999999992,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":0.46899999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.0629999999999999,"15min":1.0820000000000001},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":27.716000000000001},"last":2.5750000000000002},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.135,"15min":1.004},"min":{"1min":0.32700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2910000000000004,"5min":8.2910000000000004,"15min":18.600999999999999},"last":0.52200000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:12:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.137,"15min":1.0389999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4710000000000001,"5min":7.891,"15min":10.334},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.173,"5min":0.80300000000000005,"15min":0.96899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":12.356},"last":0.58099999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":133955584,"data_stored":133955584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":225640448,"data_stored":225640448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":173412352,"data_stored":173408280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":147783680,"data_stored":147783680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":473169920,"data_stored":473169920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":270798848,"data_stored":270794776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":491323392,"data_stored":491323392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":141365248,"data_stored":141361176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:13:08.344 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:13:08.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4098075033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:08.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4091154838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:08.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4705/25302 objects degraded (18.595%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:08.411 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4098075033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:08.411 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4091154838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:08.411 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4705/25302 objects degraded (18.595%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:08.596 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:08.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4098075033' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4091154838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4705/25302 objects degraded (18.595%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:08.703 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:08.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:08.837 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:08.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:08.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:08.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:08.999 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:09.094 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:09.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:09.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:09.295 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:09.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:09 smithi110 ceph-mon[136032]: pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 433 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4705/20832 objects degraded (22.585%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:09.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:09 smithi110 ceph-mon[136032]: from='client.35844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:09.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:09 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:13:09.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:09 smithi110 ceph-mon[136032]: failed to encode map e314 with expected crc 2023-10-09T10:13:09.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:09 smithi110 ceph-mon[136032]: failed to encode map e314 with expected crc 2023-10-09T10:13:09.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4098075033' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:09.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:09 smithi110 ceph-mon[136032]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T10:13:09.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[147926]: pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 433 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4705/20832 objects degraded (22.585%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:09.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[147926]: from='client.35844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:09.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:13:09.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[147926]: failed to encode map e314 with expected crc 2023-10-09T10:13:09.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[147926]: failed to encode map e314 with expected crc 2023-10-09T10:13:09.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4098075033' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:09.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[147926]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T10:13:09.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[152263]: pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 433 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4705/20832 objects degraded (22.585%); 0 B/s, 31 objects/s recovering 2023-10-09T10:13:09.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[152263]: from='client.35844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:09.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:13:09.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[152263]: failed to encode map e314 with expected crc 2023-10-09T10:13:09.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[152263]: failed to encode map e314 with expected crc 2023-10-09T10:13:09.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4098075033' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:09.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:09 smithi012 ceph-mon[152263]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T10:13:09.692 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:09.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:09.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:09.802 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:09.861 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:09.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:09.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:09.984 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:10.042 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:10.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:10.149 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:10.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:10.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:10.369 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:13:10.369 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:13:10.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:213850 8.c e314/271 8:3ddc640f:::benchmark_data_smithi012_271002_object105167:head v 314'13328, mlcod=314'13326)", 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:10.351680+0000", 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.014667226, 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014699204, 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.351680+0000", 2023-10-09T10:13:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.351680+0000", 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.351679+0000", 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.351688+0000", 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5299999999999996e-06 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.351704+0000", 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6588000000000001e-05 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:10.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.351713+0000", 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2889999999999998e-06 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.351774+0000", 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1860999999999996e-05 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.351795+0000", 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0893000000000001e-05 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.373 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:213853 8.f 8:f79b459d:::benchmark_data_smithi012_271002_object105170:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:10.352421+0000", 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.013926131, 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0140286, 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:10.374 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 213853 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352421+0000", 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352421+0000", 2023-10-09T10:13:10.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352420+0000", 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352425+0000", 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1800000000000004e-06 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352437+0000", 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2306999999999999e-05 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352448+0000", 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0343000000000001e-05 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:10.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352488+0000", 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0095000000000002e-05 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352640+0000", 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015244600000000001 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.352777+0000", 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000136723 2023-10-09T10:13:10.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.356458+0000", 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0036811389999999999 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.366016+0000", 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095573810000000002 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.379 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:213862 8.b 8:da3e75d6:::benchmark_data_smithi012_271002_object105179:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:10.356625+0000", 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0097221949999999994, 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0099151610000000005, 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:10.380 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 213862 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.356625+0000", 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.356625+0000", 2023-10-09T10:13:10.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.356623+0000", 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.356631+0000", 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8599999999999993e-06 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.356648+0000", 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7028e-05 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.356657+0000", 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8200000000000003e-06 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:10.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357124+0000", 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000466651 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357260+0000", 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013606600000000001 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357399+0000", 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013923000000000001 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.384 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:213854 8.8 e314/271 8:10fe7a29:::benchmark_data_smithi012_271002_object105171:head v 314'13348, mlcod=314'13346)", 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:10.357385+0000", 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0089621270000000003, 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092309460000000003, 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357385+0000", 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357385+0000", 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:10.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357384+0000", 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357397+0000", 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2867999999999999e-05 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357409+0000", 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1847e-05 2023-10-09T10:13:10.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357416+0000", 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3939999999999998e-06 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357460+0000", 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3893000000000001e-05 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.357485+0000", 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5202e-05 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:213856 8.8 e314/271 8:1f8f16a4:::benchmark_data_smithi012_271002_object105173:head v 314'13349, mlcod=314'13346)", 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:10.362167+0000", 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0041802749999999998, 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045120760000000003, 2023-10-09T10:13:10.389 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362167+0000", 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362167+0000", 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362166+0000", 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362177+0000", 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1429e-05 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362191+0000", 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3438000000000001e-05 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362200+0000", 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7499999999999992e-06 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362523+0000", 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032371800000000002 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362545+0000", 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1679e-05 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:213860 8.c e314/271 8:3367fb63:::benchmark_data_smithi012_271002_object105177:head v 314'13329, mlcod=314'13326)", 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:10.362231+0000", 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0041157779999999996, 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045085460000000004, 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362231+0000", 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362231+0000", 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362231+0000", 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362240+0000", 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0250000000000008e-06 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362251+0000", 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1421e-05 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:10.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362258+0000", 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9430000000000003e-06 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362360+0000", 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101607 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.362382+0000", 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2223000000000001e-05 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.397 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:213865 8.7 e314/271 8:e1d9f7fd:::benchmark_data_smithi012_271002_object105182:head v 314'13204, mlcod=314'13202)", 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:10.365910+0000", 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00043682299999999997, 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000890306, 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:10.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.365910+0000", 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.365910+0000", 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.365910+0000", 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.365915+0000", 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5399999999999997e-06 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.365921+0000", 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4160000000000001e-06 2023-10-09T10:13:10.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.365926+0000", 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1080000000000001e-06 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.366049+0000", 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012275799999999999 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:10.366068+0000", 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8779000000000001e-05 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 7 2023-10-09T10:13:10.402 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:13:10.403 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:10.504 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:10.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:10.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:10.694 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:10.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:13:11.089 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:11.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:11.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:11.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:11.248 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:13:11.297 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:11.474 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:11.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:11.476 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:11 smithi012 ceph-mon[147926]: pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 337 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4566/16227 objects degraded (28.138%); 0 B/s, 30 objects/s recovering 2023-10-09T10:13:11.480 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:11 smithi012 ceph-mon[152263]: pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 337 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4566/16227 objects degraded (28.138%); 0 B/s, 30 objects/s recovering 2023-10-09T10:13:11.577 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:11.608 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:11 smithi110 ceph-mon[136032]: pgmap v877: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 337 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4566/16227 objects degraded (28.138%); 0 B/s, 30 objects/s recovering 2023-10-09T10:13:11.613 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:11.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:13:11.656 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:11.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:11.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:11.765 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:11.961 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:12.038 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:12.060 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:12.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:12.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:12.175 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:12.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:12.297 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:12.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:12.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:12.344 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:12.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:12.461 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:13:12.461 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 4566/15612 objects degraded (29.247%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1109,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":314,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846114,"num_in_osds":8,"osd_in_since":1696845961,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":5204,"data_bytes":340006840,"bytes_used":4341059584,"bytes_avail":763619225600,"bytes_total":767960285184,"degraded_objects":4566,"degraded_total":15612,"degraded_ratio":0.29246733282090698,"recovering_objects_per_sec":30,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":302,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":652},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T10:12:45.433085+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true},"bec951f1-600f-43c4-a171-32bda8f84201":{"message":"Global Recovery Event (3m)\n [===========================.] (remaining: 2s)","progress":0.98876404762268066,"add_to_ceph_s":true}}} 2023-10-09T10:13:12.499 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:12.558 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:12.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:12.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:12.743 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:12.917 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:12.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:12.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:13.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:13.130 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:13.159 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:13:13.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:13:13.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:13 smithi012 ceph-mon[147926]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 324 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 4566/15612 objects degraded (29.247%); 0 B/s, 30 objects/s recovering 2023-10-09T10:13:13.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/679219834' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:13:13.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4566/15612 objects degraded (29.247%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:13.464 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:13.478 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:13.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:13.580 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:13.583 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:13 smithi012 ceph-mon[152263]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 324 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 4566/15612 objects degraded (29.247%); 0 B/s, 30 objects/s recovering 2023-10-09T10:13:13.583 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/679219834' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:13:13.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4566/15612 objects degraded (29.247%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:13.651 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:13.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:13 smithi110 ceph-mon[136032]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 324 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 4566/15612 objects degraded (29.247%); 0 B/s, 30 objects/s recovering 2023-10-09T10:13:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/679219834' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:13:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4566/15612 objects degraded (29.247%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:13.815 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:13.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:13.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:14.041 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:14.280 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:14.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:14.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:14.498 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:14.669 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:13:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:13:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:13:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:13:14.669 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:13:14.958 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:14 smithi012 ceph-mon[147926]: pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 188 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 4410/9072 objects degraded (48.611%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:14.958 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:14.959 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:14.960 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:14 smithi012 ceph-mon[152263]: pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 188 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 4410/9072 objects degraded (48.611%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:14.960 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:14.960 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:14 smithi110 ceph-mon[136032]: pgmap v879: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 188 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 4410/9072 objects degraded (48.611%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:15.928 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Removed 108680 objects 2023-10-09T10:13:15.929 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Clean up completed and total clean up time :174.255 2023-10-09T10:13:17.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:16 smithi012 ceph-mon[147926]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 125 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 4410/6024 objects degraded (73.207%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:17.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:16 smithi012 ceph-mon[152263]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 125 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 4410/6024 objects degraded (73.207%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:17.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:16 smithi110 ceph-mon[136032]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 125 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 4410/6024 objects degraded (73.207%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:13:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:13:17] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.33.4" 2023-10-09T10:13:18.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:13:18.475 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:13:18.475 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:13:18.475 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:13:18.489 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:13:18.508 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:13:18.526 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:13:18.545 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:13:18.563 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:13:18.565 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:13:18.565 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:13:18.565 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:13:18.565 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:13:18.565 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:13:18.565 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:13:18.565 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:13:18.565 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:13:18.805 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:18.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:18.900 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:18.910 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:18.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:18.969 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:18.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:18.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:19.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:19.098 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:19.132 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:19.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:18 smithi012 ceph-mon[147926]: pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 54 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 4410/2640 objects degraded (167.045%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4410/2640 objects degraded (167.045%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1262101157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:19.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:18 smithi012 ceph-mon[152263]: pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 54 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 4410/2640 objects degraded (167.045%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:19.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4410/2640 objects degraded (167.045%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:19.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1262101157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:18 smithi110 ceph-mon[136032]: pgmap v881: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 54 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 4410/2640 objects degraded (167.045%); 0 B/s, 29 objects/s recovering 2023-10-09T10:13:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4410/2640 objects degraded (167.045%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1262101157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:19.235 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:19.262 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:19.300 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:19.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:19.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:19.407 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:19.436 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:20.176 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T10:13:20.176 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T10:13:20.224 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:20.226 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:19 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:13:20.226 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:19 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:13:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:20.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:20.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:20.356 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:20.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:19 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:13:21.359 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:21.361 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:20 smithi012 ceph-mon[152263]: pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 4045/201 objects degraded (2012.438%); 0 B/s, 46 objects/s recovering 2023-10-09T10:13:21.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:20 smithi012 ceph-mon[147926]: pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 4045/201 objects degraded (2012.438%); 0 B/s, 46 objects/s recovering 2023-10-09T10:13:21.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:21.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:20 smithi110 ceph-mon[136032]: pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 4045/201 objects degraded (2012.438%); 0 B/s, 46 objects/s recovering 2023-10-09T10:13:21.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:21.490 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:21.786 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:23.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:23.476 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:23.476 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:13:23.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:22 smithi012 ceph-mon[152263]: pgmap v883: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 4045/72 objects degraded (5618.056%); 0 B/s, 43 objects/s recovering 2023-10-09T10:13:23.635 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/46606941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:23.635 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:22 smithi012 ceph-mon[147926]: pgmap v883: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 4045/72 objects degraded (5618.056%); 0 B/s, 43 objects/s recovering 2023-10-09T10:13:23.635 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/46606941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:23.636 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:23.636 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:22 smithi110 ceph-mon[136032]: pgmap v883: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 4045/72 objects degraded (5618.056%); 0 B/s, 43 objects/s recovering 2023-10-09T10:13:23.637 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/46606941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:23.664 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:23.701 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:24.290 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:24.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:24.346 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:24.346 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[147926]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.346 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/46606941' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:13:24.347 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[147926]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T10:13:24.347 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[147926]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.347 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4045/72 objects degraded (5618.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:24.347 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[147926]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.347 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[147926]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:24.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[152263]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/46606941' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:13:24.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[152263]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T10:13:24.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[152263]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4045/72 objects degraded (5618.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:24.348 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[152263]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.349 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:23 smithi012 ceph-mon[152263]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.352 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:13:24.354 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:13:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:23 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:24.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:23 smithi110 ceph-mon[136032]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:23 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/46606941' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:13:24.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:23 smithi110 ceph-mon[136032]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T10:13:24.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:23 smithi110 ceph-mon[136032]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4045/72 objects degraded (5618.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:24.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:23 smithi110 ceph-mon[136032]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:23 smithi110 ceph-mon[136032]: failed to encode map e315 with expected crc 2023-10-09T10:13:24.446 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:24.479 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:24.890 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":885,"stamp":"2023-10-09T10:13:23.455237+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3433,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":217622,"num_write_kb":6957966,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7293,"num_bytes_recovered":311175884,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142156,"ondisk_log_size":142156,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3235956,"kb_used_data":633204,"kb_used_omap":167,"kb_used_meta":2602520,"kb_avail":746725260,"statfs":{"total":767960285184,"available":764646666240,"internally_reserved":0,"allocated":648400896,"data_stored":641896224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2664981081},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-196542488,"num_objects":-3000,"num_object_clones":0,"num_object_copies":-9000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-977,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3000,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":977,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002839"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":156,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.167036+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:12:58.167036+0000","last_peered":"2023-10-09T10:12:58.167036+0000","last_clean":"2023-10-09T10:12:58.167036+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:12:58.167036+0000","last_undegraded":"2023-10-09T10:12:58.167036+0000","last_fullsized":"2023-10-09T10:12:58.167036+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":272,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155535+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:12:58.155535+0000","last_peered":"2023-10-09T10:12:58.155535+0000","last_clean":"2023-10-09T10:12:58.155535+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:12:58.155535+0000","last_undegraded":"2023-10-09T10:12:58.155535+0000","last_fullsized":"2023-10-09T10:12:58.155535+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":317,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.170207+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:12:58.170207+0000","last_peered":"2023-10-09T10:12:58.170207+0000","last_clean":"2023-10-09T10:12:58.170207+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:12:58.170207+0000","last_undegraded":"2023-10-09T10:12:58.170207+0000","last_fullsized":"2023-10-09T10:12:58.170207+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":181,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.173822+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:12:58.173822+0000","last_peered":"2023-10-09T10:12:58.173822+0000","last_clean":"2023-10-09T10:12:58.173822+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:12:58.173822+0000","last_undegraded":"2023-10-09T10:12:58.173822+0000","last_fullsized":"2023-10-09T10:12:58.173822+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":286,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.154956+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:12:58.154956+0000","last_peered":"2023-10-09T10:12:58.154956+0000","last_clean":"2023-10-09T10:12:58.154956+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:12:58.154956+0000","last_undegraded":"2023-10-09T10:12:58.154956+0000","last_fullsized":"2023-10-09T10:12:58.154956+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":133,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.287261+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:13:08.287261+0000","last_peered":"2023-10-09T10:13:08.287261+0000","last_clean":"2023-10-09T10:13:08.287261+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:13:08.287261+0000","last_undegraded":"2023-10-09T10:13:08.287261+0000","last_fullsized":"2023-10-09T10:13:08.287261+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":198,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.164646+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:12:58.164646+0000","last_peered":"2023-10-09T10:12:58.164646+0000","last_clean":"2023-10-09T10:12:58.164646+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:12:58.164646+0000","last_undegraded":"2023-10-09T10:12:58.164646+0000","last_fullsized":"2023-10-09T10:12:58.164646+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":267,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.154858+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:12:58.154858+0000","last_peered":"2023-10-09T10:12:58.154858+0000","last_clean":"2023-10-09T10:12:58.154858+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:12:58.154858+0000","last_undegraded":"2023-10-09T10:12:58.154858+0000","last_fullsized":"2023-10-09T10:12:58.154858+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":132,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.287204+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:13:08.287204+0000","last_peered":"2023-10-09T10:13:08.287204+0000","last_clean":"2023-10-09T10:13:08.287204+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:13:08.287204+0000","last_undegraded":"2023-10-09T10:13:08.287204+0000","last_fullsized":"2023-10-09T10:13:08.287204+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":228,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155119+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:12:58.155119+0000","last_peered":"2023-10-09T10:12:58.155119+0000","last_clean":"2023-10-09T10:12:58.155119+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:12:58.155119+0000","last_undegraded":"2023-10-09T10:12:58.155119+0000","last_fullsized":"2023-10-09T10:12:58.155119+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":352,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.164298+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:12:58.164298+0000","last_peered":"2023-10-09T10:12:58.164298+0000","last_clean":"2023-10-09T10:12:58.164298+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:12:58.164298+0000","last_undegraded":"2023-10-09T10:12:58.164298+0000","last_fullsized":"2023-10-09T10:12:58.164298+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":310,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:13:03.684754+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:13:03.684754+0000","last_peered":"2023-10-09T10:13:03.684754+0000","last_clean":"2023-10-09T10:13:03.684754+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:13:03.684754+0000","last_undegraded":"2023-10-09T10:13:03.684754+0000","last_fullsized":"2023-10-09T10:13:03.684754+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":201,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.165263+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:12:58.165263+0000","last_peered":"2023-10-09T10:12:58.165263+0000","last_clean":"2023-10-09T10:12:58.165263+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:12:58.165263+0000","last_undegraded":"2023-10-09T10:12:58.165263+0000","last_fullsized":"2023-10-09T10:12:58.165263+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":182,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.290243+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:13:08.290243+0000","last_peered":"2023-10-09T10:13:08.290243+0000","last_clean":"2023-10-09T10:13:08.290243+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:13:08.290243+0000","last_undegraded":"2023-10-09T10:13:08.290243+0000","last_fullsized":"2023-10-09T10:13:08.290243+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":385,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162643+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:12:58.162643+0000","last_peered":"2023-10-09T10:12:58.162643+0000","last_clean":"2023-10-09T10:12:58.162643+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:12:58.162643+0000","last_undegraded":"2023-10-09T10:12:58.162643+0000","last_fullsized":"2023-10-09T10:12:58.162643+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":254,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.156213+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:12:58.156213+0000","last_peered":"2023-10-09T10:12:58.156213+0000","last_clean":"2023-10-09T10:12:58.156213+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:12:58.156213+0000","last_undegraded":"2023-10-09T10:12:58.156213+0000","last_fullsized":"2023-10-09T10:12:58.156213+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":180,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.163025+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:12:58.163025+0000","last_peered":"2023-10-09T10:12:58.163025+0000","last_clean":"2023-10-09T10:12:58.163025+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:12:58.163025+0000","last_undegraded":"2023-10-09T10:12:58.163025+0000","last_fullsized":"2023-10-09T10:12:58.163025+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":148,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.287240+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:13:08.287240+0000","last_peered":"2023-10-09T10:13:08.287240+0000","last_clean":"2023-10-09T10:13:08.287240+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:13:08.287240+0000","last_undegraded":"2023-10-09T10:13:08.287240+0000","last_fullsized":"2023-10-09T10:13:08.287240+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162360+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:12:58.162360+0000","last_peered":"2023-10-09T10:12:58.162360+0000","last_clean":"2023-10-09T10:12:58.162360+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:12:58.162360+0000","last_undegraded":"2023-10-09T10:12:58.162360+0000","last_fullsized":"2023-10-09T10:12:58.162360+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":180,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.173887+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:12:58.173887+0000","last_peered":"2023-10-09T10:12:58.173887+0000","last_clean":"2023-10-09T10:12:58.173887+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:12:58.173887+0000","last_undegraded":"2023-10-09T10:12:58.173887+0000","last_fullsized":"2023-10-09T10:12:58.173887+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":203,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162850+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:12:58.162850+0000","last_peered":"2023-10-09T10:12:58.162850+0000","last_clean":"2023-10-09T10:12:58.162850+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:12:58.162850+0000","last_undegraded":"2023-10-09T10:12:58.162850+0000","last_fullsized":"2023-10-09T10:12:58.162850+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":267,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155067+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:12:58.155067+0000","last_peered":"2023-10-09T10:12:58.155067+0000","last_clean":"2023-10-09T10:12:58.155067+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:12:58.155067+0000","last_undegraded":"2023-10-09T10:12:58.155067+0000","last_fullsized":"2023-10-09T10:12:58.155067+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"314'13666","reported_seq":17607,"reported_epoch":315,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:13:23.039935+0000","last_change":"2023-10-09T10:12:42.183681+0000","last_active":"2023-10-09T10:13:23.039935+0000","last_peered":"2023-10-09T10:13:23.039935+0000","last_clean":"2023-10-09T10:09:45.987055+0000","last_became_active":"2023-10-09T10:09:48.407113+0000","last_became_peered":"2023-10-09T10:09:48.407113+0000","last_unstale":"2023-10-09T10:13:23.039935+0000","last_undegraded":"2023-10-09T10:09:48.036837+0000","last_fullsized":"2023-10-09T10:09:48.010544+0000","mapping_epoch":288,"log_start":"283'3600","ondisk_log_start":"283'3600","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3433,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13666,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1940,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-3433}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":346,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.163036+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:12:58.163036+0000","last_peered":"2023-10-09T10:12:58.163036+0000","last_clean":"2023-10-09T10:12:58.163036+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:12:58.163036+0000","last_undegraded":"2023-10-09T10:12:58.163036+0000","last_fullsized":"2023-10-09T10:12:58.163036+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":133,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.286993+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:13:08.286993+0000","last_peered":"2023-10-09T10:13:08.286993+0000","last_clean":"2023-10-09T10:13:08.286993+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:13:08.286993+0000","last_undegraded":"2023-10-09T10:13:08.286993+0000","last_fullsized":"2023-10-09T10:13:08.286993+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"314'13790","reported_seq":13841,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.907780+0000","last_change":"2023-10-09T10:08:43.251830+0000","last_active":"2023-10-09T10:13:15.907780+0000","last_peered":"2023-10-09T10:13:15.907780+0000","last_clean":"2023-10-09T10:13:15.907780+0000","last_became_active":"2023-10-09T10:08:43.251534+0000","last_became_peered":"2023-10-09T10:08:43.251534+0000","last_unstale":"2023-10-09T10:13:15.907780+0000","last_undegraded":"2023-10-09T10:13:15.907780+0000","last_fullsized":"2023-10-09T10:13:15.907780+0000","mapping_epoch":271,"log_start":"285'5200","ondisk_log_start":"285'5200","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8590,"log_dups_size":0,"ondisk_log_size":8590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13790,"num_write_kb":441280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":334,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162587+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:12:58.162587+0000","last_peered":"2023-10-09T10:12:58.162587+0000","last_clean":"2023-10-09T10:12:58.162587+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:12:58.162587+0000","last_undegraded":"2023-10-09T10:12:58.162587+0000","last_fullsized":"2023-10-09T10:12:58.162587+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":226,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.170114+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:12:58.170114+0000","last_peered":"2023-10-09T10:12:58.170114+0000","last_clean":"2023-10-09T10:12:58.170114+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:12:58.170114+0000","last_undegraded":"2023-10-09T10:12:58.170114+0000","last_fullsized":"2023-10-09T10:12:58.170114+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"314'13566","reported_seq":13620,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.902098+0000","last_change":"2023-10-09T10:08:43.251777+0000","last_active":"2023-10-09T10:13:15.902098+0000","last_peered":"2023-10-09T10:13:15.902098+0000","last_clean":"2023-10-09T10:13:15.902098+0000","last_became_active":"2023-10-09T10:08:43.251436+0000","last_became_peered":"2023-10-09T10:08:43.251436+0000","last_unstale":"2023-10-09T10:13:15.902098+0000","last_undegraded":"2023-10-09T10:13:15.902098+0000","last_fullsized":"2023-10-09T10:13:15.902098+0000","mapping_epoch":271,"log_start":"283'4100","ondisk_log_start":"283'4100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9466,"log_dups_size":0,"ondisk_log_size":9466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:33.216797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13566,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":137,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.287301+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:13:08.287301+0000","last_peered":"2023-10-09T10:13:08.287301+0000","last_clean":"2023-10-09T10:13:08.287301+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:13:08.287301+0000","last_undegraded":"2023-10-09T10:13:08.287301+0000","last_fullsized":"2023-10-09T10:13:08.287301+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":157,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.181197+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:12:58.181197+0000","last_peered":"2023-10-09T10:12:58.181197+0000","last_clean":"2023-10-09T10:12:58.181197+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:12:58.181197+0000","last_undegraded":"2023-10-09T10:12:58.181197+0000","last_fullsized":"2023-10-09T10:12:58.181197+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"314'13410","reported_seq":13464,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.928685+0000","last_change":"2023-10-09T10:08:43.251867+0000","last_active":"2023-10-09T10:13:15.928685+0000","last_peered":"2023-10-09T10:13:15.928685+0000","last_clean":"2023-10-09T10:13:15.928685+0000","last_became_active":"2023-10-09T10:08:43.251444+0000","last_became_peered":"2023-10-09T10:08:43.251444+0000","last_unstale":"2023-10-09T10:13:15.928685+0000","last_undegraded":"2023-10-09T10:13:15.928685+0000","last_fullsized":"2023-10-09T10:13:15.928685+0000","mapping_epoch":271,"log_start":"289'5700","ondisk_log_start":"289'5700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7710,"log_dups_size":0,"ondisk_log_size":7710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:09.772805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13410,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":148,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.290544+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:13:08.290544+0000","last_peered":"2023-10-09T10:13:08.290544+0000","last_clean":"2023-10-09T10:13:08.290544+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:13:08.290544+0000","last_undegraded":"2023-10-09T10:13:08.290544+0000","last_fullsized":"2023-10-09T10:13:08.290544+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":181,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.163179+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:12:58.163179+0000","last_peered":"2023-10-09T10:12:58.163179+0000","last_clean":"2023-10-09T10:12:58.163179+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:12:58.163179+0000","last_undegraded":"2023-10-09T10:12:58.163179+0000","last_fullsized":"2023-10-09T10:12:58.163179+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"314'13840","reported_seq":13894,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.873690+0000","last_change":"2023-10-09T10:08:43.226505+0000","last_active":"2023-10-09T10:13:15.873690+0000","last_peered":"2023-10-09T10:13:15.873690+0000","last_clean":"2023-10-09T10:13:15.873690+0000","last_became_active":"2023-10-09T10:08:43.226057+0000","last_became_peered":"2023-10-09T10:08:43.226057+0000","last_unstale":"2023-10-09T10:13:15.873690+0000","last_undegraded":"2023-10-09T10:13:15.873690+0000","last_fullsized":"2023-10-09T10:13:15.873690+0000","mapping_epoch":271,"log_start":"289'5700","ondisk_log_start":"289'5700","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:10.791004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13840,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":308,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155577+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:12:58.155577+0000","last_peered":"2023-10-09T10:12:58.155577+0000","last_clean":"2023-10-09T10:12:58.155577+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:12:58.155577+0000","last_undegraded":"2023-10-09T10:12:58.155577+0000","last_fullsized":"2023-10-09T10:12:58.155577+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":178,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.156002+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:12:58.156002+0000","last_peered":"2023-10-09T10:12:58.156002+0000","last_clean":"2023-10-09T10:12:58.156002+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:12:58.156002+0000","last_undegraded":"2023-10-09T10:12:58.156002+0000","last_fullsized":"2023-10-09T10:12:58.156002+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"314'13704","reported_seq":13758,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.913126+0000","last_change":"2023-10-09T10:08:43.324453+0000","last_active":"2023-10-09T10:13:15.913126+0000","last_peered":"2023-10-09T10:13:15.913126+0000","last_clean":"2023-10-09T10:13:15.913126+0000","last_became_active":"2023-10-09T10:08:43.324112+0000","last_became_peered":"2023-10-09T10:08:43.324112+0000","last_unstale":"2023-10-09T10:13:15.913126+0000","last_undegraded":"2023-10-09T10:13:15.913126+0000","last_fullsized":"2023-10-09T10:13:15.913126+0000","mapping_epoch":271,"log_start":"290'6000","ondisk_log_start":"290'6000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7704,"log_dups_size":0,"ondisk_log_size":7704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:48.888068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13704,"num_write_kb":438528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":240,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.166851+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:12:58.166851+0000","last_peered":"2023-10-09T10:12:58.166851+0000","last_clean":"2023-10-09T10:12:58.166851+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:12:58.166851+0000","last_undegraded":"2023-10-09T10:12:58.166851+0000","last_fullsized":"2023-10-09T10:12:58.166851+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":245,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.172116+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:12:58.172116+0000","last_peered":"2023-10-09T10:12:58.172116+0000","last_clean":"2023-10-09T10:12:58.172116+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:12:58.172116+0000","last_undegraded":"2023-10-09T10:12:58.172116+0000","last_fullsized":"2023-10-09T10:12:58.172116+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"314'13432","reported_seq":13486,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.845733+0000","last_change":"2023-10-09T10:08:43.226901+0000","last_active":"2023-10-09T10:13:15.845733+0000","last_peered":"2023-10-09T10:13:15.845733+0000","last_clean":"2023-10-09T10:13:15.845733+0000","last_became_active":"2023-10-09T10:08:43.226583+0000","last_became_peered":"2023-10-09T10:08:43.226583+0000","last_unstale":"2023-10-09T10:13:15.845733+0000","last_undegraded":"2023-10-09T10:13:15.845733+0000","last_fullsized":"2023-10-09T10:13:15.845733+0000","mapping_epoch":271,"log_start":"283'4000","ondisk_log_start":"283'4000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9432,"log_dups_size":0,"ondisk_log_size":9432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:11.239913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13432,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":208,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162284+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:12:58.162284+0000","last_peered":"2023-10-09T10:12:58.162284+0000","last_clean":"2023-10-09T10:12:58.162284+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:12:58.162284+0000","last_undegraded":"2023-10-09T10:12:58.162284+0000","last_fullsized":"2023-10-09T10:12:58.162284+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":182,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.677723+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:12:58.677723+0000","last_peered":"2023-10-09T10:12:58.677723+0000","last_clean":"2023-10-09T10:12:58.677723+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:12:58.677723+0000","last_undegraded":"2023-10-09T10:12:58.677723+0000","last_fullsized":"2023-10-09T10:12:58.677723+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"314'13598","reported_seq":13652,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.913387+0000","last_change":"2023-10-09T10:08:43.251780+0000","last_active":"2023-10-09T10:13:15.913387+0000","last_peered":"2023-10-09T10:13:15.913387+0000","last_clean":"2023-10-09T10:13:15.913387+0000","last_became_active":"2023-10-09T10:08:43.251454+0000","last_became_peered":"2023-10-09T10:08:43.251454+0000","last_unstale":"2023-10-09T10:13:15.913387+0000","last_undegraded":"2023-10-09T10:13:15.913387+0000","last_fullsized":"2023-10-09T10:13:15.913387+0000","mapping_epoch":271,"log_start":"290'5900","ondisk_log_start":"290'5900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7698,"log_dups_size":0,"ondisk_log_size":7698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:49.064684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13598,"num_write_kb":435136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":268,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.154856+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:12:58.154856+0000","last_peered":"2023-10-09T10:12:58.154856+0000","last_clean":"2023-10-09T10:12:58.154856+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:12:58.154856+0000","last_undegraded":"2023-10-09T10:12:58.154856+0000","last_fullsized":"2023-10-09T10:12:58.154856+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":318,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.167507+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:12:58.167507+0000","last_peered":"2023-10-09T10:12:58.167507+0000","last_clean":"2023-10-09T10:12:58.167507+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:12:58.167507+0000","last_undegraded":"2023-10-09T10:12:58.167507+0000","last_fullsized":"2023-10-09T10:12:58.167507+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"314'13472","reported_seq":13526,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.757703+0000","last_change":"2023-10-09T10:08:43.325443+0000","last_active":"2023-10-09T10:13:15.757703+0000","last_peered":"2023-10-09T10:13:15.757703+0000","last_clean":"2023-10-09T10:13:15.757703+0000","last_became_active":"2023-10-09T10:08:43.325138+0000","last_became_peered":"2023-10-09T10:08:43.325138+0000","last_unstale":"2023-10-09T10:13:15.757703+0000","last_undegraded":"2023-10-09T10:13:15.757703+0000","last_fullsized":"2023-10-09T10:13:15.757703+0000","mapping_epoch":271,"log_start":"283'4000","ondisk_log_start":"283'4000","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9472,"log_dups_size":0,"ondisk_log_size":9472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13472,"num_write_kb":431104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":317,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.171815+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:12:58.171815+0000","last_peered":"2023-10-09T10:12:58.171815+0000","last_clean":"2023-10-09T10:12:58.171815+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:12:58.171815+0000","last_undegraded":"2023-10-09T10:12:58.171815+0000","last_fullsized":"2023-10-09T10:12:58.171815+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":157,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.287216+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:13:08.287216+0000","last_peered":"2023-10-09T10:13:08.287216+0000","last_clean":"2023-10-09T10:13:08.287216+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:13:08.287216+0000","last_undegraded":"2023-10-09T10:13:08.287216+0000","last_fullsized":"2023-10-09T10:13:08.287216+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":274,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.167101+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:12:58.167101+0000","last_peered":"2023-10-09T10:12:58.167101+0000","last_clean":"2023-10-09T10:12:58.167101+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:12:58.167101+0000","last_undegraded":"2023-10-09T10:12:58.167101+0000","last_fullsized":"2023-10-09T10:12:58.167101+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"314'13574","reported_seq":13626,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.925692+0000","last_change":"2023-10-09T10:08:43.226148+0000","last_active":"2023-10-09T10:13:15.925692+0000","last_peered":"2023-10-09T10:13:15.925692+0000","last_clean":"2023-10-09T10:13:15.925692+0000","last_became_active":"2023-10-09T10:08:43.225889+0000","last_became_peered":"2023-10-09T10:08:43.225889+0000","last_unstale":"2023-10-09T10:13:15.925692+0000","last_undegraded":"2023-10-09T10:13:15.925692+0000","last_fullsized":"2023-10-09T10:13:15.925692+0000","mapping_epoch":271,"log_start":"283'3500","ondisk_log_start":"283'3500","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13574,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":133,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.287006+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:13:08.287006+0000","last_peered":"2023-10-09T10:13:08.287006+0000","last_clean":"2023-10-09T10:13:08.287006+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:13:08.287006+0000","last_undegraded":"2023-10-09T10:13:08.287006+0000","last_fullsized":"2023-10-09T10:13:08.287006+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":131,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162963+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:12:58.162963+0000","last_peered":"2023-10-09T10:12:58.162963+0000","last_clean":"2023-10-09T10:12:58.162963+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:12:58.162963+0000","last_undegraded":"2023-10-09T10:12:58.162963+0000","last_fullsized":"2023-10-09T10:12:58.162963+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":340,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.171744+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:12:58.171744+0000","last_peered":"2023-10-09T10:12:58.171744+0000","last_clean":"2023-10-09T10:12:58.171744+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:12:58.171744+0000","last_undegraded":"2023-10-09T10:12:58.171744+0000","last_fullsized":"2023-10-09T10:12:58.171744+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"314'13542","reported_seq":13596,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.921758+0000","last_change":"2023-10-09T10:08:43.225522+0000","last_active":"2023-10-09T10:13:15.921758+0000","last_peered":"2023-10-09T10:13:15.921758+0000","last_clean":"2023-10-09T10:13:15.921758+0000","last_became_active":"2023-10-09T10:08:43.224771+0000","last_became_peered":"2023-10-09T10:08:43.224771+0000","last_unstale":"2023-10-09T10:13:15.921758+0000","last_undegraded":"2023-10-09T10:13:15.921758+0000","last_fullsized":"2023-10-09T10:13:15.921758+0000","mapping_epoch":271,"log_start":"289'5400","ondisk_log_start":"289'5400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8142,"log_dups_size":0,"ondisk_log_size":8142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:12.133141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13542,"num_write_kb":433344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":234,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155009+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:12:58.155009+0000","last_peered":"2023-10-09T10:12:58.155009+0000","last_clean":"2023-10-09T10:12:58.155009+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:12:58.155009+0000","last_undegraded":"2023-10-09T10:12:58.155009+0000","last_fullsized":"2023-10-09T10:12:58.155009+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":310,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.170978+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:12:58.170978+0000","last_peered":"2023-10-09T10:12:58.170978+0000","last_clean":"2023-10-09T10:12:58.170978+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:12:58.170978+0000","last_undegraded":"2023-10-09T10:12:58.170978+0000","last_fullsized":"2023-10-09T10:12:58.170978+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":330,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.167427+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:12:58.167427+0000","last_peered":"2023-10-09T10:12:58.167427+0000","last_clean":"2023-10-09T10:12:58.167427+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:12:58.167427+0000","last_undegraded":"2023-10-09T10:12:58.167427+0000","last_fullsized":"2023-10-09T10:12:58.167427+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"314'13460","reported_seq":13512,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.906380+0000","last_change":"2023-10-09T10:08:43.251756+0000","last_active":"2023-10-09T10:13:15.906380+0000","last_peered":"2023-10-09T10:13:15.906380+0000","last_clean":"2023-10-09T10:13:15.906380+0000","last_became_active":"2023-10-09T10:08:43.251398+0000","last_became_peered":"2023-10-09T10:08:43.251398+0000","last_unstale":"2023-10-09T10:13:15.906380+0000","last_undegraded":"2023-10-09T10:13:15.906380+0000","last_fullsized":"2023-10-09T10:13:15.906380+0000","mapping_epoch":271,"log_start":"282'3400","ondisk_log_start":"282'3400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10060,"log_dups_size":0,"ondisk_log_size":10060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13460,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":142,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.287067+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:13:08.287067+0000","last_peered":"2023-10-09T10:13:08.287067+0000","last_clean":"2023-10-09T10:13:08.287067+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:13:08.287067+0000","last_undegraded":"2023-10-09T10:13:08.287067+0000","last_fullsized":"2023-10-09T10:13:08.287067+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":560,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162215+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:12:58.162215+0000","last_peered":"2023-10-09T10:12:58.162215+0000","last_clean":"2023-10-09T10:12:58.162215+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:12:58.162215+0000","last_undegraded":"2023-10-09T10:12:58.162215+0000","last_fullsized":"2023-10-09T10:12:58.162215+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":218,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155433+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:12:58.155433+0000","last_peered":"2023-10-09T10:12:58.155433+0000","last_clean":"2023-10-09T10:12:58.155433+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:12:58.155433+0000","last_undegraded":"2023-10-09T10:12:58.155433+0000","last_fullsized":"2023-10-09T10:12:58.155433+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":229,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.164381+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:12:58.164381+0000","last_peered":"2023-10-09T10:12:58.164381+0000","last_clean":"2023-10-09T10:12:58.164381+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:12:58.164381+0000","last_undegraded":"2023-10-09T10:12:58.164381+0000","last_fullsized":"2023-10-09T10:12:58.164381+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"314'13622","reported_seq":13676,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.883567+0000","last_change":"2023-10-09T10:08:43.325746+0000","last_active":"2023-10-09T10:13:15.883567+0000","last_peered":"2023-10-09T10:13:15.883567+0000","last_clean":"2023-10-09T10:13:15.883567+0000","last_became_active":"2023-10-09T10:08:43.324898+0000","last_became_peered":"2023-10-09T10:08:43.324898+0000","last_unstale":"2023-10-09T10:13:15.883567+0000","last_undegraded":"2023-10-09T10:13:15.883567+0000","last_fullsized":"2023-10-09T10:13:15.883567+0000","mapping_epoch":271,"log_start":"290'5900","ondisk_log_start":"290'5900","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":7722,"log_dups_size":0,"ondisk_log_size":7722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:24.239540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13622,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":290,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.171093+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:12:58.171093+0000","last_peered":"2023-10-09T10:12:58.171093+0000","last_clean":"2023-10-09T10:12:58.171093+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:12:58.171093+0000","last_undegraded":"2023-10-09T10:12:58.171093+0000","last_fullsized":"2023-10-09T10:12:58.171093+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":192,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162981+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:12:58.162981+0000","last_peered":"2023-10-09T10:12:58.162981+0000","last_clean":"2023-10-09T10:12:58.162981+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:12:58.162981+0000","last_undegraded":"2023-10-09T10:12:58.162981+0000","last_fullsized":"2023-10-09T10:12:58.162981+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":220,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.312882+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:13:08.312882+0000","last_peered":"2023-10-09T10:13:08.312882+0000","last_clean":"2023-10-09T10:13:08.312882+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:13:08.312882+0000","last_undegraded":"2023-10-09T10:13:08.312882+0000","last_fullsized":"2023-10-09T10:13:08.312882+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"314'13658","reported_seq":24296,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.928622+0000","last_change":"2023-10-09T10:12:43.942710+0000","last_active":"2023-10-09T10:13:15.928622+0000","last_peered":"2023-10-09T10:13:15.928622+0000","last_clean":"2023-10-09T10:13:15.928622+0000","last_became_active":"2023-10-09T10:12:43.912846+0000","last_became_peered":"2023-10-09T10:12:43.912846+0000","last_unstale":"2023-10-09T10:13:15.928622+0000","last_undegraded":"2023-10-09T10:13:15.928622+0000","last_fullsized":"2023-10-09T10:13:15.928622+0000","mapping_epoch":310,"log_start":"289'5500","ondisk_log_start":"289'5500","created":271,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":8158,"log_dups_size":0,"ondisk_log_size":8158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:08.472095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13658,"num_write_kb":437056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5284,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":333,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.163397+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:12:58.163397+0000","last_peered":"2023-10-09T10:12:58.163397+0000","last_clean":"2023-10-09T10:12:58.163397+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:12:58.163397+0000","last_undegraded":"2023-10-09T10:12:58.163397+0000","last_fullsized":"2023-10-09T10:12:58.163397+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":346,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.171895+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:12:58.171895+0000","last_peered":"2023-10-09T10:12:58.171895+0000","last_clean":"2023-10-09T10:12:58.171895+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:12:58.171895+0000","last_undegraded":"2023-10-09T10:12:58.171895+0000","last_fullsized":"2023-10-09T10:12:58.171895+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":184,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.163040+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:12:58.163040+0000","last_peered":"2023-10-09T10:12:58.163040+0000","last_clean":"2023-10-09T10:12:58.163040+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:12:58.163040+0000","last_undegraded":"2023-10-09T10:12:58.163040+0000","last_fullsized":"2023-10-09T10:12:58.163040+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"314'13558","reported_seq":13612,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.927333+0000","last_change":"2023-10-09T10:08:43.227498+0000","last_active":"2023-10-09T10:13:15.927333+0000","last_peered":"2023-10-09T10:13:15.927333+0000","last_clean":"2023-10-09T10:13:15.927333+0000","last_became_active":"2023-10-09T10:08:43.226571+0000","last_became_peered":"2023-10-09T10:08:43.226571+0000","last_unstale":"2023-10-09T10:13:15.927333+0000","last_undegraded":"2023-10-09T10:13:15.927333+0000","last_fullsized":"2023-10-09T10:13:15.927333+0000","mapping_epoch":271,"log_start":"283'4100","ondisk_log_start":"283'4100","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":9458,"log_dups_size":0,"ondisk_log_size":9458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:01.296008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13558,"num_write_kb":433856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":277,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155152+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:12:58.155152+0000","last_peered":"2023-10-09T10:12:58.155152+0000","last_clean":"2023-10-09T10:12:58.155152+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:12:58.155152+0000","last_undegraded":"2023-10-09T10:12:58.155152+0000","last_fullsized":"2023-10-09T10:12:58.155152+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":291,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.172336+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:12:58.172336+0000","last_peered":"2023-10-09T10:12:58.172336+0000","last_clean":"2023-10-09T10:12:58.172336+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:12:58.172336+0000","last_undegraded":"2023-10-09T10:12:58.172336+0000","last_fullsized":"2023-10-09T10:12:58.172336+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":266,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.154783+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:12:58.154783+0000","last_peered":"2023-10-09T10:12:58.154783+0000","last_clean":"2023-10-09T10:12:58.154783+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:12:58.154783+0000","last_undegraded":"2023-10-09T10:12:58.154783+0000","last_fullsized":"2023-10-09T10:12:58.154783+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"314'13470","reported_seq":13523,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:15.934398+0000","last_change":"2023-10-09T10:08:43.324273+0000","last_active":"2023-10-09T10:13:15.934398+0000","last_peered":"2023-10-09T10:13:15.934398+0000","last_clean":"2023-10-09T10:13:15.934398+0000","last_became_active":"2023-10-09T10:08:43.323980+0000","last_became_peered":"2023-10-09T10:08:43.323980+0000","last_unstale":"2023-10-09T10:13:15.934398+0000","last_undegraded":"2023-10-09T10:13:15.934398+0000","last_fullsized":"2023-10-09T10:13:15.934398+0000","mapping_epoch":271,"log_start":"282'3400","ondisk_log_start":"282'3400","created":271,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:08:42.206702+0000","last_clean_scrub_stamp":"2023-10-09T10:08:42.206702+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13470,"num_write_kb":430977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":222,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.181603+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:12:58.181603+0000","last_peered":"2023-10-09T10:12:58.181603+0000","last_clean":"2023-10-09T10:12:58.181603+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:12:58.181603+0000","last_undegraded":"2023-10-09T10:12:58.181603+0000","last_fullsized":"2023-10-09T10:12:58.181603+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":186,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.312848+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:13:08.312848+0000","last_peered":"2023-10-09T10:13:08.312848+0000","last_clean":"2023-10-09T10:13:08.312848+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:13:08.312848+0000","last_undegraded":"2023-10-09T10:13:08.312848+0000","last_fullsized":"2023-10-09T10:13:08.312848+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":168,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.286954+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:13:08.286954+0000","last_peered":"2023-10-09T10:13:08.286954+0000","last_clean":"2023-10-09T10:13:08.286954+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:13:08.286954+0000","last_undegraded":"2023-10-09T10:13:08.286954+0000","last_fullsized":"2023-10-09T10:13:08.286954+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":180,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.162864+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:12:58.162864+0000","last_peered":"2023-10-09T10:12:58.162864+0000","last_clean":"2023-10-09T10:12:58.162864+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:12:58.162864+0000","last_undegraded":"2023-10-09T10:12:58.162864+0000","last_fullsized":"2023-10-09T10:12:58.162864+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":253,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.290511+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:13:08.290511+0000","last_peered":"2023-10-09T10:13:08.290511+0000","last_clean":"2023-10-09T10:13:08.290511+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:13:08.290511+0000","last_undegraded":"2023-10-09T10:13:08.290511+0000","last_fullsized":"2023-10-09T10:13:08.290511+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":152,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.290459+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:13:08.290459+0000","last_peered":"2023-10-09T10:13:08.290459+0000","last_clean":"2023-10-09T10:13:08.290459+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:13:08.290459+0000","last_undegraded":"2023-10-09T10:13:08.290459+0000","last_fullsized":"2023-10-09T10:13:08.290459+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":193,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155494+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:12:58.155494+0000","last_peered":"2023-10-09T10:12:58.155494+0000","last_clean":"2023-10-09T10:12:58.155494+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:12:58.155494+0000","last_undegraded":"2023-10-09T10:12:58.155494+0000","last_fullsized":"2023-10-09T10:12:58.155494+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":152,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.287154+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:13:08.287154+0000","last_peered":"2023-10-09T10:13:08.287154+0000","last_clean":"2023-10-09T10:13:08.287154+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:13:08.287154+0000","last_undegraded":"2023-10-09T10:13:08.287154+0000","last_fullsized":"2023-10-09T10:13:08.287154+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":202,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155348+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:12:58.155348+0000","last_peered":"2023-10-09T10:12:58.155348+0000","last_clean":"2023-10-09T10:12:58.155348+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:12:58.155348+0000","last_undegraded":"2023-10-09T10:12:58.155348+0000","last_fullsized":"2023-10-09T10:12:58.155348+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":331,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155748+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:12:58.155748+0000","last_peered":"2023-10-09T10:12:58.155748+0000","last_clean":"2023-10-09T10:12:58.155748+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:12:58.155748+0000","last_undegraded":"2023-10-09T10:12:58.155748+0000","last_fullsized":"2023-10-09T10:12:58.155748+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":352,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.155238+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:12:58.155238+0000","last_peered":"2023-10-09T10:12:58.155238+0000","last_clean":"2023-10-09T10:12:58.155238+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:12:58.155238+0000","last_undegraded":"2023-10-09T10:12:58.155238+0000","last_fullsized":"2023-10-09T10:12:58.155238+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":262,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.164511+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:12:58.164511+0000","last_peered":"2023-10-09T10:12:58.164511+0000","last_clean":"2023-10-09T10:12:58.164511+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:12:58.164511+0000","last_undegraded":"2023-10-09T10:12:58.164511+0000","last_fullsized":"2023-10-09T10:12:58.164511+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":181,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:12:58.170796+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:12:58.170796+0000","last_peered":"2023-10-09T10:12:58.170796+0000","last_clean":"2023-10-09T10:12:58.170796+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:12:58.170796+0000","last_undegraded":"2023-10-09T10:12:58.170796+0000","last_fullsized":"2023-10-09T10:12:58.170796+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3433,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":217362,"num_write_kb":6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7224,"num_bytes_recovered":309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":619773952,"data_stored":619773952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141962,"ondisk_log_size":141962,"up":48,"acting":47,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430524,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":304636,"kb_used_data":3316,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93440516,"statfs":{"total":95995035648,"available":95683088384,"internally_reserved":0,"allocated":3395584,"data_stored":2857864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.3130000000000002,"15min":2.0539999999999998},"min":{"1min":0.27400000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":11.164,"5min":11.164,"15min":50.591999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.2770000000000001,"15min":2.0059999999999998},"min":{"1min":0.32600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.7880000000000003,"5min":15.974,"15min":50.747},"last":0.39100000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:13:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5390000000000001,"5min":2.6539999999999999,"15min":2.0910000000000002},"min":{"1min":0.26400000000000001,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":8.5890000000000004,"5min":14.311,"15min":25.338000000000001},"last":0.439},{"interface":"front","average":{"1min":4.3609999999999998,"5min":2.7679999999999998,"15min":1.9390000000000001},"min":{"1min":0.27900000000000003,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":14.5,"5min":16.526,"15min":18.27},"last":0.52300000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":3.8029999999999999,"5min":2.6030000000000002,"15min":2.4430000000000001},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":10.682,"5min":13.775,"15min":13.775},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.7200000000000002,"5min":2.4350000000000001,"15min":2.2559999999999998},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.795999999999999,"5min":12.101000000000001,"15min":12.101000000000001},"last":0.42899999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":3.6110000000000002,"5min":3.0009999999999999,"15min":2.1669999999999998},"min":{"1min":0.40300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":10.827,"5min":29.058,"15min":29.058},"last":0.54900000000000004},{"interface":"front","average":{"1min":4.0060000000000002,"5min":2.6680000000000001,"15min":2.1859999999999999},"min":{"1min":0.36699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":16.715,"5min":16.715,"15min":16.715},"last":0.55500000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.288,"15min":1.0369999999999999},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":7.7469999999999999,"5min":7.7469999999999999,"15min":7.7469999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.724,"5min":1.139,"15min":0.98499999999999999},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":0.46000000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8180000000000001,"15min":1.256},"min":{"1min":0.34000000000000002,"5min":0.219,"15min":0.219},"max":{"1min":9.2560000000000002,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.9089999999999998,"5min":1.7130000000000001,"15min":1.2150000000000001},"min":{"1min":0.38600000000000001,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":18.547999999999998,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1000000000000001,"5min":1.3620000000000001,"15min":1.1080000000000001},"min":{"1min":0.40400000000000003,"5min":0.30199999999999999,"15min":0.26100000000000001},"max":{"1min":8.8569999999999993,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":0.501},{"interface":"front","average":{"1min":2.1779999999999999,"5min":1.2090000000000001,"15min":1.026},"min":{"1min":0.33900000000000002,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":5.9660000000000002,"5min":6.4020000000000001,"15min":9.1920000000000002},"last":0.47199999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626752,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":648496,"kb_used_data":347176,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93096656,"statfs":{"total":95995035648,"available":95330975744,"internally_reserved":0,"allocated":355508224,"data_stored":354983746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8180000000000001,"5min":2.2330000000000001,"15min":1.756},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.6940000000000008,"5min":15.435,"15min":15.435},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.5129999999999999,"5min":2.1890000000000001,"15min":1.7250000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":13.385,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.42999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:13:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":3.1539999999999999,"15min":2.1749999999999998},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":9.8680000000000003,"5min":35.609999999999999,"15min":35.609999999999999},"last":0.379},{"interface":"front","average":{"1min":2.48,"5min":2.871,"15min":2.1200000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.2050000000000001,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.35499999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.8740000000000001,"15min":1.802},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":7.7359999999999998,"5min":12.898,"15min":12.898},"last":0.33800000000000002},{"interface":"front","average":{"1min":3.4609999999999999,"5min":2.6499999999999999,"15min":3.036},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.720000000000001,"5min":14.542999999999999,"15min":14.542999999999999},"last":0.504}]},{"osd":3,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.7839999999999998,"15min":1.8340000000000001},"min":{"1min":0.437,"5min":0.308,"15min":0.308},"max":{"1min":11.098000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":3.161,"5min":2.6059999999999999,"15min":2.456},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.002000000000001,"5min":11.211,"15min":11.211},"last":0.47299999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":0.94699999999999995,"15min":0.60899999999999999},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.113,"5min":10.113,"15min":10.113},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.89500000000000002,"15min":0.63100000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.948,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.52200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:12:37 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.0660000000000001,"15min":0.91800000000000004},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.23699999999999999},"max":{"1min":6.3170000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.335,"5min":1.222,"15min":0.99399999999999999},"min":{"1min":0.34699999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":14.465999999999999,"5min":14.465999999999999,"15min":14.465999999999999},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:13:06 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.0249999999999999,"15min":0.92400000000000004},"min":{"1min":0.35799999999999998,"5min":0.28599999999999998,"15min":0.252},"max":{"1min":6.601,"5min":6.601,"15min":7.5350000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0669999999999999,"15min":0.96999999999999997},"min":{"1min":0.39000000000000001,"5min":0.38100000000000001,"15min":0.30099999999999999},"max":{"1min":6.6280000000000001,"5min":6.6280000000000001,"15min":7.5570000000000004},"last":0.53700000000000003}]}]},{"osd":1,"up_from":84,"seq":360777253065,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":451136,"kb_used_data":3320,"kb_used_omap":9,"kb_used_meta":447798,"kb_avail":93294016,"statfs":{"total":95995035648,"available":95533072384,"internally_reserved":0,"allocated":3399680,"data_stored":2326677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":458546153},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.593,"15min":2.6000000000000001},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.183},"max":{"1min":13.303000000000001,"5min":31.007000000000001,"15min":31.007000000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":3.0219999999999998,"5min":3.5990000000000002,"15min":2.7650000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.192},"max":{"1min":9.3710000000000004,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.33300000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:12:52 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":3.1619999999999999,"15min":3.7450000000000001},"min":{"1min":0.22800000000000001,"5min":0.217,"15min":0.217},"max":{"1min":10.262,"5min":12.487,"15min":17.369},"last":0.251},{"interface":"front","average":{"1min":4.2009999999999996,"5min":3.1099999999999999,"15min":3.7429999999999999},"min":{"1min":0.31,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":21.645,"5min":21.645,"15min":21.645},"last":0.31900000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":3.6669999999999998,"15min":3.5779999999999998},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":21.808,"5min":21.808,"15min":21.808},"last":0.36899999999999999},{"interface":"front","average":{"1min":5.1740000000000004,"5min":3.8580000000000001,"15min":4.5540000000000003},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":21.547999999999998,"5min":21.547999999999998,"15min":21.547999999999998},"last":0.379}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.8029999999999999,"15min":1.4370000000000001},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":21.620000000000001,"5min":21.620000000000001,"15min":21.620000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.4980000000000002,"5min":2.411,"15min":1.6899999999999999},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":9.2569999999999997,"5min":12.904,"15min":12.904},"last":0.41899999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.8029999999999999,"5min":2.694,"15min":2.2080000000000002},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":9.2870000000000008,"5min":12.811,"15min":15.531000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":3.6000000000000001,"5min":2.601,"15min":2.496},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":19.513000000000002,"5min":19.513000000000002,"15min":19.706},"last":0.432}]},{"osd":6,"last update":"Mon Oct 9 10:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":2.222,"15min":2.081},"min":{"1min":0.42599999999999999,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":9.0779999999999994,"5min":21.670999999999999,"15min":21.670999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.6520000000000001,"15min":2.2599999999999998},"min":{"1min":0.39400000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":9.3160000000000007,"5min":12.868,"15min":15.481999999999999},"last":0.58199999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":2.4750000000000001,"15min":2.4609999999999999},"min":{"1min":0.42799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.1690000000000005,"5min":10.723000000000001,"15min":16.331},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.157,"5min":2.266,"15min":2.1600000000000001},"min":{"1min":0.45600000000000002,"5min":0.39100000000000001,"15min":0.378},"max":{"1min":9.125,"5min":21.593,"15min":21.593},"last":0.498}]}]},{"osd":0,"up_from":79,"seq":339302416592,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":388700,"kb_used_data":3860,"kb_used_omap":28,"kb_used_meta":384803,"kb_avail":93356452,"statfs":{"total":95995035648,"available":95597006848,"internally_reserved":0,"allocated":3952640,"data_stored":2912393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":394039120},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":2.6800000000000002,"15min":2.0209999999999999},"min":{"1min":0.26400000000000001,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":16.797000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":7.1150000000000002,"5min":4.2210000000000001,"15min":3.04},"min":{"1min":0.253,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":31.925999999999998,"5min":31.925999999999998,"15min":31.925999999999998},"last":0.35099999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":4.1130000000000004,"5min":2.7850000000000001,"15min":2.3929999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":13.483000000000001,"5min":18.003,"15min":18.003},"last":0.39400000000000002},{"interface":"front","average":{"1min":4.282,"5min":2.7949999999999999,"15min":3.0710000000000002},"min":{"1min":0.30099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":13.811,"5min":15.529,"15min":15.529},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":6.9710000000000001,"5min":5.0720000000000001,"15min":4.9340000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.013999999999999,"5min":25.734999999999999,"15min":25.734999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":4.5730000000000004,"5min":4.1059999999999999,"15min":4.3090000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":18.928999999999998,"5min":21.923999999999999,"15min":21.923999999999999},"last":0.318}]},{"osd":4,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9109999999999996,"5min":3.012,"15min":1.9319999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":0.42699999999999999},{"interface":"front","average":{"1min":4.2830000000000004,"5min":2.8639999999999999,"15min":2.246},"min":{"1min":0.34599999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.797000000000001,"5min":15.907,"15min":15.907},"last":0.44400000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.5119999999999996,"5min":3.145,"15min":2.444},"min":{"1min":0.38500000000000001,"5min":0.34599999999999997,"15min":0.28000000000000003},"max":{"1min":14.944000000000001,"5min":17.096,"15min":17.096},"last":0.38300000000000001},{"interface":"front","average":{"1min":4.1159999999999997,"5min":2.0710000000000002,"15min":1.6599999999999999},"min":{"1min":0.34799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.547000000000001,"5min":13.547000000000001,"15min":13.547000000000001},"last":0.48799999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1500000000000004,"5min":2.2210000000000001,"15min":1.8169999999999999},"min":{"1min":0.41199999999999998,"5min":0.33700000000000002,"15min":0.31900000000000001},"max":{"1min":13.51,"5min":13.51,"15min":13.51},"last":0.55800000000000005},{"interface":"front","average":{"1min":5.2859999999999996,"5min":3.726,"15min":2.7719999999999998},"min":{"1min":0.42299999999999999,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":20.446999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":0.5}]},{"osd":7,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.2040000000000002,"15min":2.601},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":15.734,"5min":16.460000000000001,"15min":26.978000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":4.5389999999999997,"5min":2.3650000000000002,"15min":1.9950000000000001},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":13.787000000000001,"5min":13.787000000000001,"15min":26.768000000000001},"last":0.58499999999999996}]}]},{"osd":2,"up_from":217,"seq":932007903325,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278364,"kb_used_data":3284,"kb_used_omap":83,"kb_used_meta":274988,"kb_avail":93466788,"statfs":{"total":95995035648,"available":95709990912,"internally_reserved":0,"allocated":3362816,"data_stored":2323558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":281588207},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3250000000000002,"5min":2.762,"15min":2.036},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17899999999999999},"max":{"1min":17.946000000000002,"5min":19.372,"15min":19.372},"last":0.27000000000000002},{"interface":"front","average":{"1min":4.7469999999999999,"5min":4.1379999999999999,"15min":3.254},"min":{"1min":0.29799999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.143000000000001,"5min":20.143000000000001,"15min":20.143000000000001},"last":0.26100000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":6.234,"5min":3.7509999999999999,"15min":2.9119999999999999},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.22800000000000001},"max":{"1min":19.283999999999999,"5min":19.283999999999999,"15min":19.283999999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":3.5550000000000002,"5min":3.6520000000000001,"15min":2.8799999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":19.027000000000001,"5min":19.027000000000001,"15min":19.027000000000001},"last":0.28599999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":5.7430000000000003,"5min":3.0430000000000001,"15min":2.363},"min":{"1min":0.45000000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":19.370999999999999,"5min":19.370999999999999,"15min":19.370999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":4.585,"5min":3.3700000000000001,"15min":3.7330000000000001},"min":{"1min":0.36799999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":19.053000000000001,"5min":19.053000000000001,"15min":19.053000000000001},"last":0.312}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":1.6200000000000001,"15min":1.0109999999999999},"min":{"1min":0.32000000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.725,"5min":12.725,"15min":12.725},"last":0.34699999999999998},{"interface":"front","average":{"1min":3.0270000000000001,"5min":1.7689999999999999,"15min":1.143},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":15.276,"5min":15.276,"15min":15.276},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.1820000000000004,"5min":2.4409999999999998,"15min":1.7669999999999999},"min":{"1min":0.438,"5min":0.32900000000000001,"15min":0.27700000000000002},"max":{"1min":15.32,"5min":15.32,"15min":15.32},"last":0.41899999999999998},{"interface":"front","average":{"1min":4.452,"5min":2.3119999999999998,"15min":1.8859999999999999},"min":{"1min":0.34300000000000003,"5min":0.315,"15min":0.315},"max":{"1min":17.882999999999999,"5min":17.882999999999999,"15min":17.882999999999999},"last":0.34000000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":3.798,"5min":2.5310000000000001,"15min":1.4590000000000001},"min":{"1min":0.435,"5min":0.374,"15min":0.34899999999999998},"max":{"1min":18.943999999999999,"5min":18.943999999999999,"15min":18.943999999999999},"last":0.434},{"interface":"front","average":{"1min":4.6109999999999998,"5min":2.6709999999999998,"15min":2.0089999999999999},"min":{"1min":0.46999999999999997,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":15.378,"5min":15.378,"15min":15.378},"last":0.39100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.452,"5min":2.7170000000000001,"15min":1.526},"min":{"1min":0.40699999999999997,"5min":0.39900000000000002,"15min":0.35899999999999999},"max":{"1min":18.972999999999999,"5min":18.972999999999999,"15min":18.972999999999999},"last":0.377},{"interface":"front","average":{"1min":5.6260000000000003,"5min":3.0609999999999999,"15min":2.0289999999999999},"min":{"1min":0.499,"5min":0.38600000000000001,"15min":0.376},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":0.45100000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824069,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":267704,"kb_used_data":3504,"kb_used_omap":10,"kb_used_meta":264181,"kb_avail":93477448,"statfs":{"total":95995035648,"available":95720906752,"internally_reserved":0,"allocated":3588096,"data_stored":2346286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":270521411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.8439999999999999,"5min":3.8610000000000002,"15min":4.423},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.373000000000001,"5min":23.381,"15min":23.381},"last":0.22900000000000001},{"interface":"front","average":{"1min":5.9340000000000002,"5min":4.9260000000000002,"15min":4.4279999999999999},"min":{"1min":0.245,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":18.141999999999999,"5min":20.239999999999998,"15min":20.239999999999998},"last":0.27900000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":6.0229999999999997,"5min":3.8959999999999999,"15min":3.9340000000000002},"min":{"1min":0.27400000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":40.618000000000002,"5min":40.618000000000002,"15min":40.618000000000002},"last":0.29799999999999999},{"interface":"front","average":{"1min":2.2490000000000001,"5min":3.4889999999999999,"15min":3.4500000000000002},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.218},"max":{"1min":6.0369999999999999,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.37}]},{"osd":2,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":6.8339999999999996,"5min":3.746,"15min":3.742},"min":{"1min":0.314,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":29.466999999999999,"5min":29.466999999999999,"15min":29.466999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.238,"5min":3.0430000000000001,"15min":2.9239999999999999},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.7510000000000003,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.42099999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1000000000000001,"5min":2.2490000000000001,"15min":1.48},"min":{"1min":0.33300000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.8700000000000001,"5min":15.676,"15min":15.676},"last":0.44500000000000001},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.6600000000000001,"15min":1.8939999999999999},"min":{"1min":0.40600000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.9890000000000008,"5min":16.478999999999999,"15min":16.478999999999999},"last":0.50900000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":4.1909999999999998,"5min":3.2410000000000001,"15min":2.823},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.307},"max":{"1min":9.0350000000000001,"5min":13.125999999999999,"15min":13.125999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":5.1120000000000001,"5min":3.2120000000000002,"15min":2.6400000000000001},"min":{"1min":0.373,"5min":0.309,"15min":0.309},"max":{"1min":17.242999999999999,"5min":17.242999999999999,"15min":17.242999999999999},"last":0.49099999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":3.0840000000000001,"15min":2.8540000000000001},"min":{"1min":0.38900000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.0660000000000007,"5min":16.504999999999999,"15min":16.504999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":4.8070000000000004,"5min":2.8650000000000002,"15min":2.6909999999999998},"min":{"1min":0.41099999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":17.286999999999999,"5min":17.286999999999999,"15min":17.286999999999999},"last":0.53300000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7850000000000001,"5min":2.4009999999999998,"15min":1.726},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.6869999999999994,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.8820000000000001,"5min":3.3140000000000001,"15min":2.8879999999999999},"min":{"1min":0.44900000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":10.241,"5min":16.367000000000001,"15min":16.367000000000001},"last":0.47699999999999998}]}]},{"osd":4,"up_from":266,"seq":1142461300795,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588788,"kb_used_data":266028,"kb_used_omap":35,"kb_used_meta":322716,"kb_avail":93156364,"statfs":{"total":95995035648,"available":95392116736,"internally_reserved":0,"allocated":272412672,"data_stored":271882530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":330461422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":1.256,"15min":0.93100000000000005},"min":{"1min":0.222,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.1319999999999997,"5min":10.881,"15min":10.881},"last":0.35699999999999998},{"interface":"front","average":{"1min":3.04,"5min":1.9790000000000001,"15min":1.2989999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":8.4740000000000002,"5min":12.246,"15min":12.246},"last":0.443}]},{"osd":1,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":3.8079999999999998,"5min":2.2530000000000001,"15min":1.2949999999999999},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":9.4700000000000006,"5min":12.161,"15min":12.161},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.986,"15min":1.512},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":13.231999999999999,"5min":13.231999999999999,"15min":13.231999999999999},"last":0.40699999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":3.1160000000000001,"5min":1.867,"15min":1.2849999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.6550000000000002,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":0.317},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.728,"15min":1.1799999999999999},"min":{"1min":0.311,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.8920000000000003,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":0.45200000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":2.1930000000000001,"15min":1.5720000000000001},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.8630000000000004,"5min":13.065,"15min":13.065},"last":0.38900000000000001},{"interface":"front","average":{"1min":4.4089999999999998,"5min":2.4449999999999998,"15min":1.4910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":14.577,"5min":14.577,"15min":14.577},"last":0.379}]},{"osd":5,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.196,"15min":0.88400000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.0589999999999993,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.66800000000000004,"15min":0.65500000000000003},"min":{"1min":0.28799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7549999999999999,"5min":5.7549999999999999,"15min":5.7549999999999999},"last":0.42799999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":0.95599999999999996,"15min":0.66900000000000004},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.851,"5min":6.851,"15min":6.851},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.0840000000000001,"15min":0.75},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.3860000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.41999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4569999999999999,"5min":1.258,"15min":0.83299999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":8.1039999999999992,"5min":12.237,"15min":12.237},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.3410000000000002,"5min":1.208,"15min":0.95499999999999996},"min":{"1min":0.314,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":0.437}]}]},{"osd":5,"up_from":36,"seq":154618822980,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308132,"kb_used_data":2716,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437020,"statfs":{"total":95995035648,"available":95679508480,"internally_reserved":0,"allocated":2781184,"data_stored":2263170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5950000000000002,"5min":1.7989999999999999,"15min":1.7090000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2639999999999993,"5min":8.2639999999999993,"15min":15.773999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.823,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.4939999999999998,"5min":10.244999999999999,"15min":15.699},"last":0.314}]},{"osd":1,"last update":"Mon Oct 9 10:13:17 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.2959999999999998,"15min":1.889},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":8.7300000000000004,"5min":11.891,"15min":15.689},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.9470000000000001,"15min":1.9079999999999999},"min":{"1min":0.29999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7450000000000001,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.34799999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":2.1739999999999999,"15min":2.4609999999999999},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.1620000000000008,"5min":10.259,"15min":15.640000000000001},"last":0.378},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.593,"15min":2.1219999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.313},"max":{"1min":6.5529999999999999,"5min":6.5529999999999999,"15min":10.656000000000001},"last":0.39300000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:12:52 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":2.3879999999999999,"15min":2.028},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.8699999999999992,"5min":9.8699999999999992,"15min":9.8699999999999992},"last":0.53600000000000003},{"interface":"front","average":{"1min":4.1420000000000003,"5min":2.706,"15min":2.9140000000000001},"min":{"1min":0.42999999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":17.193999999999999,"5min":17.34,"15min":17.34},"last":0.47099999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.0089999999999999,"15min":0.71199999999999997},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3419999999999996,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.754,"15min":0.747},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5559999999999992,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":0.42099999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.0629999999999999,"15min":1.0820000000000001},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":27.716000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.135,"15min":1.004},"min":{"1min":0.32700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2910000000000004,"5min":8.2910000000000004,"15min":18.600999999999999},"last":0.55000000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:13:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.2869999999999999,"15min":1.0820000000000001},"min":{"1min":0.36199999999999999,"5min":0.29699999999999999,"15min":0.25600000000000001},"max":{"1min":8.3409999999999993,"5min":8.3409999999999993,"15min":10.334},"last":0.442},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.82099999999999995,"15min":0.97199999999999998},"min":{"1min":0.32900000000000001,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":1.9850000000000001,"5min":8.5220000000000002,"15min":12.356},"last":0.45000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":267649024,"data_stored":267649024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":352124928,"data_stored":352124928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:13:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:24.977 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:24.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:25.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:25.111 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:25.154 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:13:25.154 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:13:25.157 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:25.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:25 smithi012 ceph-mon[147926]: pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 299 op/s; 3433/72 objects degraded (4768.056%); 0 B/s, 97 objects/s recovering 2023-10-09T10:13:25.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:25 smithi012 ceph-mon[147926]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:25.161 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:25 smithi012 ceph-mon[152263]: pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 299 op/s; 3433/72 objects degraded (4768.056%); 0 B/s, 97 objects/s recovering 2023-10-09T10:13:25.162 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:25 smithi012 ceph-mon[152263]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:25.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:25.263 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:25.291 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:25.329 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:25.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:25.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:25.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:25 smithi110 ceph-mon[136032]: pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 299 op/s; 3433/72 objects degraded (4768.056%); 0 B/s, 97 objects/s recovering 2023-10-09T10:13:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:25 smithi110 ceph-mon[136032]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:25.465 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:25.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:13:25.703 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:25.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:25.715 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:13:25.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:25.842 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:25.891 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:25.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:25.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:26.025 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:26.059 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:26.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:26.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:26.117 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:26.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:26.191 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:26.230 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:26.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:26.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:26.335 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:26.340 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:13:26.347 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:13:26.365 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:26.417 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_2' removed 2023-10-09T10:13:26.422 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:26.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:26.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:26.530 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:26.561 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:26.599 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:26.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:26.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:26.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:26.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:26.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[152263]: pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 3433/72 objects degraded (4768.056%); 0 B/s, 97 objects/s recovering 2023-10-09T10:13:26.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2383177228' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:13:26.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:13:26.619 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:26.619 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:26.619 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[147926]: pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 3433/72 objects degraded (4768.056%); 0 B/s, 97 objects/s recovering 2023-10-09T10:13:26.619 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2383177228' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:13:26.619 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:13:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:26 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:26 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:26 smithi110 ceph-mon[136032]: pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 3433/72 objects degraded (4768.056%); 0 B/s, 97 objects/s recovering 2023-10-09T10:13:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2383177228' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:13:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:13:26.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:26.733 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:26.819 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:26.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:26.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:26.923 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:26.953 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:27.047 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:27.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:27.151 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:27.180 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:27.474 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-mon[147926]: failed to encode map e316 with expected crc 2023-10-09T10:13:27.474 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:13:27.474 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-mon[147926]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T10:13:27.474 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-mon[147926]: failed to encode map e316 with expected crc 2023-10-09T10:13:27.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-mon[152263]: failed to encode map e316 with expected crc 2023-10-09T10:13:27.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:13:27.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-mon[152263]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T10:13:27.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-mon[152263]: failed to encode map e316 with expected crc 2023-10-09T10:13:27.587 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:13:27.587 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:13:27.587 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T10:13:27.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T10:13:27.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:27 smithi110 ceph-mon[136032]: failed to encode map e316 with expected crc 2023-10-09T10:13:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:13:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:27 smithi110 ceph-mon[136032]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T10:13:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:27 smithi110 ceph-mon[136032]: failed to encode map e316 with expected crc 2023-10-09T10:13:27.697 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:13:27.700 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:27.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:27.721 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:13:27.748 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:27.749 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:13:27.772 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:13:27.792 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:13:27.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:27.810 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:13:27.833 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:13:27.835 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:27.852 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:13:27.854 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:13:27.854 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:13:27.855 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:13:27.855 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:13:27.855 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:13:27.855 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:13:27.855 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:13:27.855 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:13:27.880 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:27.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:27.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:27.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:13:27.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:28.027 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:13:28.027 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:13:28.027 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:13:28.027 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:13:28.048 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:13:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:13:27] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.33.4" 2023-10-09T10:13:28.240 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:28.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:28.408 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:28.439 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:28.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:28 smithi110 ceph-mon[136032]: failed to encode map e316 with expected crc 2023-10-09T10:13:28.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:28 smithi110 ceph-mon[136032]: failed to encode map e316 with expected crc 2023-10-09T10:13:28.464 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:28 smithi110 ceph-mon[136032]: pgmap v888: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:28.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1214430711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:28.465 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3433/72 objects degraded (4768.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:28.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:13:28.479 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:28.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:28.544 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[147926]: failed to encode map e316 with expected crc 2023-10-09T10:13:28.545 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[147926]: failed to encode map e316 with expected crc 2023-10-09T10:13:28.545 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[147926]: pgmap v888: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:28.545 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1214430711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:28.545 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3433/72 objects degraded (4768.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:28.545 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[152263]: failed to encode map e316 with expected crc 2023-10-09T10:13:28.545 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[152263]: failed to encode map e316 with expected crc 2023-10-09T10:13:28.545 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[152263]: pgmap v888: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:28.546 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1214430711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:28.546 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3433/72 objects degraded (4768.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:13:28.577 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:28.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:28.610 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:28.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:28.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:28.745 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:28.771 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:28.775 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:28.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:28.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:28.906 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:28.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:28.964 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:28.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:29.067 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:29.094 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:29.121 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:13:29.126 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:29.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:29.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:29.230 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:29.256 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:29.388 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:29.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:29.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:29.495 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:29.522 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:29.555 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:29.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:29.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:29 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3433/72 objects degraded (4768.056%), 1 pg degraded, 1 pg undersized) 2023-10-09T10:13:29.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:29.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:29.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:29 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3433/72 objects degraded (4768.056%), 1 pg degraded, 1 pg undersized) 2023-10-09T10:13:29.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:29.649 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:29.659 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:29.668 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:29 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3433/72 objects degraded (4768.056%), 1 pg degraded, 1 pg undersized) 2023-10-09T10:13:29.668 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:29.669 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:29.687 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:29.871 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:29.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:29.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:29.975 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:30.003 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:30.087 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:30.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:30.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:30.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:30.225 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:30.271 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:13:30.272 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:13:30.272 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:13:30.272 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:13:30.272 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:13:30.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:13:30.483 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:30.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:30.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:30.622 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:30.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:13:30.808 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:30.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:30.912 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:30.946 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:31.031 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:31.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:31.134 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:31.161 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:31.193 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:31.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:31.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:31.201 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:31.235 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:13:31.298 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:31.324 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:31.369 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:31.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:31.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:31.430 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:13:31.431 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:13:31.471 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:31.499 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:31.532 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:31.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:31.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:31.567 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:13:31.568 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_3' created 2023-10-09T10:13:31.635 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:31.668 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[147926]: pgmap v889: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:31.668 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:13:31.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:13:31.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/578857126' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:31.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.669 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[152263]: pgmap v889: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:31.669 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.669 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.670 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:13:31.670 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:13:31.670 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/578857126' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:31.670 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:31 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:31.792 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:31 smithi110 ceph-mon[136032]: pgmap v889: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:31.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:31 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:31 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:31 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:13:31.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:31 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:13:31.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:31 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/578857126' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:31.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:31 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:31.914 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:31.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:32.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:32.055 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:32.082 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":889,"stamp":"2023-10-09T10:13:29.456629+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":1190604,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":261,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3216104,"kb_used_data":625640,"kb_used_omap":167,"kb_used_meta":2590232,"kb_avail":746745112,"statfs":{"total":767960285184,"available":764666994688,"internally_reserved":0,"allocated":640655360,"data_stored":634120316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":2652398169},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":58,"apply_latency_ms":58,"commit_latency_ns":58000000,"apply_latency_ns":58000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-4045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-217362,"num_write_kb":-6955521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-6612,"num_bytes_recovered":-309985280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001895"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":158,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182767+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:13:23.182767+0000","last_peered":"2023-10-09T10:13:23.182767+0000","last_clean":"2023-10-09T10:13:23.182767+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:13:23.182767+0000","last_undegraded":"2023-10-09T10:13:23.182767+0000","last_fullsized":"2023-10-09T10:13:23.182767+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":274,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979153+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:13:22.979153+0000","last_peered":"2023-10-09T10:13:22.979153+0000","last_clean":"2023-10-09T10:13:22.979153+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:13:22.979153+0000","last_undegraded":"2023-10-09T10:13:22.979153+0000","last_fullsized":"2023-10-09T10:13:22.979153+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":319,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182617+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:13:23.182617+0000","last_peered":"2023-10-09T10:13:23.182617+0000","last_clean":"2023-10-09T10:13:23.182617+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:13:23.182617+0000","last_undegraded":"2023-10-09T10:13:23.182617+0000","last_fullsized":"2023-10-09T10:13:23.182617+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":182,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.312921+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:13:08.312921+0000","last_peered":"2023-10-09T10:13:08.312921+0000","last_clean":"2023-10-09T10:13:08.312921+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:13:08.312921+0000","last_undegraded":"2023-10-09T10:13:08.312921+0000","last_fullsized":"2023-10-09T10:13:08.312921+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":288,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979814+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:13:22.979814+0000","last_peered":"2023-10-09T10:13:22.979814+0000","last_clean":"2023-10-09T10:13:22.979814+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:13:22.979814+0000","last_undegraded":"2023-10-09T10:13:22.979814+0000","last_fullsized":"2023-10-09T10:13:22.979814+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":134,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.990965+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:13:22.990965+0000","last_peered":"2023-10-09T10:13:22.990965+0000","last_clean":"2023-10-09T10:13:22.990965+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:13:22.990965+0000","last_undegraded":"2023-10-09T10:13:22.990965+0000","last_fullsized":"2023-10-09T10:13:22.990965+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":199,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310835+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:13:08.310835+0000","last_peered":"2023-10-09T10:13:08.310835+0000","last_clean":"2023-10-09T10:13:08.310835+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:13:08.310835+0000","last_undegraded":"2023-10-09T10:13:08.310835+0000","last_fullsized":"2023-10-09T10:13:08.310835+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":269,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979739+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:13:22.979739+0000","last_peered":"2023-10-09T10:13:22.979739+0000","last_clean":"2023-10-09T10:13:22.979739+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:13:22.979739+0000","last_undegraded":"2023-10-09T10:13:22.979739+0000","last_fullsized":"2023-10-09T10:13:22.979739+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":133,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.991005+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:13:22.991005+0000","last_peered":"2023-10-09T10:13:22.991005+0000","last_clean":"2023-10-09T10:13:22.991005+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:13:22.991005+0000","last_undegraded":"2023-10-09T10:13:22.991005+0000","last_fullsized":"2023-10-09T10:13:22.991005+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":230,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.978979+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:13:22.978979+0000","last_peered":"2023-10-09T10:13:22.978979+0000","last_clean":"2023-10-09T10:13:22.978979+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:13:22.978979+0000","last_undegraded":"2023-10-09T10:13:22.978979+0000","last_fullsized":"2023-10-09T10:13:22.978979+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":353,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.293934+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:13:08.293934+0000","last_peered":"2023-10-09T10:13:08.293934+0000","last_clean":"2023-10-09T10:13:08.293934+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:13:08.293934+0000","last_undegraded":"2023-10-09T10:13:08.293934+0000","last_fullsized":"2023-10-09T10:13:08.293934+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":312,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182669+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:13:23.182669+0000","last_peered":"2023-10-09T10:13:23.182669+0000","last_clean":"2023-10-09T10:13:23.182669+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:13:23.182669+0000","last_undegraded":"2023-10-09T10:13:23.182669+0000","last_fullsized":"2023-10-09T10:13:23.182669+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":202,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.293415+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:13:08.293415+0000","last_peered":"2023-10-09T10:13:08.293415+0000","last_clean":"2023-10-09T10:13:08.293415+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:13:08.293415+0000","last_undegraded":"2023-10-09T10:13:08.293415+0000","last_fullsized":"2023-10-09T10:13:08.293415+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":184,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182271+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:13:23.182271+0000","last_peered":"2023-10-09T10:13:23.182271+0000","last_clean":"2023-10-09T10:13:23.182271+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:13:23.182271+0000","last_undegraded":"2023-10-09T10:13:23.182271+0000","last_fullsized":"2023-10-09T10:13:23.182271+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":386,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.289892+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:13:08.289892+0000","last_peered":"2023-10-09T10:13:08.289892+0000","last_clean":"2023-10-09T10:13:08.289892+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:13:08.289892+0000","last_undegraded":"2023-10-09T10:13:08.289892+0000","last_fullsized":"2023-10-09T10:13:08.289892+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":256,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979703+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:13:22.979703+0000","last_peered":"2023-10-09T10:13:22.979703+0000","last_clean":"2023-10-09T10:13:22.979703+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:13:22.979703+0000","last_undegraded":"2023-10-09T10:13:22.979703+0000","last_fullsized":"2023-10-09T10:13:22.979703+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":181,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310565+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:13:08.310565+0000","last_peered":"2023-10-09T10:13:08.310565+0000","last_clean":"2023-10-09T10:13:08.310565+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:13:08.310565+0000","last_undegraded":"2023-10-09T10:13:08.310565+0000","last_fullsized":"2023-10-09T10:13:08.310565+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":149,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.991170+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:13:22.991170+0000","last_peered":"2023-10-09T10:13:22.991170+0000","last_clean":"2023-10-09T10:13:22.991170+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:13:22.991170+0000","last_undegraded":"2023-10-09T10:13:22.991170+0000","last_fullsized":"2023-10-09T10:13:22.991170+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":183,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.289708+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:13:08.289708+0000","last_peered":"2023-10-09T10:13:08.289708+0000","last_clean":"2023-10-09T10:13:08.289708+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:13:08.289708+0000","last_undegraded":"2023-10-09T10:13:08.289708+0000","last_fullsized":"2023-10-09T10:13:08.289708+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":181,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.312963+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:13:08.312963+0000","last_peered":"2023-10-09T10:13:08.312963+0000","last_clean":"2023-10-09T10:13:08.312963+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:13:08.312963+0000","last_undegraded":"2023-10-09T10:13:08.312963+0000","last_fullsized":"2023-10-09T10:13:08.312963+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":204,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310305+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:13:08.310305+0000","last_peered":"2023-10-09T10:13:08.310305+0000","last_clean":"2023-10-09T10:13:08.310305+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:13:08.310305+0000","last_undegraded":"2023-10-09T10:13:08.310305+0000","last_fullsized":"2023-10-09T10:13:08.310305+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":269,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979588+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:13:22.979588+0000","last_peered":"2023-10-09T10:13:22.979588+0000","last_clean":"2023-10-09T10:13:22.979588+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:13:22.979588+0000","last_undegraded":"2023-10-09T10:13:22.979588+0000","last_fullsized":"2023-10-09T10:13:22.979588+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":347,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.293260+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:13:08.293260+0000","last_peered":"2023-10-09T10:13:08.293260+0000","last_clean":"2023-10-09T10:13:08.293260+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:13:08.293260+0000","last_undegraded":"2023-10-09T10:13:08.293260+0000","last_fullsized":"2023-10-09T10:13:08.293260+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":134,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.990639+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:13:22.990639+0000","last_peered":"2023-10-09T10:13:22.990639+0000","last_clean":"2023-10-09T10:13:22.990639+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:13:22.990639+0000","last_undegraded":"2023-10-09T10:13:22.990639+0000","last_fullsized":"2023-10-09T10:13:22.990639+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":335,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.290224+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:13:08.290224+0000","last_peered":"2023-10-09T10:13:08.290224+0000","last_clean":"2023-10-09T10:13:08.290224+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:13:08.290224+0000","last_undegraded":"2023-10-09T10:13:08.290224+0000","last_fullsized":"2023-10-09T10:13:08.290224+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":228,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182517+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:13:23.182517+0000","last_peered":"2023-10-09T10:13:23.182517+0000","last_clean":"2023-10-09T10:13:23.182517+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:13:23.182517+0000","last_undegraded":"2023-10-09T10:13:23.182517+0000","last_fullsized":"2023-10-09T10:13:23.182517+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":138,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.991200+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:13:22.991200+0000","last_peered":"2023-10-09T10:13:22.991200+0000","last_clean":"2023-10-09T10:13:22.991200+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:13:22.991200+0000","last_undegraded":"2023-10-09T10:13:22.991200+0000","last_fullsized":"2023-10-09T10:13:22.991200+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":158,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.294315+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:13:08.294315+0000","last_peered":"2023-10-09T10:13:08.294315+0000","last_clean":"2023-10-09T10:13:08.294315+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:13:08.294315+0000","last_undegraded":"2023-10-09T10:13:08.294315+0000","last_fullsized":"2023-10-09T10:13:08.294315+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":149,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.992815+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:13:22.992815+0000","last_peered":"2023-10-09T10:13:22.992815+0000","last_clean":"2023-10-09T10:13:22.992815+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:13:22.992815+0000","last_undegraded":"2023-10-09T10:13:22.992815+0000","last_fullsized":"2023-10-09T10:13:22.992815+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":182,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310598+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:13:08.310598+0000","last_peered":"2023-10-09T10:13:08.310598+0000","last_clean":"2023-10-09T10:13:08.310598+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:13:08.310598+0000","last_undegraded":"2023-10-09T10:13:08.310598+0000","last_fullsized":"2023-10-09T10:13:08.310598+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":310,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979635+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:13:22.979635+0000","last_peered":"2023-10-09T10:13:22.979635+0000","last_clean":"2023-10-09T10:13:22.979635+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:13:22.979635+0000","last_undegraded":"2023-10-09T10:13:22.979635+0000","last_fullsized":"2023-10-09T10:13:22.979635+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":180,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979588+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:13:22.979588+0000","last_peered":"2023-10-09T10:13:22.979588+0000","last_clean":"2023-10-09T10:13:22.979588+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:13:22.979588+0000","last_undegraded":"2023-10-09T10:13:22.979588+0000","last_fullsized":"2023-10-09T10:13:22.979588+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":242,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182616+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:13:23.182616+0000","last_peered":"2023-10-09T10:13:23.182616+0000","last_clean":"2023-10-09T10:13:23.182616+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:13:23.182616+0000","last_undegraded":"2023-10-09T10:13:23.182616+0000","last_fullsized":"2023-10-09T10:13:23.182616+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":247,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182291+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:13:23.182291+0000","last_peered":"2023-10-09T10:13:23.182291+0000","last_clean":"2023-10-09T10:13:23.182291+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:13:23.182291+0000","last_undegraded":"2023-10-09T10:13:23.182291+0000","last_fullsized":"2023-10-09T10:13:23.182291+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":209,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.289617+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:13:08.289617+0000","last_peered":"2023-10-09T10:13:08.289617+0000","last_clean":"2023-10-09T10:13:08.289617+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:13:08.289617+0000","last_undegraded":"2023-10-09T10:13:08.289617+0000","last_fullsized":"2023-10-09T10:13:08.289617+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":184,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182330+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:13:23.182330+0000","last_peered":"2023-10-09T10:13:23.182330+0000","last_clean":"2023-10-09T10:13:23.182330+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:13:23.182330+0000","last_undegraded":"2023-10-09T10:13:23.182330+0000","last_fullsized":"2023-10-09T10:13:23.182330+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":270,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979673+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:13:22.979673+0000","last_peered":"2023-10-09T10:13:22.979673+0000","last_clean":"2023-10-09T10:13:22.979673+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:13:22.979673+0000","last_undegraded":"2023-10-09T10:13:22.979673+0000","last_fullsized":"2023-10-09T10:13:22.979673+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":320,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.183071+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:13:23.183071+0000","last_peered":"2023-10-09T10:13:23.183071+0000","last_clean":"2023-10-09T10:13:23.183071+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:13:23.183071+0000","last_undegraded":"2023-10-09T10:13:23.183071+0000","last_fullsized":"2023-10-09T10:13:23.183071+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":320,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.701264+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:13:23.701264+0000","last_peered":"2023-10-09T10:13:23.701264+0000","last_clean":"2023-10-09T10:13:23.701264+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:13:23.701264+0000","last_undegraded":"2023-10-09T10:13:23.701264+0000","last_fullsized":"2023-10-09T10:13:23.701264+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":158,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.991118+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:13:22.991118+0000","last_peered":"2023-10-09T10:13:22.991118+0000","last_clean":"2023-10-09T10:13:22.991118+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:13:22.991118+0000","last_undegraded":"2023-10-09T10:13:22.991118+0000","last_fullsized":"2023-10-09T10:13:22.991118+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":276,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182806+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:13:23.182806+0000","last_peered":"2023-10-09T10:13:23.182806+0000","last_clean":"2023-10-09T10:13:23.182806+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:13:23.182806+0000","last_undegraded":"2023-10-09T10:13:23.182806+0000","last_fullsized":"2023-10-09T10:13:23.182806+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":134,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.990807+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:13:22.990807+0000","last_peered":"2023-10-09T10:13:22.990807+0000","last_clean":"2023-10-09T10:13:22.990807+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:13:22.990807+0000","last_undegraded":"2023-10-09T10:13:22.990807+0000","last_fullsized":"2023-10-09T10:13:22.990807+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":132,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310351+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:13:08.310351+0000","last_peered":"2023-10-09T10:13:08.310351+0000","last_clean":"2023-10-09T10:13:08.310351+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:13:08.310351+0000","last_undegraded":"2023-10-09T10:13:08.310351+0000","last_fullsized":"2023-10-09T10:13:08.310351+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":343,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.701246+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:13:23.701246+0000","last_peered":"2023-10-09T10:13:23.701246+0000","last_clean":"2023-10-09T10:13:23.701246+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:13:23.701246+0000","last_undegraded":"2023-10-09T10:13:23.701246+0000","last_fullsized":"2023-10-09T10:13:23.701246+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":236,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.978879+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:13:22.978879+0000","last_peered":"2023-10-09T10:13:22.978879+0000","last_clean":"2023-10-09T10:13:22.978879+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:13:22.978879+0000","last_undegraded":"2023-10-09T10:13:22.978879+0000","last_fullsized":"2023-10-09T10:13:22.978879+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":312,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182597+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:13:23.182597+0000","last_peered":"2023-10-09T10:13:23.182597+0000","last_clean":"2023-10-09T10:13:23.182597+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:13:23.182597+0000","last_undegraded":"2023-10-09T10:13:23.182597+0000","last_fullsized":"2023-10-09T10:13:23.182597+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":332,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.183072+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:13:23.183072+0000","last_peered":"2023-10-09T10:13:23.183072+0000","last_clean":"2023-10-09T10:13:23.183072+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:13:23.183072+0000","last_undegraded":"2023-10-09T10:13:23.183072+0000","last_fullsized":"2023-10-09T10:13:23.183072+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":143,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.991067+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:13:22.991067+0000","last_peered":"2023-10-09T10:13:22.991067+0000","last_clean":"2023-10-09T10:13:22.991067+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:13:22.991067+0000","last_undegraded":"2023-10-09T10:13:22.991067+0000","last_fullsized":"2023-10-09T10:13:22.991067+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":561,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.292947+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:13:08.292947+0000","last_peered":"2023-10-09T10:13:08.292947+0000","last_clean":"2023-10-09T10:13:08.292947+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:13:08.292947+0000","last_undegraded":"2023-10-09T10:13:08.292947+0000","last_fullsized":"2023-10-09T10:13:08.292947+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":220,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.978987+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:13:22.978987+0000","last_peered":"2023-10-09T10:13:22.978987+0000","last_clean":"2023-10-09T10:13:22.978987+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:13:22.978987+0000","last_undegraded":"2023-10-09T10:13:22.978987+0000","last_fullsized":"2023-10-09T10:13:22.978987+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":230,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.294020+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:13:08.294020+0000","last_peered":"2023-10-09T10:13:08.294020+0000","last_clean":"2023-10-09T10:13:08.294020+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:13:08.294020+0000","last_undegraded":"2023-10-09T10:13:08.294020+0000","last_fullsized":"2023-10-09T10:13:08.294020+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":292,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182659+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:13:23.182659+0000","last_peered":"2023-10-09T10:13:23.182659+0000","last_clean":"2023-10-09T10:13:23.182659+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:13:23.182659+0000","last_undegraded":"2023-10-09T10:13:23.182659+0000","last_fullsized":"2023-10-09T10:13:23.182659+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":193,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310489+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:13:08.310489+0000","last_peered":"2023-10-09T10:13:08.310489+0000","last_clean":"2023-10-09T10:13:08.310489+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:13:08.310489+0000","last_undegraded":"2023-10-09T10:13:08.310489+0000","last_fullsized":"2023-10-09T10:13:08.310489+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":221,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:21.899698+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:13:21.899698+0000","last_peered":"2023-10-09T10:13:21.899698+0000","last_clean":"2023-10-09T10:13:21.899698+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:13:21.899698+0000","last_undegraded":"2023-10-09T10:13:21.899698+0000","last_fullsized":"2023-10-09T10:13:21.899698+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":334,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.293293+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:13:08.293293+0000","last_peered":"2023-10-09T10:13:08.293293+0000","last_clean":"2023-10-09T10:13:08.293293+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:13:08.293293+0000","last_undegraded":"2023-10-09T10:13:08.293293+0000","last_fullsized":"2023-10-09T10:13:08.293293+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":349,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.701274+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:13:23.701274+0000","last_peered":"2023-10-09T10:13:23.701274+0000","last_clean":"2023-10-09T10:13:23.701274+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:13:23.701274+0000","last_undegraded":"2023-10-09T10:13:23.701274+0000","last_fullsized":"2023-10-09T10:13:23.701274+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":185,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310443+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:13:08.310443+0000","last_peered":"2023-10-09T10:13:08.310443+0000","last_clean":"2023-10-09T10:13:08.310443+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:13:08.310443+0000","last_undegraded":"2023-10-09T10:13:08.310443+0000","last_fullsized":"2023-10-09T10:13:08.310443+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":279,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979357+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:13:22.979357+0000","last_peered":"2023-10-09T10:13:22.979357+0000","last_clean":"2023-10-09T10:13:22.979357+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:13:22.979357+0000","last_undegraded":"2023-10-09T10:13:22.979357+0000","last_fullsized":"2023-10-09T10:13:22.979357+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":293,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182360+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:13:23.182360+0000","last_peered":"2023-10-09T10:13:23.182360+0000","last_clean":"2023-10-09T10:13:23.182360+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:13:23.182360+0000","last_undegraded":"2023-10-09T10:13:23.182360+0000","last_fullsized":"2023-10-09T10:13:23.182360+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":268,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979583+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:13:22.979583+0000","last_peered":"2023-10-09T10:13:22.979583+0000","last_clean":"2023-10-09T10:13:22.979583+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:13:22.979583+0000","last_undegraded":"2023-10-09T10:13:22.979583+0000","last_fullsized":"2023-10-09T10:13:22.979583+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":223,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.295001+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:13:08.295001+0000","last_peered":"2023-10-09T10:13:08.295001+0000","last_clean":"2023-10-09T10:13:08.295001+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:13:08.295001+0000","last_undegraded":"2023-10-09T10:13:08.295001+0000","last_fullsized":"2023-10-09T10:13:08.295001+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":187,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:21.899685+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:13:21.899685+0000","last_peered":"2023-10-09T10:13:21.899685+0000","last_clean":"2023-10-09T10:13:21.899685+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:13:21.899685+0000","last_undegraded":"2023-10-09T10:13:21.899685+0000","last_fullsized":"2023-10-09T10:13:21.899685+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":169,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.990644+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:13:22.990644+0000","last_peered":"2023-10-09T10:13:22.990644+0000","last_clean":"2023-10-09T10:13:22.990644+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:13:22.990644+0000","last_undegraded":"2023-10-09T10:13:22.990644+0000","last_fullsized":"2023-10-09T10:13:22.990644+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":181,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310203+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:13:08.310203+0000","last_peered":"2023-10-09T10:13:08.310203+0000","last_clean":"2023-10-09T10:13:08.310203+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:13:08.310203+0000","last_undegraded":"2023-10-09T10:13:08.310203+0000","last_fullsized":"2023-10-09T10:13:08.310203+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":255,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:23.182348+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:13:23.182348+0000","last_peered":"2023-10-09T10:13:23.182348+0000","last_clean":"2023-10-09T10:13:23.182348+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:13:23.182348+0000","last_undegraded":"2023-10-09T10:13:23.182348+0000","last_fullsized":"2023-10-09T10:13:23.182348+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":153,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.992770+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:13:22.992770+0000","last_peered":"2023-10-09T10:13:22.992770+0000","last_clean":"2023-10-09T10:13:22.992770+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:13:22.992770+0000","last_undegraded":"2023-10-09T10:13:22.992770+0000","last_fullsized":"2023-10-09T10:13:22.992770+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":195,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979756+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:13:22.979756+0000","last_peered":"2023-10-09T10:13:22.979756+0000","last_clean":"2023-10-09T10:13:22.979756+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:13:22.979756+0000","last_undegraded":"2023-10-09T10:13:22.979756+0000","last_fullsized":"2023-10-09T10:13:22.979756+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":153,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.991027+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:13:22.991027+0000","last_peered":"2023-10-09T10:13:22.991027+0000","last_clean":"2023-10-09T10:13:22.991027+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:13:22.991027+0000","last_undegraded":"2023-10-09T10:13:22.991027+0000","last_fullsized":"2023-10-09T10:13:22.991027+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":204,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.978897+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:13:22.978897+0000","last_peered":"2023-10-09T10:13:22.978897+0000","last_clean":"2023-10-09T10:13:22.978897+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:13:22.978897+0000","last_undegraded":"2023-10-09T10:13:22.978897+0000","last_fullsized":"2023-10-09T10:13:22.978897+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":333,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979278+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:13:22.979278+0000","last_peered":"2023-10-09T10:13:22.979278+0000","last_clean":"2023-10-09T10:13:22.979278+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:13:22.979278+0000","last_undegraded":"2023-10-09T10:13:22.979278+0000","last_fullsized":"2023-10-09T10:13:22.979278+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":354,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:13:22.979065+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:13:22.979065+0000","last_peered":"2023-10-09T10:13:22.979065+0000","last_clean":"2023-10-09T10:13:22.979065+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:13:22.979065+0000","last_undegraded":"2023-10-09T10:13:22.979065+0000","last_fullsized":"2023-10-09T10:13:22.979065+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":263,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:13:08.310618+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:13:08.310618+0000","last_peered":"2023-10-09T10:13:08.310618+0000","last_clean":"2023-10-09T10:13:08.310618+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:13:08.310618+0000","last_undegraded":"2023-10-09T10:13:08.310618+0000","last_fullsized":"2023-10-09T10:13:08.310618+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":184,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T10:13:27.188944+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:13:27.188944+0000","last_peered":"2023-10-09T10:13:27.188944+0000","last_clean":"2023-10-09T10:13:27.188944+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:13:27.188944+0000","last_undegraded":"2023-10-09T10:13:27.188944+0000","last_fullsized":"2023-10-09T10:13:27.188944+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430525,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":304644,"kb_used_data":3324,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93440508,"statfs":{"total":95995035648,"available":95683080192,"internally_reserved":0,"allocated":3403776,"data_stored":2865789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.3130000000000002,"15min":2.0539999999999998},"min":{"1min":0.27400000000000002,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":11.164,"5min":11.164,"15min":50.591999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.2770000000000001,"15min":2.0059999999999998},"min":{"1min":0.32600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.7880000000000003,"5min":15.974,"15min":50.747},"last":0.41399999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:13:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5390000000000001,"5min":2.6539999999999999,"15min":2.0910000000000002},"min":{"1min":0.26400000000000001,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":8.5890000000000004,"5min":14.311,"15min":25.338000000000001},"last":0.434},{"interface":"front","average":{"1min":4.3609999999999998,"5min":2.7679999999999998,"15min":1.9390000000000001},"min":{"1min":0.27900000000000003,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":14.5,"5min":16.526,"15min":18.27},"last":0.45400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":3.8029999999999999,"5min":2.6030000000000002,"15min":2.4430000000000001},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":10.682,"5min":13.775,"15min":13.775},"last":0.39800000000000002},{"interface":"front","average":{"1min":3.7200000000000002,"5min":2.4350000000000001,"15min":2.2559999999999998},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.795999999999999,"5min":12.101000000000001,"15min":12.101000000000001},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":3.6110000000000002,"5min":3.0009999999999999,"15min":2.1669999999999998},"min":{"1min":0.40300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":10.827,"5min":29.058,"15min":29.058},"last":0.48199999999999998},{"interface":"front","average":{"1min":4.0060000000000002,"5min":2.6680000000000001,"15min":2.1859999999999999},"min":{"1min":0.36699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":16.715,"5min":16.715,"15min":16.715},"last":0.47399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.288,"15min":1.0369999999999999},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":7.7469999999999999,"5min":7.7469999999999999,"15min":7.7469999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.724,"5min":1.139,"15min":0.98499999999999999},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8180000000000001,"15min":1.256},"min":{"1min":0.34000000000000002,"5min":0.219,"15min":0.219},"max":{"1min":9.2560000000000002,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":2.9089999999999998,"5min":1.7130000000000001,"15min":1.2150000000000001},"min":{"1min":0.38600000000000001,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":18.547999999999998,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:13:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1000000000000001,"5min":1.3620000000000001,"15min":1.1080000000000001},"min":{"1min":0.40400000000000003,"5min":0.30199999999999999,"15min":0.26100000000000001},"max":{"1min":8.8569999999999993,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.1779999999999999,"5min":1.2090000000000001,"15min":1.026},"min":{"1min":0.33900000000000002,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":5.9660000000000002,"5min":6.4020000000000001,"15min":9.1920000000000002},"last":0.505}]}]},{"osd":6,"up_from":42,"seq":180388626753,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":648504,"kb_used_data":347184,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93096648,"statfs":{"total":95995035648,"available":95330967552,"internally_reserved":0,"allocated":355516416,"data_stored":354991671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8180000000000001,"5min":2.2330000000000001,"15min":1.756},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":9.6940000000000008,"5min":15.435,"15min":15.435},"last":0.26500000000000001},{"interface":"front","average":{"1min":2.5129999999999999,"5min":2.1890000000000001,"15min":1.7250000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":13.385,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.35099999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:13:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":3.1539999999999999,"15min":2.1749999999999998},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":9.8680000000000003,"5min":35.609999999999999,"15min":35.609999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.48,"5min":2.871,"15min":2.1200000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.2050000000000001,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.39200000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.8740000000000001,"15min":1.802},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":7.7359999999999998,"5min":12.898,"15min":12.898},"last":0.41799999999999998},{"interface":"front","average":{"1min":3.4609999999999999,"5min":2.6499999999999999,"15min":3.036},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.720000000000001,"5min":14.542999999999999,"15min":14.542999999999999},"last":0.40500000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.7839999999999998,"15min":1.8340000000000001},"min":{"1min":0.437,"5min":0.308,"15min":0.308},"max":{"1min":11.098000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":3.161,"5min":2.6059999999999999,"15min":2.456},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.002000000000001,"5min":11.211,"15min":11.211},"last":0.432}]},{"osd":4,"last update":"Mon Oct 9 10:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":0.94699999999999995,"15min":0.60899999999999999},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.113,"5min":10.113,"15min":10.113},"last":0.377},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.89500000000000002,"15min":0.63100000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.948,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.443}]},{"osd":5,"last update":"Mon Oct 9 10:12:37 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.0660000000000001,"15min":0.91800000000000004},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.23699999999999999},"max":{"1min":6.3170000000000002,"5min":7.9260000000000002,"15min":14.09},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.335,"5min":1.222,"15min":0.99399999999999999},"min":{"1min":0.34699999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":14.465999999999999,"5min":14.465999999999999,"15min":14.465999999999999},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:13:06 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.0249999999999999,"15min":0.92400000000000004},"min":{"1min":0.35799999999999998,"5min":0.28599999999999998,"15min":0.252},"max":{"1min":6.601,"5min":6.601,"15min":7.5350000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0669999999999999,"15min":0.96999999999999997},"min":{"1min":0.39000000000000001,"5min":0.38100000000000001,"15min":0.30099999999999999},"max":{"1min":6.6280000000000001,"5min":6.6280000000000001,"15min":7.5570000000000004},"last":0.51400000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253067,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":451160,"kb_used_data":3344,"kb_used_omap":9,"kb_used_meta":447798,"kb_avail":93293992,"statfs":{"total":95995035648,"available":95533047808,"internally_reserved":0,"allocated":3424256,"data_stored":2342829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":458546153},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":26,"apply_latency_ms":26,"commit_latency_ns":26000000,"apply_latency_ns":26000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.593,"15min":2.6000000000000001},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.183},"max":{"1min":13.303000000000001,"5min":31.007000000000001,"15min":31.007000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":3.0219999999999998,"5min":3.5990000000000002,"15min":2.7650000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.192},"max":{"1min":9.3710000000000004,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:12:52 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":3.1619999999999999,"15min":3.7450000000000001},"min":{"1min":0.22800000000000001,"5min":0.217,"15min":0.217},"max":{"1min":10.262,"5min":12.487,"15min":17.369},"last":0.31},{"interface":"front","average":{"1min":4.2009999999999996,"5min":3.1099999999999999,"15min":3.7429999999999999},"min":{"1min":0.31,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":21.645,"5min":21.645,"15min":21.645},"last":0.24199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.8159999999999998,"5min":3.6669999999999998,"15min":3.5779999999999998},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":21.808,"5min":21.808,"15min":21.808},"last":0.371},{"interface":"front","average":{"1min":5.1740000000000004,"5min":3.8580000000000001,"15min":4.5540000000000003},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":21.547999999999998,"5min":21.547999999999998,"15min":21.547999999999998},"last":0.33400000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.8029999999999999,"15min":1.4370000000000001},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":21.620000000000001,"5min":21.620000000000001,"15min":21.620000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.4980000000000002,"5min":2.411,"15min":1.6899999999999999},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":9.2569999999999997,"5min":12.904,"15min":12.904},"last":0.41799999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.8029999999999999,"5min":2.694,"15min":2.2080000000000002},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":9.2870000000000008,"5min":12.811,"15min":15.531000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":3.6000000000000001,"5min":2.601,"15min":2.496},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":19.513000000000002,"5min":19.513000000000002,"15min":19.706},"last":0.443}]},{"osd":6,"last update":"Mon Oct 9 10:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":2.222,"15min":2.081},"min":{"1min":0.42599999999999999,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":9.0779999999999994,"5min":21.670999999999999,"15min":21.670999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.6520000000000001,"15min":2.2599999999999998},"min":{"1min":0.39400000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":9.3160000000000007,"5min":12.868,"15min":15.481999999999999},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":2.4750000000000001,"15min":2.4609999999999999},"min":{"1min":0.42799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.1690000000000005,"5min":10.723000000000001,"15min":16.331},"last":0.503},{"interface":"front","average":{"1min":2.157,"5min":2.266,"15min":2.1600000000000001},"min":{"1min":0.45600000000000002,"5min":0.39100000000000001,"15min":0.378},"max":{"1min":9.125,"5min":21.593,"15min":21.593},"last":0.48299999999999998}]}]},{"osd":0,"up_from":79,"seq":339302416593,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":388712,"kb_used_data":3872,"kb_used_omap":28,"kb_used_meta":384803,"kb_avail":93356440,"statfs":{"total":95995035648,"available":95596994560,"internally_reserved":0,"allocated":3964928,"data_stored":2920649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":394039120},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:13:23 2023","interfaces":[{"interface":"back","average":{"1min":2.9089999999999998,"5min":2.9180000000000001,"15min":2.1899999999999999},"min":{"1min":0.23699999999999999,"5min":0.216,"15min":0.16600000000000001},"max":{"1min":10.935,"5min":16.797000000000001,"15min":16.797000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":5.2359999999999998,"5min":4.8659999999999997,"15min":3.3620000000000001},"min":{"1min":0.33400000000000002,"5min":0.253,"15min":0.21299999999999999},"max":{"1min":22.734000000000002,"5min":31.925999999999998,"15min":31.925999999999998},"last":0.46100000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":4.1130000000000004,"5min":2.7850000000000001,"15min":2.3929999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":13.483000000000001,"5min":18.003,"15min":18.003},"last":0.67800000000000005},{"interface":"front","average":{"1min":4.282,"5min":2.7949999999999999,"15min":3.0710000000000002},"min":{"1min":0.30099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":13.811,"5min":15.529,"15min":15.529},"last":0.39500000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":6.9710000000000001,"5min":5.0720000000000001,"15min":4.9340000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.013999999999999,"5min":25.734999999999999,"15min":25.734999999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":4.5730000000000004,"5min":4.1059999999999999,"15min":4.3090000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":18.928999999999998,"5min":21.923999999999999,"15min":21.923999999999999},"last":0.81899999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9109999999999996,"5min":3.012,"15min":1.9319999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":20.494,"5min":20.494,"15min":20.494},"last":0.54100000000000004},{"interface":"front","average":{"1min":4.2830000000000004,"5min":2.8639999999999999,"15min":2.246},"min":{"1min":0.34599999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":15.797000000000001,"5min":15.907,"15min":15.907},"last":0.76300000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.5119999999999996,"5min":3.145,"15min":2.444},"min":{"1min":0.38500000000000001,"5min":0.34599999999999997,"15min":0.28000000000000003},"max":{"1min":14.944000000000001,"5min":17.096,"15min":17.096},"last":0.57199999999999995},{"interface":"front","average":{"1min":4.1159999999999997,"5min":2.0710000000000002,"15min":1.6599999999999999},"min":{"1min":0.34799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.547000000000001,"5min":13.547000000000001,"15min":13.547000000000001},"last":0.747}]},{"osd":6,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1500000000000004,"5min":2.2210000000000001,"15min":1.8169999999999999},"min":{"1min":0.41199999999999998,"5min":0.33700000000000002,"15min":0.31900000000000001},"max":{"1min":13.51,"5min":13.51,"15min":13.51},"last":0.63900000000000001},{"interface":"front","average":{"1min":5.2859999999999996,"5min":3.726,"15min":2.7719999999999998},"min":{"1min":0.42299999999999999,"5min":0.35999999999999999,"15min":0.35699999999999998},"max":{"1min":20.446999999999999,"5min":23.106999999999999,"15min":23.106999999999999},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:12:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":3.2040000000000002,"15min":2.601},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":15.734,"5min":16.460000000000001,"15min":26.978000000000002},"last":0.79200000000000004},{"interface":"front","average":{"1min":4.5389999999999997,"5min":2.3650000000000002,"15min":1.9950000000000001},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":13.787000000000001,"5min":13.787000000000001,"15min":26.768000000000001},"last":0.69699999999999995}]}]},{"osd":2,"up_from":217,"seq":932007903326,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":266088,"kb_used_data":3296,"kb_used_omap":83,"kb_used_meta":262700,"kb_avail":93479064,"statfs":{"total":95995035648,"available":95722561536,"internally_reserved":0,"allocated":3375104,"data_stored":2331814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":269005295},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3250000000000002,"5min":2.762,"15min":2.036},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.17899999999999999},"max":{"1min":17.946000000000002,"5min":19.372,"15min":19.372},"last":0.26900000000000002},{"interface":"front","average":{"1min":4.7469999999999999,"5min":4.1379999999999999,"15min":3.254},"min":{"1min":0.29799999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.143000000000001,"5min":20.143000000000001,"15min":20.143000000000001},"last":0.57799999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":6.234,"5min":3.7509999999999999,"15min":2.9119999999999999},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.22800000000000001},"max":{"1min":19.283999999999999,"5min":19.283999999999999,"15min":19.283999999999999},"last":0.317},{"interface":"front","average":{"1min":3.5550000000000002,"5min":3.6520000000000001,"15min":2.8799999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":19.027000000000001,"5min":19.027000000000001,"15min":19.027000000000001},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":5.7430000000000003,"5min":3.0430000000000001,"15min":2.363},"min":{"1min":0.45000000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":19.370999999999999,"5min":19.370999999999999,"15min":19.370999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":4.585,"5min":3.3700000000000001,"15min":3.7330000000000001},"min":{"1min":0.36799999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":19.053000000000001,"5min":19.053000000000001,"15min":19.053000000000001},"last":0.51800000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:12:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":1.6200000000000001,"15min":1.0109999999999999},"min":{"1min":0.32000000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.725,"5min":12.725,"15min":12.725},"last":0.438},{"interface":"front","average":{"1min":3.0270000000000001,"5min":1.7689999999999999,"15min":1.143},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":15.276,"5min":15.276,"15min":15.276},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.1820000000000004,"5min":2.4409999999999998,"15min":1.7669999999999999},"min":{"1min":0.438,"5min":0.32900000000000001,"15min":0.27700000000000002},"max":{"1min":15.32,"5min":15.32,"15min":15.32},"last":0.374},{"interface":"front","average":{"1min":4.452,"5min":2.3119999999999998,"15min":1.8859999999999999},"min":{"1min":0.34300000000000003,"5min":0.315,"15min":0.315},"max":{"1min":17.882999999999999,"5min":17.882999999999999,"15min":17.882999999999999},"last":0.40699999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":3.798,"5min":2.5310000000000001,"15min":1.4590000000000001},"min":{"1min":0.435,"5min":0.374,"15min":0.34899999999999998},"max":{"1min":18.943999999999999,"5min":18.943999999999999,"15min":18.943999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":4.6109999999999998,"5min":2.6709999999999998,"15min":2.0089999999999999},"min":{"1min":0.46999999999999997,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":15.378,"5min":15.378,"15min":15.378},"last":0.45200000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:12:56 2023","interfaces":[{"interface":"back","average":{"1min":4.452,"5min":2.7170000000000001,"15min":1.526},"min":{"1min":0.40699999999999997,"5min":0.39900000000000002,"15min":0.35899999999999999},"max":{"1min":18.972999999999999,"5min":18.972999999999999,"15min":18.972999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":5.6260000000000003,"5min":3.0609999999999999,"15min":2.0289999999999999},"min":{"1min":0.499,"5min":0.38600000000000001,"15min":0.376},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":0.49199999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824070,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":267716,"kb_used_data":3516,"kb_used_omap":10,"kb_used_meta":264181,"kb_avail":93477436,"statfs":{"total":95995035648,"available":95720894464,"internally_reserved":0,"allocated":3600384,"data_stored":2354542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":270521411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.8439999999999999,"5min":3.8610000000000002,"15min":4.423},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.373000000000001,"5min":23.381,"15min":23.381},"last":0.33600000000000002},{"interface":"front","average":{"1min":5.9340000000000002,"5min":4.9260000000000002,"15min":4.4279999999999999},"min":{"1min":0.245,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":18.141999999999999,"5min":20.239999999999998,"15min":20.239999999999998},"last":0.34899999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":6.0229999999999997,"5min":3.8959999999999999,"15min":3.9340000000000002},"min":{"1min":0.27400000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":40.618000000000002,"5min":40.618000000000002,"15min":40.618000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.2490000000000001,"5min":3.4889999999999999,"15min":3.4500000000000002},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.218},"max":{"1min":6.0369999999999999,"5min":23.423999999999999,"15min":23.423999999999999},"last":0.378}]},{"osd":2,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":6.8339999999999996,"5min":3.746,"15min":3.742},"min":{"1min":0.314,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":29.466999999999999,"5min":29.466999999999999,"15min":29.466999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.238,"5min":3.0430000000000001,"15min":2.9239999999999999},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.7510000000000003,"5min":23.452000000000002,"15min":23.452000000000002},"last":0.41099999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:12:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1000000000000001,"5min":2.2490000000000001,"15min":1.48},"min":{"1min":0.33300000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.8700000000000001,"5min":15.676,"15min":15.676},"last":0.58199999999999996},{"interface":"front","average":{"1min":3.6269999999999998,"5min":2.6600000000000001,"15min":1.8939999999999999},"min":{"1min":0.40600000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.9890000000000008,"5min":16.478999999999999,"15min":16.478999999999999},"last":0.79600000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":4.1909999999999998,"5min":3.2410000000000001,"15min":2.823},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.307},"max":{"1min":9.0350000000000001,"5min":13.125999999999999,"15min":13.125999999999999},"last":0.754},{"interface":"front","average":{"1min":5.1120000000000001,"5min":3.2120000000000002,"15min":2.6400000000000001},"min":{"1min":0.373,"5min":0.309,"15min":0.309},"max":{"1min":17.242999999999999,"5min":17.242999999999999,"15min":17.242999999999999},"last":0.81200000000000006}]},{"osd":6,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":3.0840000000000001,"15min":2.8540000000000001},"min":{"1min":0.38900000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.0660000000000007,"5min":16.504999999999999,"15min":16.504999999999999},"last":0.84699999999999998},{"interface":"front","average":{"1min":4.8070000000000004,"5min":2.8650000000000002,"15min":2.6909999999999998},"min":{"1min":0.41099999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":17.286999999999999,"5min":17.286999999999999,"15min":17.286999999999999},"last":0.83999999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:12:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7850000000000001,"5min":2.4009999999999998,"15min":1.726},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.6869999999999994,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":3.8820000000000001,"5min":3.3140000000000001,"15min":2.8879999999999999},"min":{"1min":0.44900000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":10.241,"5min":16.367000000000001,"15min":16.367000000000001},"last":0.89000000000000001}]}]},{"osd":4,"up_from":266,"seq":1142461300796,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":581128,"kb_used_data":258368,"kb_used_omap":35,"kb_used_meta":322716,"kb_avail":93164024,"statfs":{"total":95995035648,"available":95399960576,"internally_reserved":0,"allocated":264568832,"data_stored":264034031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":330461422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":1.256,"15min":0.93100000000000005},"min":{"1min":0.222,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.1319999999999997,"5min":10.881,"15min":10.881},"last":0.57299999999999995},{"interface":"front","average":{"1min":3.04,"5min":1.9790000000000001,"15min":1.2989999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":8.4740000000000002,"5min":12.246,"15min":12.246},"last":0.66500000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":3.8079999999999998,"5min":2.2530000000000001,"15min":1.2949999999999999},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":9.4700000000000006,"5min":12.161,"15min":12.161},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.986,"15min":1.512},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":13.231999999999999,"5min":13.231999999999999,"15min":13.231999999999999},"last":0.63500000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":3.1160000000000001,"5min":1.867,"15min":1.2849999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.6550000000000002,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":0.748},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.728,"15min":1.1799999999999999},"min":{"1min":0.311,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.8920000000000003,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":0.53700000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":2.1930000000000001,"15min":1.5720000000000001},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.8630000000000004,"5min":13.065,"15min":13.065},"last":0.60099999999999998},{"interface":"front","average":{"1min":4.4089999999999998,"5min":2.4449999999999998,"15min":1.4910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":14.577,"5min":14.577,"15min":14.577},"last":0.74099999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.196,"15min":0.88400000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.0589999999999993,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.66800000000000004,"15min":0.65500000000000003},"min":{"1min":0.28799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7549999999999999,"5min":5.7549999999999999,"15min":5.7549999999999999},"last":1.115}]},{"osd":6,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":0.95599999999999996,"15min":0.66900000000000004},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.851,"5min":6.851,"15min":6.851},"last":0.88100000000000001},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.0840000000000001,"15min":0.75},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.3860000000000001,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.94499999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:12:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4569999999999999,"5min":1.258,"15min":0.83299999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":8.1039999999999992,"5min":12.237,"15min":12.237},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.3410000000000002,"5min":1.208,"15min":0.95499999999999996},"min":{"1min":0.314,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":0.78500000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822981,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308152,"kb_used_data":2736,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437000,"statfs":{"total":95995035648,"available":95679488000,"internally_reserved":0,"allocated":2801664,"data_stored":2278991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5950000000000002,"5min":1.7989999999999999,"15min":1.7090000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2639999999999993,"5min":8.2639999999999993,"15min":15.773999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.823,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.4939999999999998,"5min":10.244999999999999,"15min":15.699},"last":0.52400000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:13:17 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.2959999999999998,"15min":1.889},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":8.7300000000000004,"5min":11.891,"15min":15.689},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.9470000000000001,"15min":1.9079999999999999},"min":{"1min":0.29999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7450000000000001,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.443}]},{"osd":2,"last update":"Mon Oct 9 10:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":2.1739999999999999,"15min":2.4609999999999999},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.1620000000000008,"5min":10.259,"15min":15.640000000000001},"last":0.503},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.593,"15min":2.1219999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.313},"max":{"1min":6.5529999999999999,"5min":6.5529999999999999,"15min":10.656000000000001},"last":0.64000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:12:52 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":2.3879999999999999,"15min":2.028},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.8699999999999992,"5min":9.8699999999999992,"15min":9.8699999999999992},"last":0.51100000000000001},{"interface":"front","average":{"1min":4.1420000000000003,"5min":2.706,"15min":2.9140000000000001},"min":{"1min":0.42999999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":17.193999999999999,"5min":17.34,"15min":17.34},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:12:39 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.0089999999999999,"15min":0.71199999999999997},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3419999999999996,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.754,"15min":0.747},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.5559999999999992,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":0.38600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.0629999999999999,"15min":1.0820000000000001},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":27.716000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.135,"15min":1.004},"min":{"1min":0.32700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2910000000000004,"5min":8.2910000000000004,"15min":18.600999999999999},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:13:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.2869999999999999,"15min":1.0820000000000001},"min":{"1min":0.36199999999999999,"5min":0.29699999999999999,"15min":0.25600000000000001},"max":{"1min":8.3409999999999993,"5min":8.3409999999999993,"15min":10.334},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.82099999999999995,"15min":0.97199999999999998},"min":{"1min":0.32900000000000001,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":1.9850000000000001,"5min":8.5220000000000002,"15min":12.356},"last":0.61399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:13:32.088 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T10:13:32.088 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:13:32.089 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T10:13:32.089 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:13:32.089 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:13:32.089 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T10:13:32.094 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:32.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:32.197 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:32.242 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:32.270 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:32.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:32.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:32.409 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:84267 8.e e286/271 8:75ba2ae3:::benchmark_data_smithi012_271002_object84266:head v 286'5244, mlcod=286'5241)", 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:45.324614+0000", 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 227.21145456900001, 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020611930000000001, 2023-10-09T10:13:32.538 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.324614+0000", 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.324614+0000", 2023-10-09T10:13:32.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.324614+0000", 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031674260000000001 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.327781+0000", 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5937999999999999e-05 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.327797+0000", 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2869999999999999e-06 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.327804+0000", 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001578072 2023-10-09T10:13:32.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.329382+0000", 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3731999999999999e-05 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.329406+0000", 2023-10-09T10:13:32.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015773947999999999 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345180+0000", 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6295999999999999e-05 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:45.345226+0000", 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020611930000000001 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.543 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:136904 8.2 e297/271 8:45d8a7d7:::benchmark_data_smithi012_271002_object28221:head v 297'8455, mlcod=297'8453)", 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:11:04.816542+0000", 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 147.71952719199999, 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020184181999999998, 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.816542+0000", 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.816542+0000", 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.816541+0000", 2023-10-09T10:13:32.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4710000000000007e-06 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.816550+0000", 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6800000000000005e-06 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.816559+0000", 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9219999999999996e-06 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.816564+0000", 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8366000000000003e-05 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.816623+0000", 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8213999999999999e-05 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.816641+0000", 2023-10-09T10:13:32.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020041272999999998 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.836682+0000", 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3937999999999998e-05 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:04.836726+0000", 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020184181999999998 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.548 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:147425 8.2 e299/271 8:4bd5854b:::benchmark_data_smithi012_271002_object38742:head v 299'9104, mlcod=299'9101)", 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:11:22.433793+0000", 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 130.102276225, 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021129874, 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.433793+0000", 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.433793+0000", 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.433792+0000", 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.195e-06 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.433798+0000", 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4623e-05 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.433813+0000", 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2739999999999998e-06 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.433818+0000", 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065534130000000001 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.440372+0000", 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1024000000000001e-05 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.440393+0000", 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014493013000000001 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.454886+0000", 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6891999999999998e-05 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:22.454923+0000", 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021129874 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.553 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.554 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.554 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:148973 8.5 e300/271 8:a7f75dd7:::benchmark_data_smithi012_271002_object40290:head v 300'9423, mlcod=300'9419)", 2023-10-09T10:13:32.554 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:11:24.924577+0000", 2023-10-09T10:13:32.554 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 127.611491857, 2023-10-09T10:13:32.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020550127000000001, 2023-10-09T10:13:32.555 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.555 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.555 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.924577+0000", 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.924577+0000", 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.924577+0000", 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1200000000000004e-06 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.924581+0000", 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8560000000000003e-06 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.924587+0000", 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2139999999999998e-06 2023-10-09T10:13:32.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.924591+0000", 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006294282 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.930885+0000", 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1267000000000002e-05 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.930907+0000", 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014182093999999999 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.945089+0000", 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8574e-05 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:24.945127+0000", 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020550127000000001 2023-10-09T10:13:32.559 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:149449 8.4 e300/288 8:22b86a6f:::benchmark_data_smithi012_271002_object40766:head v 300'9437, mlcod=0'0)", 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:11:25.725204+0000", 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 126.810864612, 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023525069999999999, 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725204+0000", 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725204+0000", 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725204+0000", 2023-10-09T10:13:32.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4489999999999998e-06 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725209+0000", 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1371e-05 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725221+0000", 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.95e-06 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725226+0000", 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0683000000000003e-05 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725266+0000", 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5409e-05 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725292+0000", 2023-10-09T10:13:32.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023398059999999998 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.748690+0000", 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9641999999999998e-05 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.748729+0000", 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023525069999999999 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.564 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:158906 8.4 e302/288 8:2017a9ac:::benchmark_data_smithi012_271002_object50223:head v 302'10034, mlcod=0'0)", 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:11:41.224542+0000", 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 111.311527205, 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020047945000000001, 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.224542+0000", 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.224542+0000", 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.224541+0000", 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1930000000000004e-06 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.224548+0000", 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1557e-05 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.224560+0000", 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0889999999999996e-06 2023-10-09T10:13:32.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.224566+0000", 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0702000000000003e-05 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.224627+0000", 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3629999999999999e-05 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.224650+0000", 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019887740000000001 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.244538+0000", 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1755000000000002e-05 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:41.244590+0000", 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020047945000000001 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.569 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:174610 8.5 e305/271 8:a71863a4:::benchmark_data_smithi012_271002_object65927:head v 305'11073, mlcod=305'11069)", 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:07.073851+0000", 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 85.462218309999997, 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021680907999999999, 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.073851+0000", 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.073851+0000", 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.073850+0000", 2023-10-09T10:13:32.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4699999999999996e-06 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.073857+0000", 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0597999999999999e-05 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.073868+0000", 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9059999999999999e-06 2023-10-09T10:13:32.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.073874+0000", 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058358380000000003 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.079710+0000", 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9018999999999999e-05 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.079729+0000", 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015770421999999999 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.095499+0000", 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2459999999999998e-05 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:07.095532+0000", 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021680907999999999 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:184610 8.4 e306/288 8:2a8a1326:::benchmark_data_smithi012_271002_object75927:head v 306'11622, mlcod=0'0)", 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:23.916058+0000", 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 68.620010961000006, 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021971476, 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.575 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.916058+0000", 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.916058+0000", 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.916057+0000", 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1450000000000002e-06 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.916064+0000", 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0285999999999999e-05 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.916075+0000", 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8619999999999999e-06 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.916081+0000", 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1625999999999998e-05 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.916122+0000", 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4108999999999999e-05 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.916146+0000", 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021839450999999999 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.937986+0000", 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3732000000000001e-05 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:23.938030+0000", 2023-10-09T10:13:32.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021971476 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:184836 8.5 e306/271 8:a910306a:::benchmark_data_smithi012_271002_object76153:head v 306'11700, mlcod=306'11698)", 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:24.261702+0000", 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 68.274367085999998, 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022325889000000002, 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261702+0000", 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261702+0000", 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261701+0000", 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1691e-05 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261713+0000", 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4887e-05 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261727+0000", 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9949999999999999e-06 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261734+0000", 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1493999999999999e-05 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261776+0000", 2023-10-09T10:13:32.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9335999999999999e-05 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261795+0000", 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022185633 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.283981+0000", 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6918e-05 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.284028+0000", 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022325889000000002 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:195316 8.4 e308/288 8:2c727ca6:::benchmark_data_smithi012_271002_object86633:head v 308'12322, mlcod=0'0)", 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:40.818673+0000", 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 51.717395578999998, 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023273819000000001, 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.585 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818673+0000", 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818673+0000", 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818673+0000", 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7129999999999994e-06 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818681+0000", 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7350000000000002e-05 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818699+0000", 2023-10-09T10:13:32.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.64e-06 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818705+0000", 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6796999999999998e-05 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818762+0000", 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2611000000000001e-05 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818785+0000", 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023134630999999999 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.841919+0000", 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7971e-05 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.841947+0000", 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023273819000000001 2023-10-09T10:13:32.589 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:195367 8.e e309/271 8:722ced42:::benchmark_data_smithi012_271002_object86684:head v 309'12167, mlcod=308'12163)", 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:40.903616+0000", 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 51.632453323, 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022218882999999998, 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903616+0000", 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903616+0000", 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903615+0000", 2023-10-09T10:13:32.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.73e-06 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903624+0000", 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4894e-05 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903639+0000", 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2600000000000002e-06 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903646+0000", 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4731000000000002e-05 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903700+0000", 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5386999999999999e-05 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903726+0000", 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022070419000000001 2023-10-09T10:13:32.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.925796+0000", 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8512e-05 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.925835+0000", 2023-10-09T10:13:32.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022218882999999998 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:195702 8.4 e309/288 8:2faa2d02:::benchmark_data_smithi012_271002_object87019:head v 309'12351, mlcod=0'0)", 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:41.382004+0000", 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 51.154064521000002, 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020958648, 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.382004+0000", 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.382004+0000", 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.382004+0000", 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7739999999999999e-06 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.382011+0000", 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3988999999999999e-05 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.382025+0000", 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9739999999999999e-06 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.382031+0000", 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038641140000000001 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.385895+0000", 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4919999999999999e-05 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.385920+0000", 2023-10-09T10:13:32.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016998678999999999 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.402918+0000", 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4787999999999999e-05 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:41.402963+0000", 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020958648 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.599 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:196465 8.5 e309/271 8:a839e85e:::benchmark_data_smithi012_271002_object87782:head v 309'12423, mlcod=309'12421)", 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:42.428599+0000", 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 50.107470448999997, 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022158028, 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428599+0000", 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428599+0000", 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428597+0000", 2023-10-09T10:13:32.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1518999999999999e-05 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428609+0000", 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1316000000000001e-05 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428630+0000", 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1689999999999992e-06 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428638+0000", 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5626999999999999e-05 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428714+0000", 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1158e-05 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428735+0000", 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021970321000000001 2023-10-09T10:13:32.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.450705+0000", 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1100000000000002e-05 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.450757+0000", 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022158028 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.604 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:204399 8.4 e312/288 8:24e25026:::benchmark_data_smithi012_271002_object95716:head v 312'12896, mlcod=293'7264)", 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:55.621795+0000", 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.914273872999999, 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024465166, 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621795+0000", 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621795+0000", 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621794+0000", 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.52e-06 2023-10-09T10:13:32.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621802+0000", 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3852000000000001e-05 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621816+0000", 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4289999999999998e-06 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621822+0000", 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059956109999999996 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.627818+0000", 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5525999999999999e-05 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.627843+0000", 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018155171000000001 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.645998+0000", 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026192700000000001 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.646260+0000", 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024465166 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:206140 8.2 e313/271 8:4aaddc3b:::benchmark_data_smithi012_271002_object97457:head v 313'12708, mlcod=313'12705)", 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:58.507955+0000", 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 34.028114328999997, 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02358936, 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507955+0000", 2023-10-09T10:13:32.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507955+0000", 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507953+0000", 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1479e-05 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507965+0000", 2023-10-09T10:13:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5814000000000001e-05 2023-10-09T10:13:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507981+0000", 2023-10-09T10:13:32.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2949999999999996e-06 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507988+0000", 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00056878400000000002 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.508557+0000", 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2687000000000001e-05 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.508579+0000", 2023-10-09T10:13:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022922205000000001 2023-10-09T10:13:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.531502+0000", 2023-10-09T10:13:32.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2348000000000002e-05 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.531544+0000", 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02358936 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.615 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:206825 8.2 e313/271 8:4fa78ce4:::benchmark_data_smithi012_271002_object98142:head v 313'12745, mlcod=313'12741)", 2023-10-09T10:13:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:59.563962+0000", 2023-10-09T10:13:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.972106529999998, 2023-10-09T10:13:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023624422999999999, 2023-10-09T10:13:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563962+0000", 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563962+0000", 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563962+0000", 2023-10-09T10:13:32.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.186e-06 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563967+0000", 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6150000000000002e-06 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563974+0000", 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7530000000000001e-06 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563979+0000", 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010570637000000001 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.574549+0000", 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0530000000000002e-05 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.574570+0000", 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012928932000000001 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.587499+0000", 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8201999999999993e-05 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.587587+0000", 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023624422999999999 2023-10-09T10:13:32.620 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:212994 8.e e314/271 8:73cb14b3:::benchmark_data_smithi012_271002_object104311:head v 314'13240, mlcod=314'13237)", 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:13:09.062601+0000", 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 23.473468251, 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023475941, 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062601+0000", 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062601+0000", 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062600+0000", 2023-10-09T10:13:32.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5499999999999996e-06 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062605+0000", 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4129999999999997e-06 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062610+0000", 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4860000000000002e-06 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062614+0000", 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0072164890000000004 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.069830+0000", 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0945e-05 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.069851+0000", 2023-10-09T10:13:32.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016199209999999999 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.086051+0000", 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6112999999999999e-05 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.086077+0000", 2023-10-09T10:13:32.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023475941 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:213002 8.5 e314/271 8:aa12d4ff:::benchmark_data_smithi012_271002_object104319:head v 314'13500, mlcod=314'13498)", 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:13:09.065693+0000", 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 23.470376162000001, 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020064159000000002, 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.626 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.065693+0000", 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.065693+0000", 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.065692+0000", 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1909999999999999e-06 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.065698+0000", 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1600000000000003e-06 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.065704+0000", 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7359999999999999e-06 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.065708+0000", 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0232000000000001e-05 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.065778+0000", 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1282999999999999e-05 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.065799+0000", 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019924544999999998 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.085724+0000", 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3420000000000002e-05 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.085757+0000", 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020064159000000002 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:213041 8.5 e314/271 8:a5dccb0e:::benchmark_data_smithi012_271002_object104358:head v 314'13503, mlcod=314'13501)", 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:13:09.117461+0000", 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 23.418607667, 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021735445999999999, 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.631 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.117461+0000", 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.117461+0000", 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.117460+0000", 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8880000000000003e-06 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.117469+0000", 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0392000000000001e-05 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.117490+0000", 2023-10-09T10:13:32.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4130000000000006e-06 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.117498+0000", 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8176000000000005e-05 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.117576+0000", 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1997999999999998e-05 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.117598+0000", 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021533640999999999 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.139132+0000", 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4836000000000001e-05 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.139197+0000", 2023-10-09T10:13:32.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021735445999999999 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:215943 8.5 e314/271 8:ab7f636a:::benchmark_data_smithi012_271002_object107260:head v 314'13691, mlcod=314'13689)", 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:13:13.797567+0000", 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 18.738502050000001, 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021596344999999999, 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:32.636 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.797567+0000", 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.797567+0000", 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:13:32.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.797566+0000", 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9090000000000006e-06 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.797575+0000", 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4594e-05 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.797590+0000", 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5690000000000001e-06 2023-10-09T10:13:32.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.797596+0000", 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013960014999999999 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.811556+0000", 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1919000000000001e-05 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:13:32.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.811578+0000", 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007546461 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.819124+0000", 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8841999999999998e-05 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:13.819163+0000", 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021596344999999999 2023-10-09T10:13:32.640 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:13:32.641 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:13:32.641 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:13:32.642 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:13:32.689 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/407915212' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:13:32.689 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: from='client.56005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: pgmap v890: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/578857126' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/407915212' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[147926]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/407915212' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: from='client.56005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: pgmap v890: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/578857126' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/407915212' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:32 smithi012 ceph-mon[152263]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.709 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:32.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:32.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T10:13:32.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:32.866 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/407915212' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: from='client.56005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: pgmap v890: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/578857126' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/407915212' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T10:13:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T10:13:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:32 smithi110 ceph-mon[136032]: failed to encode map e317 with expected crc 2023-10-09T10:13:32.957 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:32.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:33.061 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:33.098 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:33.173 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:13:33.181 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:33.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:33.191 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:13:33.209 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:13:33.228 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:13:33.247 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:13:33.269 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:13:33.285 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:33.288 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:13:33.307 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:13:33.308 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:13:33.308 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:13:33.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:13:33.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:13:33.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:13:33.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:13:33.309 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:13:33.311 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:13:33.314 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:33.478 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:33.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:33.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:33.610 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:33.656 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T10:13:33.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.657 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2596781945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:33.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:13:33.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T10:13:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2596781945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:33.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: failed to encode map e317 with expected crc 2023-10-09T10:13:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T10:13:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: failed to encode map e318 with expected crc 2023-10-09T10:13:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2596781945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:33.751 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:33.871 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:33.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:33.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:33.974 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:34.002 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:34.089 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:34.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:34.192 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:34.220 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:34.253 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:34.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:34.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:34.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:34.378 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:13:34.379 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-10-09T10:13:34.379 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:13:34.390 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:34.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:13:34.619 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:34.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[152263]: pgmap v893: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:34.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[152263]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[152263]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[152263]: osdmap e319: 8 total, 8 up, 8 in 2023-10-09T10:13:34.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[152263]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:34.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[147926]: pgmap v893: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:34.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[147926]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[147926]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[147926]: osdmap e319: 8 total, 8 up, 8 in 2023-10-09T10:13:34.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[147926]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:34 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:34.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:34.626 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:34.727 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:34.755 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:34.790 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:34.792 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:34 smithi110 ceph-mon[136032]: pgmap v893: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:34.792 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:34 smithi110 ceph-mon[136032]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.792 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:34 smithi110 ceph-mon[136032]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.792 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:34 smithi110 ceph-mon[136032]: osdmap e319: 8 total, 8 up, 8 in 2023-10-09T10:13:34.792 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:34 smithi110 ceph-mon[136032]: failed to encode map e319 with expected crc 2023-10-09T10:13:34.792 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:34 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:34.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:34.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:34.895 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:34.925 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:34.957 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:34.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:34.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:35.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:35.101 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:35.189 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:35.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:35.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:35.326 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:35.579 INFO:teuthology.orchestra.run.smithi012.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T10:13:35.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:13:35.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:35 smithi012 ceph-mon[147926]: failed to encode map e319 with expected crc 2023-10-09T10:13:35.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/457389656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:13:35.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:13:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:35 smithi012 ceph-mon[152263]: failed to encode map e319 with expected crc 2023-10-09T10:13:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/457389656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:13:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:13:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:35 smithi110 ceph-mon[136032]: failed to encode map e319 with expected crc 2023-10-09T10:13:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/457389656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:13:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:13:35.957 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:35.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:36.010 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:36.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:36.095 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:36.232 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:36.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:36.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:36.337 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:36.370 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:36.406 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:36.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:36.441 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:13:36.441 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:13:36.510 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:36.539 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:36.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:36.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:36.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:36.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:36.682 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:36.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:36 smithi110 ceph-mon[136032]: pgmap v895: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:36.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:36 smithi110 ceph-mon[136032]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:13:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:36 smithi110 ceph-mon[136032]: osdmap e320: 8 total, 8 up, 8 in 2023-10-09T10:13:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:36 smithi110 ceph-mon[136032]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:36 smithi110 ceph-mon[136032]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.712 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:36.808 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[152263]: pgmap v895: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:36.808 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[152263]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.808 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:13:36.808 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[152263]: osdmap e320: 8 total, 8 up, 8 in 2023-10-09T10:13:36.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[152263]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[152263]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[147926]: pgmap v895: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:36.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[147926]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:13:36.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[147926]: osdmap e320: 8 total, 8 up, 8 in 2023-10-09T10:13:36.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[147926]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:36 smithi012 ceph-mon[147926]: failed to encode map e320 with expected crc 2023-10-09T10:13:36.987 DEBUG:teuthology.orchestra.run.smithi012:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T10:13:36.990 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:13:36.990 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:13:37.008 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:37.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:37.112 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:37.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:37.177 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:37.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:37.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:37.213 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:hints = 1 2023-10-09T10:13:37.214 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:13:37.214 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object prefix: benchmark_data_smithi012_313365 2023-10-09T10:13:37.214 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:13:37.214 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:13:37.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:37.344 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:37.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:13:37.987 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:37.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:38.097 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:38.153 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:13:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:13:37] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.33.4" 2023-10-09T10:13:38.170 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:38.214 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 1 16 3112 3096 193.488 193.5 0.00897965 0.00514088 2023-10-09T10:13:38.309 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:38.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:38.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:38.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:38.514 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:38.689 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:38.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:38.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:38.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[147926]: failed to encode map e320 with expected crc 2023-10-09T10:13:38.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[147926]: failed to encode map e320 with expected crc 2023-10-09T10:13:38.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[147926]: pgmap v897: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:38.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/415654282' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:13:38.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:13:38.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[152263]: failed to encode map e320 with expected crc 2023-10-09T10:13:38.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[152263]: failed to encode map e320 with expected crc 2023-10-09T10:13:38.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[152263]: pgmap v897: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:38.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/415654282' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:13:38.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:38 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:13:38.802 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:38.867 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:38.933 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:38.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:38.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:38 smithi110 ceph-mon[136032]: failed to encode map e320 with expected crc 2023-10-09T10:13:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:38 smithi110 ceph-mon[136032]: failed to encode map e320 with expected crc 2023-10-09T10:13:38.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:38 smithi110 ceph-mon[136032]: pgmap v897: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T10:13:38.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/415654282' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:13:38.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:38 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:13:38.960 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:39.038 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:39.039 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:13:39.074 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:39.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:13:39.116 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:13:39.123 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:39.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:39.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:39.149 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:13:39.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:13:39.205 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:13:39.214 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 2 16 5773 5757 179.887 166.312 0.0215167 0.00554598 2023-10-09T10:13:39.230 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:39.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:13:39.267 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:39.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:13:39.307 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:13:39.310 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:39.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:39.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:39.324 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:13:39.324 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:13:39.324 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:13:39.324 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:13:39.324 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:13:39.324 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:13:39.325 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:13:39.325 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:13:39.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:39.453 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:39.505 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:39.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:39.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:39.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:39.670 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:39.800 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 3, 4] 2023-10-09T10:13:39.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T10:13:39.853 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1910980327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:39.853 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:39.853 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:13:39.853 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[147926]: osdmap e321: 8 total, 8 up, 8 in 2023-10-09T10:13:39.853 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[147926]: failed to encode map e321 with expected crc 2023-10-09T10:13:39.853 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1910980327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:39.853 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:39.854 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:13:39.854 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[152263]: osdmap e321: 8 total, 8 up, 8 in 2023-10-09T10:13:39.854 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:39 smithi012 ceph-mon[152263]: failed to encode map e321 with expected crc 2023-10-09T10:13:39.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1910980327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:39 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:39 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:13:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:39 smithi110 ceph-mon[136032]: osdmap e321: 8 total, 8 up, 8 in 2023-10-09T10:13:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:39 smithi110 ceph-mon[136032]: failed to encode map e321 with expected crc 2023-10-09T10:13:40.041 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:40.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:40.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:40.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:40.214 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 3 16 8509 8493 176.914 171 0.00636177 0.00563723 2023-10-09T10:13:40.216 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:40.353 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:40.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:40.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:40.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:40.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:40.549 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:40.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:40.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:40.598 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:40 smithi012 ceph-mon[147926]: failed to encode map e321 with expected crc 2023-10-09T10:13:40.598 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:40 smithi012 ceph-mon[147926]: failed to encode map e321 with expected crc 2023-10-09T10:13:40.598 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:40 smithi012 ceph-mon[147926]: pgmap v899: 89 pgs: 89 active+clean; 108 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2023-10-09T10:13:40.653 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:40.689 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:40.731 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:40.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:40.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:40.903 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:40 smithi012 ceph-mon[152263]: failed to encode map e321 with expected crc 2023-10-09T10:13:40.903 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:40 smithi012 ceph-mon[152263]: failed to encode map e321 with expected crc 2023-10-09T10:13:40.903 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:40 smithi012 ceph-mon[152263]: pgmap v899: 89 pgs: 89 active+clean; 108 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2023-10-09T10:13:40.912 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:40.940 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:40 smithi110 ceph-mon[136032]: failed to encode map e321 with expected crc 2023-10-09T10:13:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:40 smithi110 ceph-mon[136032]: failed to encode map e321 with expected crc 2023-10-09T10:13:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:40 smithi110 ceph-mon[136032]: pgmap v899: 89 pgs: 89 active+clean; 108 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2023-10-09T10:13:40.986 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:40.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:40.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:41.038 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:13:41.038 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:13:41.038 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:13:41.038 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:13:41.038 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:13:41.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:41.134 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:41.186 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:41.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:41.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:41.219 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 4 16 11140 11124 173.791 164.438 0.00758528 0.00574942 2023-10-09T10:13:41.291 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:41.333 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:41.383 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:41.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:41.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:41.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:41.562 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:41.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:13:41.770 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:41.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:41.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:41.877 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:41.917 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:41.967 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:41.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:41.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:41.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:13:42.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:13:42.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:42.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:42.215 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 5 16 13465 13449 168.072 145.312 0.00728176 0.00594227 2023-10-09T10:13:42.364 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:42.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:42.472 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:42.513 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:42.554 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:42.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:42.660 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:42.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:42 smithi012 ceph-mon[147926]: pgmap v900: 89 pgs: 89 active+clean; 140 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 283 op/s 2023-10-09T10:13:42.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:42 smithi012 ceph-mon[152263]: pgmap v900: 89 pgs: 89 active+clean; 140 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 283 op/s 2023-10-09T10:13:42.695 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:42.737 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:42.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:42.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:42.842 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:42.851 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:42.873 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:42 smithi110 ceph-mon[136032]: pgmap v900: 89 pgs: 89 active+clean; 140 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 283 op/s 2023-10-09T10:13:42.877 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:42.924 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:42.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:43.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:43.107 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:43.215 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 6 16 15520 15504 161.465 128.438 0.00449723 0.00618757 2023-10-09T10:13:44.145 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T10:13:44.183 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:44.215 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 7 16 18233 18217 162.619 169.562 0.00602716 0.006144 2023-10-09T10:13:44.369 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:44.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:44.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1820447536' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:13:44.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1820447536' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:13:44.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:44.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1820447536' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:13:44.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:44.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:44.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:44.559 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:44.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:44.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:44.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:44.723 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:45.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T10:13:45.163 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:45.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:45.216 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 8 16 20455 20439 159.643 138.875 0.00892751 0.00625587 2023-10-09T10:13:45.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:45.357 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: pgmap v901: 89 pgs: 89 active+clean; 679 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T10:13:45.358 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.358 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.358 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1820447536' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:13:45.358 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T10:13:45.358 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.358 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.358 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.358 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.359 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:45.359 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: pgmap v901: 89 pgs: 89 active+clean; 679 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T10:13:45.359 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.359 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.359 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1820447536' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:13:45.359 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T10:13:45.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:45 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:45.363 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: pgmap v901: 89 pgs: 89 active+clean; 679 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1820447536' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: failed to encode map e322 with expected crc 2023-10-09T10:13:45.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:45 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:13:45.447 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:45.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:45.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:45.528 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:45.554 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:45.586 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:45.641 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:45.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:45.746 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:45.805 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:46.063 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:46.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:46.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:46.225 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 9 16 22412 22396 155.333 122.312 0.0166162 0.00642705 2023-10-09T10:13:46.249 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:46.313 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:46.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:46.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:46.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:46.486 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:46.738 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:46.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:46.748 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:46.783 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:13:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:13:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:13:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:13:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.783 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154067 7.e 7:729cdabe:::benchmark_data_smithi012_221426_object46289:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.944451448, 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.711856684, 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154067 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.784 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:13:46.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833144+0000", 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833150+0000", 2023-10-09T10:13:46.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0079999999999999e-06 2023-10-09T10:13:46.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833167+0000", 2023-10-09T10:13:46.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6198999999999999e-05 2023-10-09T10:13:46.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833176+0000", 2023-10-09T10:13:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4490000000000005e-06 2023-10-09T10:13:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833926+0000", 2023-10-09T10:13:46.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00074946799999999999 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833934+0000", 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1589999999999993e-06 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841253+0000", 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073188610000000003 2023-10-09T10:13:46.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841264+0000", 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1069e-05 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497295+0000", 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6560310870000001 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:13:46.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497435+0000", 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013998400000000001 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.544884+0000", 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.047449005000000002 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.545002+0000", 2023-10-09T10:13:46.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118678 2023-10-09T10:13:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154074 7.e 7:7a8c6f84:::benchmark_data_smithi012_221426_object46296:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:13:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:13:46.792 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.943717773, 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7164543729999999, 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154074 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833884+0000", 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1059999999999999e-06 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833895+0000", 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1639e-05 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833906+0000", 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1104000000000001e-05 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833999+0000", 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2387999999999997e-05 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:13:46.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841286+0000", 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072812739999999999 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841293+0000", 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7290000000000003e-06 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500331+0000", 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.659038185 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500464+0000", 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013265799999999999 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550212+0000", 2023-10-09T10:13:46.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049747711999999999 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550334+0000", 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012227200000000001 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154080 7.5 7:a9f132ab:::benchmark_data_smithi012_221426_object46302:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:13:46.799 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.94368131300001, 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5768529529999999, 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154080 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:13:46.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833916+0000", 2023-10-09T10:13:46.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833915+0000", 2023-10-09T10:13:46.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833918+0000", 2023-10-09T10:13:46.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6740000000000001e-06 2023-10-09T10:13:46.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833924+0000", 2023-10-09T10:13:46.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2179999999999996e-06 2023-10-09T10:13:46.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833929+0000", 2023-10-09T10:13:46.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2009999999999998e-06 2023-10-09T10:13:46.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834017+0000", 2023-10-09T10:13:46.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7529999999999997e-05 2023-10-09T10:13:46.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:13:46.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834024+0000", 2023-10-09T10:13:46.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2350000000000002e-06 2023-10-09T10:13:46.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837176+0000", 2023-10-09T10:13:46.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031517139999999999 2023-10-09T10:13:46.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:13:46.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837194+0000", 2023-10-09T10:13:46.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7634000000000001e-05 2023-10-09T10:13:46.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395551+0000", 2023-10-09T10:13:46.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5583572540000001 2023-10-09T10:13:46.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395681+0000", 2023-10-09T10:13:46.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130059 2023-10-09T10:13:46.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:13:46.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.395763+0000", 2023-10-09T10:13:46.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1928999999999994e-05 2023-10-09T10:13:46.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400135+0000", 2023-10-09T10:13:46.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043726360000000001 2023-10-09T10:13:46.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409675+0000", 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0095396509999999997 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409717+0000", 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2036000000000002e-05 2023-10-09T10:13:46.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410592+0000", 2023-10-09T10:13:46.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00087471800000000002 2023-10-09T10:13:46.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410648+0000", 2023-10-09T10:13:46.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6589000000000003e-05 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410769+0000", 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120383 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154083 7.e 7:757b33cc:::benchmark_data_smithi012_221426_object46305:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:13:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:13:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.94365337400001, 2023-10-09T10:13:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7033852390000002, 2023-10-09T10:13:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.815 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154083 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:13:46.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:13:46.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833943+0000", 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833946+0000", 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0000000000000001e-06 2023-10-09T10:13:46.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833952+0000", 2023-10-09T10:13:46.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9000000000000003e-06 2023-10-09T10:13:46.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833957+0000", 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9629999999999997e-06 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834051+0000", 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4141999999999994e-05 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:13:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834060+0000", 2023-10-09T10:13:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8079999999999994e-06 2023-10-09T10:13:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841319+0000", 2023-10-09T10:13:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072592009999999998 2023-10-09T10:13:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841327+0000", 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.571e-06 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500500+0000", 2023-10-09T10:13:46.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6591724889999999 2023-10-09T10:13:46.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500595+0000", 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5142000000000005e-05 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.501295+0000", 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00069987000000000001 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510695+0000", 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094008289999999994 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520701+0000", 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010005623 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534431+0000", 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013730086000000001 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534503+0000", 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2304000000000004e-05 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537329+0000", 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028256510000000002 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154085 7.5 7:a7e74dfa:::benchmark_data_smithi012_221426_object46307:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:13:46.827 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.94363547199998, 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5985369870000001, 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154085 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833962+0000", 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833961+0000", 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833964+0000", 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.294e-06 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833970+0000", 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8540000000000002e-06 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833975+0000", 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9710000000000003e-06 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834061+0000", 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6417999999999998e-05 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:13:46.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834067+0000", 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.421e-06 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837215+0000", 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031475790000000002 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837222+0000", 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4370000000000001e-06 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400164+0000", 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5629411879999999 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400262+0000", 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8307000000000006e-05 2023-10-09T10:13:46.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.400392+0000", 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013009100000000001 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409725+0000", 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093329940000000007 2023-10-09T10:13:46.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409759+0000", 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4034e-05 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.425006+0000", 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015246994 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432361+0000", 2023-10-09T10:13:46.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073547029999999998 2023-10-09T10:13:46.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432413+0000", 2023-10-09T10:13:46.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2305999999999997e-05 2023-10-09T10:13:46.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432499+0000", 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5711000000000007e-05 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154090 7.5 7:a5e0254e:::benchmark_data_smithi012_221426_object46312:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:13:46.837 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.943616217, 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.598732759, 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154090 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833981+0000", 2023-10-09T10:13:46.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833983+0000", 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.26e-06 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833988+0000", 2023-10-09T10:13:46.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4190000000000001e-06 2023-10-09T10:13:46.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833993+0000", 2023-10-09T10:13:46.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5419999999999999e-06 2023-10-09T10:13:46.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834104+0000", 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011161499999999999 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834111+0000", 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0650000000000004e-06 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837242+0000", 2023-10-09T10:13:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031316959999999998 2023-10-09T10:13:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:13:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.837250+0000", 2023-10-09T10:13:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1799999999999996e-06 2023-10-09T10:13:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.404888+0000", 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5676373880000001 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.405039+0000", 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151543 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T10:13:46.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.405181+0000", 2023-10-09T10:13:46.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014114600000000001 2023-10-09T10:13:46.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.409999+0000", 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048189210000000003 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.410086+0000", 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6949999999999999e-05 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.425038+0000", 2023-10-09T10:13:46.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014951341999999999 2023-10-09T10:13:46.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432561+0000", 2023-10-09T10:13:46.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0075235340000000001 2023-10-09T10:13:46.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432611+0000", 2023-10-09T10:13:46.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9985000000000003e-05 2023-10-09T10:13:46.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.432714+0000", 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000102497 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.850 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154097 7.e 7:7fa3f1e3:::benchmark_data_smithi012_221426_object46319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 463.94359913300002, 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.706036967, 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154097 2023-10-09T10:13:46.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834000+0000", 2023-10-09T10:13:46.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.384e-06 2023-10-09T10:13:46.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2499999999999997e-06 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834010+0000", 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8359999999999999e-06 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834090+0000", 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9957999999999996e-05 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834096+0000", 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3350000000000003e-06 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841347+0000", 2023-10-09T10:13:46.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072508479999999998 2023-10-09T10:13:46.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:13:46.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841354+0000", 2023-10-09T10:13:46.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7159999999999997e-06 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510724+0000", 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.669369753 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510903+0000", 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017934200000000001 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.511077+0000", 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017371899999999999 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520762+0000", 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0096847530000000008 2023-10-09T10:13:46.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.525146+0000", 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043842450000000002 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537388+0000", 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012242494 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537446+0000", 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7565999999999997e-05 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.540035+0000", 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025890919999999999 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:195066 8.c 8:3c6eeb6a:::benchmark_data_smithi012_271002_object86383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e308)", 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:40.472484+0000", 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 66.305112995000002, 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060124987999999997, 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195066 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.472484+0000", 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.472484+0000", 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.472484+0000", 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.472487+0000", 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9469999999999999e-06 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.472492+0000", 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7019999999999998e-06 2023-10-09T10:13:46.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.472498+0000", 2023-10-09T10:13:46.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0599999999999998e-06 2023-10-09T10:13:46.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.480411+0000", 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0079136889999999998 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.480519+0000", 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107596 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.480610+0000", 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0719999999999999e-05 2023-10-09T10:13:46.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.490746+0000", 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010135964000000001 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.499427+0000", 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0086818999999999993 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.532518+0000", 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033090807 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.532566+0000", 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7735999999999998e-05 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:40.532609+0000", 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3214999999999999e-05 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:197030 8.e 8:7abacca9:::benchmark_data_smithi012_271002_object88347:head [delete] snapc 0=[] ondisk+write+known_if_redirected e311)", 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:44.094796+0000", 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.682801292000001, 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063731010000000005, 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.869 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 197030 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.094796+0000", 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.094796+0000", 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.094796+0000", 2023-10-09T10:13:46.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.094798+0000", 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4159999999999998e-06 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.094803+0000", 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3299999999999998e-06 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.094808+0000", 2023-10-09T10:13:46.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7269999999999998e-06 2023-10-09T10:13:46.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.102685+0000", 2023-10-09T10:13:46.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0078774150000000005 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.102796+0000", 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000110312 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.102903+0000", 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107414 2023-10-09T10:13:46.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.129769+0000", 2023-10-09T10:13:46.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026865824999999999 2023-10-09T10:13:46.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.132353+0000", 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025845 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.158443+0000", 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.02608926 2023-10-09T10:13:46.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.158485+0000", 2023-10-09T10:13:46.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1987999999999998e-05 2023-10-09T10:13:46.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:44.158527+0000", 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2169000000000002e-05 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:202715 8.c 8:3ef84ed3:::benchmark_data_smithi012_271002_object94032:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:52.965702+0000", 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 53.811895675000002, 2023-10-09T10:13:46.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054512636000000003, 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 202715 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965702+0000", 2023-10-09T10:13:46.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965702+0000", 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965699+0000", 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965708+0000", 2023-10-09T10:13:46.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3909999999999998e-06 2023-10-09T10:13:46.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965720+0000", 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2561000000000001e-05 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965729+0000", 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4689999999999997e-06 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.970380+0000", 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046511119999999998 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.970544+0000", 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016453299999999999 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.970690+0000", 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014569499999999999 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.976552+0000", 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058616700000000002 2023-10-09T10:13:46.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.886 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.886 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.001332+0000", 2023-10-09T10:13:46.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024780443999999999 2023-10-09T10:13:46.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.020120+0000", 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018787626000000002 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.020170+0000", 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9840999999999999e-05 2023-10-09T10:13:46.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.020214+0000", 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4397000000000002e-05 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:202722 8.e 8:77a77751:::benchmark_data_smithi012_271002_object94039:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T10:13:46.888 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:52.965809+0000", 2023-10-09T10:13:46.970 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 53.811788467, 2023-10-09T10:13:46.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061633390000000003, 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 202722 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965809+0000", 2023-10-09T10:13:46.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965809+0000", 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965808+0000", 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965811+0000", 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8990000000000001e-06 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965817+0000", 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2079999999999997e-06 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.965823+0000", 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5539999999999997e-06 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.977470+0000", 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011646629 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.977606+0000", 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013610900000000001 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.977734+0000", 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128267 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.984642+0000", 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0069080529999999999 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:52.990494+0000", 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058517830000000002 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.027364+0000", 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.036869837000000003 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.027404+0000", 2023-10-09T10:13:46.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0086999999999998e-05 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.027442+0000", 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8389999999999997e-05 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:202800 8.c 8:31d559b2:::benchmark_data_smithi012_271002_object94117:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T10:13:46.977 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:53.135917+0000", 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 53.641680305000001, 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069697309999999998, 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 202800 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.135917+0000", 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.135917+0000", 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.135916+0000", 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:46.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.135921+0000", 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9509999999999997e-06 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.135934+0000", 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2999999999999999e-05 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.135952+0000", 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8104000000000001e-05 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.142742+0000", 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006789504 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.142860+0000", 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011866099999999999 2023-10-09T10:13:46.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.142968+0000", 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107402 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.162663+0000", 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019695430999999999 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.162672+0000", 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8240000000000006e-06 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.205539+0000", 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042866649 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.205572+0000", 2023-10-09T10:13:46.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2947000000000002e-05 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:53.205614+0000", 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2488999999999999e-05 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:203407 8.2 8:4ecf185a:::benchmark_data_smithi012_271002_object94724:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T10:13:46.984 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:54.093061+0000", 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 52.684536049000002, 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059599841000000001, 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 203407 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.093061+0000", 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.093061+0000", 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.093059+0000", 2023-10-09T10:13:46.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.093068+0000", 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4290000000000007e-06 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.093089+0000", 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0426000000000001e-05 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.093100+0000", 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0908e-05 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.104393+0000", 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011293709000000001 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.104526+0000", 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013265600000000001 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.104653+0000", 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127348 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.116471+0000", 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011817487 2023-10-09T10:13:46.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152292+0000", 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.035820773 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152570+0000", 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000278139 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152618+0000", 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8055e-05 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152661+0000", 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3111000000000001e-05 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:203411 8.2 8:4ad4177e:::benchmark_data_smithi012_271002_object94728:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:54.095145+0000", 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 52.682452552999997, 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057587639000000003, 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 203411 2023-10-09T10:13:46.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.095145+0000", 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.095145+0000", 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.095143+0000", 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.095151+0000", 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8739999999999995e-06 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:46.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.095164+0000", 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2927e-05 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.095172+0000", 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2519999999999999e-06 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.110832+0000", 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015659154000000002 2023-10-09T10:13:46.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.110982+0000", 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000150004 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.111114+0000", 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000132311 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152330+0000", 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041216524999999997 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152359+0000", 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9008000000000001e-05 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152668+0000", 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000308499 2023-10-09T10:13:46.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152702+0000", 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3602000000000001e-05 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:54.152732+0000", 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0741999999999998e-05 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:46.998 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:206821 8.2 8:4475c3ad:::benchmark_data_smithi012_271002_object98138:head [delete] snapc 0=[] ondisk+write+known_if_redirected e313)", 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:59.545216+0000", 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.232380741, 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053714280000000003, 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:46.999 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 206821 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545216+0000", 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545216+0000", 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:47.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545216+0000", 2023-10-09T10:13:47.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:13:47.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:47.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545221+0000", 2023-10-09T10:13:47.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4079999999999997e-06 2023-10-09T10:13:47.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545235+0000", 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3638000000000001e-05 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545244+0000", 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0440000000000001e-06 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.553972+0000", 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087287770000000001 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.554121+0000", 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000148641 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:13:47.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.554255+0000", 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013414500000000001 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.571065+0000", 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016810240000000001 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.586176+0000", 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015110231999999999 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.598839+0000", 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012662988999999999 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:47.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.598888+0000", 2023-10-09T10:13:47.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9293999999999999e-05 2023-10-09T10:13:47.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:47.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.598931+0000", 2023-10-09T10:13:47.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2750000000000002e-05 2023-10-09T10:13:47.007 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.008 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:47.008 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.008 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:206825 8.2 8:4fa78ce4:::benchmark_data_smithi012_271002_object98142:head [delete] snapc 0=[] ondisk+write+known_if_redirected e313)", 2023-10-09T10:13:47.008 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:12:59.545295+0000", 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 47.232301962000001, 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053788245999999998, 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 206825 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.009 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545295+0000", 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545295+0000", 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545295+0000", 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545297+0000", 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2330000000000001e-06 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545309+0000", 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1389999999999999e-05 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.545330+0000", 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1650000000000001e-05 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:47.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.561483+0000", 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016153250000000001 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.561608+0000", 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000124134 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:13:47.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.561729+0000", 2023-10-09T10:13:47.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012175 2023-10-09T10:13:47.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:47.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.571072+0000", 2023-10-09T10:13:47.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093430079999999999 2023-10-09T10:13:47.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.587660+0000", 2023-10-09T10:13:47.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016587544999999999 2023-10-09T10:13:47.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.598993+0000", 2023-10-09T10:13:47.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011333262 2023-10-09T10:13:47.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:47.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.599040+0000", 2023-10-09T10:13:47.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6709000000000003e-05 2023-10-09T10:13:47.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:47.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:12:59.599083+0000", 2023-10-09T10:13:47.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3652000000000002e-05 2023-10-09T10:13:47.025 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.025 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:47.026 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.027 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:214831 8.c 8:3f7a7a85:::benchmark_data_smithi012_271002_object106148:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T10:13:47.027 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:11.924859+0000", 2023-10-09T10:13:47.027 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 34.852738076999998, 2023-10-09T10:13:47.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.100191223, 2023-10-09T10:13:47.027 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:47.027 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:47.027 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:47.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:47.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:47.028 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 214831 2023-10-09T10:13:47.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.028 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:47.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:47.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.924859+0000", 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.924859+0000", 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:47.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.924858+0000", 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.924863+0000", 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3890000000000004e-06 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:47.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.924874+0000", 2023-10-09T10:13:47.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0691e-05 2023-10-09T10:13:47.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:47.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.924882+0000", 2023-10-09T10:13:47.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9519999999999994e-06 2023-10-09T10:13:47.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:47.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.930923+0000", 2023-10-09T10:13:47.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0060415529999999999 2023-10-09T10:13:47.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:47.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.931048+0000", 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012427 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.931168+0000", 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012039 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:47.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.942775+0000", 2023-10-09T10:13:47.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011607421999999999 2023-10-09T10:13:47.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.948255+0000", 2023-10-09T10:13:47.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054795089999999996 2023-10-09T10:13:47.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.024955+0000", 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.076700393000000006 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.025006+0000", 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0364000000000001e-05 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.025050+0000", 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4651000000000003e-05 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:214836 8.c 8:32274cf1:::benchmark_data_smithi012_271002_object106153:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:11.936596+0000", 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 34.841001036999998, 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.088565394000000006, 2023-10-09T10:13:47.036 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:47.037 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:47.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:47.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 214836 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936596+0000", 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936596+0000", 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936596+0000", 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936598+0000", 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.289e-06 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936603+0000", 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.259e-06 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936608+0000", 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6330000000000004e-06 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:47.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936735+0000", 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000126513 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936864+0000", 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012926600000000001 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.936990+0000", 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000125862 2023-10-09T10:13:47.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.948293+0000", 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011303264 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.960140+0000", 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011847096999999999 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.025088+0000", 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064947527000000005 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.025125+0000", 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7438000000000003e-05 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.025162+0000", 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6547000000000001e-05 2023-10-09T10:13:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:214853 8.c 8:3770f054:::benchmark_data_smithi012_271002_object106170:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:11.969820+0000", 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 34.807776691000001, 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055448456, 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:47.044 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 214853 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.969820+0000", 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.969820+0000", 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.969820+0000", 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:47.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.969824+0000", 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4059999999999998e-06 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.969838+0000", 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3737000000000001e-05 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.969846+0000", 2023-10-09T10:13:47.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0870000000000007e-06 2023-10-09T10:13:47.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:47.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.969890+0000", 2023-10-09T10:13:47.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3543e-05 2023-10-09T10:13:47.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.048 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:47.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.970015+0000", 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000125788 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.970137+0000", 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121688 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.976364+0000", 2023-10-09T10:13:47.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062265369999999999 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:11.983284+0000", 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0069205300000000003 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.025198+0000", 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041913961999999999 2023-10-09T10:13:47.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.025236+0000", 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7496000000000001e-05 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:12.025269+0000", 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3325999999999999e-05 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45518.0:216081 8.2 8:46f61f7f:::benchmark_data_smithi012_271002_object107398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:13:14.022869+0000", 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.754728593999999, 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053968228, 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45518", 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 216081 2023-10-09T10:13:47.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.022869+0000", 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.022869+0000", 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.022868+0000", 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.022872+0000", 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9129999999999996e-06 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:13:47.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.022881+0000", 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4569999999999994e-06 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.022888+0000", 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2219999999999999e-06 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.027377+0000", 2023-10-09T10:13:47.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0044897380000000001 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.027597+0000", 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000219748 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.027810+0000", 2023-10-09T10:13:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021302800000000001 2023-10-09T10:13:47.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.049852+0000", 2023-10-09T10:13:47.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022041535000000001 2023-10-09T10:13:47.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.049894+0000", 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2415e-05 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.076738+0000", 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026844388 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.076793+0000", 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4941000000000001e-05 2023-10-09T10:13:47.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:13:47.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:13:47.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:13:47.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:13:14.076837+0000", 2023-10-09T10:13:47.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3452999999999999e-05 2023-10-09T10:13:47.060 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.060 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:47.060 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.060 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:13:47.061 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:13:47.061 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:13:47.098 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:47.145 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:47.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:47.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:47.227 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 10 16 24352 24336 151.905 121.25 0.00561916 0.0065765 2023-10-09T10:13:47.254 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:47.293 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:47.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:47.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:47.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:47.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:47 smithi012 ceph-mon[147926]: pgmap v903: 89 pgs: 89 active+clean; 914 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-10-09T10:13:47.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:47 smithi012 ceph-mon[152263]: pgmap v903: 89 pgs: 89 active+clean; 914 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-10-09T10:13:47.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:47 smithi110 ceph-mon[136032]: pgmap v903: 89 pgs: 89 active+clean; 914 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-10-09T10:13:47.447 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:47.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:13:47.478 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:47.525 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:47.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:47.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:47.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:47.663 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:47.710 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:47.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:47.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:47.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:47.907 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:48.022 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:48.025 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:13:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:13:47] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.33.4" 2023-10-09T10:13:48.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:48.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:48.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:48.195 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:13:48.212 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:13:48.219 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:48.227 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 11 16 26076 26060 147.893 107.75 0.00847559 0.00674977 2023-10-09T10:13:48.260 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:13:48.300 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:13:48.345 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:13:48.380 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:13:48.383 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2604839053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:48.383 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:48 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:48.384 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2604839053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:48.384 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:48 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:48.417 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:13:48.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2604839053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:48.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:48 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:13:48.449 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:13:48.457 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:48.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:48.492 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:13:48.492 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:13:48.493 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:13:48.493 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:13:48.493 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:13:48.493 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:13:48.493 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:13:48.493 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:13:48.493 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:13:48.573 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:48.611 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:48.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:48.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:48.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:48.766 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:48.803 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:48.856 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:48.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:48.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:48.962 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:49.046 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:49.160 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: pgmap v904: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3362912235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.161 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.162 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.162 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[147926]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: pgmap v904: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-09T10:13:49.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:13:49.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:13:49.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3362912235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:49.162 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:49.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T10:13:49.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.163 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:49 smithi110 ceph-mon[136032]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.227 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 12 16 27875 27859 144.941 112.438 0.00340293 0.00689401 2023-10-09T10:13:49.354 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:49.355 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:49.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:49.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:49.418 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: pgmap v904: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-09T10:13:49.418 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:13:49.418 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:13:49.418 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3362912235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:49.418 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:13:49.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T10:13:49.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:49 smithi012 ceph-mon[152263]: failed to encode map e323 with expected crc 2023-10-09T10:13:49.464 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:49.523 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:49.564 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:49.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:49.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:49.751 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:50.202 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T10:13:50.228 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 13 16 29642 29626 142.288 110.438 0.00671625 0.00702179 2023-10-09T10:13:50.398 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:50.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:50.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1125662483' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:13:50.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1125662483' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:13:50.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1125662483' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:13:50.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:50.565 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:50.618 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:50.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:50.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:50.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:13:50.782 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:50.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:13:51.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[147926]: pgmap v906: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s wr, 2.80k op/s 2023-10-09T10:13:51.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[147926]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[147926]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1125662483' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:13:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[147926]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T10:13:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[147926]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[147926]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.231 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 14 15 31600 31585 140.835 122.438 0.00993636 0.00709375 2023-10-09T10:13:51.422 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:13:51.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[152263]: pgmap v906: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s wr, 2.80k op/s 2023-10-09T10:13:51.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[152263]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[152263]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1125662483' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:13:51.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[152263]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T10:13:51.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[152263]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:51 smithi012 ceph-mon[152263]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:51 smithi110 ceph-mon[136032]: pgmap v906: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s wr, 2.80k op/s 2023-10-09T10:13:51.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:51 smithi110 ceph-mon[136032]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:51 smithi110 ceph-mon[136032]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1125662483' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:13:51.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:51 smithi110 ceph-mon[136032]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T10:13:51.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:51 smithi110 ceph-mon[136032]: failed to encode map e324 with expected crc 2023-10-09T10:13:51.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:51 smithi110 ceph-mon[136032]: failed to encode map e324 with expected crc 2023-10-09T10:13:52.231 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 15 16 33500 33484 139.359 118.688 0.00656773 0.00716963 2023-10-09T10:13:52.675 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:13:52.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:52.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:13:52.786 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:52.837 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:52.899 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:52.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:53.005 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:53.050 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:53.099 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:53.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:53.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:53.206 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:53.232 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 16 16 35374 35358 137.957 117.125 0.0141528 0.00724377 2023-10-09T10:13:53.256 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:53.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T10:13:53.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:53.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:53.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:53.416 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:53.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:53 smithi110 ceph-mon[136032]: pgmap v908: 89 pgs: 89 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T10:13:53.493 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:53 smithi012 ceph-mon[147926]: pgmap v908: 89 pgs: 89 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T10:13:53.493 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:53 smithi012 ceph-mon[152263]: pgmap v908: 89 pgs: 89 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T10:13:53.495 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:53.683 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:53.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:53.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:53.786 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:53.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:53.881 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:53.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:53.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:53.987 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:13:54.018 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:54.067 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:13:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:54.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:13:54.173 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:54.230 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:54.234 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 17 16 37009 36993 135.845 102.188 0.0122442 0.00735583 2023-10-09T10:13:54.437 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:54.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:54.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:54.465 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:54.469 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:54.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:54.568 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:54.627 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:54.821 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:54.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:54.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:54.929 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:54.979 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:55.035 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:55.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:55.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:55.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:13:55.214 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:55.234 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 18 16 38632 38616 133.935 101.438 0.00697131 0.00746154 2023-10-09T10:13:55.353 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:55 smithi012 ceph-mon[147926]: pgmap v909: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-10-09T10:13:55.354 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:55 smithi012 ceph-mon[152263]: pgmap v909: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-10-09T10:13:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:55 smithi110 ceph-mon[136032]: pgmap v909: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-10-09T10:13:55.533 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:13:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:55.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:13:55.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:55.706 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:55.836 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:13:55.836 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:13:56.234 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 19 16 40208 40192 132.071 98.5 0.0102826 0.00756631 2023-10-09T10:13:56.650 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:56.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:56.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:56.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:56.826 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:57.170 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:57.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:57.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:57.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:13:57.235 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:13:57.234620+0000 min lat: 0.00139647 max lat: 0.100282 avg lat: 0.0076623 2023-10-09T10:13:57.235 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:13:57.235 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 20 16 41792 41776 130.419 99 0.0146945 0.0076623 2023-10-09T10:13:57.277 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:57.312 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:57.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:57 smithi012 ceph-mon[147926]: pgmap v910: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T10:13:57.342 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:57 smithi012 ceph-mon[152263]: pgmap v910: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T10:13:57.351 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:57.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:57.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:57.488 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:57 smithi110 ceph-mon[136032]: pgmap v910: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T10:13:57.490 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:57.530 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:57.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:57.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:57.636 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:57.674 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:57.725 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:57.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:57.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:57.830 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:13:57.866 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:57.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:13:57.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:13:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:57.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:13:57.939 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:13:57.972 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:13:58.011 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:58.012 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:13:58.045 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:58.054 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:13:58.094 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:58.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:58.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:13:58.148 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:13:58.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:13:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:13:57] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.33.4" 2023-10-09T10:13:58.185 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:13:58.197 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:13:58.197 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:13:58.197 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:13:58.197 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:13:58.197 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:13:58.197 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:13:58.198 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:13:58.198 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:13:58.198 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:13:58.212 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:13:58.238 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 21 16 43077 43061 128.019 80.3125 0.0136791 0.00780546 2023-10-09T10:13:58.239 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:58.262 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T10:13:58.272 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:13:58.286 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:13:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:58.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:13:58.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:13:58.475 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/375165091' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:13:58.475 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:13:58.475 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/821758279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:58.475 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3686335336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:58.475 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:58.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/375165091' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:13:58.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:13:58.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/821758279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:58.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3686335336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:58.477 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:58.479 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/375165091' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:13:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:13:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/821758279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:13:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3686335336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:13:58.982 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:13:58.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:58.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:13:59.072 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T10:13:59.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:13:59.206 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:59.237 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 22 16 44584 44568 126.483 94.1875 0.0155344 0.00789965 2023-10-09T10:13:59.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: pgmap v911: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:13:59.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:59.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:13:59.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:13:59.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T10:13:59.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: pgmap v911: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:13:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:59.637 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:13:59.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:59.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:13:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: pgmap v911: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:13:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:13:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:13:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:13:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T10:13:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: failed to encode map e325 with expected crc 2023-10-09T10:13:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:13:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:13:59.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:13:59.795 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:13:59.853 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:13:59.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:13:59.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:13:59.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:00.002 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:00.054 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:00.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:00.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:00.225 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:00.240 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 23 16 46156 46140 125.24 98.25 0.00578744 0.0079803 2023-10-09T10:14:00.520 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:00.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:00.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:00.707 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:00.736 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:00.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:14:00.904 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:00.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:00.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:01.018 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:01.145 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:01.177 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:14:01.178 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:14:01.178 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:14:01.178 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:14:01.178 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:14:01.237 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:01.240 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 24 16 47479 47463 123.468 82.6875 0.0125532 0.00809333 2023-10-09T10:14:01.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:01.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:01.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:01.414 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:01 smithi012 ceph-mon[147926]: pgmap v913: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T10:14:01.415 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:01 smithi012 ceph-mon[152263]: pgmap v913: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T10:14:01.422 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:01.538 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:01.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:01.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:01 smithi110 ceph-mon[136032]: pgmap v913: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T10:14:01.717 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:01.787 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:01.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:01.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:14:01.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:01.972 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:02.063 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:02.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:02.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:02.079 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:02.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:02.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:02.241 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 25 16 48812 48796 121.858 83.3125 0.012886 0.00820038 2023-10-09T10:14:02.261 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:02.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:02.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:14:02.367 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:02.452 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:02.510 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:02.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:02.618 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:02.734 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:02.796 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:02.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:02.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:02.902 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:02.940 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:02.985 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:02.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:02.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:03.090 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:03.169 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:03.203 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:03.245 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 26 16 50005 49989 120.023 74.5625 0.0336598 0.00832547 2023-10-09T10:14:03.261 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:03.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:03.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:03.381 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:03.418 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:03.424 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:03 smithi012 ceph-mon[152263]: pgmap v914: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:14:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3710550909' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:14:03.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:03 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:14:03.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:03 smithi012 ceph-mon[147926]: pgmap v914: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:14:03.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3710550909' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:14:03.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:03 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:14:03.472 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:03.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:03.576 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:03.612 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:03 smithi110 ceph-mon[136032]: pgmap v914: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T10:14:03.612 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3710550909' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:14:03.612 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:03 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:14:03.614 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:03.661 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:03.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:03.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:03.845 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:03.989 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:14:04.051 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:14:04.099 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:14:04.129 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:04.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:04.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:04.139 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:14:04.179 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T10:14:04.191 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:14:04.234 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:14:04.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:04.245 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 27 16 51293 51277 118.56 80.5 0.0142704 0.00842832 2023-10-09T10:14:04.269 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:04.280 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:14:04.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:04.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:04.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:04.321 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:14:04.334 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:14:04.334 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:14:04.334 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:14:04.334 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:14:04.335 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:14:04.335 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:14:04.335 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:14:04.335 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:14:04.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1321641260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:04.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:04 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:14:04.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:04 smithi012 ceph-mon[147926]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T10:14:04.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1321641260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:04.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:04 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:14:04.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:04 smithi012 ceph-mon[152263]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T10:14:04.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:04.501 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:04.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1321641260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:04.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:04 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:14:04.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:04 smithi110 ceph-mon[136032]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T10:14:05.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T10:14:05.190 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:05.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:05.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:05.251 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 28 16 52700 52684 117.452 87.9375 0.0180735 0.0085081 2023-10-09T10:14:05.297 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:05.341 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:05.416 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:05.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:05.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:05 smithi012 ceph-mon[147926]: pgmap v915: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T10:14:05.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:05 smithi012 ceph-mon[147926]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:05 smithi012 ceph-mon[147926]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:05 smithi012 ceph-mon[147926]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:05 smithi012 ceph-mon[152263]: pgmap v915: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T10:14:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:05 smithi012 ceph-mon[152263]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:05 smithi012 ceph-mon[152263]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:05 smithi012 ceph-mon[152263]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:05.568 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:05.616 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:05.622 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:05 smithi110 ceph-mon[136032]: pgmap v915: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T10:14:05.622 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:05 smithi110 ceph-mon[136032]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.623 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:05 smithi110 ceph-mon[136032]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.623 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:05 smithi110 ceph-mon[136032]: failed to encode map e326 with expected crc 2023-10-09T10:14:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:05.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:05.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:05.802 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:06.002 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:06.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:06.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:06.108 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:06.182 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:06.254 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 29 16 54116 54100 116.443 88.5 0.0166037 0.00858281 2023-10-09T10:14:06.276 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:06.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:06.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:06.400 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:06.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:06.522 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:06.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:06.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:06.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:06.683 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:06.736 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:06.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:06.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:06.921 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:07.251 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:07.252 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 30 16 55534 55518 115.516 88.625 0.00600716 0.00865103 2023-10-09T10:14:07.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:07.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:07.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:14:07.364 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:07.394 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:07.435 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:07.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:07.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:07.542 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:07.564 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:07.564 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:07.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:07 smithi012 ceph-mon[147926]: pgmap v917: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-10-09T10:14:07.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:07 smithi012 ceph-mon[152263]: pgmap v917: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-10-09T10:14:07.592 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:07 smithi110 ceph-mon[136032]: pgmap v917: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-10-09T10:14:07.594 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:07.639 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:07.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:07.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:07.745 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:07.796 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:07.851 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:07.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:07.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:07.898 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:07.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:07.995 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:08.053 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:08.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:08.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:08.157 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:14:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:14:07] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.33.4" 2023-10-09T10:14:08.201 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:08.248 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:08.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:08.252 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 31 16 56705 56689 114.152 73.1875 0.0107863 0.00875476 2023-10-09T10:14:08.345 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:14:08.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:08.468 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:08.546 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:08.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:08.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:08.647 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:08.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:08.742 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:09.015 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:09.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:09.125 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:09.194 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:09.252 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 32 16 57979 57963 113.074 79.625 0.0227665 0.00883786 2023-10-09T10:14:09.390 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:14:09.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:09 smithi012 ceph-mon[147926]: pgmap v918: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T10:14:09.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3138940601' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:09.608 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:09 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:09.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:09 smithi012 ceph-mon[152263]: pgmap v918: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T10:14:09.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3138940601' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:09.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:09 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:09.637 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:09 smithi110 ceph-mon[136032]: pgmap v918: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T10:14:09.638 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3138940601' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:09.638 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:09 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:09.640 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:10.193 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:10.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:10.252 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 33 16 59222 59206 112.003 77.6875 0.00814219 0.00892319 2023-10-09T10:14:10.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:10.426 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:10.494 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:10.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:10.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:10.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:10.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[147926]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:10.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[147926]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T10:14:10.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[147926]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[147926]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[147926]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[147926]: pgmap v920: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T10:14:10.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:10.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[152263]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:10.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[152263]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T10:14:10.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[152263]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[152263]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[152263]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:10 smithi012 ceph-mon[152263]: pgmap v920: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T10:14:10.668 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:10 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:10 smithi110 ceph-mon[136032]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:10 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:10 smithi110 ceph-mon[136032]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T10:14:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:10 smithi110 ceph-mon[136032]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:10 smithi110 ceph-mon[136032]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:10 smithi110 ceph-mon[136032]: failed to encode map e327 with expected crc 2023-10-09T10:14:10.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:10 smithi110 ceph-mon[136032]: pgmap v920: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T10:14:10.783 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:10.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:10.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:10.892 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:10.945 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:10.964 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:14:10.997 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:14:11.032 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:14:11.073 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:14:11.103 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:11.103 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:14:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:11.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:11.140 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:14:11.181 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:14:11.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:11.211 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:14:11.212 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:14:11.212 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:14:11.213 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:14:11.213 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:14:11.213 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:14:11.213 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:14:11.213 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:14:11.213 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:14:11.252 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 34 16 60427 60411 110.924 75.3125 0.0280601 0.00900978 2023-10-09T10:14:11.279 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:11.365 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:11.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:11.409 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T10:14:11.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:11.553 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1365200201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:14:11.553 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:11 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:14:11.553 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3338615571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:11.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1365200201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:14:11.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:11 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:14:11.554 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3338615571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:11.564 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:11.576 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1365200201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:14:11.576 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:11 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:14:11.576 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3338615571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:11.929 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:11.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:11.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:11.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T10:14:12.040 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:12.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:14:12.115 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:12.209 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:12.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:12.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:12.260 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 35 16 61681 61665 109.972 78.375 0.0119578 0.00908636 2023-10-09T10:14:12.319 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:12.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[147926]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:14:12.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[147926]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T10:14:12.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:12 smithi110 ceph-mon[136032]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:12 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:14:12.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:12 smithi110 ceph-mon[136032]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T10:14:12.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:12 smithi110 ceph-mon[136032]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.413 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:12 smithi110 ceph-mon[136032]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.414 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:12 smithi110 ceph-mon[136032]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.414 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:12 smithi110 ceph-mon[136032]: pgmap v922: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-10-09T10:14:12.592 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[147926]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[147926]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[147926]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[147926]: pgmap v922: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-10-09T10:14:12.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[152263]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:14:12.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[152263]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T10:14:12.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[152263]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[152263]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[152263]: failed to encode map e328 with expected crc 2023-10-09T10:14:12.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:12 smithi012 ceph-mon[152263]: pgmap v922: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-10-09T10:14:12.909 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:13.262 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 36 16 62925 62909 109.078 77.75 0.0130984 0.0091629 2023-10-09T10:14:13.569 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:13.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:13.647 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:14:13.696 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:13.790 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:13.866 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:13.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:13.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:14.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:14.111 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:14.155 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:14.262 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 37 16 64192 64176 108.264 79.1875 0.0182553 0.00923188 2023-10-09T10:14:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:14.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:14.476 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:14.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:14.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:14.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:14.640 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:14.946 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:14:14.946 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:14:14.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.946 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65070 9.9 e328/317 9:93cf2890:::benchmark_data_smithi012_313365_object65069:head v 328'4256, mlcod=328'4251)", 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.938606+0000", 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0054872790000000003, 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055072430000000002, 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938606+0000", 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938606+0000", 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938605+0000", 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021780599999999999 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.940783+0000", 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4498000000000001e-05 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.940798+0000", 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2530000000000001e-06 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.940804+0000", 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00056751700000000002 2023-10-09T10:14:14.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:14.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941372+0000", 2023-10-09T10:14:14.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7863000000000002e-05 2023-10-09T10:14:14.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:14.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941389+0000", 2023-10-09T10:14:14.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027832960000000002 2023-10-09T10:14:14.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.951 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.951 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65075 9.f 9:fd999df4:::benchmark_data_smithi012_313365_object65074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.941761+0000", 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0023328149999999998, 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023991630000000002, 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65075 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941761+0000", 2023-10-09T10:14:14.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941761+0000", 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941759+0000", 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7302999999999998e-05 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941797+0000", 2023-10-09T10:14:14.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2039999999999997e-06 2023-10-09T10:14:14.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941805+0000", 2023-10-09T10:14:14.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6299999999999997e-06 2023-10-09T10:14:14.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941809+0000", 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3275999999999997e-05 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941853+0000", 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011671900000000001 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941969+0000", 2023-10-09T10:14:14.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000148732 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.942118+0000", 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000357554 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.957 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65076 9.a 9.8286ccaa (undecoded) ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.941821+0000", 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0022724899999999998, 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023841270000000002, 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65076 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941821+0000", 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941821+0000", 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941820+0000", 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5806999999999998e-05 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941856+0000", 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5370000000000003e-06 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941863+0000", 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0049999999999998e-06 2023-10-09T10:14:14.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941867+0000", 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5862999999999998e-05 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65077 9.a 9.16fdcdba (undecoded) ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.941871+0000", 2023-10-09T10:14:14.961 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.002222098, 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023655070000000002, 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65077 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941871+0000", 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941871+0000", 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941871+0000", 2023-10-09T10:14:14.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9329e-05 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941900+0000", 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9449999999999998e-06 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941906+0000", 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5319999999999998e-06 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941910+0000", 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8285999999999999e-05 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65078 9.f 9.5cad798f (undecoded) ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.941913+0000", 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0021799570000000002, 2023-10-09T10:14:14.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002355841, 2023-10-09T10:14:14.966 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.966 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:14:14.966 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:14.966 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:14.966 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:14.966 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65078 2023-10-09T10:14:14.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.967 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941913+0000", 2023-10-09T10:14:14.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941913+0000", 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941913+0000", 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6498000000000002e-05 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941950+0000", 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3330000000000002e-06 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941956+0000", 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8600000000000003e-06 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941960+0000", 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6267999999999998e-05 2023-10-09T10:14:14.969 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.970 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.970 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.970 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65079 9.6 9:60d0453e:::benchmark_data_smithi012_313365_object65078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:14.970 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.941968+0000", 2023-10-09T10:14:14.970 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0021258549999999998, 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023362410000000002, 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65079 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941968+0000", 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941968+0000", 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941967+0000", 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6358e-05 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.942003+0000", 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9609999999999999e-06 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.942010+0000", 2023-10-09T10:14:14.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3370000000000001e-06 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.942015+0000", 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8688e-05 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.942053+0000", 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000123041 2023-10-09T10:14:14.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.942177+0000", 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014088600000000001 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.942317+0000", 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034983600000000002 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.975 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.45908.0:65071 9.f e328/317)", 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.943445+0000", 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00064831600000000002, 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00089890100000000002, 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943445+0000", 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943445+0000", 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943444+0000", 2023-10-09T10:14:14.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4989999999999996e-06 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943449+0000", 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4003e-05 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943463+0000", 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.207e-06 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943469+0000", 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4023e-05 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop_reply(client.45908.0:65069 9.a e328/317)", 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.943478+0000", 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00061577500000000005, 2023-10-09T10:14:14.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00089299699999999997, 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943478+0000", 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943478+0000", 2023-10-09T10:14:14.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943477+0000", 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2850000000000001e-06 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943480+0000", 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3510000000000001e-06 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943485+0000", 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7239999999999998e-06 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943489+0000", 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1089e-05 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.982 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65066 9.b e328/317 9:d360c4ca:::benchmark_data_smithi012_313365_object65065:head v 328'4003, mlcod=328'4000)", 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.934632+0000", 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0094614060000000003, 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097867679999999995, 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934632+0000", 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934632+0000", 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934632+0000", 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4721000000000002e-05 2023-10-09T10:14:14.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934666+0000", 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1670000000000002e-06 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934674+0000", 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9720000000000003e-06 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934678+0000", 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00095527600000000002 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.935633+0000", 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9675999999999998e-05 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.935662+0000", 2023-10-09T10:14:14.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010304629999999999 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65067 9.5 e328/317 9:ad7e373f:::benchmark_data_smithi012_313365_object65066:head v 328'4076, mlcod=328'4073)", 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.934699+0000", 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0093939739999999994, 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097894890000000002, 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.987 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934699+0000", 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934699+0000", 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934699+0000", 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012990899999999999 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934829+0000", 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2165e-05 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934841+0000", 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2789999999999998e-06 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934846+0000", 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8878999999999997e-05 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934895+0000", 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2059999999999999e-05 2023-10-09T10:14:14.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.934917+0000", 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021794300000000001 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65062 9.9 e328/317 9:94c678f3:::benchmark_data_smithi012_313365_object65061:head v 328'4255, mlcod=328'4250)", 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.935745+0000", 2023-10-09T10:14:14.991 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0083482529999999999, 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0088232250000000005, 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.935745+0000", 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.935745+0000", 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.935744+0000", 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9245000000000005e-05 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.935814+0000", 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.419e-05 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.935828+0000", 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.879e-06 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.935834+0000", 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00083865399999999998 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.936672+0000", 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2252999999999999e-05 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.936695+0000", 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00094955500000000004 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65065 9.8 e328/317 9:1db94562:::benchmark_data_smithi012_313365_object65064:head v 328'4209, mlcod=328'4206)", 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.938619+0000", 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0054742640000000004, 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060206929999999997, 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938619+0000", 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938619+0000", 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938619+0000", 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6770999999999997e-05 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:14.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938656+0000", 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5169999999999996e-06 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938662+0000", 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6219999999999997e-06 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938668+0000", 2023-10-09T10:14:14.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6093000000000001e-05 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938714+0000", 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3973999999999997e-05 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938748+0000", 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012860000000000001 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:14.999 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65069 9.a 9:599b1671:::benchmark_data_smithi012_313365_object65068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.938688+0000", 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0054051020000000002, 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060248619999999998, 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:15.000 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65069 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938688+0000", 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938688+0000", 2023-10-09T10:14:15.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938687+0000", 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2101000000000003e-05 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938719+0000", 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1020000000000003e-06 2023-10-09T10:14:15.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938728+0000", 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8269999999999997e-06 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938733+0000", 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7209000000000002e-05 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938770+0000", 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000150111 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938920+0000", 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0001706 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939091+0000", 2023-10-09T10:14:15.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00040218600000000001 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65071 9.f 9:f9adb4cb:::benchmark_data_smithi012_313365_object65070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.938745+0000", 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0053489039999999998, 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060530990000000002, 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:14:15.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65071 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938745+0000", 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938745+0000", 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938744+0000", 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3957e-05 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938778+0000", 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1929999999999993e-06 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938786+0000", 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.848e-06 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:15.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938791+0000", 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5466000000000001e-05 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938846+0000", 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010967999999999999 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938956+0000", 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014443999999999999 2023-10-09T10:14:15.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939101+0000", 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026908219999999998 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941791+0000", 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030468159999999999 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.010 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65072 9.a 9:5feabaf9:::benchmark_data_smithi012_313365_object65071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.938807+0000", 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0052861030000000003, 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060892209999999997, 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:15.011 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65072 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938807+0000", 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938807+0000", 2023-10-09T10:14:15.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938807+0000", 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1980999999999998e-05 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938839+0000", 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7800000000000001e-06 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938846+0000", 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5299999999999998e-06 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938851+0000", 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024020809999999999 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:15.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941253+0000", 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012490799999999999 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941378+0000", 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017093699999999999 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.941549+0000", 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002741626 2023-10-09T10:14:15.015 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65073 9.a 9:58e4a2c7:::benchmark_data_smithi012_313365_object65072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.938856+0000", 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0052373350000000001, 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061273209999999998, 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:15.016 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65073 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938856+0000", 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:15.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938856+0000", 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938855+0000", 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9986000000000001e-05 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938885+0000", 2023-10-09T10:14:15.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9330000000000004e-06 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938892+0000", 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7570000000000004e-06 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.938897+0000", 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047092430000000001 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943606+0000", 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013845599999999999 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943745+0000", 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000188763 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:14:15.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.943934+0000", 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050774979999999997 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65074 9.7 9:e099214d:::benchmark_data_smithi012_313365_object65073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:14.939203+0000", 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0048904639999999997, 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058548749999999998, 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:15.021 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65074 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939203+0000", 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939203+0000", 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939202+0000", 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0521000000000001e-05 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:15.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939233+0000", 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1130000000000001e-06 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939241+0000", 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6840000000000001e-06 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939246+0000", 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9669000000000003e-05 2023-10-09T10:14:15.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939286+0000", 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011509400000000001 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939401+0000", 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000160958 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:15.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:14.939562+0000", 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035912999999999999 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 17 2023-10-09T10:14:15.026 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:14:15.051 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:15.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:15.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:15.156 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:15.262 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 38 16 65251 65235 107.158 66.1875 0.00268747 0.00932282 2023-10-09T10:14:15.284 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:15 smithi012 ceph-mon[147926]: pgmap v923: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T10:14:15.284 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:15 smithi012 ceph-mon[152263]: pgmap v923: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T10:14:15.349 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:15 smithi110 ceph-mon[136032]: pgmap v923: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T10:14:15.350 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:15.402 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:15.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:15.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:15.508 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:15.563 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:15.729 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:15.735 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:14:15.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:15.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:15.775 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:15.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:15.902 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:16.262 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 39 16 66278 66262 106.057 64.1875 0.0188501 0.00942337 2023-10-09T10:14:16.462 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:16.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:16.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:16.644 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:16.644 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:16.724 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:16.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:16.838 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:16.902 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:16.962 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:16.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:17.067 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:17.168 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T10:14:17.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:14:17.263183+0000 min lat: 0.00139647 max lat: 0.202238 avg lat: 0.00949141 2023-10-09T10:14:17.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:14:17.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 40 16 67199 67183 104.846 57.5625 0.00515421 0.00949141 2023-10-09T10:14:17.374 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:17.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:17 smithi012 ceph-mon[147926]: pgmap v924: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T10:14:17.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:17 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2061807215' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:14:17.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:17 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:14:17.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:17 smithi012 ceph-mon[152263]: pgmap v924: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T10:14:17.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:17 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2061807215' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:14:17.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:17 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:14:17.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:17 smithi110 ceph-mon[136032]: pgmap v924: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T10:14:17.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:17 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2061807215' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:14:17.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:17 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:14:17.533 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:17.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:17.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:17.643 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:17.689 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:17.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T10:14:18.181 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:14:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:14:17] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.33.4" 2023-10-09T10:14:18.215 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:18.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:18.243 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:14:18.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 41 16 68180 68164 103.785 61.3125 0.0197379 0.00962923 2023-10-09T10:14:18.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:18.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:14:18.425 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:18.437 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:14:18.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:18 smithi110 ceph-mon[136032]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:18 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:14:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:18 smithi110 ceph-mon[136032]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T10:14:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:18 smithi110 ceph-mon[136032]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:18 smithi110 ceph-mon[136032]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:18 smithi110 ceph-mon[136032]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:18 smithi110 ceph-mon[136032]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2804647452' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:18.461 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[147926]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.461 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:14:18.461 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[147926]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T10:14:18.461 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[147926]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.461 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[147926]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[147926]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[147926]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2804647452' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[152263]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[152263]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[152263]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[152263]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[152263]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.462 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[152263]: failed to encode map e329 with expected crc 2023-10-09T10:14:18.463 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2804647452' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:18.481 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:14:18.538 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:14:18.562 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:14:18.611 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:14:18.661 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:14:18.700 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:14:18.721 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:14:18.721 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:14:18.721 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:14:18.721 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:14:18.721 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:14:18.721 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:14:18.721 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:14:18.721 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:14:18.732 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:18.735 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:18.735 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:18.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:18.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:18.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:18.913 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:19.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:14:19.126 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:19.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:19.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:19.223 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:19.237 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:19.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 42 16 69278 69262 102.948 68.625 0.0140422 0.00970849 2023-10-09T10:14:19.290 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:19.370 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:19.376 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: pgmap v926: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:14:19.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:19.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2804647452' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:14:19.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T10:14:19.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.377 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1047123746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:19.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:19.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: pgmap v926: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:14:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2804647452' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1047123746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: pgmap v926: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2804647452' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:14:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T10:14:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: failed to encode map e330 with expected crc 2023-10-09T10:14:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1047123746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:19.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:19.555 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:19.635 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:19.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:19.742 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:19.794 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:19.853 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:19.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:19.898 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:19.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:20.030 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:20.112 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:20.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:20.221 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:20.265 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 43 16 70414 70398 102.206 71 0.0145386 0.00977888 2023-10-09T10:14:20.283 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:20.582 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:20.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:20.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:20.688 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:20.751 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:20.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:14:20.808 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:20.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:20.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:21.038 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:21.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 44 16 71519 71503 101.453 69.0625 0.0232757 0.0098515 2023-10-09T10:14:21.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:14:21.313 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:21.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:21.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:21.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:21.481 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:21 smithi110 ceph-mon[136032]: pgmap v928: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:14:21.602 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:21 smithi012 ceph-mon[147926]: pgmap v928: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:14:21.606 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:21 smithi012 ceph-mon[152263]: pgmap v928: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T10:14:21.611 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:21.667 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:21.668 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:21.675 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:21.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:21.715 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:21.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:21.955 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:22.065 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:22.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:22.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:22.173 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:22.206 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:22.210 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T10:14:22.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 45 16 72441 72425 100.48 57.625 0.10727 0.00993302 2023-10-09T10:14:22.390 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:14:22.398 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:22.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:22.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:22.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/746930058' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:14:22.454 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:14:22.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:22.562 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/746930058' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:14:22.562 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:14:22.563 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/746930058' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:14:22.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:14:22.579 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:22.664 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:22.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:22.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:22.833 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:22.862 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T10:14:23.086 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:23.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:23.149 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:23.195 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:23.211 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:23.239 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:23.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 46 16 73479 73463 99.7065 64.875 0.0198634 0.0100239 2023-10-09T10:14:23.361 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:23.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:23.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:23.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[147926]: pgmap v929: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T10:14:23.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[147926]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:14:23.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[147926]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T10:14:23.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[147926]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[147926]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[147926]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[152263]: pgmap v929: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T10:14:23.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[152263]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:14:23.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[152263]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T10:14:23.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[152263]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[152263]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:23 smithi012 ceph-mon[152263]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.469 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:23.515 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:23 smithi110 ceph-mon[136032]: pgmap v929: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T10:14:23.515 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:23 smithi110 ceph-mon[136032]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.515 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:14:23.515 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:23 smithi110 ceph-mon[136032]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T10:14:23.515 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:23 smithi110 ceph-mon[136032]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.515 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:23 smithi110 ceph-mon[136032]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.515 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:23 smithi110 ceph-mon[136032]: failed to encode map e331 with expected crc 2023-10-09T10:14:23.545 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:23.784 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:23.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:23.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:23.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:23.940 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:24.253 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:14:24.266 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 47 16 74526 74510 98.972 65.4375 0.0225108 0.0100979 2023-10-09T10:14:24.357 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:24.534 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:24.538 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2419099116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:24.538 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:24.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:24.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:24.643 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2419099116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:24.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2419099116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:24.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:24.706 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:24.801 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:24.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:24.909 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:24.975 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:25.031 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:25.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:25.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:25.188 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:25.228 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:25.228 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:25.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: pgmap v931: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T10:14:25.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:14:25.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:14:25.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.252 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:25.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T10:14:25.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[147926]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.253 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: pgmap v931: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T10:14:25.253 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:14:25.253 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:14:25.253 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.254 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:25.254 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T10:14:25.254 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.254 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.254 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:25 smithi012 ceph-mon[152263]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.256 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:25.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:25.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:25.267 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 48 16 75593 75577 98.2984 66.6875 0.00859973 0.0101676 2023-10-09T10:14:25.316 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:14:25.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:14:25.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:25.405 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:14:25.437 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:25.439 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:14:25.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: pgmap v931: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T10:14:25.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:14:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:14:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T10:14:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:25 smithi110 ceph-mon[136032]: failed to encode map e332 with expected crc 2023-10-09T10:14:25.480 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:14:25.521 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:14:25.559 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:14:25.596 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:14:25.602 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:14:25.602 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:14:25.602 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:14:25.602 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:14:25.603 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:14:25.603 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:14:25.603 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:14:25.603 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:14:25.612 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:14:25.939 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:25.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:25.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:26.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:26.130 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:26.250 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:26.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:26.267 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 49 16 76652 76636 97.6436 66.1875 0.0258222 0.0102361 2023-10-09T10:14:26.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:26.420 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:26.428 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:26.485 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:26.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:26.591 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/893020456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:26.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/893020456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:26.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:26.653 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/893020456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:26.669 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:26.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:14:27.093 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:27.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:27.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:27.276 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 50 16 77710 77694 97.0014 66.125 0.0219665 0.0103027 2023-10-09T10:14:27.276 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:27 smithi110 ceph-mon[136032]: pgmap v933: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-10-09T10:14:27.276 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:27 smithi012 ceph-mon[147926]: pgmap v933: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-10-09T10:14:27.283 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:27.368 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:27.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:27.478 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:27.534 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:27 smithi012 ceph-mon[152263]: pgmap v933: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-10-09T10:14:27.538 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:27.610 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:27.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:27.705 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:27.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:27.773 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:27.824 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:27.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:27.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:27.928 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:27.974 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:28.038 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:28.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:28.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:28.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:14:28.129 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:14:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:14:27] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.33.4" 2023-10-09T10:14:28.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:28.228 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:28.277 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 51 16 78642 78626 96.2353 58.25 0.0345837 0.0103856 2023-10-09T10:14:28.330 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T10:14:28.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2695431184' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:14:28.431 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2695431184' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:14:28.505 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:28.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:28.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:28.599 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:14:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:14:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:14:28.599 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:14:28.600 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:14:28.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:28.632 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2695431184' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:14:28.685 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:28.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:28.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:28.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:28.956 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T10:14:31.258 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:14:31.259 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:31.261 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: pgmap v934: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-10-09T10:14:31.261 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.261 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2695431184' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:14:31.261 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T10:14:31.261 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.262 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.262 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.262 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.262 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:31.262 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:31.263 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 52 16 79554 79538 95.4816 57 0.0095566 0.0104681 2023-10-09T10:14:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: pgmap v934: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-10-09T10:14:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2695431184' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:14:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T10:14:31.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:31.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:31.266 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: pgmap v934: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-10-09T10:14:31.266 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.267 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2695431184' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:14:31.267 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T10:14:31.267 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.267 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: failed to encode map e333 with expected crc 2023-10-09T10:14:31.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:31.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:31.269 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:31.270 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 53 16 80609 80593 94.9246 65.9375 0.0125623 0.0105295 2023-10-09T10:14:31.270 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:14:31.271 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:14:31.272 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4129271533' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:14:31.272 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:14:31.272 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4129271533' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:14:31.272 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:14:31.274 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4129271533' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:14:31.274 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:14:31.280 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 54 16 81630 81614 94.349 63.8125 0.013264 0.010594 2023-10-09T10:14:31.379 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:31.396 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:14:31.452 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:14:31.480 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:14:31.487 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:31.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:31.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:31.528 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:14:31.586 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:14:31.598 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:31.627 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:14:31.645 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:31 smithi110 ceph-mon[136032]: pgmap v936: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:14:31.645 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:31 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:31.645 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:31 smithi110 ceph-mon[136032]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.645 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:31 smithi110 ceph-mon[136032]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:31 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:14:31.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:31 smithi110 ceph-mon[136032]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T10:14:31.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:31 smithi110 ceph-mon[136032]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:31 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:14:31.648 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[147926]: pgmap v936: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:14:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[147926]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[147926]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:14:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[147926]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T10:14:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[147926]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:14:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[152263]: pgmap v936: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:14:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[152263]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[152263]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:14:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[152263]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T10:14:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[152263]: failed to encode map e334 with expected crc 2023-10-09T10:14:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:31 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:14:31.679 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:14:31.709 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:31.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:31.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:31.718 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:14:31.742 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:14:31.742 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:14:31.742 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:14:31.743 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:14:31.743 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:14:31.743 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:14:31.743 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:14:31.743 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:14:31.818 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:31.863 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:31.951 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:31.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:32.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:32.083 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:32.119 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:32.278 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 55 16 82558 82542 93.6887 58 0.00474311 0.0106697 2023-10-09T10:14:32.340 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2564848458' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:32.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2564848458' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:32.341 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2564848458' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:33.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:14:33.278 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 56 16 83577 83561 93.1533 63.6875 0.0173163 0.0107299 2023-10-09T10:14:33.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:33 smithi012 ceph-mon[147926]: pgmap v938: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-09T10:14:33.683 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:33 smithi012 ceph-mon[152263]: pgmap v938: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-09T10:14:33.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:33 smithi110 ceph-mon[136032]: pgmap v938: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-09T10:14:33.787 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:33.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:33.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:33.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:33.999 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:34.001 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:34.001 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:34.278 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 57 16 84545 84529 92.5809 60.5 0.0153142 0.0107973 2023-10-09T10:14:34.439 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:34.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:34.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:34.547 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:34.553 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:14:34.597 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:34.644 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:34.653 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:34.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:34.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:34.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:34.830 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:34.912 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:34.952 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:34.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:34.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:35.063 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:35.135 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:35.241 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:35.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:35.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:35.278 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 58 16 85505 85489 92.0197 60 0.0197779 0.0108627 2023-10-09T10:14:35.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:35.408 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:35.440 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:35 smithi012 ceph-mon[147926]: pgmap v939: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T10:14:35.440 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:35 smithi012 ceph-mon[152263]: pgmap v939: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T10:14:35.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:35 smithi110 ceph-mon[136032]: pgmap v939: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T10:14:35.444 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:35.684 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:35.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:35.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:35.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:35.865 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:35.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:14:36.162 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:36.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:36.272 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:36.278 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 59 16 86402 86386 91.4107 56.0625 0.0356759 0.0109329 2023-10-09T10:14:36.307 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:36.382 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:36 smithi110 ceph-mon[136032]: pgmap v940: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:14:36.422 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:36.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:36.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:36.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:36.589 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:36.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:36 smithi012 ceph-mon[147926]: pgmap v940: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:14:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:36 smithi012 ceph-mon[152263]: pgmap v940: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:14:36.679 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:36.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:36.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:36.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:36.892 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:37.031 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:37.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:37.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:37.073 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:37.073 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:37.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:14:37.196 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:14:37.221 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:37.244 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:37.287 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:14:37.287324+0000 min lat: 0.00139647 max lat: 0.278978 avg lat: 0.0110179 2023-10-09T10:14:37.287 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:14:37.287 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 60 16 87201 87185 90.7076 49.9375 0.0247685 0.0110179 2023-10-09T10:14:37.372 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:37.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:37.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:37.389 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:37 smithi110 ceph-mon[136032]: from='client.56122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:37.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:37.570 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:37.608 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:14:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:37 smithi012 ceph-mon[147926]: from='client.56122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:37.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:37 smithi012 ceph-mon[152263]: from='client.56122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:37.761 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:37.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:37.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:37.798 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":940,"stamp":"2023-10-09T10:14:35.493457+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5445588896,"num_objects":83108,"num_object_clones":0,"num_object_copies":249324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83108,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":83344,"num_write_kb":5319821,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":1190604,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83278,"ondisk_log_size":83278,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20080288,"kb_used_data":16380256,"kb_used_omap":167,"kb_used_meta":3699800,"kb_avail":729880928,"statfs":{"total":767960285184,"available":747398070272,"internally_reserved":0,"allocated":16773382144,"data_stored":16766549650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":3788595801},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,10,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":510001152,"num_objects":7782,"num_object_clones":0,"num_object_copies":23346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7782,"num_write_kb":498048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003043"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":177,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312458+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:14:28.312458+0000","last_peered":"2023-10-09T10:14:28.312458+0000","last_clean":"2023-10-09T10:14:28.312458+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:14:28.312458+0000","last_undegraded":"2023-10-09T10:14:28.312458+0000","last_fullsized":"2023-10-09T10:14:28.312458+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":292,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304160+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:14:28.304160+0000","last_peered":"2023-10-09T10:14:28.304160+0000","last_clean":"2023-10-09T10:14:28.304160+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:14:28.304160+0000","last_undegraded":"2023-10-09T10:14:28.304160+0000","last_fullsized":"2023-10-09T10:14:28.304160+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":338,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312271+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:14:28.312271+0000","last_peered":"2023-10-09T10:14:28.312271+0000","last_clean":"2023-10-09T10:14:28.312271+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:14:28.312271+0000","last_undegraded":"2023-10-09T10:14:28.312271+0000","last_fullsized":"2023-10-09T10:14:28.312271+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":201,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303884+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:14:28.303884+0000","last_peered":"2023-10-09T10:14:28.303884+0000","last_clean":"2023-10-09T10:14:28.303884+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:14:28.303884+0000","last_undegraded":"2023-10-09T10:14:28.303884+0000","last_fullsized":"2023-10-09T10:14:28.303884+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":306,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305808+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:14:28.305808+0000","last_peered":"2023-10-09T10:14:28.305808+0000","last_clean":"2023-10-09T10:14:28.305808+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:14:28.305808+0000","last_undegraded":"2023-10-09T10:14:28.305808+0000","last_fullsized":"2023-10-09T10:14:28.305808+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":152,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.331628+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:14:28.331628+0000","last_peered":"2023-10-09T10:14:28.331628+0000","last_clean":"2023-10-09T10:14:28.331628+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:14:28.331628+0000","last_undegraded":"2023-10-09T10:14:28.331628+0000","last_fullsized":"2023-10-09T10:14:28.331628+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":218,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304462+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:14:28.304462+0000","last_peered":"2023-10-09T10:14:28.304462+0000","last_clean":"2023-10-09T10:14:28.304462+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:14:28.304462+0000","last_undegraded":"2023-10-09T10:14:28.304462+0000","last_fullsized":"2023-10-09T10:14:28.304462+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":287,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305709+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:14:28.305709+0000","last_peered":"2023-10-09T10:14:28.305709+0000","last_clean":"2023-10-09T10:14:28.305709+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:14:28.305709+0000","last_undegraded":"2023-10-09T10:14:28.305709+0000","last_fullsized":"2023-10-09T10:14:28.305709+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":151,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325267+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:14:28.325267+0000","last_peered":"2023-10-09T10:14:28.325267+0000","last_clean":"2023-10-09T10:14:28.325267+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:14:28.325267+0000","last_undegraded":"2023-10-09T10:14:28.325267+0000","last_fullsized":"2023-10-09T10:14:28.325267+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":248,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306763+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:14:28.306763+0000","last_peered":"2023-10-09T10:14:28.306763+0000","last_clean":"2023-10-09T10:14:28.306763+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:14:28.306763+0000","last_undegraded":"2023-10-09T10:14:28.306763+0000","last_fullsized":"2023-10-09T10:14:28.306763+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":372,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324290+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:14:28.324290+0000","last_peered":"2023-10-09T10:14:28.324290+0000","last_clean":"2023-10-09T10:14:28.324290+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:14:28.324290+0000","last_undegraded":"2023-10-09T10:14:28.324290+0000","last_fullsized":"2023-10-09T10:14:28.324290+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":330,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312361+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:14:28.312361+0000","last_peered":"2023-10-09T10:14:28.312361+0000","last_clean":"2023-10-09T10:14:28.312361+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:14:28.312361+0000","last_undegraded":"2023-10-09T10:14:28.312361+0000","last_fullsized":"2023-10-09T10:14:28.312361+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":221,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324242+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:14:28.324242+0000","last_peered":"2023-10-09T10:14:28.324242+0000","last_clean":"2023-10-09T10:14:28.324242+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:14:28.324242+0000","last_undegraded":"2023-10-09T10:14:28.324242+0000","last_fullsized":"2023-10-09T10:14:28.324242+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":202,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312790+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:14:28.312790+0000","last_peered":"2023-10-09T10:14:28.312790+0000","last_clean":"2023-10-09T10:14:28.312790+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:14:28.312790+0000","last_undegraded":"2023-10-09T10:14:28.312790+0000","last_fullsized":"2023-10-09T10:14:28.312790+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":405,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324014+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:14:28.324014+0000","last_peered":"2023-10-09T10:14:28.324014+0000","last_clean":"2023-10-09T10:14:28.324014+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:14:28.324014+0000","last_undegraded":"2023-10-09T10:14:28.324014+0000","last_fullsized":"2023-10-09T10:14:28.324014+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":274,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.310915+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:14:28.310915+0000","last_peered":"2023-10-09T10:14:28.310915+0000","last_clean":"2023-10-09T10:14:28.310915+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:14:28.310915+0000","last_undegraded":"2023-10-09T10:14:28.310915+0000","last_fullsized":"2023-10-09T10:14:28.310915+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":201,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303485+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:14:28.303485+0000","last_peered":"2023-10-09T10:14:28.303485+0000","last_clean":"2023-10-09T10:14:28.303485+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:14:28.303485+0000","last_undegraded":"2023-10-09T10:14:28.303485+0000","last_fullsized":"2023-10-09T10:14:28.303485+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":167,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325418+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:14:28.325418+0000","last_peered":"2023-10-09T10:14:28.325418+0000","last_clean":"2023-10-09T10:14:28.325418+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:14:28.325418+0000","last_undegraded":"2023-10-09T10:14:28.325418+0000","last_fullsized":"2023-10-09T10:14:28.325418+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":202,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.322278+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:14:28.322278+0000","last_peered":"2023-10-09T10:14:28.322278+0000","last_clean":"2023-10-09T10:14:28.322278+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:14:28.322278+0000","last_undegraded":"2023-10-09T10:14:28.322278+0000","last_fullsized":"2023-10-09T10:14:28.322278+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":200,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304001+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:14:28.304001+0000","last_peered":"2023-10-09T10:14:28.304001+0000","last_clean":"2023-10-09T10:14:28.304001+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:14:28.304001+0000","last_undegraded":"2023-10-09T10:14:28.304001+0000","last_fullsized":"2023-10-09T10:14:28.304001+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":224,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303445+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:14:28.303445+0000","last_peered":"2023-10-09T10:14:28.303445+0000","last_clean":"2023-10-09T10:14:28.303445+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:14:28.303445+0000","last_undegraded":"2023-10-09T10:14:28.303445+0000","last_fullsized":"2023-10-09T10:14:28.303445+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":287,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305372+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:14:28.305372+0000","last_peered":"2023-10-09T10:14:28.305372+0000","last_clean":"2023-10-09T10:14:28.305372+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:14:28.305372+0000","last_undegraded":"2023-10-09T10:14:28.305372+0000","last_fullsized":"2023-10-09T10:14:28.305372+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"334'5156","reported_seq":5182,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:31.876368+0000","last_change":"2023-10-09T10:13:32.575493+0000","last_active":"2023-10-09T10:14:31.876368+0000","last_peered":"2023-10-09T10:14:31.876368+0000","last_clean":"2023-10-09T10:14:31.876368+0000","last_became_active":"2023-10-09T10:13:32.575194+0000","last_became_peered":"2023-10-09T10:13:32.575194+0000","last_unstale":"2023-10-09T10:14:31.876368+0000","last_undegraded":"2023-10-09T10:14:31.876368+0000","last_fullsized":"2023-10-09T10:14:31.876368+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5156,"log_dups_size":0,"ondisk_log_size":5156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:21.292749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337903616,"num_objects":5156,"num_object_clones":0,"num_object_copies":15468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5156,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":366,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324725+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:14:28.324725+0000","last_peered":"2023-10-09T10:14:28.324725+0000","last_clean":"2023-10-09T10:14:28.324725+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:14:28.324725+0000","last_undegraded":"2023-10-09T10:14:28.324725+0000","last_fullsized":"2023-10-09T10:14:28.324725+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":152,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.323896+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:14:28.323896+0000","last_peered":"2023-10-09T10:14:28.323896+0000","last_clean":"2023-10-09T10:14:28.323896+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:14:28.323896+0000","last_undegraded":"2023-10-09T10:14:28.323896+0000","last_fullsized":"2023-10-09T10:14:28.323896+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"334'5197","reported_seq":5223,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:33.752805+0000","last_change":"2023-10-09T10:13:32.571424+0000","last_active":"2023-10-09T10:14:33.752805+0000","last_peered":"2023-10-09T10:14:33.752805+0000","last_clean":"2023-10-09T10:14:33.752805+0000","last_became_active":"2023-10-09T10:13:32.571091+0000","last_became_peered":"2023-10-09T10:13:32.571091+0000","last_unstale":"2023-10-09T10:14:33.752805+0000","last_undegraded":"2023-10-09T10:14:33.752805+0000","last_fullsized":"2023-10-09T10:14:33.752805+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5197,"log_dups_size":0,"ondisk_log_size":5197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:33.565620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340590592,"num_objects":5197,"num_object_clones":0,"num_object_copies":15591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5197,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":354,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324225+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:14:28.324225+0000","last_peered":"2023-10-09T10:14:28.324225+0000","last_clean":"2023-10-09T10:14:28.324225+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:14:28.324225+0000","last_undegraded":"2023-10-09T10:14:28.324225+0000","last_fullsized":"2023-10-09T10:14:28.324225+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":246,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312183+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:14:28.312183+0000","last_peered":"2023-10-09T10:14:28.312183+0000","last_clean":"2023-10-09T10:14:28.312183+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:14:28.312183+0000","last_undegraded":"2023-10-09T10:14:28.312183+0000","last_fullsized":"2023-10-09T10:14:28.312183+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"334'5158","reported_seq":5183,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:33.017138+0000","last_change":"2023-10-09T10:13:32.575491+0000","last_active":"2023-10-09T10:14:33.017138+0000","last_peered":"2023-10-09T10:14:33.017138+0000","last_clean":"2023-10-09T10:14:33.017138+0000","last_became_active":"2023-10-09T10:13:32.575164+0000","last_became_peered":"2023-10-09T10:13:32.575164+0000","last_unstale":"2023-10-09T10:14:33.017138+0000","last_undegraded":"2023-10-09T10:14:33.017138+0000","last_fullsized":"2023-10-09T10:14:33.017138+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5158,"log_dups_size":0,"ondisk_log_size":5158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338034688,"num_objects":5158,"num_object_clones":0,"num_object_copies":15474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5158,"num_write_kb":330112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":156,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325560+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:14:28.325560+0000","last_peered":"2023-10-09T10:14:28.325560+0000","last_clean":"2023-10-09T10:14:28.325560+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:14:28.325560+0000","last_undegraded":"2023-10-09T10:14:28.325560+0000","last_fullsized":"2023-10-09T10:14:28.325560+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":177,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.313983+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:14:28.313983+0000","last_peered":"2023-10-09T10:14:28.313983+0000","last_clean":"2023-10-09T10:14:28.313983+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:14:28.313983+0000","last_undegraded":"2023-10-09T10:14:28.313983+0000","last_fullsized":"2023-10-09T10:14:28.313983+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"334'5261","reported_seq":5286,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:33.063809+0000","last_change":"2023-10-09T10:13:32.857374+0000","last_active":"2023-10-09T10:14:33.063809+0000","last_peered":"2023-10-09T10:14:33.063809+0000","last_clean":"2023-10-09T10:14:33.063809+0000","last_became_active":"2023-10-09T10:13:32.857082+0000","last_became_peered":"2023-10-09T10:13:32.857082+0000","last_unstale":"2023-10-09T10:14:33.063809+0000","last_undegraded":"2023-10-09T10:14:33.063809+0000","last_fullsized":"2023-10-09T10:14:33.063809+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5261,"log_dups_size":0,"ondisk_log_size":5261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344784896,"num_objects":5261,"num_object_clones":0,"num_object_copies":15783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5261,"num_write_kb":336704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":167,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325352+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:14:28.325352+0000","last_peered":"2023-10-09T10:14:28.325352+0000","last_clean":"2023-10-09T10:14:28.325352+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:14:28.325352+0000","last_undegraded":"2023-10-09T10:14:28.325352+0000","last_fullsized":"2023-10-09T10:14:28.325352+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":202,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303824+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:14:28.303824+0000","last_peered":"2023-10-09T10:14:28.303824+0000","last_clean":"2023-10-09T10:14:28.303824+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:14:28.303824+0000","last_undegraded":"2023-10-09T10:14:28.303824+0000","last_fullsized":"2023-10-09T10:14:28.303824+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"334'5177","reported_seq":5203,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:31.615751+0000","last_change":"2023-10-09T10:13:32.695943+0000","last_active":"2023-10-09T10:14:31.615751+0000","last_peered":"2023-10-09T10:14:31.615751+0000","last_clean":"2023-10-09T10:14:31.615751+0000","last_became_active":"2023-10-09T10:13:32.695608+0000","last_became_peered":"2023-10-09T10:13:32.695608+0000","last_unstale":"2023-10-09T10:14:31.615751+0000","last_undegraded":"2023-10-09T10:14:31.615751+0000","last_fullsized":"2023-10-09T10:14:31.615751+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5177,"log_dups_size":0,"ondisk_log_size":5177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:05.782965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339279872,"num_objects":5177,"num_object_clones":0,"num_object_copies":15531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5177,"num_write_kb":331328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":329,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304820+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:14:28.304820+0000","last_peered":"2023-10-09T10:14:28.304820+0000","last_clean":"2023-10-09T10:14:28.304820+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:14:28.304820+0000","last_undegraded":"2023-10-09T10:14:28.304820+0000","last_fullsized":"2023-10-09T10:14:28.304820+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":199,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.310743+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:14:28.310743+0000","last_peered":"2023-10-09T10:14:28.310743+0000","last_clean":"2023-10-09T10:14:28.310743+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:14:28.310743+0000","last_undegraded":"2023-10-09T10:14:28.310743+0000","last_fullsized":"2023-10-09T10:14:28.310743+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"334'5190","reported_seq":5216,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:33.760192+0000","last_change":"2023-10-09T10:13:32.857224+0000","last_active":"2023-10-09T10:14:33.760192+0000","last_peered":"2023-10-09T10:14:33.760192+0000","last_clean":"2023-10-09T10:14:33.760192+0000","last_became_active":"2023-10-09T10:13:32.856912+0000","last_became_peered":"2023-10-09T10:13:32.856912+0000","last_unstale":"2023-10-09T10:14:33.760192+0000","last_undegraded":"2023-10-09T10:14:33.760192+0000","last_fullsized":"2023-10-09T10:14:33.760192+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5190,"log_dups_size":0,"ondisk_log_size":5190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:25.963375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340131840,"num_objects":5190,"num_object_clones":0,"num_object_copies":15570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5190,"num_write_kb":332160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":260,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312270+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:14:28.312270+0000","last_peered":"2023-10-09T10:14:28.312270+0000","last_clean":"2023-10-09T10:14:28.312270+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:14:28.312270+0000","last_undegraded":"2023-10-09T10:14:28.312270+0000","last_fullsized":"2023-10-09T10:14:28.312270+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":266,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312129+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:14:28.312129+0000","last_peered":"2023-10-09T10:14:28.312129+0000","last_clean":"2023-10-09T10:14:28.312129+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:14:28.312129+0000","last_undegraded":"2023-10-09T10:14:28.312129+0000","last_fullsized":"2023-10-09T10:14:28.312129+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"334'5272","reported_seq":5297,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:35.171812+0000","last_change":"2023-10-09T10:13:32.577948+0000","last_active":"2023-10-09T10:14:35.171812+0000","last_peered":"2023-10-09T10:14:35.171812+0000","last_clean":"2023-10-09T10:14:35.171812+0000","last_became_active":"2023-10-09T10:13:32.577597+0000","last_became_peered":"2023-10-09T10:13:32.577597+0000","last_unstale":"2023-10-09T10:14:35.171812+0000","last_undegraded":"2023-10-09T10:14:35.171812+0000","last_fullsized":"2023-10-09T10:14:35.171812+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345505792,"num_objects":5272,"num_object_clones":0,"num_object_copies":15816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5272,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":228,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.322229+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:14:28.322229+0000","last_peered":"2023-10-09T10:14:28.322229+0000","last_clean":"2023-10-09T10:14:28.322229+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:14:28.322229+0000","last_undegraded":"2023-10-09T10:14:28.322229+0000","last_fullsized":"2023-10-09T10:14:28.322229+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":202,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312585+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:14:28.312585+0000","last_peered":"2023-10-09T10:14:28.312585+0000","last_clean":"2023-10-09T10:14:28.312585+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:14:28.312585+0000","last_undegraded":"2023-10-09T10:14:28.312585+0000","last_fullsized":"2023-10-09T10:14:28.312585+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"334'5031","reported_seq":5057,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:31.588566+0000","last_change":"2023-10-09T10:13:32.857205+0000","last_active":"2023-10-09T10:14:31.588566+0000","last_peered":"2023-10-09T10:14:31.588566+0000","last_clean":"2023-10-09T10:14:31.588566+0000","last_became_active":"2023-10-09T10:13:32.856928+0000","last_became_peered":"2023-10-09T10:13:32.856928+0000","last_unstale":"2023-10-09T10:14:31.588566+0000","last_undegraded":"2023-10-09T10:14:31.588566+0000","last_fullsized":"2023-10-09T10:14:31.588566+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5031,"log_dups_size":0,"ondisk_log_size":5031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:43:02.753960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329711616,"num_objects":5031,"num_object_clones":0,"num_object_copies":15093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5031,"num_write_kb":321984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":289,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305607+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:14:28.305607+0000","last_peered":"2023-10-09T10:14:28.305607+0000","last_clean":"2023-10-09T10:14:28.305607+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:14:28.305607+0000","last_undegraded":"2023-10-09T10:14:28.305607+0000","last_fullsized":"2023-10-09T10:14:28.305607+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":338,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312688+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:14:28.312688+0000","last_peered":"2023-10-09T10:14:28.312688+0000","last_clean":"2023-10-09T10:14:28.312688+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:14:28.312688+0000","last_undegraded":"2023-10-09T10:14:28.312688+0000","last_fullsized":"2023-10-09T10:14:28.312688+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"334'5244","reported_seq":5269,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:33.047636+0000","last_change":"2023-10-09T10:13:32.573670+0000","last_active":"2023-10-09T10:14:33.047636+0000","last_peered":"2023-10-09T10:14:33.047636+0000","last_clean":"2023-10-09T10:14:33.047636+0000","last_became_active":"2023-10-09T10:13:32.573356+0000","last_became_peered":"2023-10-09T10:13:32.573356+0000","last_unstale":"2023-10-09T10:14:33.047636+0000","last_undegraded":"2023-10-09T10:14:33.047636+0000","last_fullsized":"2023-10-09T10:14:33.047636+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5244,"log_dups_size":0,"ondisk_log_size":5244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343670784,"num_objects":5244,"num_object_clones":0,"num_object_copies":15732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5244,"num_write_kb":335616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":338,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312613+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:14:28.312613+0000","last_peered":"2023-10-09T10:14:28.312613+0000","last_clean":"2023-10-09T10:14:28.312613+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:14:28.312613+0000","last_undegraded":"2023-10-09T10:14:28.312613+0000","last_fullsized":"2023-10-09T10:14:28.312613+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325300+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:14:28.325300+0000","last_peered":"2023-10-09T10:14:28.325300+0000","last_clean":"2023-10-09T10:14:28.325300+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:14:28.325300+0000","last_undegraded":"2023-10-09T10:14:28.325300+0000","last_fullsized":"2023-10-09T10:14:28.325300+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":294,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.314831+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:14:28.314831+0000","last_peered":"2023-10-09T10:14:28.314831+0000","last_clean":"2023-10-09T10:14:28.314831+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:14:28.314831+0000","last_undegraded":"2023-10-09T10:14:28.314831+0000","last_fullsized":"2023-10-09T10:14:28.314831+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"334'5328","reported_seq":5353,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:31.880059+0000","last_change":"2023-10-09T10:13:32.576627+0000","last_active":"2023-10-09T10:14:31.880059+0000","last_peered":"2023-10-09T10:14:31.880059+0000","last_clean":"2023-10-09T10:14:31.880059+0000","last_became_active":"2023-10-09T10:13:32.576321+0000","last_became_peered":"2023-10-09T10:13:32.576321+0000","last_unstale":"2023-10-09T10:14:31.880059+0000","last_undegraded":"2023-10-09T10:14:31.880059+0000","last_fullsized":"2023-10-09T10:14:31.880059+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5328,"log_dups_size":0,"ondisk_log_size":5328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:04.882671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349175808,"num_objects":5328,"num_object_clones":0,"num_object_copies":15984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5328,"num_write_kb":340992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":152,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324942+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:14:28.324942+0000","last_peered":"2023-10-09T10:14:28.324942+0000","last_clean":"2023-10-09T10:14:28.324942+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:14:28.324942+0000","last_undegraded":"2023-10-09T10:14:28.324942+0000","last_fullsized":"2023-10-09T10:14:28.324942+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303656+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:14:28.303656+0000","last_peered":"2023-10-09T10:14:28.303656+0000","last_clean":"2023-10-09T10:14:28.303656+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:14:28.303656+0000","last_undegraded":"2023-10-09T10:14:28.303656+0000","last_fullsized":"2023-10-09T10:14:28.303656+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":361,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312494+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:14:28.312494+0000","last_peered":"2023-10-09T10:14:28.312494+0000","last_clean":"2023-10-09T10:14:28.312494+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:14:28.312494+0000","last_undegraded":"2023-10-09T10:14:28.312494+0000","last_fullsized":"2023-10-09T10:14:28.312494+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"334'5043","reported_seq":5068,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:32.995662+0000","last_change":"2023-10-09T10:13:32.574493+0000","last_active":"2023-10-09T10:14:32.995662+0000","last_peered":"2023-10-09T10:14:32.995662+0000","last_clean":"2023-10-09T10:14:32.995662+0000","last_became_active":"2023-10-09T10:13:32.574198+0000","last_became_peered":"2023-10-09T10:13:32.574198+0000","last_unstale":"2023-10-09T10:14:32.995662+0000","last_undegraded":"2023-10-09T10:14:32.995662+0000","last_fullsized":"2023-10-09T10:14:32.995662+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5043,"log_dups_size":0,"ondisk_log_size":5043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330498048,"num_objects":5043,"num_object_clones":0,"num_object_copies":15129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5043,"num_write_kb":322752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":255,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306632+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:14:28.306632+0000","last_peered":"2023-10-09T10:14:28.306632+0000","last_clean":"2023-10-09T10:14:28.306632+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:14:28.306632+0000","last_undegraded":"2023-10-09T10:14:28.306632+0000","last_fullsized":"2023-10-09T10:14:28.306632+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":330,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.315402+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:14:28.315402+0000","last_peered":"2023-10-09T10:14:28.315402+0000","last_clean":"2023-10-09T10:14:28.315402+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:14:28.315402+0000","last_undegraded":"2023-10-09T10:14:28.315402+0000","last_fullsized":"2023-10-09T10:14:28.315402+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":351,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312601+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:14:28.312601+0000","last_peered":"2023-10-09T10:14:28.312601+0000","last_clean":"2023-10-09T10:14:28.312601+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:14:28.312601+0000","last_undegraded":"2023-10-09T10:14:28.312601+0000","last_fullsized":"2023-10-09T10:14:28.312601+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"334'5264","reported_seq":5290,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:31.538719+0000","last_change":"2023-10-09T10:13:32.572705+0000","last_active":"2023-10-09T10:14:31.538719+0000","last_peered":"2023-10-09T10:14:31.538719+0000","last_clean":"2023-10-09T10:14:31.538719+0000","last_became_active":"2023-10-09T10:13:32.571805+0000","last_became_peered":"2023-10-09T10:13:32.571805+0000","last_unstale":"2023-10-09T10:14:31.538719+0000","last_undegraded":"2023-10-09T10:14:31.538719+0000","last_fullsized":"2023-10-09T10:14:31.538719+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:13.430092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344981504,"num_objects":5264,"num_object_clones":0,"num_object_copies":15792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5264,"num_write_kb":336896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":161,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325082+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:14:28.325082+0000","last_peered":"2023-10-09T10:14:28.325082+0000","last_clean":"2023-10-09T10:14:28.325082+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:14:28.325082+0000","last_undegraded":"2023-10-09T10:14:28.325082+0000","last_fullsized":"2023-10-09T10:14:28.325082+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":580,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.319932+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:14:28.319932+0000","last_peered":"2023-10-09T10:14:28.319932+0000","last_clean":"2023-10-09T10:14:28.319932+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:14:28.319932+0000","last_undegraded":"2023-10-09T10:14:28.319932+0000","last_fullsized":"2023-10-09T10:14:28.319932+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":239,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306710+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:14:28.306710+0000","last_peered":"2023-10-09T10:14:28.306710+0000","last_clean":"2023-10-09T10:14:28.306710+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:14:28.306710+0000","last_undegraded":"2023-10-09T10:14:28.306710+0000","last_fullsized":"2023-10-09T10:14:28.306710+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":249,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324388+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:14:28.324388+0000","last_peered":"2023-10-09T10:14:28.324388+0000","last_clean":"2023-10-09T10:14:28.324388+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:14:28.324388+0000","last_undegraded":"2023-10-09T10:14:28.324388+0000","last_fullsized":"2023-10-09T10:14:28.324388+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"334'5110","reported_seq":5136,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:31.876331+0000","last_change":"2023-10-09T10:13:32.694210+0000","last_active":"2023-10-09T10:14:31.876331+0000","last_peered":"2023-10-09T10:14:31.876331+0000","last_clean":"2023-10-09T10:14:31.876331+0000","last_became_active":"2023-10-09T10:13:32.693883+0000","last_became_peered":"2023-10-09T10:13:32.693883+0000","last_unstale":"2023-10-09T10:14:31.876331+0000","last_undegraded":"2023-10-09T10:14:31.876331+0000","last_fullsized":"2023-10-09T10:14:31.876331+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5110,"log_dups_size":0,"ondisk_log_size":5110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:35.108481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334888960,"num_objects":5110,"num_object_clones":0,"num_object_copies":15330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5110,"num_write_kb":327040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":311,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.747839+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:14:28.747839+0000","last_peered":"2023-10-09T10:14:28.747839+0000","last_clean":"2023-10-09T10:14:28.747839+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:14:28.747839+0000","last_undegraded":"2023-10-09T10:14:28.747839+0000","last_fullsized":"2023-10-09T10:14:28.747839+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":212,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303707+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:14:28.303707+0000","last_peered":"2023-10-09T10:14:28.303707+0000","last_clean":"2023-10-09T10:14:28.303707+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:14:28.303707+0000","last_undegraded":"2023-10-09T10:14:28.303707+0000","last_fullsized":"2023-10-09T10:14:28.303707+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":240,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303807+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:14:28.303807+0000","last_peered":"2023-10-09T10:14:28.303807+0000","last_clean":"2023-10-09T10:14:28.303807+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:14:28.303807+0000","last_undegraded":"2023-10-09T10:14:28.303807+0000","last_fullsized":"2023-10-09T10:14:28.303807+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"334'5217","reported_seq":5243,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:33.726576+0000","last_change":"2023-10-09T10:13:32.576814+0000","last_active":"2023-10-09T10:14:33.726576+0000","last_peered":"2023-10-09T10:14:33.726576+0000","last_clean":"2023-10-09T10:14:33.726576+0000","last_became_active":"2023-10-09T10:13:32.576567+0000","last_became_peered":"2023-10-09T10:13:32.576567+0000","last_unstale":"2023-10-09T10:14:33.726576+0000","last_undegraded":"2023-10-09T10:14:33.726576+0000","last_fullsized":"2023-10-09T10:14:33.726576+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5217,"log_dups_size":0,"ondisk_log_size":5217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:37.032579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341901312,"num_objects":5217,"num_object_clones":0,"num_object_copies":15651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5217,"num_write_kb":333888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":353,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324196+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:14:28.324196+0000","last_peered":"2023-10-09T10:14:28.324196+0000","last_clean":"2023-10-09T10:14:28.324196+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:14:28.324196+0000","last_undegraded":"2023-10-09T10:14:28.324196+0000","last_fullsized":"2023-10-09T10:14:28.324196+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":367,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312717+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:14:28.312717+0000","last_peered":"2023-10-09T10:14:28.312717+0000","last_clean":"2023-10-09T10:14:28.312717+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:14:28.312717+0000","last_undegraded":"2023-10-09T10:14:28.312717+0000","last_fullsized":"2023-10-09T10:14:28.312717+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":204,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303732+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:14:28.303732+0000","last_peered":"2023-10-09T10:14:28.303732+0000","last_clean":"2023-10-09T10:14:28.303732+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:14:28.303732+0000","last_undegraded":"2023-10-09T10:14:28.303732+0000","last_fullsized":"2023-10-09T10:14:28.303732+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"334'5340","reported_seq":5365,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:33.760035+0000","last_change":"2023-10-09T10:13:32.575359+0000","last_active":"2023-10-09T10:14:33.760035+0000","last_peered":"2023-10-09T10:14:33.760035+0000","last_clean":"2023-10-09T10:14:33.760035+0000","last_became_active":"2023-10-09T10:13:32.575050+0000","last_became_peered":"2023-10-09T10:13:32.575050+0000","last_unstale":"2023-10-09T10:14:33.760035+0000","last_undegraded":"2023-10-09T10:14:33.760035+0000","last_fullsized":"2023-10-09T10:14:33.760035+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:15:48.495574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349962240,"num_objects":5340,"num_object_clones":0,"num_object_copies":16020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5340,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":298,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305477+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:14:28.305477+0000","last_peered":"2023-10-09T10:14:28.305477+0000","last_clean":"2023-10-09T10:14:28.305477+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:14:28.305477+0000","last_undegraded":"2023-10-09T10:14:28.305477+0000","last_fullsized":"2023-10-09T10:14:28.305477+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":312,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312688+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:14:28.312688+0000","last_peered":"2023-10-09T10:14:28.312688+0000","last_clean":"2023-10-09T10:14:28.312688+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:14:28.312688+0000","last_undegraded":"2023-10-09T10:14:28.312688+0000","last_fullsized":"2023-10-09T10:14:28.312688+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":287,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305497+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:14:28.305497+0000","last_peered":"2023-10-09T10:14:28.305497+0000","last_clean":"2023-10-09T10:14:28.305497+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:14:28.305497+0000","last_undegraded":"2023-10-09T10:14:28.305497+0000","last_fullsized":"2023-10-09T10:14:28.305497+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"334'5096","reported_seq":5122,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:31.582701+0000","last_change":"2023-10-09T10:13:32.577001+0000","last_active":"2023-10-09T10:14:31.582701+0000","last_peered":"2023-10-09T10:14:31.582701+0000","last_clean":"2023-10-09T10:14:31.582701+0000","last_became_active":"2023-10-09T10:13:32.576694+0000","last_became_peered":"2023-10-09T10:13:32.576694+0000","last_unstale":"2023-10-09T10:14:31.582701+0000","last_undegraded":"2023-10-09T10:14:31.582701+0000","last_fullsized":"2023-10-09T10:14:31.582701+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:36.747060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333971456,"num_objects":5096,"num_object_clones":0,"num_object_copies":15288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5096,"num_write_kb":326144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":242,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.314451+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:14:28.314451+0000","last_peered":"2023-10-09T10:14:28.314451+0000","last_clean":"2023-10-09T10:14:28.314451+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:14:28.314451+0000","last_undegraded":"2023-10-09T10:14:28.314451+0000","last_fullsized":"2023-10-09T10:14:28.314451+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":206,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303720+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:14:28.303720+0000","last_peered":"2023-10-09T10:14:28.303720+0000","last_clean":"2023-10-09T10:14:28.303720+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:14:28.303720+0000","last_undegraded":"2023-10-09T10:14:28.303720+0000","last_fullsized":"2023-10-09T10:14:28.303720+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":187,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324902+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:14:28.324902+0000","last_peered":"2023-10-09T10:14:28.324902+0000","last_clean":"2023-10-09T10:14:28.324902+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:14:28.324902+0000","last_undegraded":"2023-10-09T10:14:28.324902+0000","last_fullsized":"2023-10-09T10:14:28.324902+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":200,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303571+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:14:28.303571+0000","last_peered":"2023-10-09T10:14:28.303571+0000","last_clean":"2023-10-09T10:14:28.303571+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:14:28.303571+0000","last_undegraded":"2023-10-09T10:14:28.303571+0000","last_fullsized":"2023-10-09T10:14:28.303571+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":273,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.312800+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:14:28.312800+0000","last_peered":"2023-10-09T10:14:28.312800+0000","last_clean":"2023-10-09T10:14:28.312800+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:14:28.312800+0000","last_undegraded":"2023-10-09T10:14:28.312800+0000","last_fullsized":"2023-10-09T10:14:28.312800+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":171,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325290+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:14:28.325290+0000","last_peered":"2023-10-09T10:14:28.325290+0000","last_clean":"2023-10-09T10:14:28.325290+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:14:28.325290+0000","last_undegraded":"2023-10-09T10:14:28.325290+0000","last_fullsized":"2023-10-09T10:14:28.325290+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":213,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304802+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:14:28.304802+0000","last_peered":"2023-10-09T10:14:28.304802+0000","last_clean":"2023-10-09T10:14:28.304802+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:14:28.304802+0000","last_undegraded":"2023-10-09T10:14:28.304802+0000","last_fullsized":"2023-10-09T10:14:28.304802+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":171,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325214+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:14:28.325214+0000","last_peered":"2023-10-09T10:14:28.325214+0000","last_clean":"2023-10-09T10:14:28.325214+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:14:28.325214+0000","last_undegraded":"2023-10-09T10:14:28.325214+0000","last_fullsized":"2023-10-09T10:14:28.325214+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":222,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306627+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:14:28.306627+0000","last_peered":"2023-10-09T10:14:28.306627+0000","last_clean":"2023-10-09T10:14:28.306627+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:14:28.306627+0000","last_undegraded":"2023-10-09T10:14:28.306627+0000","last_fullsized":"2023-10-09T10:14:28.306627+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":352,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304718+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:14:28.304718+0000","last_peered":"2023-10-09T10:14:28.304718+0000","last_clean":"2023-10-09T10:14:28.304718+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:14:28.304718+0000","last_undegraded":"2023-10-09T10:14:28.304718+0000","last_fullsized":"2023-10-09T10:14:28.304718+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":372,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306971+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:14:28.306971+0000","last_peered":"2023-10-09T10:14:28.306971+0000","last_clean":"2023-10-09T10:14:28.306971+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:14:28.306971+0000","last_undegraded":"2023-10-09T10:14:28.306971+0000","last_fullsized":"2023-10-09T10:14:28.306971+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":282,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304148+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:14:28.304148+0000","last_peered":"2023-10-09T10:14:28.304148+0000","last_clean":"2023-10-09T10:14:28.304148+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:14:28.304148+0000","last_undegraded":"2023-10-09T10:14:28.304148+0000","last_fullsized":"2023-10-09T10:14:28.304148+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":202,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.311991+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:14:28.311991+0000","last_peered":"2023-10-09T10:14:28.311991+0000","last_clean":"2023-10-09T10:14:28.311991+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:14:28.311991+0000","last_undegraded":"2023-10-09T10:14:28.311991+0000","last_fullsized":"2023-10-09T10:14:28.311991+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5444993024,"num_objects":83084,"num_object_clones":0,"num_object_copies":249252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":83084,"num_write_kb":5317376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16465985536,"data_stored":16465985536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83084,"ondisk_log_size":83084,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430539,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3659616,"kb_used_data":3069912,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":90085536,"statfs":{"total":95995035648,"available":92247588864,"internally_reserved":0,"allocated":3143589888,"data_stored":3143042162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2269999999999999,"5min":2.472,"15min":2.1669999999999998},"min":{"1min":0.22900000000000001,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":8.5429999999999993,"5min":11.164,"15min":50.591999999999999},"last":1.2529999999999999},{"interface":"front","average":{"1min":2.2130000000000001,"5min":2.2839999999999998,"15min":2.1150000000000002},"min":{"1min":0.308,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.7629999999999999,"5min":15.974,"15min":50.747},"last":0.69899999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:14:16 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":2.6840000000000002,"15min":2.1419999999999999},"min":{"1min":0.35299999999999998,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":4.6890000000000001,"5min":14.311,"15min":25.338000000000001},"last":1.292},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.6040000000000001,"15min":1.9650000000000001},"min":{"1min":0.30399999999999999,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":2.7040000000000002,"5min":16.526,"15min":18.27},"last":0.66200000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6080000000000001,"5min":2.613,"15min":2.3809999999999998},"min":{"1min":0.32100000000000001,"5min":0.27400000000000002,"15min":0.246},"max":{"1min":7.3479999999999999,"5min":13.775,"15min":13.775},"last":1.321},{"interface":"front","average":{"1min":2.1200000000000001,"5min":2.5960000000000001,"15min":2.2490000000000001},"min":{"1min":0.38300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.6200000000000001,"5min":12.101000000000001,"15min":12.101000000000001},"last":0.75700000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":2.8759999999999999,"15min":2.125},"min":{"1min":0.46100000000000002,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":3.306,"5min":29.058,"15min":29.058},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.6579999999999999,"15min":2.1829999999999998},"min":{"1min":0.313,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":10.666,"5min":16.715,"15min":16.715},"last":0.52000000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:13:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":1.54,"15min":1.121},"min":{"1min":0.27500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":12.699999999999999,"5min":12.699999999999999,"15min":12.699999999999999},"last":1.349},{"interface":"front","average":{"1min":1.744,"5min":1.306,"15min":1.0409999999999999},"min":{"1min":0.32900000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.003,"5min":12.003,"15min":12.003},"last":0.77900000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.8979999999999999,"15min":1.29},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.219},"max":{"1min":6.766,"5min":25.277999999999999,"15min":25.277999999999999},"last":1.375},{"interface":"front","average":{"1min":0.70599999999999996,"5min":1.7190000000000001,"15min":1.2190000000000001},"min":{"1min":0.34499999999999997,"5min":0.29099999999999998,"15min":0.22600000000000001},"max":{"1min":2.6680000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.876}]},{"osd":6,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.401,"15min":1.1180000000000001},"min":{"1min":0.32800000000000001,"5min":0.31900000000000001,"15min":0.26100000000000001},"max":{"1min":3.085,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.625,"5min":1.4330000000000001,"15min":1.0920000000000001},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":10.625,"5min":10.625,"15min":10.625},"last":6.702}]}]},{"osd":6,"up_from":42,"seq":180388626767,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1818900,"kb_used_data":1233292,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":91926252,"statfs":{"total":95995035648,"available":94132482048,"internally_reserved":0,"allocated":1262891008,"data_stored":1262356524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":2.3370000000000002,"15min":1.8169999999999999},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.222},"max":{"1min":8.8079999999999998,"5min":15.435,"15min":15.435},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.988,"15min":1.7649999999999999},"min":{"1min":0.251,"5min":0.22,"15min":0.22},"max":{"1min":4.375,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.38100000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:14:09 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":2.9089999999999998,"15min":2.226},"min":{"1min":0.34399999999999997,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.8780000000000001,"5min":35.609999999999999,"15min":35.609999999999999},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.7869999999999999,"15min":2.1539999999999999},"min":{"1min":0.28499999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.2069999999999999,"5min":32.396000000000001,"15min":32.396000000000001},"last":1.873}]},{"osd":2,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.9199999999999999,"15min":1.778},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":4.4729999999999999,"5min":12.898,"15min":12.898},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.4359999999999999,"5min":2.4369999999999998,"15min":2.899},"min":{"1min":0.33100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.7669999999999995,"5min":14.542999999999999,"15min":14.542999999999999},"last":0.89900000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":2.766,"15min":1.8280000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":4.1520000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":4.8079999999999998},{"interface":"front","average":{"1min":2.1040000000000001,"5min":2.5510000000000002,"15min":2.4369999999999998},"min":{"1min":0.432,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3599999999999994,"5min":11.211,"15min":11.211},"last":0.34300000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.0089999999999999,"15min":0.629},"min":{"1min":0.29999999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.069,"5min":10.113,"15min":10.113},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.101,"5min":1.0149999999999999,"15min":0.67100000000000004},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":9.218,"5min":9.218,"15min":9.218},"last":0.30499999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.1619999999999999,"15min":0.93899999999999995},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.23699999999999999},"max":{"1min":4.1210000000000004,"5min":7.9260000000000002,"15min":14.09},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.3600000000000001,"15min":1.046},"min":{"1min":0.36099999999999999,"5min":0.27500000000000002,"15min":0.254},"max":{"1min":9.4890000000000008,"5min":14.465999999999999,"15min":14.465999999999999},"last":0.46300000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:14:09 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.214,"15min":0.97499999999999998},"min":{"1min":0.434,"5min":0.32800000000000001,"15min":0.252},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.254,"15min":1.02},"min":{"1min":0.46100000000000002,"5min":0.38100000000000001,"15min":0.30099999999999999},"max":{"1min":8.8550000000000004,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":0.44500000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253080,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3557724,"kb_used_data":3001620,"kb_used_omap":9,"kb_used_meta":556086,"kb_avail":90187428,"statfs":{"total":95995035648,"available":92351926272,"internally_reserved":0,"allocated":3073658880,"data_stored":3072524260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":569433065},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":2.6480000000000001,"15min":2.6890000000000001},"min":{"1min":0.20699999999999999,"5min":0.183,"15min":0.183},"max":{"1min":9.3040000000000003,"5min":31.007000000000001,"15min":31.007000000000001},"last":0.219},{"interface":"front","average":{"1min":2.0059999999999998,"5min":3.75,"15min":2.8759999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.192},"max":{"1min":9.5429999999999993,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.254}]},{"osd":2,"last update":"Mon Oct 9 10:13:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.9329999999999998,"15min":3.6219999999999999},"min":{"1min":0.251,"5min":0.217,"15min":0.217},"max":{"1min":11.140000000000001,"5min":11.140000000000001,"15min":17.369},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.621,"5min":3.0699999999999998,"15min":3.6269999999999998},"min":{"1min":0.24199999999999999,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":28.024999999999999,"5min":28.024999999999999,"15min":28.024999999999999},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:13:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":3.4289999999999998,"15min":3.4990000000000001},"min":{"1min":0.28399999999999997,"5min":0.27400000000000002,"15min":0.23400000000000001},"max":{"1min":9.0069999999999997,"5min":21.808,"15min":21.808},"last":0.37},{"interface":"front","average":{"1min":1.4339999999999999,"5min":3.1640000000000001,"15min":4.3220000000000001},"min":{"1min":0.28499999999999998,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":6.3659999999999997,"5min":21.547999999999998,"15min":21.547999999999998},"last":0.33800000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.7949999999999999,"15min":1.4350000000000001},"min":{"1min":0.34599999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.8799999999999999,"5min":21.620000000000001,"15min":21.620000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.2410000000000001,"5min":2.593,"15min":1.7509999999999999},"min":{"1min":0.41799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.4030000000000005,"5min":12.904,"15min":12.904},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":2.802,"15min":2.2639999999999998},"min":{"1min":0.40000000000000002,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":4.5670000000000002,"5min":12.811,"15min":15.531000000000001},"last":92.177999999999997},{"interface":"front","average":{"1min":2.3639999999999999,"5min":2.806,"15min":2.6139999999999999},"min":{"1min":0.443,"5min":0.32400000000000001,"15min":0.29099999999999998},"max":{"1min":9.9629999999999992,"5min":19.513000000000002,"15min":19.706},"last":91.869}]},{"osd":6,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":2.1840000000000002,"15min":2.1259999999999999},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":5.3120000000000003,"5min":21.670999999999999,"15min":21.670999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.5109999999999999,"5min":2.7799999999999998,"15min":2.3199999999999998},"min":{"1min":0.45800000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":4.9390000000000001,"5min":12.868,"15min":15.481999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.6970000000000001,"15min":2.589},"min":{"1min":0.48799999999999999,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":10.004,"5min":10.723000000000001,"15min":16.331},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.96799999999999997,"5min":2.1429999999999998,"15min":2.1760000000000002},"min":{"1min":0.45900000000000002,"5min":0.39200000000000002,"15min":0.378},"max":{"1min":2.8399999999999999,"5min":21.593,"15min":21.593},"last":0.56699999999999995}]}]},{"osd":0,"up_from":79,"seq":339302416606,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1741060,"kb_used_data":1332412,"kb_used_omap":28,"kb_used_meta":408611,"kb_avail":92004092,"statfs":{"total":95995035648,"available":94212190208,"internally_reserved":0,"allocated":1364389888,"data_stored":1363276753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":418418512},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.8919999999999999,"15min":2.2210000000000001},"min":{"1min":0.25900000000000001,"5min":0.216,"15min":0.16600000000000001},"max":{"1min":2.7410000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":1.2470000000000001},{"interface":"front","average":{"1min":1.25,"5min":4.6740000000000004,"15min":3.4199999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.21299999999999999},"max":{"1min":7.0060000000000002,"5min":31.925999999999998,"15min":31.925999999999998},"last":7.8879999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.7010000000000001,"15min":2.3300000000000001},"min":{"1min":0.26800000000000002,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":4.9109999999999996,"5min":18.003,"15min":18.003},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.1379999999999999,"5min":2.5339999999999998,"15min":2.9140000000000001},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.976,"5min":15.529,"15min":15.529},"last":0.372}]},{"osd":3,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":4.6779999999999999,"15min":4.8019999999999996},"min":{"1min":0.28399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":22.684000000000001,"5min":25.013999999999999,"15min":25.734999999999999},"last":5.6589999999999998},{"interface":"front","average":{"1min":1.367,"5min":3.4969999999999999,"15min":4.1059999999999999},"min":{"1min":0.318,"5min":0.221,"15min":0.221},"max":{"1min":4.0220000000000002,"5min":18.928999999999998,"15min":21.923999999999999},"last":0.54500000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":3.0259999999999998,"15min":1.9359999999999999},"min":{"1min":0.41799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":4.8070000000000004,"5min":20.494,"15min":20.494},"last":3.7400000000000002},{"interface":"front","average":{"1min":1.4119999999999999,"5min":2.7589999999999999,"15min":2.2109999999999999},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.109,"5min":15.907,"15min":15.907},"last":1.016}]},{"osd":5,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":3.1389999999999998,"15min":2.5550000000000002},"min":{"1min":0.371,"5min":0.36099999999999999,"15min":0.28000000000000003},"max":{"1min":18.620000000000001,"5min":18.620000000000001,"15min":18.620000000000001},"last":1.208},{"interface":"front","average":{"1min":2.653,"5min":2.3599999999999999,"15min":1.7929999999999999},"min":{"1min":0.45200000000000001,"5min":0.30499999999999999,"15min":0.27000000000000002},"max":{"1min":19.798999999999999,"5min":19.798999999999999,"15min":19.798999999999999},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":3.262,"5min":2.5750000000000002,"15min":1.99},"min":{"1min":0.40699999999999997,"5min":0.33700000000000002,"15min":0.31900000000000001},"max":{"1min":19.875,"5min":19.875,"15min":19.875},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.8049999999999999,"5min":3.5369999999999999,"15min":2.847},"min":{"1min":0.378,"5min":0.378,"15min":0.35699999999999998},"max":{"1min":6.0259999999999998,"5min":23.106999999999999,"15min":23.106999999999999},"last":5.492}]},{"osd":7,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":3.1659999999999999,"15min":2.71},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33900000000000002},"max":{"1min":18.904,"5min":18.904,"15min":26.978000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.8119999999999998,"5min":2.544,"15min":2.137},"min":{"1min":0.45800000000000002,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":19.843,"5min":19.843,"15min":26.768000000000001},"last":0.61099999999999999}]}]},{"osd":2,"up_from":217,"seq":932007903340,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1321664,"kb_used_data":1010488,"kb_used_omap":83,"kb_used_meta":311084,"kb_avail":92423488,"statfs":{"total":95995035648,"available":94641651712,"internally_reserved":0,"allocated":1034739712,"data_stored":1033631315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":318550511},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":2.8940000000000001,"15min":2.0430000000000001},"min":{"1min":0.185,"5min":0.185,"15min":0.17899999999999999},"max":{"1min":11.327,"5min":19.372,"15min":19.372},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.488,"5min":3.4940000000000002,"15min":3.157},"min":{"1min":0.23599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.7479999999999993,"5min":20.143000000000001,"15min":20.143000000000001},"last":1.1819999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":3.7999999999999998,"15min":2.8340000000000001},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.22800000000000001},"max":{"1min":11.398,"5min":19.283999999999999,"15min":19.283999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.423,"5min":3.234,"15min":2.7949999999999999},"min":{"1min":0.28599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":8.7739999999999991,"5min":19.027000000000001,"15min":19.027000000000001},"last":1.222}]},{"osd":3,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.9009999999999998,"15min":2.3159999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.368,"5min":19.370999999999999,"15min":19.370999999999999},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.661,"5min":2.919,"15min":3.5830000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.7959999999999994,"5min":19.053000000000001,"15min":19.053000000000001},"last":1.321}]},{"osd":4,"last update":"Mon Oct 9 10:13:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8839999999999999,"5min":2.056,"15min":1.1559999999999999},"min":{"1min":0.34699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":18.873999999999999,"5min":18.873999999999999,"15min":18.873999999999999},"last":1.2549999999999999},{"interface":"front","average":{"1min":2.4300000000000002,"5min":2.089,"15min":1.2490000000000001},"min":{"1min":0.41699999999999998,"5min":0.314,"15min":0.314},"max":{"1min":11.993,"5min":15.276,"15min":15.276},"last":0.36199999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":2.2309999999999999,"15min":1.7729999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":8.8140000000000001,"5min":15.32,"15min":15.32},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.802,"5min":2.5150000000000001,"15min":1.8879999999999999},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":9.4789999999999992,"5min":17.882999999999999,"15min":17.882999999999999},"last":0.45800000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":2.3839999999999999,"15min":1.542},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":12.257,"5min":18.943999999999999,"15min":18.943999999999999},"last":1.3460000000000001},{"interface":"front","average":{"1min":1.4139999999999999,"5min":2.4249999999999998,"15min":1.9870000000000001},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":4.5979999999999999,"5min":15.378,"15min":15.378},"last":0.63500000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.5529999999999999,"15min":1.6020000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.35899999999999999},"max":{"1min":12.186,"5min":18.972999999999999,"15min":18.972999999999999},"last":1.2969999999999999},{"interface":"front","average":{"1min":1.8029999999999999,"5min":2.8879999999999999,"15min":2.0449999999999999},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.376},"max":{"1min":9.9440000000000008,"5min":15.353,"15min":15.353},"last":0.60299999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824083,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2955244,"kb_used_data":2614820,"kb_used_omap":10,"kb_used_meta":340405,"kb_avail":90789908,"statfs":{"total":95995035648,"available":92968865792,"internally_reserved":0,"allocated":2677575680,"data_stored":2676249716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":348574787},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":3.1840000000000002,"15min":4.1970000000000001},"min":{"1min":0.21299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.5470000000000006,"5min":17.373000000000001,"15min":23.381},"last":3.2530000000000001},{"interface":"front","average":{"1min":1.788,"5min":4.4470000000000001,"15min":4.2679999999999998},"min":{"1min":0.23499999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.56,"5min":20.239999999999998,"15min":20.239999999999998},"last":5.3730000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":3.5569999999999999,"15min":3.8210000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7409999999999997,"5min":40.618000000000002,"15min":40.618000000000002},"last":9.9009999999999998},{"interface":"front","average":{"1min":2.0350000000000001,"5min":3.21,"15min":3.3570000000000002},"min":{"1min":0.27400000000000002,"5min":0.218,"15min":0.218},"max":{"1min":8.5139999999999993,"5min":15.704000000000001,"15min":23.423999999999999},"last":7.6820000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":3.347,"15min":3.609},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.6399999999999997,"5min":29.466999999999999,"15min":29.466999999999999},"last":3.2320000000000002},{"interface":"front","average":{"1min":1.5800000000000001,"5min":2.786,"15min":2.8380000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":8.5790000000000006,"5min":15.73,"15min":23.452000000000002},"last":7.6399999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:13:42 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":2.306,"15min":1.4990000000000001},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.6669999999999998,"5min":15.676,"15min":15.676},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.917,"5min":2.7410000000000001,"15min":1.921},"min":{"1min":0.379,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.673,"5min":16.478999999999999,"15min":16.478999999999999},"last":5.3029999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":3.0979999999999999,"15min":2.7749999999999999},"min":{"1min":0.33200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.638,"5min":13.638,"15min":13.638},"last":5.2309999999999999},{"interface":"front","average":{"1min":1.986,"5min":3.1379999999999999,"15min":2.6150000000000002},"min":{"1min":0.373,"5min":0.318,"15min":0.309},"max":{"1min":9.4860000000000007,"5min":17.242999999999999,"15min":17.242999999999999},"last":3.286}]},{"osd":6,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":2.9329999999999998,"15min":2.8029999999999999},"min":{"1min":0.45500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":13.705,"5min":16.504999999999999,"15min":16.504999999999999},"last":5.3289999999999997},{"interface":"front","average":{"1min":1.714,"5min":2.6869999999999998,"15min":2.6309999999999998},"min":{"1min":0.39800000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.6820000000000004,"5min":17.286999999999999,"15min":17.286999999999999},"last":6.3739999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":2.3740000000000001,"15min":1.7170000000000001},"min":{"1min":0.433,"5min":0.35199999999999998,"15min":0.34000000000000002},"max":{"1min":8.4079999999999995,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.732,"5min":3.125,"15min":2.8250000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":13.601000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":5.274}]}]},{"osd":4,"up_from":266,"seq":1142461300809,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1434904,"kb_used_data":1116240,"kb_used_omap":35,"kb_used_meta":318620,"kb_avail":92310248,"statfs":{"total":95995035648,"available":94525693952,"internally_reserved":0,"allocated":1143029760,"data_stored":1142489612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":326267118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.3720000000000001,"15min":0.96999999999999997},"min":{"1min":0.252,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.6970000000000001,"5min":10.881,"15min":10.881},"last":1.4410000000000001},{"interface":"front","average":{"1min":1.7989999999999999,"5min":2.1469999999999998,"15min":1.355},"min":{"1min":0.318,"5min":0.224,"15min":0.224},"max":{"1min":8.2050000000000001,"5min":12.246,"15min":12.246},"last":0.27100000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":3.0569999999999999,"5min":2.7010000000000001,"15min":1.4450000000000001},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":18.303000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.488,"5min":2.2280000000000002,"15min":1.5920000000000001},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":18.036000000000001,"5min":18.036000000000001,"15min":18.036000000000001},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.9950000000000001,"15min":1.327},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.6689999999999996,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":0.376},{"interface":"front","average":{"1min":1.9950000000000001,"5min":1.9450000000000001,"15min":1.252},"min":{"1min":0.28100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.778,"5min":11.778,"15min":11.778},"last":1.5249999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":3.105,"5min":2.5619999999999998,"15min":1.6950000000000001},"min":{"1min":0.38900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":14.323,"5min":14.323,"15min":14.323},"last":1.5029999999999999},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.847,"15min":1.625},"min":{"1min":0.379,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":25.550000000000001,"5min":25.550000000000001,"15min":25.550000000000001},"last":0.32200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.198,"15min":0.88500000000000001},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.7509999999999999,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":2.8439999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.72399999999999998,"15min":0.67400000000000004},"min":{"1min":0.29999999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.6689999999999996,"5min":5.7549999999999999,"15min":5.7549999999999999},"last":4.8719999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0620000000000001,"15min":0.70399999999999996},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.6449999999999996,"5min":6.851,"15min":6.851},"last":1.4830000000000001},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.1699999999999999,"15min":0.77900000000000003},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.9080000000000004,"5min":12.255000000000001,"15min":12.255000000000001},"last":2.8010000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.397,"15min":0.879},"min":{"1min":0.36799999999999999,"5min":0.312,"15min":0.312},"max":{"1min":6.6630000000000003,"5min":12.237,"15min":12.237},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.2350000000000001,"15min":0.96399999999999997},"min":{"1min":0.33300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.3879999999999999,"5min":14.964,"15min":14.964},"last":0.53700000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822994,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3591176,"kb_used_data":3001472,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":90153976,"statfs":{"total":95995035648,"available":92317671424,"internally_reserved":0,"allocated":3073507328,"data_stored":3072979308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:36 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.845,"15min":1.7689999999999999},"min":{"1min":0.23300000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.432,"5min":10.432,"15min":15.773999999999999},"last":2.0630000000000002},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.6040000000000001,"15min":1.796},"min":{"1min":0.29499999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.8339999999999996,"5min":8.4939999999999998,"15min":15.699},"last":0.35699999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:14:19 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":2.3580000000000001,"15min":1.9219999999999999},"min":{"1min":0.251,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":6.274,"5min":11.891,"15min":15.689},"last":2.1299999999999999},{"interface":"front","average":{"1min":2.153,"5min":2.0089999999999999,"15min":2},"min":{"1min":0.33900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":23.099,"5min":23.099,"15min":23.099},"last":0.32300000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":2.1859999999999999,"15min":2.3479999999999999},"min":{"1min":0.29999999999999999,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.5030000000000001,"5min":10.259,"15min":15.640000000000001},"last":2.1520000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.4550000000000001,"15min":2.0179999999999998},"min":{"1min":0.36299999999999999,"5min":0.32200000000000001,"15min":0.313},"max":{"1min":3.6640000000000001,"5min":6.5529999999999999,"15min":10.656000000000001},"last":0.40500000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":2.3109999999999999,"15min":2.0030000000000001},"min":{"1min":0.42399999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.1059999999999999,"5min":9.8699999999999992,"15min":9.8699999999999992},"last":0.373},{"interface":"front","average":{"1min":0.80100000000000005,"5min":2.2629999999999999,"15min":2.7669999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9830000000000001,"5min":17.193999999999999,"15min":17.34},"last":0.379}]},{"osd":4,"last update":"Mon Oct 9 10:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.0760000000000001,"15min":0.73399999999999999},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.2679999999999998,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.72199999999999998,"15min":0.73599999999999999},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.3599999999999999,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":1.7669999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:13:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.2070000000000001,"15min":1.1180000000000001},"min":{"1min":0.33200000000000002,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":4.0380000000000003,"5min":9.5519999999999996,"15min":27.716000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.2450000000000001,"15min":1.024},"min":{"1min":0.34000000000000002,"5min":0.26300000000000001,"15min":0.22600000000000001},"max":{"1min":3.448,"5min":8.2910000000000004,"15min":18.600999999999999},"last":2.093}]},{"osd":7,"last update":"Mon Oct 9 10:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.3360000000000001,"15min":1.0860000000000001},"min":{"1min":0.35499999999999998,"5min":0.312,"15min":0.25600000000000001},"max":{"1min":3.4790000000000001,"5min":8.3409999999999993,"15min":10.334},"last":2.1099999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.81799999999999995,"15min":0.95899999999999996},"min":{"1min":0.34599999999999997,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":0.67500000000000004,"5min":8.5220000000000002,"15min":12.356},"last":0.54400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1368391680,"data_stored":1368391680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3076521984,"data_stored":3076521984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1037762560,"data_stored":1037762560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2679242752,"data_stored":2679242752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1018036224,"data_stored":1018036224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3091922944,"data_stored":3091922944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1045954560,"data_stored":1045954560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3148152832,"data_stored":3148152832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:14:37.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:14:37.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:37.934 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:38.006 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:38.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:38.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:38.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:38.141 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:14:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:14:37] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.33.4" 2023-10-09T10:14:38.210 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:38.287 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 61 16 88060 88044 90.1013 53.6875 0.0302475 0.0110938 2023-10-09T10:14:38.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:38 smithi012 ceph-mon[147926]: pgmap v941: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-10-09T10:14:38.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/868147191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:38.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:38 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:38.429 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:14:38.588 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:38.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:38 smithi012 ceph-mon[152263]: pgmap v941: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-10-09T10:14:38.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/868147191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:38.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:38 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:38.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:38 smithi110 ceph-mon[136032]: pgmap v941: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-10-09T10:14:38.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/868147191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:38.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:38 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:38.780 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:38.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:38.887 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:38.945 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:39.005 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:39.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:39.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:39.113 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:39.230 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:39.287 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 62 16 88929 88913 89.5246 54.3125 0.0217044 0.0111656 2023-10-09T10:14:39.348 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:39.367 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:39.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:39.478 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:39.500 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:39.500 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:39.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.502 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:39.502 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T10:14:39.502 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.502 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.502 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.502 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.502 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:39.502 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:39 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:39.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:39.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:39.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T10:14:39.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.541 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.541 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.541 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: failed to encode map e335 with expected crc 2023-10-09T10:14:39.541 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:39.541 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:39 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:39.541 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:39.708 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:39.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:39.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:39.877 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:39.932 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:39.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:39.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:39.981 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:39.982 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:40.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:40.138 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:40.290 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 63 16 89889 89873 89.0564 60 0.0145184 0.0112243 2023-10-09T10:14:40.418 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:40 smithi110 ceph-mon[136032]: pgmap v943: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T10:14:40.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:14:40.600 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:14:40.600 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:14:40.600 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:14:40.600 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:14:40.619 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:40.628 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:14:40.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:40.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:40.664 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:14:40.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:40 smithi012 ceph-mon[147926]: pgmap v943: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T10:14:40.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:40 smithi012 ceph-mon[152263]: pgmap v943: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T10:14:40.698 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:14:40.730 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:40.736 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:14:40.741 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:14:40.741 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:14:40.741 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:14:40.741 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:14:40.741 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:14:40.742 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:14:40.742 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:14:40.742 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:14:40.814 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:40.862 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:40.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:40.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:14:40.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:41.106 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:41.201 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:41.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:41.237 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:14:41.287 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 64 16 90770 90754 88.5256 55.0625 0.0247097 0.0112907 2023-10-09T10:14:41.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:41.353 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:41.424 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:41.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:41.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:41.529 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:41.581 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:41.600 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/777923274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:41.600 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:41 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:41.600 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:41 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:41.646 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:41.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:41.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/777923274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:41 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:41 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/777923274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:41 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:41 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:41.755 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:41.800 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:42.102 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:42.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:42.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:42.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:42.292 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:42.292 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 65 16 91661 91645 88.0138 55.6875 0.013193 0.0113567 2023-10-09T10:14:42.309 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:42.428 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:42 smithi110 ceph-mon[136032]: pgmap v944: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T10:14:42.428 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:14:42.428 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:14:42.428 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:42 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:42.469 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:42.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:42.579 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:42.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:42 smithi012 ceph-mon[147926]: pgmap v944: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T10:14:42.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:14:42.608 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:14:42.608 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:42 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:42.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:42 smithi012 ceph-mon[152263]: pgmap v944: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T10:14:42.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:14:42.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:14:42.609 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:42 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:42.618 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:42.766 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:14:42.785 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:14:43.292 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 66 16 92541 92525 87.5141 55 0.0161909 0.0114218 2023-10-09T10:14:43.403 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":944,"stamp":"2023-10-09T10:14:41.495135+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5745743776,"num_objects":87688,"num_object_clones":0,"num_object_copies":263064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87688,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":87924,"num_write_kb":5612941,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":1190604,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87858,"ondisk_log_size":87858,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20970440,"kb_used_data":17240072,"kb_used_omap":167,"kb_used_meta":3730136,"kb_avail":728990776,"statfs":{"total":767960285184,"available":746486554624,"internally_reserved":0,"allocated":17653833728,"data_stored":17646992520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":3819659865},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":622002176,"num_objects":9491,"num_object_clones":0,"num_object_copies":28473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9491,"num_write_kb":607424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002751"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":178,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.336943+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:14:30.336943+0000","last_peered":"2023-10-09T10:14:30.336943+0000","last_clean":"2023-10-09T10:14:30.336943+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:14:30.336943+0000","last_undegraded":"2023-10-09T10:14:30.336943+0000","last_fullsized":"2023-10-09T10:14:30.336943+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":292,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304160+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:14:28.304160+0000","last_peered":"2023-10-09T10:14:28.304160+0000","last_clean":"2023-10-09T10:14:28.304160+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:14:28.304160+0000","last_undegraded":"2023-10-09T10:14:28.304160+0000","last_fullsized":"2023-10-09T10:14:28.304160+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":339,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.336465+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:14:30.336465+0000","last_peered":"2023-10-09T10:14:30.336465+0000","last_clean":"2023-10-09T10:14:30.336465+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:14:30.336465+0000","last_undegraded":"2023-10-09T10:14:30.336465+0000","last_fullsized":"2023-10-09T10:14:30.336465+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":202,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.347867+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:14:30.347867+0000","last_peered":"2023-10-09T10:14:30.347867+0000","last_clean":"2023-10-09T10:14:30.347867+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:14:30.347867+0000","last_undegraded":"2023-10-09T10:14:30.347867+0000","last_fullsized":"2023-10-09T10:14:30.347867+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":306,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305808+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:14:28.305808+0000","last_peered":"2023-10-09T10:14:28.305808+0000","last_clean":"2023-10-09T10:14:28.305808+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:14:28.305808+0000","last_undegraded":"2023-10-09T10:14:28.305808+0000","last_fullsized":"2023-10-09T10:14:28.305808+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":152,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.331628+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:14:28.331628+0000","last_peered":"2023-10-09T10:14:28.331628+0000","last_clean":"2023-10-09T10:14:28.331628+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:14:28.331628+0000","last_undegraded":"2023-10-09T10:14:28.331628+0000","last_fullsized":"2023-10-09T10:14:28.331628+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":219,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.348404+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:14:30.348404+0000","last_peered":"2023-10-09T10:14:30.348404+0000","last_clean":"2023-10-09T10:14:30.348404+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:14:30.348404+0000","last_undegraded":"2023-10-09T10:14:30.348404+0000","last_fullsized":"2023-10-09T10:14:30.348404+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":287,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305709+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:14:28.305709+0000","last_peered":"2023-10-09T10:14:28.305709+0000","last_clean":"2023-10-09T10:14:28.305709+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:14:28.305709+0000","last_undegraded":"2023-10-09T10:14:28.305709+0000","last_fullsized":"2023-10-09T10:14:28.305709+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":151,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325267+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:14:28.325267+0000","last_peered":"2023-10-09T10:14:28.325267+0000","last_clean":"2023-10-09T10:14:28.325267+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:14:28.325267+0000","last_undegraded":"2023-10-09T10:14:28.325267+0000","last_fullsized":"2023-10-09T10:14:28.325267+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":248,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306763+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:14:28.306763+0000","last_peered":"2023-10-09T10:14:28.306763+0000","last_clean":"2023-10-09T10:14:28.306763+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:14:28.306763+0000","last_undegraded":"2023-10-09T10:14:28.306763+0000","last_fullsized":"2023-10-09T10:14:28.306763+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":373,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.341296+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:14:30.341296+0000","last_peered":"2023-10-09T10:14:30.341296+0000","last_clean":"2023-10-09T10:14:30.341296+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:14:30.341296+0000","last_undegraded":"2023-10-09T10:14:30.341296+0000","last_fullsized":"2023-10-09T10:14:30.341296+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":331,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.336785+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:14:30.336785+0000","last_peered":"2023-10-09T10:14:30.336785+0000","last_clean":"2023-10-09T10:14:30.336785+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:14:30.336785+0000","last_undegraded":"2023-10-09T10:14:30.336785+0000","last_fullsized":"2023-10-09T10:14:30.336785+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":222,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.340464+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:14:30.340464+0000","last_peered":"2023-10-09T10:14:30.340464+0000","last_clean":"2023-10-09T10:14:30.340464+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:14:30.340464+0000","last_undegraded":"2023-10-09T10:14:30.340464+0000","last_fullsized":"2023-10-09T10:14:30.340464+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":203,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.337078+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:14:30.337078+0000","last_peered":"2023-10-09T10:14:30.337078+0000","last_clean":"2023-10-09T10:14:30.337078+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:14:30.337078+0000","last_undegraded":"2023-10-09T10:14:30.337078+0000","last_fullsized":"2023-10-09T10:14:30.337078+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":406,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.340292+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:14:30.340292+0000","last_peered":"2023-10-09T10:14:30.340292+0000","last_clean":"2023-10-09T10:14:30.340292+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:14:30.340292+0000","last_undegraded":"2023-10-09T10:14:30.340292+0000","last_fullsized":"2023-10-09T10:14:30.340292+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":274,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.310915+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:14:28.310915+0000","last_peered":"2023-10-09T10:14:28.310915+0000","last_clean":"2023-10-09T10:14:28.310915+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:14:28.310915+0000","last_undegraded":"2023-10-09T10:14:28.310915+0000","last_fullsized":"2023-10-09T10:14:28.310915+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":201,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303485+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:14:28.303485+0000","last_peered":"2023-10-09T10:14:28.303485+0000","last_clean":"2023-10-09T10:14:28.303485+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:14:28.303485+0000","last_undegraded":"2023-10-09T10:14:28.303485+0000","last_fullsized":"2023-10-09T10:14:28.303485+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":167,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325418+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:14:28.325418+0000","last_peered":"2023-10-09T10:14:28.325418+0000","last_clean":"2023-10-09T10:14:28.325418+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:14:28.325418+0000","last_undegraded":"2023-10-09T10:14:28.325418+0000","last_fullsized":"2023-10-09T10:14:28.325418+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":203,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.340712+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:14:30.340712+0000","last_peered":"2023-10-09T10:14:30.340712+0000","last_clean":"2023-10-09T10:14:30.340712+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:14:30.340712+0000","last_undegraded":"2023-10-09T10:14:30.340712+0000","last_fullsized":"2023-10-09T10:14:30.340712+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":201,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.347970+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:14:30.347970+0000","last_peered":"2023-10-09T10:14:30.347970+0000","last_clean":"2023-10-09T10:14:30.347970+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:14:30.347970+0000","last_undegraded":"2023-10-09T10:14:30.347970+0000","last_fullsized":"2023-10-09T10:14:30.347970+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":224,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303445+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:14:28.303445+0000","last_peered":"2023-10-09T10:14:28.303445+0000","last_clean":"2023-10-09T10:14:28.303445+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:14:28.303445+0000","last_undegraded":"2023-10-09T10:14:28.303445+0000","last_fullsized":"2023-10-09T10:14:28.303445+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":287,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305372+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:14:28.305372+0000","last_peered":"2023-10-09T10:14:28.305372+0000","last_clean":"2023-10-09T10:14:28.305372+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:14:28.305372+0000","last_undegraded":"2023-10-09T10:14:28.305372+0000","last_fullsized":"2023-10-09T10:14:28.305372+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"334'5417","reported_seq":5443,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:36.893480+0000","last_change":"2023-10-09T10:13:32.575493+0000","last_active":"2023-10-09T10:14:36.893480+0000","last_peered":"2023-10-09T10:14:36.893480+0000","last_clean":"2023-10-09T10:14:36.893480+0000","last_became_active":"2023-10-09T10:13:32.575194+0000","last_became_peered":"2023-10-09T10:13:32.575194+0000","last_unstale":"2023-10-09T10:14:36.893480+0000","last_undegraded":"2023-10-09T10:14:36.893480+0000","last_fullsized":"2023-10-09T10:14:36.893480+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5417,"log_dups_size":0,"ondisk_log_size":5417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:21.292749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355008512,"num_objects":5417,"num_object_clones":0,"num_object_copies":16251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5417,"num_write_kb":346688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":367,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.340248+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:14:30.340248+0000","last_peered":"2023-10-09T10:14:30.340248+0000","last_clean":"2023-10-09T10:14:30.340248+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:14:30.340248+0000","last_undegraded":"2023-10-09T10:14:30.340248+0000","last_fullsized":"2023-10-09T10:14:30.340248+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":152,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.323896+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:14:28.323896+0000","last_peered":"2023-10-09T10:14:28.323896+0000","last_clean":"2023-10-09T10:14:28.323896+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:14:28.323896+0000","last_undegraded":"2023-10-09T10:14:28.323896+0000","last_fullsized":"2023-10-09T10:14:28.323896+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"335'5466","reported_seq":5490,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.756716+0000","last_change":"2023-10-09T10:13:32.571424+0000","last_active":"2023-10-09T10:14:38.756716+0000","last_peered":"2023-10-09T10:14:38.756716+0000","last_clean":"2023-10-09T10:14:38.756716+0000","last_became_active":"2023-10-09T10:13:32.571091+0000","last_became_peered":"2023-10-09T10:13:32.571091+0000","last_unstale":"2023-10-09T10:14:38.756716+0000","last_undegraded":"2023-10-09T10:14:38.756716+0000","last_fullsized":"2023-10-09T10:14:38.756716+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:33.565620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358219776,"num_objects":5466,"num_object_clones":0,"num_object_copies":16398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5466,"num_write_kb":349824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":355,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.340895+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:14:30.340895+0000","last_peered":"2023-10-09T10:14:30.340895+0000","last_clean":"2023-10-09T10:14:30.340895+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:14:30.340895+0000","last_undegraded":"2023-10-09T10:14:30.340895+0000","last_fullsized":"2023-10-09T10:14:30.340895+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":247,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.336353+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:14:30.336353+0000","last_peered":"2023-10-09T10:14:30.336353+0000","last_clean":"2023-10-09T10:14:30.336353+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:14:30.336353+0000","last_undegraded":"2023-10-09T10:14:30.336353+0000","last_fullsized":"2023-10-09T10:14:30.336353+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"334'5430","reported_seq":5455,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.067054+0000","last_change":"2023-10-09T10:13:32.575491+0000","last_active":"2023-10-09T10:14:38.067054+0000","last_peered":"2023-10-09T10:14:38.067054+0000","last_clean":"2023-10-09T10:14:38.067054+0000","last_became_active":"2023-10-09T10:13:32.575164+0000","last_became_peered":"2023-10-09T10:13:32.575164+0000","last_unstale":"2023-10-09T10:14:38.067054+0000","last_undegraded":"2023-10-09T10:14:38.067054+0000","last_fullsized":"2023-10-09T10:14:38.067054+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5430,"log_dups_size":0,"ondisk_log_size":5430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355860480,"num_objects":5430,"num_object_clones":0,"num_object_copies":16290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5430,"num_write_kb":347520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":156,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325560+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:14:28.325560+0000","last_peered":"2023-10-09T10:14:28.325560+0000","last_clean":"2023-10-09T10:14:28.325560+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:14:28.325560+0000","last_undegraded":"2023-10-09T10:14:28.325560+0000","last_fullsized":"2023-10-09T10:14:28.325560+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":178,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.346717+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:14:30.346717+0000","last_peered":"2023-10-09T10:14:30.346717+0000","last_clean":"2023-10-09T10:14:30.346717+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:14:30.346717+0000","last_undegraded":"2023-10-09T10:14:30.346717+0000","last_fullsized":"2023-10-09T10:14:30.346717+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"334'5544","reported_seq":5569,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.069266+0000","last_change":"2023-10-09T10:13:32.857374+0000","last_active":"2023-10-09T10:14:38.069266+0000","last_peered":"2023-10-09T10:14:38.069266+0000","last_clean":"2023-10-09T10:14:38.069266+0000","last_became_active":"2023-10-09T10:13:32.857082+0000","last_became_peered":"2023-10-09T10:13:32.857082+0000","last_unstale":"2023-10-09T10:14:38.069266+0000","last_undegraded":"2023-10-09T10:14:38.069266+0000","last_fullsized":"2023-10-09T10:14:38.069266+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5544,"log_dups_size":0,"ondisk_log_size":5544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363331584,"num_objects":5544,"num_object_clones":0,"num_object_copies":16632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5544,"num_write_kb":354816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":167,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325352+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:14:28.325352+0000","last_peered":"2023-10-09T10:14:28.325352+0000","last_clean":"2023-10-09T10:14:28.325352+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:14:28.325352+0000","last_undegraded":"2023-10-09T10:14:28.325352+0000","last_fullsized":"2023-10-09T10:14:28.325352+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":202,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303824+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:14:28.303824+0000","last_peered":"2023-10-09T10:14:28.303824+0000","last_clean":"2023-10-09T10:14:28.303824+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:14:28.303824+0000","last_undegraded":"2023-10-09T10:14:28.303824+0000","last_fullsized":"2023-10-09T10:14:28.303824+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"334'5503","reported_seq":5529,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:36.593553+0000","last_change":"2023-10-09T10:13:32.695943+0000","last_active":"2023-10-09T10:14:36.593553+0000","last_peered":"2023-10-09T10:14:36.593553+0000","last_clean":"2023-10-09T10:14:36.593553+0000","last_became_active":"2023-10-09T10:13:32.695608+0000","last_became_peered":"2023-10-09T10:13:32.695608+0000","last_unstale":"2023-10-09T10:14:36.593553+0000","last_undegraded":"2023-10-09T10:14:36.593553+0000","last_fullsized":"2023-10-09T10:14:36.593553+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:05.782965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360644608,"num_objects":5503,"num_object_clones":0,"num_object_copies":16509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5503,"num_write_kb":352192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":329,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304820+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:14:28.304820+0000","last_peered":"2023-10-09T10:14:28.304820+0000","last_clean":"2023-10-09T10:14:28.304820+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:14:28.304820+0000","last_undegraded":"2023-10-09T10:14:28.304820+0000","last_fullsized":"2023-10-09T10:14:28.304820+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":199,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.310743+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:14:28.310743+0000","last_peered":"2023-10-09T10:14:28.310743+0000","last_clean":"2023-10-09T10:14:28.310743+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:14:28.310743+0000","last_undegraded":"2023-10-09T10:14:28.310743+0000","last_fullsized":"2023-10-09T10:14:28.310743+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"335'5466","reported_seq":5493,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.749684+0000","last_change":"2023-10-09T10:13:32.857224+0000","last_active":"2023-10-09T10:14:38.749684+0000","last_peered":"2023-10-09T10:14:38.749684+0000","last_clean":"2023-10-09T10:14:38.749684+0000","last_became_active":"2023-10-09T10:13:32.856912+0000","last_became_peered":"2023-10-09T10:13:32.856912+0000","last_unstale":"2023-10-09T10:14:38.749684+0000","last_undegraded":"2023-10-09T10:14:38.749684+0000","last_fullsized":"2023-10-09T10:14:38.749684+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:25.963375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358219776,"num_objects":5466,"num_object_clones":0,"num_object_copies":16398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5466,"num_write_kb":349824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":261,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.336706+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:14:30.336706+0000","last_peered":"2023-10-09T10:14:30.336706+0000","last_clean":"2023-10-09T10:14:30.336706+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:14:30.336706+0000","last_undegraded":"2023-10-09T10:14:30.336706+0000","last_fullsized":"2023-10-09T10:14:30.336706+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":267,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.337074+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:14:30.337074+0000","last_peered":"2023-10-09T10:14:30.337074+0000","last_clean":"2023-10-09T10:14:30.337074+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:14:30.337074+0000","last_undegraded":"2023-10-09T10:14:30.337074+0000","last_fullsized":"2023-10-09T10:14:30.337074+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"335'5545","reported_seq":5571,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:40.149581+0000","last_change":"2023-10-09T10:13:32.577948+0000","last_active":"2023-10-09T10:14:40.149581+0000","last_peered":"2023-10-09T10:14:40.149581+0000","last_clean":"2023-10-09T10:14:40.149581+0000","last_became_active":"2023-10-09T10:13:32.577597+0000","last_became_peered":"2023-10-09T10:13:32.577597+0000","last_unstale":"2023-10-09T10:14:40.149581+0000","last_undegraded":"2023-10-09T10:14:40.149581+0000","last_fullsized":"2023-10-09T10:14:40.149581+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363397120,"num_objects":5545,"num_object_clones":0,"num_object_copies":16635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5545,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":229,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.340640+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:14:30.340640+0000","last_peered":"2023-10-09T10:14:30.340640+0000","last_clean":"2023-10-09T10:14:30.340640+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:14:30.340640+0000","last_undegraded":"2023-10-09T10:14:30.340640+0000","last_fullsized":"2023-10-09T10:14:30.340640+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":203,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.338911+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:14:30.338911+0000","last_peered":"2023-10-09T10:14:30.338911+0000","last_clean":"2023-10-09T10:14:30.338911+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:14:30.338911+0000","last_undegraded":"2023-10-09T10:14:30.338911+0000","last_fullsized":"2023-10-09T10:14:30.338911+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"334'5326","reported_seq":5352,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:36.593486+0000","last_change":"2023-10-09T10:13:32.857205+0000","last_active":"2023-10-09T10:14:36.593486+0000","last_peered":"2023-10-09T10:14:36.593486+0000","last_clean":"2023-10-09T10:14:36.593486+0000","last_became_active":"2023-10-09T10:13:32.856928+0000","last_became_peered":"2023-10-09T10:13:32.856928+0000","last_unstale":"2023-10-09T10:14:36.593486+0000","last_undegraded":"2023-10-09T10:14:36.593486+0000","last_fullsized":"2023-10-09T10:14:36.593486+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5326,"log_dups_size":0,"ondisk_log_size":5326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:43:02.753960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349044736,"num_objects":5326,"num_object_clones":0,"num_object_copies":15978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5326,"num_write_kb":340864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":289,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305607+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:14:28.305607+0000","last_peered":"2023-10-09T10:14:28.305607+0000","last_clean":"2023-10-09T10:14:28.305607+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:14:28.305607+0000","last_undegraded":"2023-10-09T10:14:28.305607+0000","last_fullsized":"2023-10-09T10:14:28.305607+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":339,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.337568+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:14:30.337568+0000","last_peered":"2023-10-09T10:14:30.337568+0000","last_clean":"2023-10-09T10:14:30.337568+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:14:30.337568+0000","last_undegraded":"2023-10-09T10:14:30.337568+0000","last_fullsized":"2023-10-09T10:14:30.337568+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"334'5541","reported_seq":5565,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.056287+0000","last_change":"2023-10-09T10:13:32.573670+0000","last_active":"2023-10-09T10:14:38.056287+0000","last_peered":"2023-10-09T10:14:38.056287+0000","last_clean":"2023-10-09T10:14:38.056287+0000","last_became_active":"2023-10-09T10:13:32.573356+0000","last_became_peered":"2023-10-09T10:13:32.573356+0000","last_unstale":"2023-10-09T10:14:38.056287+0000","last_undegraded":"2023-10-09T10:14:38.056287+0000","last_fullsized":"2023-10-09T10:14:38.056287+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5541,"log_dups_size":0,"ondisk_log_size":5541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363134976,"num_objects":5541,"num_object_clones":0,"num_object_copies":16623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5541,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":339,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.339016+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:14:30.339016+0000","last_peered":"2023-10-09T10:14:30.339016+0000","last_clean":"2023-10-09T10:14:30.339016+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:14:30.339016+0000","last_undegraded":"2023-10-09T10:14:30.339016+0000","last_fullsized":"2023-10-09T10:14:30.339016+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325300+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:14:28.325300+0000","last_peered":"2023-10-09T10:14:28.325300+0000","last_clean":"2023-10-09T10:14:28.325300+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:14:28.325300+0000","last_undegraded":"2023-10-09T10:14:28.325300+0000","last_fullsized":"2023-10-09T10:14:28.325300+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":295,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.337034+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:14:30.337034+0000","last_peered":"2023-10-09T10:14:30.337034+0000","last_clean":"2023-10-09T10:14:30.337034+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:14:30.337034+0000","last_undegraded":"2023-10-09T10:14:30.337034+0000","last_fullsized":"2023-10-09T10:14:30.337034+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"334'5614","reported_seq":5640,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:36.873167+0000","last_change":"2023-10-09T10:13:32.576627+0000","last_active":"2023-10-09T10:14:36.873167+0000","last_peered":"2023-10-09T10:14:36.873167+0000","last_clean":"2023-10-09T10:14:36.873167+0000","last_became_active":"2023-10-09T10:13:32.576321+0000","last_became_peered":"2023-10-09T10:13:32.576321+0000","last_unstale":"2023-10-09T10:14:36.873167+0000","last_undegraded":"2023-10-09T10:14:36.873167+0000","last_fullsized":"2023-10-09T10:14:36.873167+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5614,"log_dups_size":0,"ondisk_log_size":5614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:04.882671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367919104,"num_objects":5614,"num_object_clones":0,"num_object_copies":16842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5614,"num_write_kb":359296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":152,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324942+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:14:28.324942+0000","last_peered":"2023-10-09T10:14:28.324942+0000","last_clean":"2023-10-09T10:14:28.324942+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:14:28.324942+0000","last_undegraded":"2023-10-09T10:14:28.324942+0000","last_fullsized":"2023-10-09T10:14:28.324942+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":152,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.342787+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:14:30.342787+0000","last_peered":"2023-10-09T10:14:30.342787+0000","last_clean":"2023-10-09T10:14:30.342787+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:14:30.342787+0000","last_undegraded":"2023-10-09T10:14:30.342787+0000","last_fullsized":"2023-10-09T10:14:30.342787+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":362,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.338932+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:14:30.338932+0000","last_peered":"2023-10-09T10:14:30.338932+0000","last_clean":"2023-10-09T10:14:30.338932+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:14:30.338932+0000","last_undegraded":"2023-10-09T10:14:30.338932+0000","last_fullsized":"2023-10-09T10:14:30.338932+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"334'5297","reported_seq":5322,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.040547+0000","last_change":"2023-10-09T10:13:32.574493+0000","last_active":"2023-10-09T10:14:38.040547+0000","last_peered":"2023-10-09T10:14:38.040547+0000","last_clean":"2023-10-09T10:14:38.040547+0000","last_became_active":"2023-10-09T10:13:32.574198+0000","last_became_peered":"2023-10-09T10:13:32.574198+0000","last_unstale":"2023-10-09T10:14:38.040547+0000","last_undegraded":"2023-10-09T10:14:38.040547+0000","last_fullsized":"2023-10-09T10:14:38.040547+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5297,"log_dups_size":0,"ondisk_log_size":5297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347144192,"num_objects":5297,"num_object_clones":0,"num_object_copies":15891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5297,"num_write_kb":339008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":255,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306632+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:14:28.306632+0000","last_peered":"2023-10-09T10:14:28.306632+0000","last_clean":"2023-10-09T10:14:28.306632+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:14:28.306632+0000","last_undegraded":"2023-10-09T10:14:28.306632+0000","last_fullsized":"2023-10-09T10:14:28.306632+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":331,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.337433+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:14:30.337433+0000","last_peered":"2023-10-09T10:14:30.337433+0000","last_clean":"2023-10-09T10:14:30.337433+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:14:30.337433+0000","last_undegraded":"2023-10-09T10:14:30.337433+0000","last_fullsized":"2023-10-09T10:14:30.337433+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":352,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.337469+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:14:30.337469+0000","last_peered":"2023-10-09T10:14:30.337469+0000","last_clean":"2023-10-09T10:14:30.337469+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:14:30.337469+0000","last_undegraded":"2023-10-09T10:14:30.337469+0000","last_fullsized":"2023-10-09T10:14:30.337469+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"334'5589","reported_seq":5615,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:36.549895+0000","last_change":"2023-10-09T10:13:32.572705+0000","last_active":"2023-10-09T10:14:36.549895+0000","last_peered":"2023-10-09T10:14:36.549895+0000","last_clean":"2023-10-09T10:14:36.549895+0000","last_became_active":"2023-10-09T10:13:32.571805+0000","last_became_peered":"2023-10-09T10:13:32.571805+0000","last_unstale":"2023-10-09T10:14:36.549895+0000","last_undegraded":"2023-10-09T10:14:36.549895+0000","last_fullsized":"2023-10-09T10:14:36.549895+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5589,"log_dups_size":0,"ondisk_log_size":5589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:13.430092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366280704,"num_objects":5589,"num_object_clones":0,"num_object_copies":16767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5589,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":161,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325082+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:14:28.325082+0000","last_peered":"2023-10-09T10:14:28.325082+0000","last_clean":"2023-10-09T10:14:28.325082+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:14:28.325082+0000","last_undegraded":"2023-10-09T10:14:28.325082+0000","last_fullsized":"2023-10-09T10:14:28.325082+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":581,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.339833+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:14:30.339833+0000","last_peered":"2023-10-09T10:14:30.339833+0000","last_clean":"2023-10-09T10:14:30.339833+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:14:30.339833+0000","last_undegraded":"2023-10-09T10:14:30.339833+0000","last_fullsized":"2023-10-09T10:14:30.339833+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":239,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306710+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:14:28.306710+0000","last_peered":"2023-10-09T10:14:28.306710+0000","last_clean":"2023-10-09T10:14:28.306710+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:14:28.306710+0000","last_undegraded":"2023-10-09T10:14:28.306710+0000","last_fullsized":"2023-10-09T10:14:28.306710+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":250,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.341367+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:14:30.341367+0000","last_peered":"2023-10-09T10:14:30.341367+0000","last_clean":"2023-10-09T10:14:30.341367+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:14:30.341367+0000","last_undegraded":"2023-10-09T10:14:30.341367+0000","last_fullsized":"2023-10-09T10:14:30.341367+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"334'5424","reported_seq":5450,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:36.899984+0000","last_change":"2023-10-09T10:13:32.694210+0000","last_active":"2023-10-09T10:14:36.899984+0000","last_peered":"2023-10-09T10:14:36.899984+0000","last_clean":"2023-10-09T10:14:36.899984+0000","last_became_active":"2023-10-09T10:13:32.693883+0000","last_became_peered":"2023-10-09T10:13:32.693883+0000","last_unstale":"2023-10-09T10:14:36.899984+0000","last_undegraded":"2023-10-09T10:14:36.899984+0000","last_fullsized":"2023-10-09T10:14:36.899984+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5424,"log_dups_size":0,"ondisk_log_size":5424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:35.108481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355467264,"num_objects":5424,"num_object_clones":0,"num_object_copies":16272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5424,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":312,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.337561+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:14:30.337561+0000","last_peered":"2023-10-09T10:14:30.337561+0000","last_clean":"2023-10-09T10:14:30.337561+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:14:30.337561+0000","last_undegraded":"2023-10-09T10:14:30.337561+0000","last_fullsized":"2023-10-09T10:14:30.337561+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":212,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303707+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:14:28.303707+0000","last_peered":"2023-10-09T10:14:28.303707+0000","last_clean":"2023-10-09T10:14:28.303707+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:14:28.303707+0000","last_undegraded":"2023-10-09T10:14:28.303707+0000","last_fullsized":"2023-10-09T10:14:28.303707+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":240,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303807+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:14:28.303807+0000","last_peered":"2023-10-09T10:14:28.303807+0000","last_clean":"2023-10-09T10:14:28.303807+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:14:28.303807+0000","last_undegraded":"2023-10-09T10:14:28.303807+0000","last_fullsized":"2023-10-09T10:14:28.303807+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"335'5485","reported_seq":5512,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.756014+0000","last_change":"2023-10-09T10:13:32.576814+0000","last_active":"2023-10-09T10:14:38.756014+0000","last_peered":"2023-10-09T10:14:38.756014+0000","last_clean":"2023-10-09T10:14:38.756014+0000","last_became_active":"2023-10-09T10:13:32.576567+0000","last_became_peered":"2023-10-09T10:13:32.576567+0000","last_unstale":"2023-10-09T10:14:38.756014+0000","last_undegraded":"2023-10-09T10:14:38.756014+0000","last_fullsized":"2023-10-09T10:14:38.756014+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5485,"log_dups_size":0,"ondisk_log_size":5485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:37.032579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359464960,"num_objects":5485,"num_object_clones":0,"num_object_copies":16455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5485,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":354,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.341284+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:14:30.341284+0000","last_peered":"2023-10-09T10:14:30.341284+0000","last_clean":"2023-10-09T10:14:30.341284+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:14:30.341284+0000","last_undegraded":"2023-10-09T10:14:30.341284+0000","last_fullsized":"2023-10-09T10:14:30.341284+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":368,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.339062+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:14:30.339062+0000","last_peered":"2023-10-09T10:14:30.339062+0000","last_clean":"2023-10-09T10:14:30.339062+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:14:30.339062+0000","last_undegraded":"2023-10-09T10:14:30.339062+0000","last_fullsized":"2023-10-09T10:14:30.339062+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":204,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303732+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:14:28.303732+0000","last_peered":"2023-10-09T10:14:28.303732+0000","last_clean":"2023-10-09T10:14:28.303732+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:14:28.303732+0000","last_undegraded":"2023-10-09T10:14:28.303732+0000","last_fullsized":"2023-10-09T10:14:28.303732+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"335'5614","reported_seq":5641,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.740125+0000","last_change":"2023-10-09T10:13:32.575359+0000","last_active":"2023-10-09T10:14:38.740125+0000","last_peered":"2023-10-09T10:14:38.740125+0000","last_clean":"2023-10-09T10:14:38.740125+0000","last_became_active":"2023-10-09T10:13:32.575050+0000","last_became_peered":"2023-10-09T10:13:32.575050+0000","last_unstale":"2023-10-09T10:14:38.740125+0000","last_undegraded":"2023-10-09T10:14:38.740125+0000","last_fullsized":"2023-10-09T10:14:38.740125+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5614,"log_dups_size":0,"ondisk_log_size":5614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:15:48.495574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367919104,"num_objects":5614,"num_object_clones":0,"num_object_copies":16842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5614,"num_write_kb":359296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":298,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305477+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:14:28.305477+0000","last_peered":"2023-10-09T10:14:28.305477+0000","last_clean":"2023-10-09T10:14:28.305477+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:14:28.305477+0000","last_undegraded":"2023-10-09T10:14:28.305477+0000","last_fullsized":"2023-10-09T10:14:28.305477+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":313,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.338991+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:14:30.338991+0000","last_peered":"2023-10-09T10:14:30.338991+0000","last_clean":"2023-10-09T10:14:30.338991+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:14:30.338991+0000","last_undegraded":"2023-10-09T10:14:30.338991+0000","last_fullsized":"2023-10-09T10:14:30.338991+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":287,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.305497+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:14:28.305497+0000","last_peered":"2023-10-09T10:14:28.305497+0000","last_clean":"2023-10-09T10:14:28.305497+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:14:28.305497+0000","last_undegraded":"2023-10-09T10:14:28.305497+0000","last_fullsized":"2023-10-09T10:14:28.305497+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"334'5403","reported_seq":5429,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:36.593711+0000","last_change":"2023-10-09T10:13:32.577001+0000","last_active":"2023-10-09T10:14:36.593711+0000","last_peered":"2023-10-09T10:14:36.593711+0000","last_clean":"2023-10-09T10:14:36.593711+0000","last_became_active":"2023-10-09T10:13:32.576694+0000","last_became_peered":"2023-10-09T10:13:32.576694+0000","last_unstale":"2023-10-09T10:14:36.593711+0000","last_undegraded":"2023-10-09T10:14:36.593711+0000","last_fullsized":"2023-10-09T10:14:36.593711+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5403,"log_dups_size":0,"ondisk_log_size":5403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:36.747060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354091008,"num_objects":5403,"num_object_clones":0,"num_object_copies":16209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5403,"num_write_kb":345792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":243,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.347393+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:14:30.347393+0000","last_peered":"2023-10-09T10:14:30.347393+0000","last_clean":"2023-10-09T10:14:30.347393+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:14:30.347393+0000","last_undegraded":"2023-10-09T10:14:30.347393+0000","last_fullsized":"2023-10-09T10:14:30.347393+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":206,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303720+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:14:28.303720+0000","last_peered":"2023-10-09T10:14:28.303720+0000","last_clean":"2023-10-09T10:14:28.303720+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:14:28.303720+0000","last_undegraded":"2023-10-09T10:14:28.303720+0000","last_fullsized":"2023-10-09T10:14:28.303720+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":187,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.324902+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:14:28.324902+0000","last_peered":"2023-10-09T10:14:28.324902+0000","last_clean":"2023-10-09T10:14:28.324902+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:14:28.324902+0000","last_undegraded":"2023-10-09T10:14:28.324902+0000","last_fullsized":"2023-10-09T10:14:28.324902+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":200,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.303571+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:14:28.303571+0000","last_peered":"2023-10-09T10:14:28.303571+0000","last_clean":"2023-10-09T10:14:28.303571+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:14:28.303571+0000","last_undegraded":"2023-10-09T10:14:28.303571+0000","last_fullsized":"2023-10-09T10:14:28.303571+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":274,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.339123+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:14:30.339123+0000","last_peered":"2023-10-09T10:14:30.339123+0000","last_clean":"2023-10-09T10:14:30.339123+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:14:30.339123+0000","last_undegraded":"2023-10-09T10:14:30.339123+0000","last_fullsized":"2023-10-09T10:14:30.339123+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":171,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325290+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:14:28.325290+0000","last_peered":"2023-10-09T10:14:28.325290+0000","last_clean":"2023-10-09T10:14:28.325290+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:14:28.325290+0000","last_undegraded":"2023-10-09T10:14:28.325290+0000","last_fullsized":"2023-10-09T10:14:28.325290+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":213,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304802+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:14:28.304802+0000","last_peered":"2023-10-09T10:14:28.304802+0000","last_clean":"2023-10-09T10:14:28.304802+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:14:28.304802+0000","last_undegraded":"2023-10-09T10:14:28.304802+0000","last_fullsized":"2023-10-09T10:14:28.304802+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":171,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.325214+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:14:28.325214+0000","last_peered":"2023-10-09T10:14:28.325214+0000","last_clean":"2023-10-09T10:14:28.325214+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:14:28.325214+0000","last_undegraded":"2023-10-09T10:14:28.325214+0000","last_fullsized":"2023-10-09T10:14:28.325214+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":222,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306627+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:14:28.306627+0000","last_peered":"2023-10-09T10:14:28.306627+0000","last_clean":"2023-10-09T10:14:28.306627+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:14:28.306627+0000","last_undegraded":"2023-10-09T10:14:28.306627+0000","last_fullsized":"2023-10-09T10:14:28.306627+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":352,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304718+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:14:28.304718+0000","last_peered":"2023-10-09T10:14:28.304718+0000","last_clean":"2023-10-09T10:14:28.304718+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:14:28.304718+0000","last_undegraded":"2023-10-09T10:14:28.304718+0000","last_fullsized":"2023-10-09T10:14:28.304718+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":372,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.306971+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:14:28.306971+0000","last_peered":"2023-10-09T10:14:28.306971+0000","last_clean":"2023-10-09T10:14:28.306971+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:14:28.306971+0000","last_undegraded":"2023-10-09T10:14:28.306971+0000","last_fullsized":"2023-10-09T10:14:28.306971+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":282,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T10:14:28.304148+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:14:28.304148+0000","last_peered":"2023-10-09T10:14:28.304148+0000","last_clean":"2023-10-09T10:14:28.304148+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:14:28.304148+0000","last_undegraded":"2023-10-09T10:14:28.304148+0000","last_fullsized":"2023-10-09T10:14:28.304148+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":203,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T10:14:30.337215+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:14:30.337215+0000","last_peered":"2023-10-09T10:14:30.337215+0000","last_clean":"2023-10-09T10:14:30.337215+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:14:30.337215+0000","last_undegraded":"2023-10-09T10:14:30.337215+0000","last_fullsized":"2023-10-09T10:14:30.337215+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5745147904,"num_objects":87664,"num_object_clones":0,"num_object_copies":262992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87664,"num_write_kb":5610496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17359634432,"data_stored":17359634432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87664,"ondisk_log_size":87664,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430540,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3822120,"kb_used_data":3224224,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":89923032,"statfs":{"total":95995035648,"available":92081184768,"internally_reserved":0,"allocated":3301605376,"data_stored":3301057354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2269999999999999,"5min":2.472,"15min":2.1669999999999998},"min":{"1min":0.22900000000000001,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":8.5429999999999993,"5min":11.164,"15min":50.591999999999999},"last":7.806},{"interface":"front","average":{"1min":2.2130000000000001,"5min":2.2839999999999998,"15min":2.1150000000000002},"min":{"1min":0.308,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.7629999999999999,"5min":15.974,"15min":50.747},"last":0.48099999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:14:16 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":2.6840000000000002,"15min":2.1419999999999999},"min":{"1min":0.35299999999999998,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":4.6890000000000001,"5min":14.311,"15min":25.338000000000001},"last":7.827},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.6040000000000001,"15min":1.9650000000000001},"min":{"1min":0.30399999999999999,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":2.7040000000000002,"5min":16.526,"15min":18.27},"last":3.1429999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6080000000000001,"5min":2.613,"15min":2.3809999999999998},"min":{"1min":0.32100000000000001,"5min":0.27400000000000002,"15min":0.246},"max":{"1min":7.3479999999999999,"5min":13.775,"15min":13.775},"last":7.7279999999999998},{"interface":"front","average":{"1min":2.1200000000000001,"5min":2.5960000000000001,"15min":2.2490000000000001},"min":{"1min":0.38300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.6200000000000001,"5min":12.101000000000001,"15min":12.101000000000001},"last":3.528}]},{"osd":3,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":2.8759999999999999,"15min":2.125},"min":{"1min":0.46100000000000002,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":3.306,"5min":29.058,"15min":29.058},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.6579999999999999,"15min":2.1829999999999998},"min":{"1min":0.313,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":10.666,"5min":16.715,"15min":16.715},"last":4.734}]},{"osd":4,"last update":"Mon Oct 9 10:13:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":1.54,"15min":1.121},"min":{"1min":0.27500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":12.699999999999999,"5min":12.699999999999999,"15min":12.699999999999999},"last":7.7619999999999996},{"interface":"front","average":{"1min":1.744,"5min":1.306,"15min":1.0409999999999999},"min":{"1min":0.32900000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.003,"5min":12.003,"15min":12.003},"last":1.0429999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.8979999999999999,"15min":1.29},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.219},"max":{"1min":6.766,"5min":25.277999999999999,"15min":25.277999999999999},"last":7.7850000000000001},{"interface":"front","average":{"1min":0.70599999999999996,"5min":1.7190000000000001,"15min":1.2190000000000001},"min":{"1min":0.34499999999999997,"5min":0.29099999999999998,"15min":0.22600000000000001},"max":{"1min":2.6680000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":2.911}]},{"osd":6,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.401,"15min":1.1180000000000001},"min":{"1min":0.32800000000000001,"5min":0.31900000000000001,"15min":0.26100000000000001},"max":{"1min":3.085,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":2.9489999999999998},{"interface":"front","average":{"1min":1.625,"5min":1.4330000000000001,"15min":1.0920000000000001},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":10.625,"5min":10.625,"15min":10.625},"last":3.4580000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626768,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1867804,"kb_used_data":1278100,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":91877348,"statfs":{"total":95995035648,"available":94082404352,"internally_reserved":0,"allocated":1308774400,"data_stored":1308239620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":2.3370000000000002,"15min":1.8169999999999999},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.222},"max":{"1min":8.8079999999999998,"5min":15.435,"15min":15.435},"last":2.024},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.988,"15min":1.7649999999999999},"min":{"1min":0.251,"5min":0.22,"15min":0.22},"max":{"1min":4.375,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.72399999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:14:09 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":2.9089999999999998,"15min":2.226},"min":{"1min":0.34399999999999997,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.8780000000000001,"5min":35.609999999999999,"15min":35.609999999999999},"last":1.768},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.7869999999999999,"15min":2.1539999999999999},"min":{"1min":0.28499999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.2069999999999999,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.74099999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.9199999999999999,"15min":1.778},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":4.4729999999999999,"5min":12.898,"15min":12.898},"last":2.6139999999999999},{"interface":"front","average":{"1min":1.4359999999999999,"5min":2.4369999999999998,"15min":2.899},"min":{"1min":0.33100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.7669999999999995,"5min":14.542999999999999,"15min":14.542999999999999},"last":0.48499999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":2.766,"15min":1.8280000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":4.1520000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":2.6499999999999999},{"interface":"front","average":{"1min":2.1040000000000001,"5min":2.5510000000000002,"15min":2.4369999999999998},"min":{"1min":0.432,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3599999999999994,"5min":11.211,"15min":11.211},"last":5.3659999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.0089999999999999,"15min":0.629},"min":{"1min":0.29999999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.069,"5min":10.113,"15min":10.113},"last":2.6949999999999998},{"interface":"front","average":{"1min":1.101,"5min":1.0149999999999999,"15min":0.67100000000000004},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":9.218,"5min":9.218,"15min":9.218},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.1619999999999999,"15min":0.93899999999999995},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.23699999999999999},"max":{"1min":4.1210000000000004,"5min":7.9260000000000002,"15min":14.09},"last":2.6720000000000002},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.3600000000000001,"15min":1.046},"min":{"1min":0.36099999999999999,"5min":0.27500000000000002,"15min":0.254},"max":{"1min":9.4890000000000008,"5min":14.465999999999999,"15min":14.465999999999999},"last":0.53800000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:14:09 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.214,"15min":0.97499999999999998},"min":{"1min":0.434,"5min":0.32800000000000001,"15min":0.252},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":5.2839999999999998},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.254,"15min":1.02},"min":{"1min":0.46100000000000002,"5min":0.38100000000000001,"15min":0.30099999999999999},"max":{"1min":8.8550000000000004,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":5.319}]}]},{"osd":1,"up_from":84,"seq":360777253081,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3709864,"kb_used_data":3161952,"kb_used_omap":9,"kb_used_meta":547894,"kb_avail":90035288,"statfs":{"total":95995035648,"available":92196134912,"internally_reserved":0,"allocated":3237838848,"data_stored":3236700167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":561044457},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":2.6480000000000001,"15min":2.6890000000000001},"min":{"1min":0.20699999999999999,"5min":0.183,"15min":0.183},"max":{"1min":9.3040000000000003,"5min":31.007000000000001,"15min":31.007000000000001},"last":1.476},{"interface":"front","average":{"1min":2.0059999999999998,"5min":3.75,"15min":2.8759999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.192},"max":{"1min":9.5429999999999993,"5min":20.501999999999999,"15min":20.501999999999999},"last":2.5499999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:13:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.9329999999999998,"15min":3.6219999999999999},"min":{"1min":0.251,"5min":0.217,"15min":0.217},"max":{"1min":11.140000000000001,"5min":11.140000000000001,"15min":17.369},"last":0.755},{"interface":"front","average":{"1min":2.621,"5min":3.0699999999999998,"15min":3.6269999999999998},"min":{"1min":0.24199999999999999,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":28.024999999999999,"5min":28.024999999999999,"15min":28.024999999999999},"last":1.853}]},{"osd":3,"last update":"Mon Oct 9 10:13:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":3.4289999999999998,"15min":3.4990000000000001},"min":{"1min":0.28399999999999997,"5min":0.27400000000000002,"15min":0.23400000000000001},"max":{"1min":9.0069999999999997,"5min":21.808,"15min":21.808},"last":3.6349999999999998},{"interface":"front","average":{"1min":1.4339999999999999,"5min":3.1640000000000001,"15min":4.3220000000000001},"min":{"1min":0.28499999999999998,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":6.3659999999999997,"5min":21.547999999999998,"15min":21.547999999999998},"last":1.5069999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.7949999999999999,"15min":1.4350000000000001},"min":{"1min":0.34599999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.8799999999999999,"5min":21.620000000000001,"15min":21.620000000000001},"last":1.5609999999999999},{"interface":"front","average":{"1min":2.2410000000000001,"5min":2.593,"15min":1.7509999999999999},"min":{"1min":0.41799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.4030000000000005,"5min":12.904,"15min":12.904},"last":2.6349999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":2.802,"15min":2.2639999999999998},"min":{"1min":0.40000000000000002,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":4.5670000000000002,"5min":12.811,"15min":15.531000000000001},"last":2.6669999999999998},{"interface":"front","average":{"1min":2.3639999999999999,"5min":2.806,"15min":2.6139999999999999},"min":{"1min":0.443,"5min":0.32400000000000001,"15min":0.29099999999999998},"max":{"1min":9.9629999999999992,"5min":19.513000000000002,"15min":19.706},"last":0.69999999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":2.1840000000000002,"15min":2.1259999999999999},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":5.3120000000000003,"5min":21.670999999999999,"15min":21.670999999999999},"last":1.591},{"interface":"front","average":{"1min":1.5109999999999999,"5min":2.7799999999999998,"15min":2.3199999999999998},"min":{"1min":0.45800000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":4.9390000000000001,"5min":12.868,"15min":15.481999999999999},"last":2.7000000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.6970000000000001,"15min":2.589},"min":{"1min":0.48799999999999999,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":10.004,"5min":10.723000000000001,"15min":16.331},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.96799999999999997,"5min":2.1429999999999998,"15min":2.1760000000000002},"min":{"1min":0.45900000000000002,"5min":0.39200000000000002,"15min":0.378},"max":{"1min":2.8399999999999999,"5min":21.593,"15min":21.593},"last":1.8109999999999999}]}]},{"osd":0,"up_from":79,"seq":339302416607,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1820868,"kb_used_data":1408124,"kb_used_omap":28,"kb_used_meta":412707,"kb_avail":91924284,"statfs":{"total":95995035648,"available":94130466816,"internally_reserved":0,"allocated":1441918976,"data_stored":1440805841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":422612816},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.8919999999999999,"15min":2.2210000000000001},"min":{"1min":0.25900000000000001,"5min":0.216,"15min":0.16600000000000001},"max":{"1min":2.7410000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":1.236},{"interface":"front","average":{"1min":1.25,"5min":4.6740000000000004,"15min":3.4199999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.21299999999999999},"max":{"1min":7.0060000000000002,"5min":31.925999999999998,"15min":31.925999999999998},"last":0.65600000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.7010000000000001,"15min":2.3300000000000001},"min":{"1min":0.26800000000000002,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":4.9109999999999996,"5min":18.003,"15min":18.003},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.1379999999999999,"5min":2.5339999999999998,"15min":2.9140000000000001},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.976,"5min":15.529,"15min":15.529},"last":2.258}]},{"osd":3,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":4.6779999999999999,"15min":4.8019999999999996},"min":{"1min":0.28399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":22.684000000000001,"5min":25.013999999999999,"15min":25.734999999999999},"last":2.6080000000000001},{"interface":"front","average":{"1min":1.367,"5min":3.4969999999999999,"15min":4.1059999999999999},"min":{"1min":0.318,"5min":0.221,"15min":0.221},"max":{"1min":4.0220000000000002,"5min":18.928999999999998,"15min":21.923999999999999},"last":3.4300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":3.0259999999999998,"15min":1.9359999999999999},"min":{"1min":0.41799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":4.8070000000000004,"5min":20.494,"15min":20.494},"last":0.76200000000000001},{"interface":"front","average":{"1min":1.4119999999999999,"5min":2.7589999999999999,"15min":2.2109999999999999},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.109,"5min":15.907,"15min":15.907},"last":1.833}]},{"osd":5,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":3.1389999999999998,"15min":2.5550000000000002},"min":{"1min":0.371,"5min":0.36099999999999999,"15min":0.28000000000000003},"max":{"1min":18.620000000000001,"5min":18.620000000000001,"15min":18.620000000000001},"last":1.879},{"interface":"front","average":{"1min":2.653,"5min":2.3599999999999999,"15min":1.7929999999999999},"min":{"1min":0.45200000000000001,"5min":0.30499999999999999,"15min":0.27000000000000002},"max":{"1min":19.798999999999999,"5min":19.798999999999999,"15min":19.798999999999999},"last":0.53100000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":3.262,"5min":2.5750000000000002,"15min":1.99},"min":{"1min":0.40699999999999997,"5min":0.33700000000000002,"15min":0.31900000000000001},"max":{"1min":19.875,"5min":19.875,"15min":19.875},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.8049999999999999,"5min":3.5369999999999999,"15min":2.847},"min":{"1min":0.378,"5min":0.378,"15min":0.35699999999999998},"max":{"1min":6.0259999999999998,"5min":23.106999999999999,"15min":23.106999999999999},"last":0.83299999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":3.1659999999999999,"15min":2.71},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33900000000000002},"max":{"1min":18.904,"5min":18.904,"15min":26.978000000000002},"last":1.917},{"interface":"front","average":{"1min":2.8119999999999998,"5min":2.544,"15min":2.137},"min":{"1min":0.45800000000000002,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":19.843,"5min":19.843,"15min":26.768000000000001},"last":0.74299999999999999}]}]},{"osd":2,"up_from":217,"seq":932007903341,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375116,"kb_used_data":1063940,"kb_used_omap":83,"kb_used_meta":311084,"kb_avail":92370036,"statfs":{"total":95995035648,"available":94586916864,"internally_reserved":0,"allocated":1089474560,"data_stored":1088362102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":318550511},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":2.8940000000000001,"15min":2.0430000000000001},"min":{"1min":0.185,"5min":0.185,"15min":0.17899999999999999},"max":{"1min":11.327,"5min":19.372,"15min":19.372},"last":7.641},{"interface":"front","average":{"1min":1.488,"5min":3.4940000000000002,"15min":3.157},"min":{"1min":0.23599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.7479999999999993,"5min":20.143000000000001,"15min":20.143000000000001},"last":0.55100000000000005}]},{"osd":1,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":3.7999999999999998,"15min":2.8340000000000001},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.22800000000000001},"max":{"1min":11.398,"5min":19.283999999999999,"15min":19.283999999999999},"last":16.684000000000001},{"interface":"front","average":{"1min":1.423,"5min":3.234,"15min":2.7949999999999999},"min":{"1min":0.28599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":8.7739999999999991,"5min":19.027000000000001,"15min":19.027000000000001},"last":10.585000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.9009999999999998,"15min":2.3159999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.368,"5min":19.370999999999999,"15min":19.370999999999999},"last":7.8490000000000002},{"interface":"front","average":{"1min":1.661,"5min":2.919,"15min":3.5830000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.7959999999999994,"5min":19.053000000000001,"15min":19.053000000000001},"last":1.071}]},{"osd":4,"last update":"Mon Oct 9 10:13:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8839999999999999,"5min":2.056,"15min":1.1559999999999999},"min":{"1min":0.34699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":18.873999999999999,"5min":18.873999999999999,"15min":18.873999999999999},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.4300000000000002,"5min":2.089,"15min":1.2490000000000001},"min":{"1min":0.41699999999999998,"5min":0.314,"15min":0.314},"max":{"1min":11.993,"5min":15.276,"15min":15.276},"last":0.56999999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":2.2309999999999999,"15min":1.7729999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":8.8140000000000001,"5min":15.32,"15min":15.32},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.802,"5min":2.5150000000000001,"15min":1.8879999999999999},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":9.4789999999999992,"5min":17.882999999999999,"15min":17.882999999999999},"last":6.7919999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":2.3839999999999999,"15min":1.542},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":12.257,"5min":18.943999999999999,"15min":18.943999999999999},"last":4.577},{"interface":"front","average":{"1min":1.4139999999999999,"5min":2.4249999999999998,"15min":1.9870000000000001},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":4.5979999999999999,"5min":15.378,"15min":15.378},"last":0.68500000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.5529999999999999,"15min":1.6020000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.35899999999999999},"max":{"1min":12.186,"5min":18.972999999999999,"15min":18.972999999999999},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.8029999999999999,"5min":2.8879999999999999,"15min":2.0449999999999999},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.376},"max":{"1min":9.9440000000000008,"5min":15.353,"15min":15.353},"last":8.0739999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824084,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3122284,"kb_used_data":2772004,"kb_used_omap":10,"kb_used_meta":350261,"kb_avail":90622868,"statfs":{"total":95995035648,"available":92797816832,"internally_reserved":0,"allocated":2838532096,"data_stored":2837206132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":358667331},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":3.1840000000000002,"15min":4.1970000000000001},"min":{"1min":0.21299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.5470000000000006,"5min":17.373000000000001,"15min":23.381},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.788,"5min":4.4470000000000001,"15min":4.2679999999999998},"min":{"1min":0.23499999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.56,"5min":20.239999999999998,"15min":20.239999999999998},"last":5.2169999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":3.5569999999999999,"15min":3.8210000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7409999999999997,"5min":40.618000000000002,"15min":40.618000000000002},"last":2.3799999999999999},{"interface":"front","average":{"1min":2.0350000000000001,"5min":3.21,"15min":3.3570000000000002},"min":{"1min":0.27400000000000002,"5min":0.218,"15min":0.218},"max":{"1min":8.5139999999999993,"5min":15.704000000000001,"15min":23.423999999999999},"last":0.49399999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":3.347,"15min":3.609},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.6399999999999997,"5min":29.466999999999999,"15min":29.466999999999999},"last":5.0449999999999999},{"interface":"front","average":{"1min":1.5800000000000001,"5min":2.786,"15min":2.8380000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":8.5790000000000006,"5min":15.73,"15min":23.452000000000002},"last":0.95099999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:13:42 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":2.306,"15min":1.4990000000000001},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.6669999999999998,"5min":15.676,"15min":15.676},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.917,"5min":2.7410000000000001,"15min":1.921},"min":{"1min":0.379,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.673,"5min":16.478999999999999,"15min":16.478999999999999},"last":5.343}]},{"osd":5,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":3.0979999999999999,"15min":2.7749999999999999},"min":{"1min":0.33200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.638,"5min":13.638,"15min":13.638},"last":5.4160000000000004},{"interface":"front","average":{"1min":1.986,"5min":3.1379999999999999,"15min":2.6150000000000002},"min":{"1min":0.373,"5min":0.318,"15min":0.309},"max":{"1min":9.4860000000000007,"5min":17.242999999999999,"15min":17.242999999999999},"last":2.5600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":2.9329999999999998,"15min":2.8029999999999999},"min":{"1min":0.45500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":13.705,"5min":16.504999999999999,"15min":16.504999999999999},"last":5.3769999999999998},{"interface":"front","average":{"1min":1.714,"5min":2.6869999999999998,"15min":2.6309999999999998},"min":{"1min":0.39800000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.6820000000000004,"5min":17.286999999999999,"15min":17.286999999999999},"last":2.52}]},{"osd":7,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":2.3740000000000001,"15min":1.7170000000000001},"min":{"1min":0.433,"5min":0.35199999999999998,"15min":0.34000000000000002},"max":{"1min":8.4079999999999995,"5min":15.635999999999999,"15min":15.635999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.732,"5min":3.125,"15min":2.8250000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":13.601000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":5.4610000000000003}]}]},{"osd":4,"up_from":266,"seq":1142461300810,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1486808,"kb_used_data":1164048,"kb_used_omap":35,"kb_used_meta":322716,"kb_avail":92258344,"statfs":{"total":95995035648,"available":94472544256,"internally_reserved":0,"allocated":1191985152,"data_stored":1191445004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":330461422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.3720000000000001,"15min":0.96999999999999997},"min":{"1min":0.252,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.6970000000000001,"5min":10.881,"15min":10.881},"last":1.6200000000000001},{"interface":"front","average":{"1min":1.7989999999999999,"5min":2.1469999999999998,"15min":1.355},"min":{"1min":0.318,"5min":0.224,"15min":0.224},"max":{"1min":8.2050000000000001,"5min":12.246,"15min":12.246},"last":1.3049999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":3.0569999999999999,"5min":2.7010000000000001,"15min":1.4450000000000001},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":18.303000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":4.3490000000000002},{"interface":"front","average":{"1min":2.488,"5min":2.2280000000000002,"15min":1.5920000000000001},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":18.036000000000001,"5min":18.036000000000001,"15min":18.036000000000001},"last":2.0419999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.9950000000000001,"15min":1.327},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.6689999999999996,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":3.0259999999999998},{"interface":"front","average":{"1min":1.9950000000000001,"5min":1.9450000000000001,"15min":1.252},"min":{"1min":0.28100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.778,"5min":11.778,"15min":11.778},"last":1.52}]},{"osd":3,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":3.105,"5min":2.5619999999999998,"15min":1.6950000000000001},"min":{"1min":0.38900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":14.323,"5min":14.323,"15min":14.323},"last":1.651},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.847,"15min":1.625},"min":{"1min":0.379,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":25.550000000000001,"5min":25.550000000000001,"15min":25.550000000000001},"last":1.6100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.198,"15min":0.88500000000000001},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.7509999999999999,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":2.0680000000000001},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.72399999999999998,"15min":0.67400000000000004},"min":{"1min":0.29999999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.6689999999999996,"5min":5.7549999999999999,"15min":5.7549999999999999},"last":1.55}]},{"osd":6,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0620000000000001,"15min":0.70399999999999996},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.6449999999999996,"5min":6.851,"15min":6.851},"last":1.573},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.1699999999999999,"15min":0.77900000000000003},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.9080000000000004,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.39100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.397,"15min":0.879},"min":{"1min":0.36799999999999999,"5min":0.312,"15min":0.312},"max":{"1min":6.6630000000000003,"5min":12.237,"15min":12.237},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.2350000000000001,"15min":0.96399999999999997},"min":{"1min":0.33300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.3879999999999999,"5min":14.964,"15min":14.964},"last":2.0960000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822995,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3765576,"kb_used_data":3167680,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":89979576,"statfs":{"total":95995035648,"available":92139085824,"internally_reserved":0,"allocated":3243704320,"data_stored":3243176300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:14:37 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.1230000000000002,"15min":1.853},"min":{"1min":0.32100000000000001,"5min":0.23300000000000001,"15min":0.21099999999999999},"max":{"1min":7.1879999999999997,"5min":10.432,"15min":15.773999999999999},"last":2.5939999999999999},{"interface":"front","average":{"1min":1.55,"5min":1.704,"15min":1.8580000000000001},"min":{"1min":0.27000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4119999999999999,"5min":8.4939999999999998,"15min":15.699},"last":2.9430000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:14:19 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":2.3580000000000001,"15min":1.9219999999999999},"min":{"1min":0.251,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":6.274,"5min":11.891,"15min":15.689},"last":2.0070000000000001},{"interface":"front","average":{"1min":2.153,"5min":2.0089999999999999,"15min":2},"min":{"1min":0.33900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":23.099,"5min":23.099,"15min":23.099},"last":2.3809999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":2.1859999999999999,"15min":2.3479999999999999},"min":{"1min":0.29999999999999999,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.5030000000000001,"5min":10.259,"15min":15.640000000000001},"last":2.4039999999999999},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.4550000000000001,"15min":2.0179999999999998},"min":{"1min":0.36299999999999999,"5min":0.32200000000000001,"15min":0.313},"max":{"1min":3.6640000000000001,"5min":6.5529999999999999,"15min":10.656000000000001},"last":2.1560000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":2.3109999999999999,"15min":2.0030000000000001},"min":{"1min":0.42399999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.1059999999999999,"5min":9.8699999999999992,"15min":9.8699999999999992},"last":2.1819999999999999},{"interface":"front","average":{"1min":0.80100000000000005,"5min":2.2629999999999999,"15min":2.7669999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9830000000000001,"5min":17.193999999999999,"15min":17.34},"last":2.4569999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.0760000000000001,"15min":0.73399999999999999},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.2679999999999998,"5min":7.3460000000000001,"15min":7.3460000000000001},"last":6.1619999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.72199999999999998,"15min":0.73599999999999999},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.3599999999999999,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":3.141}]},{"osd":6,"last update":"Mon Oct 9 10:14:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":1.3939999999999999,"15min":1.1759999999999999},"min":{"1min":0.33600000000000002,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":10.222,"5min":10.222,"15min":27.716000000000001},"last":3.0249999999999999},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.3759999999999999,"15min":1.0740000000000001},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.22600000000000001},"max":{"1min":7.1219999999999999,"5min":8.2910000000000004,"15min":18.600999999999999},"last":1.913}]},{"osd":7,"last update":"Mon Oct 9 10:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.3360000000000001,"15min":1.0860000000000001},"min":{"1min":0.35499999999999998,"5min":0.312,"15min":0.25600000000000001},"max":{"1min":3.4790000000000001,"5min":8.3409999999999993,"15min":10.334},"last":1.859},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.81799999999999995,"15min":0.95899999999999996},"min":{"1min":0.34599999999999997,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":0.67500000000000004,"5min":8.5220000000000002,"15min":12.356},"last":1.885}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1445920768,"data_stored":1445920768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3239510016,"data_stored":3239510016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1094254592,"data_stored":1094254592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2842099712,"data_stored":2842099712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1071841280,"data_stored":1071841280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3256942592,"data_stored":3256942592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1101856768,"data_stored":1101856768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3307208704,"data_stored":3307208704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:14:43.410 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T10:14:43.410 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:14:43.410 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T10:14:43.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:14:43.668 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:43.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:43.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:43.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:43.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:14:43.858 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:44.075 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:44.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:44.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:44.185 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:44.280 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:44.293 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 67 16 93397 93381 87.0069 53.5 0.0101661 0.0114884 2023-10-09T10:14:44.335 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:14:44.336 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:14:44.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:14:44.336 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:14:44.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.336 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190193 7.8 7:1a8121c5:::benchmark_data_smithi012_221426_object82415:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e240)", 2023-10-09T10:14:44.336 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 453.93340469899999, 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.051354506, 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190193 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397656+0000", 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7679999999999997e-06 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397662+0000", 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0439999999999996e-06 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397670+0000", 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0260000000000004e-06 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397676+0000", 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00094881100000000003 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398625+0000", 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2380000000000003e-06 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398634+0000", 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0429888860000001 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441623+0000", 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011069399999999999 2023-10-09T10:14:44.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441733+0000", 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4486000000000003e-05 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441828+0000", 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033881190000000002 2023-10-09T10:14:44.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445216+0000", 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036284030000000001 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.448844+0000", 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.923e-05 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.448904+0000", 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010786 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449012+0000", 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.051354506 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190203 7.8 7:1d4612d0:::benchmark_data_smithi012_221426_object82425:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 453.93338161899999, 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0571886909999999, 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190203 2023-10-09T10:14:44.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0150000000000002e-06 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397682+0000", 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.647e-06 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397685+0000", 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7410000000000001e-06 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397688+0000", 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099381899999999995 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398682+0000", 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5570000000000003e-06 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398688+0000", 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.046572745 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445260+0000", 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012410700000000001 2023-10-09T10:14:44.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445384+0000", 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000154804 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445539+0000", 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058752240000000001 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451414+0000", 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033171709999999998 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454732+0000", 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5908000000000001e-05 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454788+0000", 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1254999999999994e-05 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454869+0000", 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0571886909999999 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.352 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190210 7.8 7:1b3a4cee:::benchmark_data_smithi012_221426_object82432:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 453.93337033799997, 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0644897120000001, 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190210 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.353 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:14:44.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7489999999999999e-06 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397693+0000", 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0630000000000002e-06 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397696+0000", 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2269999999999996e-06 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397703+0000", 2023-10-09T10:14:44.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001020782 2023-10-09T10:14:44.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398724+0000", 2023-10-09T10:14:44.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3509999999999996e-06 2023-10-09T10:14:44.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:14:44.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398731+0000", 2023-10-09T10:14:44.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0503301599999999 2023-10-09T10:14:44.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449062+0000", 2023-10-09T10:14:44.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013289700000000001 2023-10-09T10:14:44.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449194+0000", 2023-10-09T10:14:44.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018222 2023-10-09T10:14:44.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:14:44.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449377+0000", 2023-10-09T10:14:44.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012586909 2023-10-09T10:14:44.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.461964+0000", 2023-10-09T10:14:44.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4403e-05 2023-10-09T10:14:44.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462028+0000", 2023-10-09T10:14:44.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3278e-05 2023-10-09T10:14:44.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462081+0000", 2023-10-09T10:14:44.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9912000000000002e-05 2023-10-09T10:14:44.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462181+0000", 2023-10-09T10:14:44.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0644897120000001 2023-10-09T10:14:44.363 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.364 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.364 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.364 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190212 7.8 7:1e06d44a:::benchmark_data_smithi012_221426_object82434:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:14:44.364 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:14:44.364 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 453.93335600400002, 2023-10-09T10:14:44.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.071536394, 2023-10-09T10:14:44.365 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.365 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.365 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.365 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:14:44.365 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:14:44.365 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190212 2023-10-09T10:14:44.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.366 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:14:44.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:14:44.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397705+0000", 2023-10-09T10:14:44.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7659999999999999e-06 2023-10-09T10:14:44.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397707+0000", 2023-10-09T10:14:44.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8750000000000002e-06 2023-10-09T10:14:44.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397711+0000", 2023-10-09T10:14:44.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4600000000000002e-06 2023-10-09T10:14:44.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397714+0000", 2023-10-09T10:14:44.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010809909999999999 2023-10-09T10:14:44.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398795+0000", 2023-10-09T10:14:44.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5300000000000004e-06 2023-10-09T10:14:44.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:14:44.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398800+0000", 2023-10-09T10:14:44.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0526496919999999 2023-10-09T10:14:44.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451450+0000", 2023-10-09T10:14:44.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013340300000000001 2023-10-09T10:14:44.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451583+0000", 2023-10-09T10:14:44.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017290599999999999 2023-10-09T10:14:44.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:14:44.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451756+0000", 2023-10-09T10:14:44.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010213186000000001 2023-10-09T10:14:44.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.461969+0000", 2023-10-09T10:14:44.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071399239999999997 2023-10-09T10:14:44.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469109+0000", 2023-10-09T10:14:44.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9132000000000003e-05 2023-10-09T10:14:44.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469158+0000", 2023-10-09T10:14:44.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3792e-05 2023-10-09T10:14:44.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469242+0000", 2023-10-09T10:14:44.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.071536394 2023-10-09T10:14:44.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.379 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.380 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190219 7.8 7:155dd6d1:::benchmark_data_smithi012_221426_object82441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:14:44.380 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:14:44.380 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 453.93334500700001, 2023-10-09T10:14:44.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0716708749999999, 2023-10-09T10:14:44.380 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.381 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.381 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.381 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:14:44.381 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:14:44.381 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190219 2023-10-09T10:14:44.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.381 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:14:44.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:14:44.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397716+0000", 2023-10-09T10:14:44.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6100000000000002e-06 2023-10-09T10:14:44.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397723+0000", 2023-10-09T10:14:44.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5250000000000001e-06 2023-10-09T10:14:44.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.385 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397727+0000", 2023-10-09T10:14:44.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.711e-06 2023-10-09T10:14:44.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.385 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397729+0000", 2023-10-09T10:14:44.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001098705 2023-10-09T10:14:44.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398828+0000", 2023-10-09T10:14:44.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5500000000000002e-06 2023-10-09T10:14:44.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:14:44.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398834+0000", 2023-10-09T10:14:44.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0560779309999999 2023-10-09T10:14:44.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454912+0000", 2023-10-09T10:14:44.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012238900000000001 2023-10-09T10:14:44.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.455034+0000", 2023-10-09T10:14:44.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015768999999999999 2023-10-09T10:14:44.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:14:44.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.455192+0000", 2023-10-09T10:14:44.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010377422000000001 2023-10-09T10:14:44.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.390 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.390 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465569+0000", 2023-10-09T10:14:44.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037032409999999999 2023-10-09T10:14:44.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469272+0000", 2023-10-09T10:14:44.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1624e-05 2023-10-09T10:14:44.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469314+0000", 2023-10-09T10:14:44.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3724000000000001e-05 2023-10-09T10:14:44.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469388+0000", 2023-10-09T10:14:44.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0716708749999999 2023-10-09T10:14:44.392 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.393 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.393 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.393 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190227 7.8 7:129c86af:::benchmark_data_smithi012_221426_object82449:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:14:44.393 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:14:44.394 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 453.93333005, 2023-10-09T10:14:44.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0717717900000001, 2023-10-09T10:14:44.394 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.394 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.394 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.394 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:14:44.395 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:14:44.395 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190227 2023-10-09T10:14:44.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.395 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:14:44.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:14:44.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.397 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397731+0000", 2023-10-09T10:14:44.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5820000000000001e-06 2023-10-09T10:14:44.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.397 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397733+0000", 2023-10-09T10:14:44.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8680000000000001e-06 2023-10-09T10:14:44.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397737+0000", 2023-10-09T10:14:44.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4420000000000001e-06 2023-10-09T10:14:44.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397739+0000", 2023-10-09T10:14:44.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011216900000000001 2023-10-09T10:14:44.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398861+0000", 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4060000000000004e-06 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398866+0000", 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0599802039999999 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.458847+0000", 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9543000000000006e-05 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.458946+0000", 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5651999999999999e-05 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.459032+0000", 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010030136 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469062+0000", 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036131999999999999 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469423+0000", 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0482000000000001e-05 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469454+0000", 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9725000000000002e-05 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469503+0000", 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0717717900000001 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190233 7.8 7:13f66138:::benchmark_data_smithi012_221426_object82455:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:14:44.404 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 453.93332026500002, 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.076501538, 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190233 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.405 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.406 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4050000000000001e-06 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397743+0000", 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.63e-06 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397746+0000", 2023-10-09T10:14:44.407 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.255e-06 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397749+0000", 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011455700000000001 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398894+0000", 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3720000000000004e-06 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.408 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398899+0000", 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0633114340000001 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462211+0000", 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8008000000000001e-05 2023-10-09T10:14:44.409 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462309+0000", 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013975599999999999 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462449+0000", 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068008950000000004 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.410 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469250+0000", 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048359240000000001 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474086+0000", 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7553999999999998e-05 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.411 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474143+0000", 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9965999999999998e-05 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474243+0000", 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.076501538 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.412 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.413 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190240 7.8 7:1bb20cb0:::benchmark_data_smithi012_221426_object82462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:14:44.413 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:14:44.413 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 453.93331127599998, 2023-10-09T10:14:44.413 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0788701549999999, 2023-10-09T10:14:44.413 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.413 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.413 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.414 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:14:44.414 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:14:44.414 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190240 2023-10-09T10:14:44.414 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.414 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.414 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:14:44.415 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.415 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.415 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.415 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.415 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:14:44.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:14:44.416 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.416 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.416 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.416 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:14:44.416 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.508e-06 2023-10-09T10:14:44.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.417 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.417 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397752+0000", 2023-10-09T10:14:44.417 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5329999999999999e-06 2023-10-09T10:14:44.417 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.417 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.418 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397755+0000", 2023-10-09T10:14:44.418 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1100000000000001e-06 2023-10-09T10:14:44.418 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.418 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.418 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397757+0000", 2023-10-09T10:14:44.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001159413 2023-10-09T10:14:44.419 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.419 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.419 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.419 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398917+0000", 2023-10-09T10:14:44.419 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4650000000000002e-06 2023-10-09T10:14:44.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.420 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.420 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:14:44.420 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398922+0000", 2023-10-09T10:14:44.420 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.066675222 2023-10-09T10:14:44.420 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.421 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.421 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465597+0000", 2023-10-09T10:14:44.421 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5288000000000001e-05 2023-10-09T10:14:44.421 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.421 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465683+0000", 2023-10-09T10:14:44.422 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107876 2023-10-09T10:14:44.422 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.422 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.422 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:14:44.422 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465791+0000", 2023-10-09T10:14:44.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036041200000000002 2023-10-09T10:14:44.423 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.423 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.423 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.423 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469395+0000", 2023-10-09T10:14:44.423 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071257839999999996 2023-10-09T10:14:44.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.424 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.424 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476521+0000", 2023-10-09T10:14:44.424 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2398999999999998e-05 2023-10-09T10:14:44.424 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.424 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.425 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476563+0000", 2023-10-09T10:14:44.425 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7669e-05 2023-10-09T10:14:44.425 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.425 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.425 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476621+0000", 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0788701549999999 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65159 9.3 9:cf47ceeb:::benchmark_data_smithi012_313365_object65158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:14:44.426 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 29.32279608, 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19414071499999999, 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65159 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.427 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008264+0000", 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8745000000000001e-05 2023-10-09T10:14:44.428 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008323+0000", 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1172000000000001e-05 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008334+0000", 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4099999999999996e-06 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.429 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008341+0000", 2023-10-09T10:14:44.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6387999999999998e-05 2023-10-09T10:14:44.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.430 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.430 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.430 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008377+0000", 2023-10-09T10:14:44.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015405 2023-10-09T10:14:44.430 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008531+0000", 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016722800000000001 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008698+0000", 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023079260000000001 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.011006+0000", 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19127018400000001 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202277+0000", 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9379999999999999e-05 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202316+0000", 2023-10-09T10:14:44.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0695999999999997e-05 2023-10-09T10:14:44.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202367+0000", 2023-10-09T10:14:44.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9728000000000003e-05 2023-10-09T10:14:44.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202406+0000", 2023-10-09T10:14:44.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19414071499999999 2023-10-09T10:14:44.434 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.434 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.434 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.435 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65155 9.e e328/317 9:70ee838f:::benchmark_data_smithi012_313365_object65154:head v 328'4057, mlcod=328'4055)", 2023-10-09T10:14:44.435 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:15.015416+0000", 2023-10-09T10:14:44.435 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 29.315645406000002, 2023-10-09T10:14:44.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18207646599999999, 2023-10-09T10:14:44.435 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.435 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015416+0000", 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015416+0000", 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015415+0000", 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1175999999999998e-05 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015487+0000", 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.313e-05 2023-10-09T10:14:44.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015500+0000", 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.366e-06 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015505+0000", 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5562000000000001e-05 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015561+0000", 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7224e-05 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015578+0000", 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18186355000000001 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197441+0000", 2023-10-09T10:14:44.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1319000000000001e-05 2023-10-09T10:14:44.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197493+0000", 2023-10-09T10:14:44.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18207646599999999 2023-10-09T10:14:44.441 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.441 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.441 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.442 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65165 9.d e328/317 9:bf9ec36c:::benchmark_data_smithi012_313365_object65164:head v 328'4139, mlcod=328'4137)", 2023-10-09T10:14:44.442 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:15.015534+0000", 2023-10-09T10:14:44.442 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 29.315527400000001, 2023-10-09T10:14:44.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.191150345, 2023-10-09T10:14:44.442 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.443 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.443 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015534+0000", 2023-10-09T10:14:44.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.444 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015534+0000", 2023-10-09T10:14:44.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015534+0000", 2023-10-09T10:14:44.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6866000000000006e-05 2023-10-09T10:14:44.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015611+0000", 2023-10-09T10:14:44.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8059999999999995e-06 2023-10-09T10:14:44.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015619+0000", 2023-10-09T10:14:44.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4700000000000001e-06 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015624+0000", 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18184143899999999 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197465+0000", 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8736999999999999e-05 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197494+0000", 2023-10-09T10:14:44.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091424269999999998 2023-10-09T10:14:44.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.206637+0000", 2023-10-09T10:14:44.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8059000000000002e-05 2023-10-09T10:14:44.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.449 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.449 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.206685+0000", 2023-10-09T10:14:44.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.191150345 2023-10-09T10:14:44.450 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.450 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.450 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.450 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65167 9.3 9:c0760dd7:::benchmark_data_smithi012_313365_object65166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:14:44.450 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:15.016568+0000", 2023-10-09T10:14:44.451 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 29.314493951999999, 2023-10-09T10:14:44.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18863607600000001, 2023-10-09T10:14:44.451 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.451 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.451 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:14:44.451 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:14:44.451 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:14:44.451 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65167 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.016568+0000", 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.016568+0000", 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:14:44.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.016566+0000", 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1714e-05 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.016618+0000", 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18080917699999999 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197427+0000", 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9647e-05 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197447+0000", 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0683000000000002e-05 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197507+0000", 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015766900000000001 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197665+0000", 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000192587 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197858+0000", 2023-10-09T10:14:44.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00455909 2023-10-09T10:14:44.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:14:44.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202417+0000", 2023-10-09T10:14:44.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5722999999999997e-05 2023-10-09T10:14:44.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202452+0000", 2023-10-09T10:14:44.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026472190000000001 2023-10-09T10:14:44.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:14:44.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.205100+0000", 2023-10-09T10:14:44.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5439999999999998e-05 2023-10-09T10:14:44.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.205155+0000", 2023-10-09T10:14:44.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8745999999999997e-05 2023-10-09T10:14:44.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.205204+0000", 2023-10-09T10:14:44.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18863607600000001 2023-10-09T10:14:44.459 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.460 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.460 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.460 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65168 9.d e328/317 9:bb48a428:::benchmark_data_smithi012_313365_object65167:head v 328'4140, mlcod=328'4137)", 2023-10-09T10:14:44.460 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:15.018742+0000", 2023-10-09T10:14:44.460 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 29.312320019000001, 2023-10-09T10:14:44.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18799660300000001, 2023-10-09T10:14:44.461 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.461 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.461 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.018742+0000", 2023-10-09T10:14:44.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.018742+0000", 2023-10-09T10:14:44.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999938 2023-10-09T10:14:44.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.018735+0000", 2023-10-09T10:14:44.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3792999999999998e-05 2023-10-09T10:14:44.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.018789+0000", 2023-10-09T10:14:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178633772 2023-10-09T10:14:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197423+0000", 2023-10-09T10:14:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2351e-05 2023-10-09T10:14:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197445+0000", 2023-10-09T10:14:44.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033238149999999999 2023-10-09T10:14:44.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.466 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.466 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.466 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.200769+0000", 2023-10-09T10:14:44.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7859e-05 2023-10-09T10:14:44.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.200787+0000", 2023-10-09T10:14:44.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059328810000000001 2023-10-09T10:14:44.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.206720+0000", 2023-10-09T10:14:44.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8391999999999999e-05 2023-10-09T10:14:44.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.206738+0000", 2023-10-09T10:14:44.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18799660300000001 2023-10-09T10:14:44.469 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.469 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.469 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.469 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:79101 9.7 e333/317 9:eec59131:::benchmark_data_smithi012_313365_object79100:head v 333'4902, mlcod=333'4897)", 2023-10-09T10:14:44.470 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:28.746150+0000", 2023-10-09T10:14:44.470 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.584911565000001, 2023-10-09T10:14:44.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.050210313999999999, 2023-10-09T10:14:44.470 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.470 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.470 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.746150+0000", 2023-10-09T10:14:44.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.746150+0000", 2023-10-09T10:14:44.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:44.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.746149+0000", 2023-10-09T10:14:44.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7978999999999997e-05 2023-10-09T10:14:44.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.746187+0000", 2023-10-09T10:14:44.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040296184999999998 2023-10-09T10:14:44.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.786483+0000", 2023-10-09T10:14:44.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.677e-06 2023-10-09T10:14:44.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.786489+0000", 2023-10-09T10:14:44.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3948e-05 2023-10-09T10:14:44.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.786533+0000", 2023-10-09T10:14:44.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1131000000000001e-05 2023-10-09T10:14:44.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.786554+0000", 2023-10-09T10:14:44.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0097702359999999999 2023-10-09T10:14:44.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.796324+0000", 2023-10-09T10:14:44.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6136000000000001e-05 2023-10-09T10:14:44.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.477 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:28.796361+0000", 2023-10-09T10:14:44.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.050210313999999999 2023-10-09T10:14:44.478 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.478 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.478 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.478 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:92299 9.a e335/317 9:5eaf83e1:::benchmark_data_smithi012_313365_object92298:head v 335'5571, mlcod=335'5569)", 2023-10-09T10:14:44.478 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:43.026953+0000", 2023-10-09T10:14:44.479 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.3041086019999999, 2023-10-09T10:14:44.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.062011528000000003, 2023-10-09T10:14:44.479 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.479 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.479 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.026953+0000", 2023-10-09T10:14:44.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.026953+0000", 2023-10-09T10:14:44.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.026952+0000", 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2595999999999998e-05 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.026975+0000", 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0165e-05 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.026985+0000", 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6019999999999999e-06 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.026991+0000", 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3140999999999999e-05 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.027034+0000", 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.277e-05 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.027067+0000", 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.061821139999999997 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088888+0000", 2023-10-09T10:14:44.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6798000000000006e-05 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088965+0000", 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.062011528000000003 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:92303 9.7 e335/317 9:e2e79ffc:::benchmark_data_smithi012_313365_object92302:head v 335'5699, mlcod=335'5696)", 2023-10-09T10:14:44.484 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:43.031626+0000", 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.2994360330000001, 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057285049999999997, 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031626+0000", 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.485 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031626+0000", 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031625+0000", 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6658000000000001e-05 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031652+0000", 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0316000000000001e-05 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031662+0000", 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5940000000000001e-06 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031667+0000", 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9187000000000002e-05 2023-10-09T10:14:44.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031717+0000", 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8301e-05 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031735+0000", 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025694020000000001 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034304+0000", 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054606466999999999 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088911+0000", 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057285049999999997 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:92300 9.a e335/317 9:547c1065:::benchmark_data_smithi012_313365_object92299:head v 335'5572, mlcod=335'5569)", 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:43.031679+0000", 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.299382612, 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057305746999999997, 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031679+0000", 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031679+0000", 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031679+0000", 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8397999999999999e-05 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031697+0000", 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2419999999999997e-06 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031703+0000", 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3309999999999997e-06 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.031708+0000", 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001499848 2023-10-09T10:14:44.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.033208+0000", 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0018e-05 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.033228+0000", 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055742115000000002 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088970+0000", 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5146e-05 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088985+0000", 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057305746999999997 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.494 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:92304 9.1 e335/317 9:82da9355:::benchmark_data_smithi012_313365_object92303:head v 335'5857, mlcod=335'5855)", 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:43.034105+0000", 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.296956475, 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060997688000000001, 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034105+0000", 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034105+0000", 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034105+0000", 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8572000000000002e-05 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034153+0000", 2023-10-09T10:14:44.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1395e-05 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034164+0000", 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5889999999999999e-06 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034171+0000", 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7159e-05 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034218+0000", 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3176999999999999e-05 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034231+0000", 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060836319 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.095068+0000", 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5250000000000003e-05 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.095103+0000", 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060997688000000001 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:92307 9.c e335/317 9:395669b0:::benchmark_data_smithi012_313365_object92306:head v 335'5753, mlcod=335'5751)", 2023-10-09T10:14:44.499 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:43.034688+0000", 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.2963737150000001, 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060279734000000001, 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034688+0000", 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034688+0000", 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034687+0000", 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7126000000000002e-05 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.034745+0000", 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054135560999999999 2023-10-09T10:14:44.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088880+0000", 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4139999999999999e-05 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088904+0000", 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2230999999999993e-05 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088977+0000", 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3023e-05 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.089000+0000", 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059170539999999997 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.094917+0000", 2023-10-09T10:14:44.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.117e-05 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.094968+0000", 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060279734000000001 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:92309 9.9 e335/317 9:958a778d:::benchmark_data_smithi012_313365_object92308:head v 335'5966, mlcod=335'5964)", 2023-10-09T10:14:44.504 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:43.035208+0000", 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.295853562, 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.059923354999999998, 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.035208+0000", 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.035208+0000", 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.035207+0000", 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1059999999999997e-05 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.035239+0000", 2023-10-09T10:14:44.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.053602178 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088841+0000", 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.58e-05 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088867+0000", 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7701000000000005e-05 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088964+0000", 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1832e-05 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.088986+0000", 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061196820000000004 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:14:44.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.095106+0000", 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5826e-05 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:43.095132+0000", 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.059923354999999998 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:14:44.509 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:14:44.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:44.510 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:44 smithi110 ceph-mon[136032]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:44.511 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:44.511 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:44.511 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:44.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:44.542 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:44.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:44.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:44 smithi012 ceph-mon[147926]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:44.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:44.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:44.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:44 smithi012 ceph-mon[152263]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:44.682 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:44.720 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:44.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:14:44.874 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:44.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:44.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:45.060 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:45.225 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:45 smithi012 ceph-mon[147926]: pgmap v945: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T10:14:45.225 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1701103642' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:45.225 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:45 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:45.226 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:45 smithi012 ceph-mon[152263]: pgmap v945: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T10:14:45.226 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1701103642' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:45.226 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:45 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:45.233 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:45.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:45.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:45.242 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:14:45.293 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 68 16 94183 94167 86.4503 49.125 0.0135465 0.0115628 2023-10-09T10:14:45.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:45.401 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:45.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:45 smithi110 ceph-mon[136032]: pgmap v945: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T10:14:45.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1701103642' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:45.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:45 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:14:45.473 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:45.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:45.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:45.581 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:45.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:45.700 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:45.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:45.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:45.867 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:46.293 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 69 16 95181 95165 86.1011 62.375 0.00914653 0.0116096 2023-10-09T10:14:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:46 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:46 smithi110 ceph-mon[136032]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:46 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:14:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:46 smithi110 ceph-mon[136032]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T10:14:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:46 smithi110 ceph-mon[136032]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:46 smithi110 ceph-mon[136032]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:46 smithi110 ceph-mon[136032]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:46.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[152263]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:14:46.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[152263]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T10:14:46.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[152263]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[152263]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[152263]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:14:46.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[147926]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:14:46.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[147926]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T10:14:46.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[147926]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[147926]: failed to encode map e336 with expected crc 2023-10-09T10:14:46.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:46 smithi012 ceph-mon[147926]: failed to encode map e336 with expected crc 2023-10-09T10:14:47.293 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 70 16 96228 96212 85.8061 65.4375 0.0157141 0.0116491 2023-10-09T10:14:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:47 smithi012 ceph-mon[147926]: pgmap v947: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T10:14:47.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:47 smithi012 ceph-mon[152263]: pgmap v947: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T10:14:47.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:47 smithi110 ceph-mon[136032]: pgmap v947: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T10:14:48.022 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:48.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:48.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:48.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:48.175 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:14:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:14:47] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-10-09T10:14:48.245 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:48.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:48.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:48.293 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 71 16 97160 97144 85.4183 58.25 0.0105111 0.0117027 2023-10-09T10:14:48.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:48.412 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:48.763 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:48.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:48.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:48.870 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:48.945 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:49.294 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 72 16 98103 98087 85.0508 58.9375 0.0140151 0.011753 2023-10-09T10:14:49.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:49 smithi012 ceph-mon[147926]: pgmap v948: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T10:14:49.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:49 smithi012 ceph-mon[152263]: pgmap v948: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T10:14:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:49 smithi110 ceph-mon[136032]: pgmap v948: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T10:14:49.839 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:49.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:49.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:49.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:49.998 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:50.023 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:50.294 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 73 16 99086 99070 84.7275 61.4375 0.0192266 0.0117979 2023-10-09T10:14:50.494 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:14:50.999 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:51.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:51.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:51.235 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:51.295 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 74 16 99972 99956 84.3302 55.375 0.0400521 0.0118536 2023-10-09T10:14:51.343 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:51.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:51.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:51.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:51 smithi012 ceph-mon[147926]: pgmap v949: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T10:14:51.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:51 smithi012 ceph-mon[152263]: pgmap v949: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T10:14:51.546 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:14:51.548 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:51.575 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:14:51.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:51 smithi110 ceph-mon[136032]: pgmap v949: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T10:14:51.711 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:14:51.734 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:51.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:51.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:51.763 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:14:51.798 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:14:51.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:51.847 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:14:51.884 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:14:51.910 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:51.926 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:14:51.958 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:14:52.014 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:14:52.014 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:14:52.014 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:14:52.014 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:14:52.014 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:14:52.015 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:14:52.015 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:14:52.015 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:14:52.015 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:14:52.080 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:52.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:52.186 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:52.189 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":950,"stamp":"2023-10-09T10:14:51.499648+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6334715808,"num_objects":96675,"num_object_clones":0,"num_object_copies":290025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":96675,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":96911,"num_write_kb":6188109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":1190604,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96845,"ondisk_log_size":96845,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22674432,"kb_used_data":18944064,"kb_used_omap":167,"kb_used_meta":3730136,"kb_avail":727286784,"statfs":{"total":767960285184,"available":744741666816,"internally_reserved":0,"allocated":19398721536,"data_stored":19391854186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":171431,"internal_metadata":3819659865},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,5,3,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":588972032,"num_objects":8987,"num_object_clones":0,"num_object_copies":26961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8987,"num_write_kb":575168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004512"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":179,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417662+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:14:38.417662+0000","last_peered":"2023-10-09T10:14:38.417662+0000","last_clean":"2023-10-09T10:14:38.417662+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:14:38.417662+0000","last_undegraded":"2023-10-09T10:14:38.417662+0000","last_fullsized":"2023-10-09T10:14:38.417662+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":294,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.411877+0000","last_change":"2023-10-09T10:08:35.091737+0000","last_active":"2023-10-09T10:14:38.411877+0000","last_peered":"2023-10-09T10:14:38.411877+0000","last_clean":"2023-10-09T10:14:38.411877+0000","last_became_active":"2023-10-09T10:08:35.091452+0000","last_became_peered":"2023-10-09T10:08:35.091452+0000","last_unstale":"2023-10-09T10:14:38.411877+0000","last_undegraded":"2023-10-09T10:14:38.411877+0000","last_fullsized":"2023-10-09T10:14:38.411877+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.686183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":340,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.416940+0000","last_change":"2023-10-09T10:08:35.098964+0000","last_active":"2023-10-09T10:14:38.416940+0000","last_peered":"2023-10-09T10:14:38.416940+0000","last_clean":"2023-10-09T10:14:38.416940+0000","last_became_active":"2023-10-09T10:08:35.098219+0000","last_became_peered":"2023-10-09T10:08:35.098219+0000","last_unstale":"2023-10-09T10:14:38.416940+0000","last_undegraded":"2023-10-09T10:14:38.416940+0000","last_fullsized":"2023-10-09T10:14:38.416940+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:07.928300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":204,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419559+0000","last_change":"2023-10-09T10:07:42.903840+0000","last_active":"2023-10-09T10:14:38.419559+0000","last_peered":"2023-10-09T10:14:38.419559+0000","last_clean":"2023-10-09T10:14:38.419559+0000","last_became_active":"2023-10-09T10:07:42.903418+0000","last_became_peered":"2023-10-09T10:07:42.903418+0000","last_unstale":"2023-10-09T10:14:38.419559+0000","last_undegraded":"2023-10-09T10:14:38.419559+0000","last_fullsized":"2023-10-09T10:14:38.419559+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:22.250116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":308,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.409729+0000","last_change":"2023-10-09T10:07:42.932864+0000","last_active":"2023-10-09T10:14:38.409729+0000","last_peered":"2023-10-09T10:14:38.409729+0000","last_clean":"2023-10-09T10:14:38.409729+0000","last_became_active":"2023-10-09T10:07:42.928283+0000","last_became_peered":"2023-10-09T10:07:42.928283+0000","last_unstale":"2023-10-09T10:14:38.409729+0000","last_undegraded":"2023-10-09T10:14:38.409729+0000","last_fullsized":"2023-10-09T10:14:38.409729+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:23.809675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":154,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419603+0000","last_change":"2023-10-09T10:03:30.090803+0000","last_active":"2023-10-09T10:14:38.419603+0000","last_peered":"2023-10-09T10:14:38.419603+0000","last_clean":"2023-10-09T10:14:38.419603+0000","last_became_active":"2023-10-09T10:03:30.090482+0000","last_became_peered":"2023-10-09T10:03:30.090482+0000","last_unstale":"2023-10-09T10:14:38.419603+0000","last_undegraded":"2023-10-09T10:14:38.419603+0000","last_fullsized":"2023-10-09T10:14:38.419603+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":221,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419842+0000","last_change":"2023-10-09T10:07:42.957100+0000","last_active":"2023-10-09T10:14:38.419842+0000","last_peered":"2023-10-09T10:14:38.419842+0000","last_clean":"2023-10-09T10:14:38.419842+0000","last_became_active":"2023-10-09T10:07:42.956753+0000","last_became_peered":"2023-10-09T10:07:42.956753+0000","last_unstale":"2023-10-09T10:14:38.419842+0000","last_undegraded":"2023-10-09T10:14:38.419842+0000","last_fullsized":"2023-10-09T10:14:38.419842+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:31.184853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":290,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:41.630443+0000","last_change":"2023-10-09T10:07:42.932694+0000","last_active":"2023-10-09T10:14:41.630443+0000","last_peered":"2023-10-09T10:14:41.630443+0000","last_clean":"2023-10-09T10:14:41.630443+0000","last_became_active":"2023-10-09T10:07:42.927966+0000","last_became_peered":"2023-10-09T10:07:42.927966+0000","last_unstale":"2023-10-09T10:14:41.630443+0000","last_undegraded":"2023-10-09T10:14:41.630443+0000","last_fullsized":"2023-10-09T10:14:41.630443+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:58.311812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":153,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419738+0000","last_change":"2023-10-09T10:03:30.081963+0000","last_active":"2023-10-09T10:14:38.419738+0000","last_peered":"2023-10-09T10:14:38.419738+0000","last_clean":"2023-10-09T10:14:38.419738+0000","last_became_active":"2023-10-09T10:03:30.081541+0000","last_became_peered":"2023-10-09T10:03:30.081541+0000","last_unstale":"2023-10-09T10:14:38.419738+0000","last_undegraded":"2023-10-09T10:14:38.419738+0000","last_fullsized":"2023-10-09T10:14:38.419738+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":251,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:41.630495+0000","last_change":"2023-10-09T10:07:42.910438+0000","last_active":"2023-10-09T10:14:41.630495+0000","last_peered":"2023-10-09T10:14:41.630495+0000","last_clean":"2023-10-09T10:14:41.630495+0000","last_became_active":"2023-10-09T10:07:42.910102+0000","last_became_peered":"2023-10-09T10:07:42.910102+0000","last_unstale":"2023-10-09T10:14:41.630495+0000","last_undegraded":"2023-10-09T10:14:41.630495+0000","last_fullsized":"2023-10-09T10:14:41.630495+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.505610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":374,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.422124+0000","last_change":"2023-10-09T10:08:35.099710+0000","last_active":"2023-10-09T10:14:38.422124+0000","last_peered":"2023-10-09T10:14:38.422124+0000","last_clean":"2023-10-09T10:14:38.422124+0000","last_became_active":"2023-10-09T10:08:35.099207+0000","last_became_peered":"2023-10-09T10:08:35.099207+0000","last_unstale":"2023-10-09T10:14:38.422124+0000","last_undegraded":"2023-10-09T10:14:38.422124+0000","last_fullsized":"2023-10-09T10:14:38.422124+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":332,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417530+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:14:38.417530+0000","last_peered":"2023-10-09T10:14:38.417530+0000","last_clean":"2023-10-09T10:14:38.417530+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:14:38.417530+0000","last_undegraded":"2023-10-09T10:14:38.417530+0000","last_fullsized":"2023-10-09T10:14:38.417530+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":223,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.420411+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:14:38.420411+0000","last_peered":"2023-10-09T10:14:38.420411+0000","last_clean":"2023-10-09T10:14:38.420411+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:14:38.420411+0000","last_undegraded":"2023-10-09T10:14:38.420411+0000","last_fullsized":"2023-10-09T10:14:38.420411+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":204,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417347+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:14:38.417347+0000","last_peered":"2023-10-09T10:14:38.417347+0000","last_clean":"2023-10-09T10:14:38.417347+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:14:38.417347+0000","last_undegraded":"2023-10-09T10:14:38.417347+0000","last_fullsized":"2023-10-09T10:14:38.417347+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":407,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419999+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:14:38.419999+0000","last_peered":"2023-10-09T10:14:38.419999+0000","last_clean":"2023-10-09T10:14:38.419999+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:14:38.419999+0000","last_undegraded":"2023-10-09T10:14:38.419999+0000","last_fullsized":"2023-10-09T10:14:38.419999+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":277,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:41.630606+0000","last_change":"2023-10-09T10:07:42.902755+0000","last_active":"2023-10-09T10:14:41.630606+0000","last_peered":"2023-10-09T10:14:41.630606+0000","last_clean":"2023-10-09T10:14:41.630606+0000","last_became_active":"2023-10-09T10:07:42.902478+0000","last_became_peered":"2023-10-09T10:07:42.902478+0000","last_unstale":"2023-10-09T10:14:41.630606+0000","last_undegraded":"2023-10-09T10:14:41.630606+0000","last_fullsized":"2023-10-09T10:14:41.630606+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:38.359700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":203,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419213+0000","last_change":"2023-10-09T10:07:42.903510+0000","last_active":"2023-10-09T10:14:38.419213+0000","last_peered":"2023-10-09T10:14:38.419213+0000","last_clean":"2023-10-09T10:14:38.419213+0000","last_became_active":"2023-10-09T10:07:42.903151+0000","last_became_peered":"2023-10-09T10:07:42.903151+0000","last_unstale":"2023-10-09T10:14:38.419213+0000","last_undegraded":"2023-10-09T10:14:38.419213+0000","last_fullsized":"2023-10-09T10:14:38.419213+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:12.616133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":169,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419567+0000","last_change":"2023-10-09T10:07:42.889533+0000","last_active":"2023-10-09T10:14:38.419567+0000","last_peered":"2023-10-09T10:14:38.419567+0000","last_clean":"2023-10-09T10:14:38.419567+0000","last_became_active":"2023-10-09T10:07:42.889189+0000","last_became_peered":"2023-10-09T10:07:42.889189+0000","last_unstale":"2023-10-09T10:14:38.419567+0000","last_undegraded":"2023-10-09T10:14:38.419567+0000","last_fullsized":"2023-10-09T10:14:38.419567+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":204,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.418213+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:14:38.418213+0000","last_peered":"2023-10-09T10:14:38.418213+0000","last_clean":"2023-10-09T10:14:38.418213+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:14:38.418213+0000","last_undegraded":"2023-10-09T10:14:38.418213+0000","last_fullsized":"2023-10-09T10:14:38.418213+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":203,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419587+0000","last_change":"2023-10-09T10:07:42.903741+0000","last_active":"2023-10-09T10:14:38.419587+0000","last_peered":"2023-10-09T10:14:38.419587+0000","last_clean":"2023-10-09T10:14:38.419587+0000","last_became_active":"2023-10-09T10:07:42.902996+0000","last_became_peered":"2023-10-09T10:07:42.902996+0000","last_unstale":"2023-10-09T10:14:38.419587+0000","last_undegraded":"2023-10-09T10:14:38.419587+0000","last_fullsized":"2023-10-09T10:14:38.419587+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:55.244692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":226,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.418655+0000","last_change":"2023-10-09T10:03:30.129058+0000","last_active":"2023-10-09T10:14:38.418655+0000","last_peered":"2023-10-09T10:14:38.418655+0000","last_clean":"2023-10-09T10:14:38.418655+0000","last_became_active":"2023-10-09T10:03:30.091806+0000","last_became_peered":"2023-10-09T10:03:30.091806+0000","last_unstale":"2023-10-09T10:14:38.418655+0000","last_undegraded":"2023-10-09T10:14:38.418655+0000","last_fullsized":"2023-10-09T10:14:38.418655+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:46.053172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":289,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.413462+0000","last_change":"2023-10-09T10:07:42.917909+0000","last_active":"2023-10-09T10:14:38.413462+0000","last_peered":"2023-10-09T10:14:38.413462+0000","last_clean":"2023-10-09T10:14:38.413462+0000","last_became_active":"2023-10-09T10:07:42.917580+0000","last_became_peered":"2023-10-09T10:07:42.917580+0000","last_unstale":"2023-10-09T10:14:38.413462+0000","last_undegraded":"2023-10-09T10:14:38.413462+0000","last_fullsized":"2023-10-09T10:14:38.413462+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:26.080598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"336'6007","reported_seq":6035,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:46.936386+0000","last_change":"2023-10-09T10:13:32.575493+0000","last_active":"2023-10-09T10:14:46.936386+0000","last_peered":"2023-10-09T10:14:46.936386+0000","last_clean":"2023-10-09T10:14:46.936386+0000","last_became_active":"2023-10-09T10:13:32.575194+0000","last_became_peered":"2023-10-09T10:13:32.575194+0000","last_unstale":"2023-10-09T10:14:46.936386+0000","last_undegraded":"2023-10-09T10:14:46.936386+0000","last_fullsized":"2023-10-09T10:14:46.936386+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6007,"log_dups_size":0,"ondisk_log_size":6007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:21.292749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393674752,"num_objects":6007,"num_object_clones":0,"num_object_copies":18021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6007,"num_write_kb":384448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":368,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.420325+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:14:38.420325+0000","last_peered":"2023-10-09T10:14:38.420325+0000","last_clean":"2023-10-09T10:14:38.420325+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:14:38.420325+0000","last_undegraded":"2023-10-09T10:14:38.420325+0000","last_fullsized":"2023-10-09T10:14:38.420325+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":154,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419174+0000","last_change":"2023-10-09T10:03:30.089832+0000","last_active":"2023-10-09T10:14:38.419174+0000","last_peered":"2023-10-09T10:14:38.419174+0000","last_clean":"2023-10-09T10:14:38.419174+0000","last_became_active":"2023-10-09T10:03:30.089563+0000","last_became_peered":"2023-10-09T10:03:30.089563+0000","last_unstale":"2023-10-09T10:14:38.419174+0000","last_undegraded":"2023-10-09T10:14:38.419174+0000","last_fullsized":"2023-10-09T10:14:38.419174+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"336'6025","reported_seq":6053,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:48.770627+0000","last_change":"2023-10-09T10:13:32.571424+0000","last_active":"2023-10-09T10:14:48.770627+0000","last_peered":"2023-10-09T10:14:48.770627+0000","last_clean":"2023-10-09T10:14:48.770627+0000","last_became_active":"2023-10-09T10:13:32.571091+0000","last_became_peered":"2023-10-09T10:13:32.571091+0000","last_unstale":"2023-10-09T10:14:48.770627+0000","last_undegraded":"2023-10-09T10:14:48.770627+0000","last_fullsized":"2023-10-09T10:14:48.770627+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6025,"log_dups_size":0,"ondisk_log_size":6025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:33.565620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394854400,"num_objects":6025,"num_object_clones":0,"num_object_copies":18075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6025,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":356,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.423400+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:14:38.423400+0000","last_peered":"2023-10-09T10:14:38.423400+0000","last_clean":"2023-10-09T10:14:38.423400+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:14:38.423400+0000","last_undegraded":"2023-10-09T10:14:38.423400+0000","last_fullsized":"2023-10-09T10:14:38.423400+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":248,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.416851+0000","last_change":"2023-10-09T10:08:35.099101+0000","last_active":"2023-10-09T10:14:38.416851+0000","last_peered":"2023-10-09T10:14:38.416851+0000","last_clean":"2023-10-09T10:14:38.416851+0000","last_became_active":"2023-10-09T10:08:35.098651+0000","last_became_peered":"2023-10-09T10:08:35.098651+0000","last_unstale":"2023-10-09T10:14:38.416851+0000","last_undegraded":"2023-10-09T10:14:38.416851+0000","last_fullsized":"2023-10-09T10:14:38.416851+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:24.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"336'6021","reported_seq":6048,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:48.042691+0000","last_change":"2023-10-09T10:13:32.575491+0000","last_active":"2023-10-09T10:14:48.042691+0000","last_peered":"2023-10-09T10:14:48.042691+0000","last_clean":"2023-10-09T10:14:48.042691+0000","last_became_active":"2023-10-09T10:13:32.575164+0000","last_became_peered":"2023-10-09T10:13:32.575164+0000","last_unstale":"2023-10-09T10:14:48.042691+0000","last_undegraded":"2023-10-09T10:14:48.042691+0000","last_fullsized":"2023-10-09T10:14:48.042691+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394592256,"num_objects":6021,"num_object_clones":0,"num_object_copies":18063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6021,"num_write_kb":385344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":158,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419650+0000","last_change":"2023-10-09T10:03:30.118590+0000","last_active":"2023-10-09T10:14:38.419650+0000","last_peered":"2023-10-09T10:14:38.419650+0000","last_clean":"2023-10-09T10:14:38.419650+0000","last_became_active":"2023-10-09T10:03:30.092139+0000","last_became_peered":"2023-10-09T10:03:30.092139+0000","last_unstale":"2023-10-09T10:14:38.419650+0000","last_undegraded":"2023-10-09T10:14:38.419650+0000","last_fullsized":"2023-10-09T10:14:38.419650+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.438874+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:14:38.438874+0000","last_peered":"2023-10-09T10:14:38.438874+0000","last_clean":"2023-10-09T10:14:38.438874+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:14:38.438874+0000","last_undegraded":"2023-10-09T10:14:38.438874+0000","last_fullsized":"2023-10-09T10:14:38.438874+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"336'6103","reported_seq":6130,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:48.027972+0000","last_change":"2023-10-09T10:13:32.857374+0000","last_active":"2023-10-09T10:14:48.027972+0000","last_peered":"2023-10-09T10:14:48.027972+0000","last_clean":"2023-10-09T10:14:48.027972+0000","last_became_active":"2023-10-09T10:13:32.857082+0000","last_became_peered":"2023-10-09T10:13:32.857082+0000","last_unstale":"2023-10-09T10:14:48.027972+0000","last_undegraded":"2023-10-09T10:14:48.027972+0000","last_fullsized":"2023-10-09T10:14:48.027972+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6103,"log_dups_size":0,"ondisk_log_size":6103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399966208,"num_objects":6103,"num_object_clones":0,"num_object_copies":18309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6103,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":169,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419475+0000","last_change":"2023-10-09T10:07:42.915817+0000","last_active":"2023-10-09T10:14:38.419475+0000","last_peered":"2023-10-09T10:14:38.419475+0000","last_clean":"2023-10-09T10:14:38.419475+0000","last_became_active":"2023-10-09T10:07:42.915470+0000","last_became_peered":"2023-10-09T10:07:42.915470+0000","last_unstale":"2023-10-09T10:14:38.419475+0000","last_undegraded":"2023-10-09T10:14:38.419475+0000","last_fullsized":"2023-10-09T10:14:38.419475+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":204,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.423690+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:14:38.423690+0000","last_peered":"2023-10-09T10:14:38.423690+0000","last_clean":"2023-10-09T10:14:38.423690+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:14:38.423690+0000","last_undegraded":"2023-10-09T10:14:38.423690+0000","last_fullsized":"2023-10-09T10:14:38.423690+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"336'6087","reported_seq":6115,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:46.601865+0000","last_change":"2023-10-09T10:13:32.695943+0000","last_active":"2023-10-09T10:14:46.601865+0000","last_peered":"2023-10-09T10:14:46.601865+0000","last_clean":"2023-10-09T10:14:46.601865+0000","last_became_active":"2023-10-09T10:13:32.695608+0000","last_became_peered":"2023-10-09T10:13:32.695608+0000","last_unstale":"2023-10-09T10:14:46.601865+0000","last_undegraded":"2023-10-09T10:14:46.601865+0000","last_fullsized":"2023-10-09T10:14:46.601865+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6087,"log_dups_size":0,"ondisk_log_size":6087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:05.782965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398917632,"num_objects":6087,"num_object_clones":0,"num_object_copies":18261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6087,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":331,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.412513+0000","last_change":"2023-10-09T10:08:35.096589+0000","last_active":"2023-10-09T10:14:38.412513+0000","last_peered":"2023-10-09T10:14:38.412513+0000","last_clean":"2023-10-09T10:14:38.412513+0000","last_became_active":"2023-10-09T10:08:35.096190+0000","last_became_peered":"2023-10-09T10:08:35.096190+0000","last_unstale":"2023-10-09T10:14:38.412513+0000","last_undegraded":"2023-10-09T10:14:38.412513+0000","last_fullsized":"2023-10-09T10:14:38.412513+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:56.297445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":201,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.412730+0000","last_change":"2023-10-09T10:08:35.086081+0000","last_active":"2023-10-09T10:14:38.412730+0000","last_peered":"2023-10-09T10:14:38.412730+0000","last_clean":"2023-10-09T10:14:38.412730+0000","last_became_active":"2023-10-09T10:08:35.085826+0000","last_became_peered":"2023-10-09T10:08:35.085826+0000","last_unstale":"2023-10-09T10:14:38.412730+0000","last_undegraded":"2023-10-09T10:14:38.412730+0000","last_fullsized":"2023-10-09T10:14:38.412730+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:11.131739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"336'6019","reported_seq":6047,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:48.762759+0000","last_change":"2023-10-09T10:13:32.857224+0000","last_active":"2023-10-09T10:14:48.762759+0000","last_peered":"2023-10-09T10:14:48.762759+0000","last_clean":"2023-10-09T10:14:48.762759+0000","last_became_active":"2023-10-09T10:13:32.856912+0000","last_became_peered":"2023-10-09T10:13:32.856912+0000","last_unstale":"2023-10-09T10:14:48.762759+0000","last_undegraded":"2023-10-09T10:14:48.762759+0000","last_fullsized":"2023-10-09T10:14:48.762759+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6019,"log_dups_size":0,"ondisk_log_size":6019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:25.963375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394461184,"num_objects":6019,"num_object_clones":0,"num_object_copies":18057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6019,"num_write_kb":385216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":262,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.416986+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:14:38.416986+0000","last_peered":"2023-10-09T10:14:38.416986+0000","last_clean":"2023-10-09T10:14:38.416986+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:14:38.416986+0000","last_undegraded":"2023-10-09T10:14:38.416986+0000","last_fullsized":"2023-10-09T10:14:38.416986+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":268,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417075+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:14:38.417075+0000","last_peered":"2023-10-09T10:14:38.417075+0000","last_clean":"2023-10-09T10:14:38.417075+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:14:38.417075+0000","last_undegraded":"2023-10-09T10:14:38.417075+0000","last_fullsized":"2023-10-09T10:14:38.417075+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"336'6106","reported_seq":6133,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:50.194259+0000","last_change":"2023-10-09T10:13:32.577948+0000","last_active":"2023-10-09T10:14:50.194259+0000","last_peered":"2023-10-09T10:14:50.194259+0000","last_clean":"2023-10-09T10:14:50.194259+0000","last_became_active":"2023-10-09T10:13:32.577597+0000","last_became_peered":"2023-10-09T10:13:32.577597+0000","last_unstale":"2023-10-09T10:14:50.194259+0000","last_undegraded":"2023-10-09T10:14:50.194259+0000","last_fullsized":"2023-10-09T10:14:50.194259+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6106,"log_dups_size":0,"ondisk_log_size":6106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400162816,"num_objects":6106,"num_object_clones":0,"num_object_copies":18318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6106,"num_write_kb":390784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":230,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.418182+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:14:38.418182+0000","last_peered":"2023-10-09T10:14:38.418182+0000","last_clean":"2023-10-09T10:14:38.418182+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:14:38.418182+0000","last_undegraded":"2023-10-09T10:14:38.418182+0000","last_fullsized":"2023-10-09T10:14:38.418182+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":204,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.418952+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:14:38.418952+0000","last_peered":"2023-10-09T10:14:38.418952+0000","last_clean":"2023-10-09T10:14:38.418952+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:14:38.418952+0000","last_undegraded":"2023-10-09T10:14:38.418952+0000","last_fullsized":"2023-10-09T10:14:38.418952+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"336'5868","reported_seq":5895,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:46.567867+0000","last_change":"2023-10-09T10:13:32.857205+0000","last_active":"2023-10-09T10:14:46.567867+0000","last_peered":"2023-10-09T10:14:46.567867+0000","last_clean":"2023-10-09T10:14:46.567867+0000","last_became_active":"2023-10-09T10:13:32.856928+0000","last_became_peered":"2023-10-09T10:13:32.856928+0000","last_unstale":"2023-10-09T10:14:46.567867+0000","last_undegraded":"2023-10-09T10:14:46.567867+0000","last_fullsized":"2023-10-09T10:14:46.567867+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5868,"log_dups_size":0,"ondisk_log_size":5868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:43:02.753960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384565248,"num_objects":5868,"num_object_clones":0,"num_object_copies":17604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5868,"num_write_kb":375552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":291,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.409594+0000","last_change":"2023-10-09T10:07:42.933058+0000","last_active":"2023-10-09T10:14:38.409594+0000","last_peered":"2023-10-09T10:14:38.409594+0000","last_clean":"2023-10-09T10:14:38.409594+0000","last_became_active":"2023-10-09T10:07:42.930282+0000","last_became_peered":"2023-10-09T10:07:42.930282+0000","last_unstale":"2023-10-09T10:14:38.409594+0000","last_undegraded":"2023-10-09T10:14:38.409594+0000","last_fullsized":"2023-10-09T10:14:38.409594+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:58.706680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":341,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:45.243210+0000","last_change":"2023-10-09T10:08:35.094230+0000","last_active":"2023-10-09T10:14:45.243210+0000","last_peered":"2023-10-09T10:14:45.243210+0000","last_clean":"2023-10-09T10:14:45.243210+0000","last_became_active":"2023-10-09T10:08:35.092513+0000","last_became_peered":"2023-10-09T10:08:35.092513+0000","last_unstale":"2023-10-09T10:14:45.243210+0000","last_undegraded":"2023-10-09T10:14:45.243210+0000","last_fullsized":"2023-10-09T10:14:45.243210+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:27.652270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"336'6105","reported_seq":6132,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:48.056125+0000","last_change":"2023-10-09T10:13:32.573670+0000","last_active":"2023-10-09T10:14:48.056125+0000","last_peered":"2023-10-09T10:14:48.056125+0000","last_clean":"2023-10-09T10:14:48.056125+0000","last_became_active":"2023-10-09T10:13:32.573356+0000","last_became_peered":"2023-10-09T10:13:32.573356+0000","last_unstale":"2023-10-09T10:14:48.056125+0000","last_undegraded":"2023-10-09T10:14:48.056125+0000","last_fullsized":"2023-10-09T10:14:48.056125+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6105,"log_dups_size":0,"ondisk_log_size":6105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400097280,"num_objects":6105,"num_object_clones":0,"num_object_copies":18315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6105,"num_write_kb":390720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":340,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417372+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:14:38.417372+0000","last_peered":"2023-10-09T10:14:38.417372+0000","last_clean":"2023-10-09T10:14:38.417372+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:14:38.417372+0000","last_undegraded":"2023-10-09T10:14:38.417372+0000","last_fullsized":"2023-10-09T10:14:38.417372+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419471+0000","last_change":"2023-10-09T10:07:42.915812+0000","last_active":"2023-10-09T10:14:38.419471+0000","last_peered":"2023-10-09T10:14:38.419471+0000","last_clean":"2023-10-09T10:14:38.419471+0000","last_became_active":"2023-10-09T10:07:42.915472+0000","last_became_peered":"2023-10-09T10:07:42.915472+0000","last_unstale":"2023-10-09T10:14:38.419471+0000","last_undegraded":"2023-10-09T10:14:38.419471+0000","last_fullsized":"2023-10-09T10:14:38.419471+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":297,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:45.249128+0000","last_change":"2023-10-09T10:08:35.095289+0000","last_active":"2023-10-09T10:14:45.249128+0000","last_peered":"2023-10-09T10:14:45.249128+0000","last_clean":"2023-10-09T10:14:45.249128+0000","last_became_active":"2023-10-09T10:08:35.093131+0000","last_became_peered":"2023-10-09T10:08:35.093131+0000","last_unstale":"2023-10-09T10:14:45.249128+0000","last_undegraded":"2023-10-09T10:14:45.249128+0000","last_fullsized":"2023-10-09T10:14:45.249128+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:47.127898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"336'6159","reported_seq":6187,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:46.923872+0000","last_change":"2023-10-09T10:13:32.576627+0000","last_active":"2023-10-09T10:14:46.923872+0000","last_peered":"2023-10-09T10:14:46.923872+0000","last_clean":"2023-10-09T10:14:46.923872+0000","last_became_active":"2023-10-09T10:13:32.576321+0000","last_became_peered":"2023-10-09T10:13:32.576321+0000","last_unstale":"2023-10-09T10:14:46.923872+0000","last_undegraded":"2023-10-09T10:14:46.923872+0000","last_fullsized":"2023-10-09T10:14:46.923872+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6159,"log_dups_size":0,"ondisk_log_size":6159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:04.882671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403636224,"num_objects":6159,"num_object_clones":0,"num_object_copies":18477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6159,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":154,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.421932+0000","last_change":"2023-10-09T10:03:30.090185+0000","last_active":"2023-10-09T10:14:38.421932+0000","last_peered":"2023-10-09T10:14:38.421932+0000","last_clean":"2023-10-09T10:14:38.421932+0000","last_became_active":"2023-10-09T10:03:30.089859+0000","last_became_peered":"2023-10-09T10:03:30.089859+0000","last_unstale":"2023-10-09T10:14:38.421932+0000","last_undegraded":"2023-10-09T10:14:38.421932+0000","last_fullsized":"2023-10-09T10:14:38.421932+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":154,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.423591+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:14:38.423591+0000","last_peered":"2023-10-09T10:14:38.423591+0000","last_clean":"2023-10-09T10:14:38.423591+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:14:38.423591+0000","last_undegraded":"2023-10-09T10:14:38.423591+0000","last_fullsized":"2023-10-09T10:14:38.423591+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":363,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417284+0000","last_change":"2023-10-09T10:07:42.919297+0000","last_active":"2023-10-09T10:14:38.417284+0000","last_peered":"2023-10-09T10:14:38.417284+0000","last_clean":"2023-10-09T10:14:38.417284+0000","last_became_active":"2023-10-09T10:07:42.919026+0000","last_became_peered":"2023-10-09T10:07:42.919026+0000","last_unstale":"2023-10-09T10:14:38.417284+0000","last_undegraded":"2023-10-09T10:14:38.417284+0000","last_fullsized":"2023-10-09T10:14:38.417284+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:33.657009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"336'5861","reported_seq":5888,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:48.080498+0000","last_change":"2023-10-09T10:13:32.574493+0000","last_active":"2023-10-09T10:14:48.080498+0000","last_peered":"2023-10-09T10:14:48.080498+0000","last_clean":"2023-10-09T10:14:48.080498+0000","last_became_active":"2023-10-09T10:13:32.574198+0000","last_became_peered":"2023-10-09T10:13:32.574198+0000","last_unstale":"2023-10-09T10:14:48.080498+0000","last_undegraded":"2023-10-09T10:14:48.080498+0000","last_fullsized":"2023-10-09T10:14:48.080498+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384106496,"num_objects":5861,"num_object_clones":0,"num_object_copies":17583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5861,"num_write_kb":375104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":257,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.410690+0000","last_change":"2023-10-09T10:09:47.008444+0000","last_active":"2023-10-09T10:14:38.410690+0000","last_peered":"2023-10-09T10:14:38.410690+0000","last_clean":"2023-10-09T10:14:38.410690+0000","last_became_active":"2023-10-09T10:09:47.008097+0000","last_became_peered":"2023-10-09T10:09:47.008097+0000","last_unstale":"2023-10-09T10:14:38.410690+0000","last_undegraded":"2023-10-09T10:14:38.410690+0000","last_fullsized":"2023-10-09T10:14:38.410690+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:58.454544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":333,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:45.243021+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:14:45.243021+0000","last_peered":"2023-10-09T10:14:45.243021+0000","last_clean":"2023-10-09T10:14:45.243021+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:14:45.243021+0000","last_undegraded":"2023-10-09T10:14:45.243021+0000","last_fullsized":"2023-10-09T10:14:45.243021+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":353,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417200+0000","last_change":"2023-10-09T10:03:30.080902+0000","last_active":"2023-10-09T10:14:38.417200+0000","last_peered":"2023-10-09T10:14:38.417200+0000","last_clean":"2023-10-09T10:14:38.417200+0000","last_became_active":"2023-10-09T10:03:30.080259+0000","last_became_peered":"2023-10-09T10:03:30.080259+0000","last_unstale":"2023-10-09T10:14:38.417200+0000","last_undegraded":"2023-10-09T10:14:38.417200+0000","last_fullsized":"2023-10-09T10:14:38.417200+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:32:10.506512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"336'6118","reported_seq":6146,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:46.597809+0000","last_change":"2023-10-09T10:13:32.572705+0000","last_active":"2023-10-09T10:14:46.597809+0000","last_peered":"2023-10-09T10:14:46.597809+0000","last_clean":"2023-10-09T10:14:46.597809+0000","last_became_active":"2023-10-09T10:13:32.571805+0000","last_became_peered":"2023-10-09T10:13:32.571805+0000","last_unstale":"2023-10-09T10:14:46.597809+0000","last_undegraded":"2023-10-09T10:14:46.597809+0000","last_fullsized":"2023-10-09T10:14:46.597809+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6118,"log_dups_size":0,"ondisk_log_size":6118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:13.430092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400949248,"num_objects":6118,"num_object_clones":0,"num_object_copies":18354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6118,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":163,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419952+0000","last_change":"2023-10-09T10:06:02.846244+0000","last_active":"2023-10-09T10:14:38.419952+0000","last_peered":"2023-10-09T10:14:38.419952+0000","last_clean":"2023-10-09T10:14:38.419952+0000","last_became_active":"2023-10-09T10:06:02.845858+0000","last_became_peered":"2023-10-09T10:06:02.845858+0000","last_unstale":"2023-10-09T10:14:38.419952+0000","last_undegraded":"2023-10-09T10:14:38.419952+0000","last_fullsized":"2023-10-09T10:14:38.419952+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":582,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.418113+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:14:38.418113+0000","last_peered":"2023-10-09T10:14:38.418113+0000","last_clean":"2023-10-09T10:14:38.418113+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:14:38.418113+0000","last_undegraded":"2023-10-09T10:14:38.418113+0000","last_fullsized":"2023-10-09T10:14:38.418113+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":241,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.409269+0000","last_change":"2023-10-09T10:07:42.910134+0000","last_active":"2023-10-09T10:14:38.409269+0000","last_peered":"2023-10-09T10:14:38.409269+0000","last_clean":"2023-10-09T10:14:38.409269+0000","last_became_active":"2023-10-09T10:07:42.909354+0000","last_became_peered":"2023-10-09T10:07:42.909354+0000","last_unstale":"2023-10-09T10:14:38.409269+0000","last_undegraded":"2023-10-09T10:14:38.409269+0000","last_fullsized":"2023-10-09T10:14:38.409269+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:49.600396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":251,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.422208+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:14:38.422208+0000","last_peered":"2023-10-09T10:14:38.422208+0000","last_clean":"2023-10-09T10:14:38.422208+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:14:38.422208+0000","last_undegraded":"2023-10-09T10:14:38.422208+0000","last_fullsized":"2023-10-09T10:14:38.422208+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"336'5980","reported_seq":6008,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:46.929978+0000","last_change":"2023-10-09T10:13:32.694210+0000","last_active":"2023-10-09T10:14:46.929978+0000","last_peered":"2023-10-09T10:14:46.929978+0000","last_clean":"2023-10-09T10:14:46.929978+0000","last_became_active":"2023-10-09T10:13:32.693883+0000","last_became_peered":"2023-10-09T10:13:32.693883+0000","last_unstale":"2023-10-09T10:14:46.929978+0000","last_undegraded":"2023-10-09T10:14:46.929978+0000","last_fullsized":"2023-10-09T10:14:46.929978+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5980,"log_dups_size":0,"ondisk_log_size":5980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:35.108481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391905280,"num_objects":5980,"num_object_clones":0,"num_object_copies":17940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5980,"num_write_kb":382720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":313,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417402+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:14:38.417402+0000","last_peered":"2023-10-09T10:14:38.417402+0000","last_clean":"2023-10-09T10:14:38.417402+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:14:38.417402+0000","last_undegraded":"2023-10-09T10:14:38.417402+0000","last_fullsized":"2023-10-09T10:14:38.417402+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":214,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419600+0000","last_change":"2023-10-09T10:08:35.097954+0000","last_active":"2023-10-09T10:14:38.419600+0000","last_peered":"2023-10-09T10:14:38.419600+0000","last_clean":"2023-10-09T10:14:38.419600+0000","last_became_active":"2023-10-09T10:08:35.097661+0000","last_became_peered":"2023-10-09T10:08:35.097661+0000","last_unstale":"2023-10-09T10:14:38.419600+0000","last_undegraded":"2023-10-09T10:14:38.419600+0000","last_fullsized":"2023-10-09T10:14:38.419600+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:36.807469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":242,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419475+0000","last_change":"2023-10-09T10:09:47.008841+0000","last_active":"2023-10-09T10:14:38.419475+0000","last_peered":"2023-10-09T10:14:38.419475+0000","last_clean":"2023-10-09T10:14:38.419475+0000","last_became_active":"2023-10-09T10:09:47.008535+0000","last_became_peered":"2023-10-09T10:09:47.008535+0000","last_unstale":"2023-10-09T10:14:38.419475+0000","last_undegraded":"2023-10-09T10:14:38.419475+0000","last_fullsized":"2023-10-09T10:14:38.419475+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:02.279780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"336'6046","reported_seq":6074,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:48.765585+0000","last_change":"2023-10-09T10:13:32.576814+0000","last_active":"2023-10-09T10:14:48.765585+0000","last_peered":"2023-10-09T10:14:48.765585+0000","last_clean":"2023-10-09T10:14:48.765585+0000","last_became_active":"2023-10-09T10:13:32.576567+0000","last_became_peered":"2023-10-09T10:13:32.576567+0000","last_unstale":"2023-10-09T10:14:48.765585+0000","last_undegraded":"2023-10-09T10:14:48.765585+0000","last_fullsized":"2023-10-09T10:14:48.765585+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6046,"log_dups_size":0,"ondisk_log_size":6046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:37.032579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396230656,"num_objects":6046,"num_object_clones":0,"num_object_copies":18138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6046,"num_write_kb":386944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":355,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.422010+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:14:38.422010+0000","last_peered":"2023-10-09T10:14:38.422010+0000","last_clean":"2023-10-09T10:14:38.422010+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:14:38.422010+0000","last_undegraded":"2023-10-09T10:14:38.422010+0000","last_fullsized":"2023-10-09T10:14:38.422010+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":369,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417459+0000","last_change":"2023-10-09T10:09:47.006092+0000","last_active":"2023-10-09T10:14:38.417459+0000","last_peered":"2023-10-09T10:14:38.417459+0000","last_clean":"2023-10-09T10:14:38.417459+0000","last_became_active":"2023-10-09T10:09:47.005735+0000","last_became_peered":"2023-10-09T10:09:47.005735+0000","last_unstale":"2023-10-09T10:14:38.417459+0000","last_undegraded":"2023-10-09T10:14:38.417459+0000","last_fullsized":"2023-10-09T10:14:38.417459+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:03.615850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":206,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.423617+0000","last_change":"2023-10-09T10:03:30.082191+0000","last_active":"2023-10-09T10:14:38.423617+0000","last_peered":"2023-10-09T10:14:38.423617+0000","last_clean":"2023-10-09T10:14:38.423617+0000","last_became_active":"2023-10-09T10:03:30.081876+0000","last_became_peered":"2023-10-09T10:03:30.081876+0000","last_unstale":"2023-10-09T10:14:38.423617+0000","last_undegraded":"2023-10-09T10:14:38.423617+0000","last_fullsized":"2023-10-09T10:14:38.423617+0000","mapping_epoch":191,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:57.192799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"336'6197","reported_seq":6225,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:48.753726+0000","last_change":"2023-10-09T10:13:32.575359+0000","last_active":"2023-10-09T10:14:48.753726+0000","last_peered":"2023-10-09T10:14:48.753726+0000","last_clean":"2023-10-09T10:14:48.753726+0000","last_became_active":"2023-10-09T10:13:32.575050+0000","last_became_peered":"2023-10-09T10:13:32.575050+0000","last_unstale":"2023-10-09T10:14:48.753726+0000","last_undegraded":"2023-10-09T10:14:48.753726+0000","last_fullsized":"2023-10-09T10:14:48.753726+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6197,"log_dups_size":0,"ondisk_log_size":6197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:15:48.495574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406126592,"num_objects":6197,"num_object_clones":0,"num_object_copies":18591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6197,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":300,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.413551+0000","last_change":"2023-10-09T10:07:42.917593+0000","last_active":"2023-10-09T10:14:38.413551+0000","last_peered":"2023-10-09T10:14:38.413551+0000","last_clean":"2023-10-09T10:14:38.413551+0000","last_became_active":"2023-10-09T10:07:42.915009+0000","last_became_peered":"2023-10-09T10:07:42.915009+0000","last_unstale":"2023-10-09T10:14:38.413551+0000","last_undegraded":"2023-10-09T10:14:38.413551+0000","last_fullsized":"2023-10-09T10:14:38.413551+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:56.699957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":314,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419058+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:14:38.419058+0000","last_peered":"2023-10-09T10:14:38.419058+0000","last_clean":"2023-10-09T10:14:38.419058+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:14:38.419058+0000","last_undegraded":"2023-10-09T10:14:38.419058+0000","last_fullsized":"2023-10-09T10:14:38.419058+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":289,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.409524+0000","last_change":"2023-10-09T10:07:42.936383+0000","last_active":"2023-10-09T10:14:38.409524+0000","last_peered":"2023-10-09T10:14:38.409524+0000","last_clean":"2023-10-09T10:14:38.409524+0000","last_became_active":"2023-10-09T10:07:42.934499+0000","last_became_peered":"2023-10-09T10:07:42.934499+0000","last_unstale":"2023-10-09T10:14:38.409524+0000","last_undegraded":"2023-10-09T10:14:38.409524+0000","last_fullsized":"2023-10-09T10:14:38.409524+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:00.267558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"336'5949","reported_seq":5977,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:14:46.623791+0000","last_change":"2023-10-09T10:13:32.577001+0000","last_active":"2023-10-09T10:14:46.623791+0000","last_peered":"2023-10-09T10:14:46.623791+0000","last_clean":"2023-10-09T10:14:46.623791+0000","last_became_active":"2023-10-09T10:13:32.576694+0000","last_became_peered":"2023-10-09T10:13:32.576694+0000","last_unstale":"2023-10-09T10:14:46.623791+0000","last_undegraded":"2023-10-09T10:14:46.623791+0000","last_fullsized":"2023-10-09T10:14:46.623791+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5949,"log_dups_size":0,"ondisk_log_size":5949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:36.747060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389873664,"num_objects":5949,"num_object_clones":0,"num_object_copies":17847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5949,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":244,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.438768+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:14:38.438768+0000","last_peered":"2023-10-09T10:14:38.438768+0000","last_clean":"2023-10-09T10:14:38.438768+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:14:38.438768+0000","last_undegraded":"2023-10-09T10:14:38.438768+0000","last_fullsized":"2023-10-09T10:14:38.438768+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":208,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419392+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:14:38.419392+0000","last_peered":"2023-10-09T10:14:38.419392+0000","last_clean":"2023-10-09T10:14:38.419392+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:14:38.419392+0000","last_undegraded":"2023-10-09T10:14:38.419392+0000","last_fullsized":"2023-10-09T10:14:38.419392+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":189,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419125+0000","last_change":"2023-10-09T10:08:35.085261+0000","last_active":"2023-10-09T10:14:38.419125+0000","last_peered":"2023-10-09T10:14:38.419125+0000","last_clean":"2023-10-09T10:14:38.419125+0000","last_became_active":"2023-10-09T10:08:35.084935+0000","last_became_peered":"2023-10-09T10:08:35.084935+0000","last_unstale":"2023-10-09T10:14:38.419125+0000","last_undegraded":"2023-10-09T10:14:38.419125+0000","last_fullsized":"2023-10-09T10:14:38.419125+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":202,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.423539+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:14:38.423539+0000","last_peered":"2023-10-09T10:14:38.423539+0000","last_clean":"2023-10-09T10:14:38.423539+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:14:38.423539+0000","last_undegraded":"2023-10-09T10:14:38.423539+0000","last_fullsized":"2023-10-09T10:14:38.423539+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":275,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.417532+0000","last_change":"2023-10-09T10:09:47.011234+0000","last_active":"2023-10-09T10:14:38.417532+0000","last_peered":"2023-10-09T10:14:38.417532+0000","last_clean":"2023-10-09T10:14:38.417532+0000","last_became_active":"2023-10-09T10:09:47.010933+0000","last_became_peered":"2023-10-09T10:09:47.010933+0000","last_unstale":"2023-10-09T10:14:38.417532+0000","last_undegraded":"2023-10-09T10:14:38.417532+0000","last_fullsized":"2023-10-09T10:14:38.417532+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:07.103134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":173,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419394+0000","last_change":"2023-10-09T10:08:35.090008+0000","last_active":"2023-10-09T10:14:38.419394+0000","last_peered":"2023-10-09T10:14:38.419394+0000","last_clean":"2023-10-09T10:14:38.419394+0000","last_became_active":"2023-10-09T10:08:35.089658+0000","last_became_peered":"2023-10-09T10:08:35.089658+0000","last_unstale":"2023-10-09T10:14:38.419394+0000","last_undegraded":"2023-10-09T10:14:38.419394+0000","last_fullsized":"2023-10-09T10:14:38.419394+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":215,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.412426+0000","last_change":"2023-10-09T10:08:35.091367+0000","last_active":"2023-10-09T10:14:38.412426+0000","last_peered":"2023-10-09T10:14:38.412426+0000","last_clean":"2023-10-09T10:14:38.412426+0000","last_became_active":"2023-10-09T10:08:35.090940+0000","last_became_peered":"2023-10-09T10:08:35.090940+0000","last_unstale":"2023-10-09T10:14:38.412426+0000","last_undegraded":"2023-10-09T10:14:38.412426+0000","last_fullsized":"2023-10-09T10:14:38.412426+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:54.564264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":173,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419389+0000","last_change":"2023-10-09T10:08:35.086307+0000","last_active":"2023-10-09T10:14:38.419389+0000","last_peered":"2023-10-09T10:14:38.419389+0000","last_clean":"2023-10-09T10:14:38.419389+0000","last_became_active":"2023-10-09T10:08:35.085960+0000","last_became_peered":"2023-10-09T10:08:35.085960+0000","last_unstale":"2023-10-09T10:14:38.419389+0000","last_undegraded":"2023-10-09T10:14:38.419389+0000","last_fullsized":"2023-10-09T10:14:38.419389+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":224,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.409193+0000","last_change":"2023-10-09T10:08:35.094547+0000","last_active":"2023-10-09T10:14:38.409193+0000","last_peered":"2023-10-09T10:14:38.409193+0000","last_clean":"2023-10-09T10:14:38.409193+0000","last_became_active":"2023-10-09T10:08:35.094271+0000","last_became_peered":"2023-10-09T10:08:35.094271+0000","last_unstale":"2023-10-09T10:14:38.409193+0000","last_undegraded":"2023-10-09T10:14:38.409193+0000","last_fullsized":"2023-10-09T10:14:38.409193+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:36.599951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":354,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.412241+0000","last_change":"2023-10-09T10:09:47.009217+0000","last_active":"2023-10-09T10:14:38.412241+0000","last_peered":"2023-10-09T10:14:38.412241+0000","last_clean":"2023-10-09T10:14:38.412241+0000","last_became_active":"2023-10-09T10:09:47.008848+0000","last_became_peered":"2023-10-09T10:09:47.008848+0000","last_unstale":"2023-10-09T10:14:38.412241+0000","last_undegraded":"2023-10-09T10:14:38.412241+0000","last_fullsized":"2023-10-09T10:14:38.412241+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:23.839707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":374,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.410917+0000","last_change":"2023-10-09T10:07:42.894222+0000","last_active":"2023-10-09T10:14:38.410917+0000","last_peered":"2023-10-09T10:14:38.410917+0000","last_clean":"2023-10-09T10:14:38.410917+0000","last_became_active":"2023-10-09T10:07:42.893805+0000","last_became_peered":"2023-10-09T10:07:42.893805+0000","last_unstale":"2023-10-09T10:14:38.410917+0000","last_undegraded":"2023-10-09T10:14:38.410917+0000","last_fullsized":"2023-10-09T10:14:38.410917+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:06.104625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":284,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.419658+0000","last_change":"2023-10-09T10:09:47.014012+0000","last_active":"2023-10-09T10:14:38.419658+0000","last_peered":"2023-10-09T10:14:38.419658+0000","last_clean":"2023-10-09T10:14:38.419658+0000","last_became_active":"2023-10-09T10:09:47.007974+0000","last_became_peered":"2023-10-09T10:09:47.007974+0000","last_unstale":"2023-10-09T10:14:38.419658+0000","last_undegraded":"2023-10-09T10:14:38.419658+0000","last_fullsized":"2023-10-09T10:14:38.419658+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:57.029545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":204,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:14:38.416735+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:14:38.416735+0000","last_peered":"2023-10-09T10:14:38.416735+0000","last_clean":"2023-10-09T10:14:38.416735+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:14:38.416735+0000","last_undegraded":"2023-10-09T10:14:38.416735+0000","last_fullsized":"2023-10-09T10:14:38.416735+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":9868,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6334119936,"num_objects":96651,"num_object_clones":0,"num_object_copies":289953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":96651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":96651,"num_write_kb":6185664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19143000064,"data_stored":19143000064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96651,"ondisk_log_size":96651,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430542,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4156976,"kb_used_data":3559080,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":89588176,"statfs":{"total":95995035648,"available":91738292224,"internally_reserved":0,"allocated":3644497920,"data_stored":3643949602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2269999999999999,"5min":2.472,"15min":2.1669999999999998},"min":{"1min":0.22900000000000001,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":8.5429999999999993,"5min":11.164,"15min":50.591999999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.2130000000000001,"5min":2.2839999999999998,"15min":2.1150000000000002},"min":{"1min":0.308,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.7629999999999999,"5min":15.974,"15min":50.747},"last":3.0030000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:14:16 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":2.6840000000000002,"15min":2.1419999999999999},"min":{"1min":0.35299999999999998,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":4.6890000000000001,"5min":14.311,"15min":25.338000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.6040000000000001,"15min":1.9650000000000001},"min":{"1min":0.30399999999999999,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":2.7040000000000002,"5min":16.526,"15min":18.27},"last":3.0209999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6080000000000001,"5min":2.613,"15min":2.3809999999999998},"min":{"1min":0.32100000000000001,"5min":0.27400000000000002,"15min":0.246},"max":{"1min":7.3479999999999999,"5min":13.775,"15min":13.775},"last":2.2879999999999998},{"interface":"front","average":{"1min":2.1200000000000001,"5min":2.5960000000000001,"15min":2.2490000000000001},"min":{"1min":0.38300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.6200000000000001,"5min":12.101000000000001,"15min":12.101000000000001},"last":0.40600000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":2.8759999999999999,"15min":2.125},"min":{"1min":0.46100000000000002,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":3.306,"5min":29.058,"15min":29.058},"last":3.3690000000000002},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.6579999999999999,"15min":2.1829999999999998},"min":{"1min":0.313,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":10.666,"5min":16.715,"15min":16.715},"last":2.4729999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:14:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.6160000000000001,"15min":1.1459999999999999},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":7.7619999999999996,"5min":12.699999999999999,"15min":12.699999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.872,"5min":1.2989999999999999,"15min":1.0389999999999999},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":4.2279999999999998,"5min":12.003,"15min":12.003},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.8979999999999999,"15min":1.29},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.219},"max":{"1min":6.766,"5min":25.277999999999999,"15min":25.277999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.70599999999999996,"5min":1.7190000000000001,"15min":1.2190000000000001},"min":{"1min":0.34499999999999997,"5min":0.29099999999999998,"15min":0.22600000000000001},"max":{"1min":2.6680000000000001,"5min":25.754000000000001,"15min":25.754000000000001},"last":0.56699999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.401,"15min":1.1180000000000001},"min":{"1min":0.32800000000000001,"5min":0.31900000000000001,"15min":0.26100000000000001},"max":{"1min":3.085,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.625,"5min":1.4330000000000001,"15min":1.0920000000000001},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":10.625,"5min":10.625,"15min":10.625},"last":0.433}]}]},{"osd":6,"up_from":42,"seq":180388626770,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1963044,"kb_used_data":1369244,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91782108,"statfs":{"total":95995035648,"available":93984878592,"internally_reserved":0,"allocated":1402105856,"data_stored":1401570780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":2.3370000000000002,"15min":1.8169999999999999},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.222},"max":{"1min":8.8079999999999998,"5min":15.435,"15min":15.435},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.988,"15min":1.7649999999999999},"min":{"1min":0.251,"5min":0.22,"15min":0.22},"max":{"1min":4.375,"5min":15.308999999999999,"15min":15.308999999999999},"last":0.36899999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:14:09 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":2.9089999999999998,"15min":2.226},"min":{"1min":0.34399999999999997,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":8.8780000000000001,"5min":35.609999999999999,"15min":35.609999999999999},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.7869999999999999,"15min":2.1539999999999999},"min":{"1min":0.28499999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.2069999999999999,"5min":32.396000000000001,"15min":32.396000000000001},"last":0.40999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.9199999999999999,"15min":1.778},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":4.4729999999999999,"5min":12.898,"15min":12.898},"last":0.316},{"interface":"front","average":{"1min":1.4359999999999999,"5min":2.4369999999999998,"15min":2.899},"min":{"1min":0.33100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.7669999999999995,"5min":14.542999999999999,"15min":14.542999999999999},"last":0.48199999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":2.766,"15min":1.8280000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":4.1520000000000001,"5min":19.222000000000001,"15min":19.222000000000001},"last":6.9640000000000004},{"interface":"front","average":{"1min":2.1040000000000001,"5min":2.5510000000000002,"15min":2.4369999999999998},"min":{"1min":0.432,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3599999999999994,"5min":11.211,"15min":11.211},"last":0.441}]},{"osd":4,"last update":"Mon Oct 9 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.131,"15min":0.67000000000000004},"min":{"1min":0.27200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2539999999999996,"5min":10.113,"15min":10.113},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.036,"15min":0.67800000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28699999999999998},"max":{"1min":2.343,"5min":9.218,"15min":9.218},"last":0.57199999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:14:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.2490000000000001,"15min":0.97199999999999998},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.23699999999999999},"max":{"1min":4.4299999999999997,"5min":7.9260000000000002,"15min":14.09},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.359,"15min":1.0469999999999999},"min":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.254},"max":{"1min":2.3799999999999999,"5min":14.465999999999999,"15min":14.465999999999999},"last":0.54500000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:14:09 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.214,"15min":0.97499999999999998},"min":{"1min":0.434,"5min":0.32800000000000001,"15min":0.252},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.254,"15min":1.02},"min":{"1min":0.46100000000000002,"5min":0.38100000000000001,"15min":0.30099999999999999},"max":{"1min":8.8550000000000004,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":0.59999999999999998}]}]},{"osd":1,"up_from":84,"seq":360777253083,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4034292,"kb_used_data":3490476,"kb_used_omap":9,"kb_used_meta":543798,"kb_avail":89710860,"statfs":{"total":95995035648,"available":91863920640,"internally_reserved":0,"allocated":3574247424,"data_stored":3573104682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":556850153},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":2.6480000000000001,"15min":2.6890000000000001},"min":{"1min":0.20699999999999999,"5min":0.183,"15min":0.183},"max":{"1min":9.3040000000000003,"5min":31.007000000000001,"15min":31.007000000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.0059999999999998,"5min":3.75,"15min":2.8759999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.192},"max":{"1min":9.5429999999999993,"5min":20.501999999999999,"15min":20.501999999999999},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:13:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.9329999999999998,"15min":3.6219999999999999},"min":{"1min":0.251,"5min":0.217,"15min":0.217},"max":{"1min":11.140000000000001,"5min":11.140000000000001,"15min":17.369},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.621,"5min":3.0699999999999998,"15min":3.6269999999999998},"min":{"1min":0.24199999999999999,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":28.024999999999999,"5min":28.024999999999999,"15min":28.024999999999999},"last":0.439}]},{"osd":3,"last update":"Mon Oct 9 10:13:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":3.4289999999999998,"15min":3.4990000000000001},"min":{"1min":0.28399999999999997,"5min":0.27400000000000002,"15min":0.23400000000000001},"max":{"1min":9.0069999999999997,"5min":21.808,"15min":21.808},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.4339999999999999,"5min":3.1640000000000001,"15min":4.3220000000000001},"min":{"1min":0.28499999999999998,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":6.3659999999999997,"5min":21.547999999999998,"15min":21.547999999999998},"last":0.53700000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.7410000000000001,"15min":1.417},"min":{"1min":0.32300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8199999999999998,"5min":21.620000000000001,"15min":21.620000000000001},"last":0.497},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.7559999999999998,"15min":1.8049999999999999},"min":{"1min":0.439,"5min":0.308,"15min":0.308},"max":{"1min":6.4800000000000004,"5min":12.904,"15min":12.904},"last":2.0409999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":2.802,"15min":2.2639999999999998},"min":{"1min":0.40000000000000002,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":4.5670000000000002,"5min":12.811,"15min":15.531000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.3639999999999999,"5min":2.806,"15min":2.6139999999999999},"min":{"1min":0.443,"5min":0.32400000000000001,"15min":0.29099999999999998},"max":{"1min":9.9629999999999992,"5min":19.513000000000002,"15min":19.706},"last":0.52500000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":2.1840000000000002,"15min":2.1259999999999999},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":5.3120000000000003,"5min":21.670999999999999,"15min":21.670999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.5109999999999999,"5min":2.7799999999999998,"15min":2.3199999999999998},"min":{"1min":0.45800000000000002,"5min":0.35499999999999998,"15min":0.35099999999999998},"max":{"1min":4.9390000000000001,"5min":12.868,"15min":15.481999999999999},"last":0.42599999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:14:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.6970000000000001,"15min":2.589},"min":{"1min":0.48799999999999999,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":10.004,"5min":10.723000000000001,"15min":16.331},"last":4.0990000000000002},{"interface":"front","average":{"1min":0.96799999999999997,"5min":2.1429999999999998,"15min":2.1760000000000002},"min":{"1min":0.45900000000000002,"5min":0.39200000000000002,"15min":0.378},"max":{"1min":2.8399999999999999,"5min":21.593,"15min":21.593},"last":2.0830000000000002}]}]},{"osd":0,"up_from":79,"seq":339302416609,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1973020,"kb_used_data":1552084,"kb_used_omap":28,"kb_used_meta":420899,"kb_avail":91772132,"statfs":{"total":95995035648,"available":93974663168,"internally_reserved":0,"allocated":1589334016,"data_stored":1588212759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":431001424},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.8919999999999999,"15min":2.2210000000000001},"min":{"1min":0.25900000000000001,"5min":0.216,"15min":0.16600000000000001},"max":{"1min":2.7410000000000001,"5min":16.797000000000001,"15min":16.797000000000001},"last":24.399999999999999},{"interface":"front","average":{"1min":1.25,"5min":4.6740000000000004,"15min":3.4199999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.21299999999999999},"max":{"1min":7.0060000000000002,"5min":31.925999999999998,"15min":31.925999999999998},"last":6.2910000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.7010000000000001,"15min":2.3300000000000001},"min":{"1min":0.26800000000000002,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":4.9109999999999996,"5min":18.003,"15min":18.003},"last":24.43},{"interface":"front","average":{"1min":1.1379999999999999,"5min":2.5339999999999998,"15min":2.9140000000000001},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.976,"5min":15.529,"15min":15.529},"last":24.459}]},{"osd":3,"last update":"Mon Oct 9 10:13:58 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":4.6779999999999999,"15min":4.8019999999999996},"min":{"1min":0.28399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":22.684000000000001,"5min":25.013999999999999,"15min":25.734999999999999},"last":11.478999999999999},{"interface":"front","average":{"1min":1.367,"5min":3.4969999999999999,"15min":4.1059999999999999},"min":{"1min":0.318,"5min":0.221,"15min":0.221},"max":{"1min":4.0220000000000002,"5min":18.928999999999998,"15min":21.923999999999999},"last":7.3700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":3.0259999999999998,"15min":1.9359999999999999},"min":{"1min":0.41799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":4.8070000000000004,"5min":20.494,"15min":20.494},"last":6.3440000000000003},{"interface":"front","average":{"1min":1.4119999999999999,"5min":2.7589999999999999,"15min":2.2109999999999999},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.109,"5min":15.907,"15min":15.907},"last":3.891}]},{"osd":5,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":3.1389999999999998,"15min":2.5550000000000002},"min":{"1min":0.371,"5min":0.36099999999999999,"15min":0.28000000000000003},"max":{"1min":18.620000000000001,"5min":18.620000000000001,"15min":18.620000000000001},"last":7.2960000000000003},{"interface":"front","average":{"1min":2.653,"5min":2.3599999999999999,"15min":1.7929999999999999},"min":{"1min":0.45200000000000001,"5min":0.30499999999999999,"15min":0.27000000000000002},"max":{"1min":19.798999999999999,"5min":19.798999999999999,"15min":19.798999999999999},"last":24.295000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":3.262,"5min":2.5750000000000002,"15min":1.99},"min":{"1min":0.40699999999999997,"5min":0.33700000000000002,"15min":0.31900000000000001},"max":{"1min":19.875,"5min":19.875,"15min":19.875},"last":24.341999999999999},{"interface":"front","average":{"1min":1.8049999999999999,"5min":3.5369999999999999,"15min":2.847},"min":{"1min":0.378,"5min":0.378,"15min":0.35699999999999998},"max":{"1min":6.0259999999999998,"5min":23.106999999999999,"15min":23.106999999999999},"last":6.3200000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:13:52 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":3.1659999999999999,"15min":2.71},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33900000000000002},"max":{"1min":18.904,"5min":18.904,"15min":26.978000000000002},"last":7.335},{"interface":"front","average":{"1min":2.8119999999999998,"5min":2.544,"15min":2.137},"min":{"1min":0.45800000000000002,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":19.843,"5min":19.843,"15min":26.768000000000001},"last":24.372}]}]},{"osd":2,"up_from":217,"seq":932007903343,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1475096,"kb_used_data":1172112,"kb_used_omap":83,"kb_used_meta":302892,"kb_avail":92270056,"statfs":{"total":95995035648,"available":94484537344,"internally_reserved":0,"allocated":1200242688,"data_stored":1199126169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":310161903},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":2.8940000000000001,"15min":2.0430000000000001},"min":{"1min":0.185,"5min":0.185,"15min":0.17899999999999999},"max":{"1min":11.327,"5min":19.372,"15min":19.372},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.488,"5min":3.4940000000000002,"15min":3.157},"min":{"1min":0.23599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.7479999999999993,"5min":20.143000000000001,"15min":20.143000000000001},"last":0.33800000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":3.7999999999999998,"15min":2.8340000000000001},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.22800000000000001},"max":{"1min":11.398,"5min":19.283999999999999,"15min":19.283999999999999},"last":2.7050000000000001},{"interface":"front","average":{"1min":1.423,"5min":3.234,"15min":2.7949999999999999},"min":{"1min":0.28599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":8.7739999999999991,"5min":19.027000000000001,"15min":19.027000000000001},"last":2.831}]},{"osd":3,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.9009999999999998,"15min":2.3159999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.368,"5min":19.370999999999999,"15min":19.370999999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.661,"5min":2.919,"15min":3.5830000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.7959999999999994,"5min":19.053000000000001,"15min":19.053000000000001},"last":2.4350000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":2.2389999999999999,"15min":1.2170000000000001},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.2530000000000001,"5min":18.873999999999999,"15min":18.873999999999999},"last":3.6859999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":2.2839999999999998,"15min":1.3140000000000001},"min":{"1min":0.36199999999999999,"5min":0.314,"15min":0.314},"max":{"1min":5.7690000000000001,"5min":15.276,"15min":15.276},"last":2.4660000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":2.2309999999999999,"15min":1.7729999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":8.8140000000000001,"5min":15.32,"15min":15.32},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.802,"5min":2.5150000000000001,"15min":1.8879999999999999},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":9.4789999999999992,"5min":17.882999999999999,"15min":17.882999999999999},"last":3.105}]},{"osd":6,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":2.3839999999999999,"15min":1.542},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":12.257,"5min":18.943999999999999,"15min":18.943999999999999},"last":2.395},{"interface":"front","average":{"1min":1.4139999999999999,"5min":2.4249999999999998,"15min":1.9870000000000001},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.33300000000000002},"max":{"1min":4.5979999999999999,"5min":15.378,"15min":15.378},"last":0.59799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:13:56 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.5529999999999999,"15min":1.6020000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.35899999999999999},"max":{"1min":12.186,"5min":18.972999999999999,"15min":18.972999999999999},"last":3.5670000000000002},{"interface":"front","average":{"1min":1.8029999999999999,"5min":2.8879999999999999,"15min":2.0449999999999999},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.376},"max":{"1min":9.9440000000000008,"5min":15.353,"15min":15.353},"last":7.6829999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824086,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3399620,"kb_used_data":3053436,"kb_used_omap":10,"kb_used_meta":346165,"kb_avail":90345532,"statfs":{"total":95995035648,"available":92513824768,"internally_reserved":0,"allocated":3126718464,"data_stored":3125384378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11197,"internal_metadata":354473027},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":3.1840000000000002,"15min":4.1970000000000001},"min":{"1min":0.21299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.5470000000000006,"5min":17.373000000000001,"15min":23.381},"last":2.5830000000000002},{"interface":"front","average":{"1min":1.788,"5min":4.4470000000000001,"15min":4.2679999999999998},"min":{"1min":0.23499999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.56,"5min":20.239999999999998,"15min":20.239999999999998},"last":2.1869999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":3.5569999999999999,"15min":3.8210000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7409999999999997,"5min":40.618000000000002,"15min":40.618000000000002},"last":6.5149999999999997},{"interface":"front","average":{"1min":2.0350000000000001,"5min":3.21,"15min":3.3570000000000002},"min":{"1min":0.27400000000000002,"5min":0.218,"15min":0.218},"max":{"1min":8.5139999999999993,"5min":15.704000000000001,"15min":23.423999999999999},"last":1.3500000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":3.347,"15min":3.609},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.6399999999999997,"5min":29.466999999999999,"15min":29.466999999999999},"last":6.4699999999999998},{"interface":"front","average":{"1min":1.5800000000000001,"5min":2.786,"15min":2.8380000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":8.5790000000000006,"5min":15.73,"15min":23.452000000000002},"last":1.321}]},{"osd":4,"last update":"Mon Oct 9 10:13:42 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":2.306,"15min":1.4990000000000001},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.6669999999999998,"5min":15.676,"15min":15.676},"last":1.381},{"interface":"front","average":{"1min":1.917,"5min":2.7410000000000001,"15min":1.921},"min":{"1min":0.379,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.673,"5min":16.478999999999999,"15min":16.478999999999999},"last":2.1070000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":3.0979999999999999,"15min":2.7749999999999999},"min":{"1min":0.33200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":13.638,"5min":13.638,"15min":13.638},"last":2.0630000000000002},{"interface":"front","average":{"1min":1.986,"5min":3.1379999999999999,"15min":2.6150000000000002},"min":{"1min":0.373,"5min":0.318,"15min":0.309},"max":{"1min":9.4860000000000007,"5min":17.242999999999999,"15min":17.242999999999999},"last":2.6400000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":2.9329999999999998,"15min":2.8029999999999999},"min":{"1min":0.45500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":13.705,"5min":16.504999999999999,"15min":16.504999999999999},"last":2.1619999999999999},{"interface":"front","average":{"1min":1.714,"5min":2.6869999999999998,"15min":2.6309999999999998},"min":{"1min":0.39800000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.6820000000000004,"5min":17.286999999999999,"15min":17.286999999999999},"last":2.7309999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:14:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":2.3740000000000001,"15min":1.7170000000000001},"min":{"1min":0.433,"5min":0.35199999999999998,"15min":0.34000000000000002},"max":{"1min":8.4079999999999995,"5min":15.635999999999999,"15min":15.635999999999999},"last":3.851},{"interface":"front","average":{"1min":1.732,"5min":3.125,"15min":2.8250000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":13.601000000000001,"5min":16.367000000000001,"15min":16.367000000000001},"last":2.1389999999999998}]}]},{"osd":4,"up_from":266,"seq":1142461300812,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1576104,"kb_used_data":1249248,"kb_used_omap":35,"kb_used_meta":326812,"kb_avail":92169048,"statfs":{"total":95995035648,"available":94381105152,"internally_reserved":0,"allocated":1279229952,"data_stored":1278689212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":334655726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.3720000000000001,"15min":0.96999999999999997},"min":{"1min":0.252,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.6970000000000001,"5min":10.881,"15min":10.881},"last":0.75},{"interface":"front","average":{"1min":1.7989999999999999,"5min":2.1469999999999998,"15min":1.355},"min":{"1min":0.318,"5min":0.224,"15min":0.224},"max":{"1min":8.2050000000000001,"5min":12.246,"15min":12.246},"last":1.0329999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":3.0569999999999999,"5min":2.7010000000000001,"15min":1.4450000000000001},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":18.303000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":0.90300000000000002},{"interface":"front","average":{"1min":2.488,"5min":2.2280000000000002,"15min":1.5920000000000001},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":18.036000000000001,"5min":18.036000000000001,"15min":18.036000000000001},"last":0.39500000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.9950000000000001,"15min":1.327},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.6689999999999996,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.9950000000000001,"5min":1.9450000000000001,"15min":1.252},"min":{"1min":0.28100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.778,"5min":11.778,"15min":11.778},"last":3.7429999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":3.105,"5min":2.5619999999999998,"15min":1.6950000000000001},"min":{"1min":0.38900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":14.323,"5min":14.323,"15min":14.323},"last":0.46500000000000002},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.847,"15min":1.625},"min":{"1min":0.379,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":25.550000000000001,"5min":25.550000000000001,"15min":25.550000000000001},"last":1.1040000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.198,"15min":0.88500000000000001},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.7509999999999999,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.72399999999999998,"15min":0.67400000000000004},"min":{"1min":0.29999999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.6689999999999996,"5min":5.7549999999999999,"15min":5.7549999999999999},"last":0.312}]},{"osd":6,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0620000000000001,"15min":0.70399999999999996},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.6449999999999996,"5min":6.851,"15min":6.851},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.1699999999999999,"15min":0.77900000000000003},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.9080000000000004,"5min":12.255000000000001,"15min":12.255000000000001},"last":0.30099999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.397,"15min":0.879},"min":{"1min":0.36799999999999999,"5min":0.312,"15min":0.312},"max":{"1min":6.6630000000000003,"5min":12.237,"15min":12.237},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.2350000000000001,"15min":0.96399999999999997},"min":{"1min":0.33300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.3879999999999999,"5min":14.964,"15min":14.964},"last":0.41799999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822997,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4096280,"kb_used_data":3498384,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":89648872,"statfs":{"total":95995035648,"available":91800444928,"internally_reserved":0,"allocated":3582345216,"data_stored":3581816604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:14:37 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.1230000000000002,"15min":1.853},"min":{"1min":0.32100000000000001,"5min":0.23300000000000001,"15min":0.21099999999999999},"max":{"1min":7.1879999999999997,"5min":10.432,"15min":15.773999999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.55,"5min":1.704,"15min":1.8580000000000001},"min":{"1min":0.27000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4119999999999999,"5min":8.4939999999999998,"15min":15.699},"last":0.42999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:14:19 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":2.3580000000000001,"15min":1.9219999999999999},"min":{"1min":0.251,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":6.274,"5min":11.891,"15min":15.689},"last":0.99199999999999999},{"interface":"front","average":{"1min":2.153,"5min":2.0089999999999999,"15min":2},"min":{"1min":0.33900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":23.099,"5min":23.099,"15min":23.099},"last":0.315}]},{"osd":2,"last update":"Mon Oct 9 10:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":2.1859999999999999,"15min":2.3479999999999999},"min":{"1min":0.29999999999999999,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.5030000000000001,"5min":10.259,"15min":15.640000000000001},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.4550000000000001,"15min":2.0179999999999998},"min":{"1min":0.36299999999999999,"5min":0.32200000000000001,"15min":0.313},"max":{"1min":3.6640000000000001,"5min":6.5529999999999999,"15min":10.656000000000001},"last":0.501}]},{"osd":3,"last update":"Mon Oct 9 10:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":2.3109999999999999,"15min":2.0030000000000001},"min":{"1min":0.42399999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.1059999999999999,"5min":9.8699999999999992,"15min":9.8699999999999992},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.80100000000000005,"5min":2.2629999999999999,"15min":2.7669999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9830000000000001,"5min":17.193999999999999,"15min":17.34},"last":0.34100000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:14:43 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.3300000000000001,"15min":0.81899999999999995},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.9819999999999993,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.73399999999999999,"15min":0.73999999999999999},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.141,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":0.47099999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:14:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":1.3939999999999999,"15min":1.1759999999999999},"min":{"1min":0.33600000000000002,"5min":0.26200000000000001,"15min":0.25},"max":{"1min":10.222,"5min":10.222,"15min":27.716000000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.3759999999999999,"15min":1.0740000000000001},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.22600000000000001},"max":{"1min":7.1219999999999999,"5min":8.2910000000000004,"15min":18.600999999999999},"last":3.0419999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.3360000000000001,"15min":1.0860000000000001},"min":{"1min":0.35499999999999998,"5min":0.312,"15min":0.25600000000000001},"max":{"1min":3.4790000000000001,"5min":8.3409999999999993,"15min":10.334},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.81799999999999995,"15min":0.95899999999999996},"min":{"1min":0.34599999999999997,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":0.67500000000000004,"5min":8.5220000000000002,"15min":12.356},"last":0.39400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3621,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1591607296,"data_stored":1591607296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3572563968,"data_stored":3572563968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1203503104,"data_stored":1203503104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3132030976,"data_stored":3132030976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1181941760,"data_stored":1181941760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3595042816,"data_stored":3595042816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1215823872,"data_stored":1215823872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3650486272,"data_stored":3650486272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:14:52.196 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:14:52.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-10-09T10:14:52.250 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:52.296 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 75 16 100759 100743 83.8611 49.1875 0.0389762 0.0119179 2023-10-09T10:14:52.296 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:52.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:52.302 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4211995929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:52.302 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4211995929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:52.325 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:52.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:52.506 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:52.639 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:52.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:52.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:52.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4211995929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:52.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:52.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:52.849 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:52.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:52.955 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:14:53.006 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:53.070 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:14:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:53.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:14:53.176 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:53.278 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:53.299 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 76 16 101573 101557 83.424 50.875 0.0415302 0.011981 2023-10-09T10:14:53.299 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:53.300 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:53.335 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:53.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:53 smithi110 ceph-mon[136032]: pgmap v950: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T10:14:53.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:53 smithi110 ceph-mon[136032]: from='client.36042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:53.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:53 smithi012 ceph-mon[147926]: pgmap v950: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T10:14:53.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:53 smithi012 ceph-mon[147926]: from='client.36042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:53.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:53 smithi012 ceph-mon[152263]: pgmap v950: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T10:14:53.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:53 smithi012 ceph-mon[152263]: from='client.36042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:14:53.782 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:53.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:53.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:53.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:14:53.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:14:53.962 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:54.100 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:14:54.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:54.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:14:54.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:54.286 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:54.299 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 77 16 102452 102436 83.0544 54.9375 0.0471133 0.0120355 2023-10-09T10:14:54.373 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:54.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:54.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:54.553 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:54.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:14:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:55.153 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:55.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:55.253 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:55.255 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:55.288 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:55.301 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 78 16 103286 103270 82.6562 52.125 0.0119842 0.0120926 2023-10-09T10:14:55.338 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:55.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:55.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:55.482 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:55.529 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:55.531 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:55 smithi110 ceph-mon[136032]: pgmap v951: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-10-09T10:14:55.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:55.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:55.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:55.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:55 smithi012 ceph-mon[147926]: pgmap v951: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-10-09T10:14:55.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:55 smithi012 ceph-mon[152263]: pgmap v951: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-10-09T10:14:55.751 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:55.995 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:56.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:56.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:56.188 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:56.301 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 79 16 104007 103991 82.1808 45.0625 0.00659652 0.0121626 2023-10-09T10:14:56.316 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:14:56.340 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 9 pg_num to 26 2023-10-09T10:14:56.381 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:14:56.382 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:56.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:56.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:14:56.490 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:56.551 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1381845063' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:56.551 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3892182136' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:14:56.551 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1381845063' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:56.551 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3892182136' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:14:56.555 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:56.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1381845063' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:14:56.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3892182136' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:14:56.860 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:56.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:56.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:56.974 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:57.010 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:57.073 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:57.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:57.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:57.186 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:57.220 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:57.282 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: pgmap v952: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T10:14:57.282 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:14:57.282 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1381845063' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3892182136' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:57 smithi110 ceph-mon[136032]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.283 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:57.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:57.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:57.301 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:14:57.301668+0000 min lat: 0.00139647 max lat: 0.278978 avg lat: 0.0122157 2023-10-09T10:14:57.301 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:14:57.302 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 80 16 104867 104851 81.8257 53.75 0.0278207 0.0122157 2023-10-09T10:14:57.387 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:57.428 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:57.486 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:57.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:57.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:57.595 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:57.631 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:57.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: pgmap v952: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T10:14:57.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:14:57.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:14:57.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1381845063' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3892182136' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[147926]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: pgmap v952: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T10:14:57.646 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1381845063' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3892182136' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.647 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-mon[152263]: failed to encode map e337 with expected crc 2023-10-09T10:14:57.681 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:14:57.682 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:14:57.684 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:57.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:57.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:57.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:57.833 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:57.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:57.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:57.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:57.916 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:14:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:14:57] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-10-09T10:14:58.014 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:58.121 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:58.309 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 81 16 105726 105710 81.471 53.6875 0.0179125 0.0122676 2023-10-09T10:14:58.325 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:58.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:58.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:58.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:14:58.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:14:58.528 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:58.567 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:14:58.569 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:58 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:14:58.569 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:58 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:14:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:58.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:14:58.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:58 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:14:58.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:58 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:14:58.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:58 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:14:58.629 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:58 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:14:58.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:14:58.713 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:58.802 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:14:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:58.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:14:58.886 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:14:58.905 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:14:58.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:58.923 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:14:58.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:58.943 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:14:58.963 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:14:58.970 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:58.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:58.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:58.981 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:14:59.000 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:14:59.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:14:59.020 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:14:59.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:14:59.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:14:59.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:14:59.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:14:59.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:14:59.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:14:59.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:14:59.022 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:14:59.074 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:14:59.101 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:59.134 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:14:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:59.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:14:59.238 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:14:59.265 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:59.309 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:14:59.310 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 82 16 105741 105725 80.4899 0.9375 0.00468211 0.0122786 2023-10-09T10:14:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:59.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:14:59.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: pgmap v954: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T10:14:59.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:14:59.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T10:14:59.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3644275368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:59.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:59.394 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: pgmap v954: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3644275368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:59.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:14:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:59.415 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:14:59.427 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:14:59.494 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:59.590 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:14:59.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:14:59.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:14:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: pgmap v954: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T10:14:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:14:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T10:14:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: failed to encode map e338 with expected crc 2023-10-09T10:14:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3644275368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:14:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:14:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:14:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:14:59.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:14:59.771 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:14:59.944 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:00.245 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:00.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:00.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.303 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[147926]: 9.7 scrub starts 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.304 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.305 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.305 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T10:15:00.305 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.305 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.305 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.305 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.305 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:00 smithi012 ceph-mon[152263]: 9.7 scrub starts 2023-10-09T10:15:00.317 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 83 16 106815 106799 80.329 67.125 0.010937 0.0124438 2023-10-09T10:15:00.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:15:00.445 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:00.536 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:15:00.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:00.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:15:00.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:00.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T10:15:00.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: failed to encode map e339 with expected crc 2023-10-09T10:15:00.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:00 smithi110 ceph-mon[136032]: 9.7 scrub starts 2023-10-09T10:15:00.721 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:01.310 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 84 16 107990 107974 80.2463 73.4375 0.0215713 0.0124568 2023-10-09T10:15:01.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:15:01.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:01 smithi012 ceph-mon[147926]: pgmap v957: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T10:15:01.673 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:01 smithi012 ceph-mon[152263]: pgmap v957: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T10:15:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:01 smithi110 ceph-mon[136032]: pgmap v957: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T10:15:01.932 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:15:01.932 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:15:01.932 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:15:01.932 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:15:01.932 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154065 7.8 7.6bf60448 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.45060011999999, 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86727346999999999, 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154065 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.933 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969555+0000", 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.934 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969553+0000", 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969561+0000", 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7109999999999995e-06 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969572+0000", 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.115e-05 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969582+0000", 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5100000000000004e-06 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969631+0000", 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9395999999999999e-05 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969640+0000", 2023-10-09T10:15:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8540000000000003e-06 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836819+0000", 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86717872399999996 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836828+0000", 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6709999999999994e-06 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154067 7.e 7.7d5b394e (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.450557828, 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86689926799999995, 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154067 2023-10-09T10:15:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969597+0000", 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969600+0000", 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8399999999999999e-06 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969607+0000", 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.922e-06 2023-10-09T10:15:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969612+0000", 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7830000000000004e-06 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976392+0000", 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067794639999999998 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976404+0000", 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1769e-05 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836486+0000", 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008206399999998 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836497+0000", 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0947e-05 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154074 7.e 7.21f6315e (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.45052792000001, 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86689908999999998, 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154074 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969627+0000", 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969629+0000", 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4509999999999999e-06 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969636+0000", 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.216e-06 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969642+0000", 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0129999999999999e-06 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976425+0000", 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006782649 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976431+0000", 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8719999999999999e-06 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836518+0000", 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008733299999995 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836526+0000", 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9610000000000005e-06 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154080 7.5 7.d54c8f95 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.45050725800002, 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.868647155, 2023-10-09T10:15:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154080 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969648+0000", 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969647+0000", 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969650+0000", 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9440000000000001e-06 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969656+0000", 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2160000000000003e-06 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969663+0000", 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5780000000000004e-06 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969744+0000", 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0678000000000002e-05 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969751+0000", 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3719999999999999e-06 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838286+0000", 2023-10-09T10:15:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.868534837 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838295+0000", 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0669999999999998e-06 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154083 7.e 7.33ccdeae (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:15:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.45047877599995, 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86687670800000005, 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154083 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969676+0000", 2023-10-09T10:15:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969679+0000", 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.892e-06 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969686+0000", 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0500000000000003e-06 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969692+0000", 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7699999999999998e-06 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976451+0000", 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067598859999999997 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.976457+0000", 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1880000000000001e-06 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836546+0000", 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86008883199999997 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836553+0000", 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5619999999999998e-06 2023-10-09T10:15:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154085 7.5 7.5fb2e7e5 (undecoded) ondisk+retry+write+known_if_redirected e221)", 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.45045875100004, 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86864703600000004, 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154085 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969696+0000", 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969698+0000", 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2050000000000001e-06 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969704+0000", 2023-10-09T10:15:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6799999999999998e-06 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969709+0000", 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5639999999999996e-06 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969770+0000", 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0999000000000003e-05 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.969775+0000", 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.95e-06 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838335+0000", 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.86855936700000003 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838343+0000", 2023-10-09T10:15:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6770000000000001e-06 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154089 7.8 7.cb7923e8 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.441070223, 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.8577726, 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154089 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979085+0000", 2023-10-09T10:15:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979083+0000", 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979093+0000", 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5319999999999995e-06 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979109+0000", 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.677e-05 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979118+0000", 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979202+0000", 2023-10-09T10:15:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3975999999999997e-05 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979214+0000", 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1324e-05 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836849+0000", 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85763573500000001 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836858+0000", 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0099999999999995e-06 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154090 7.5 7.72a407a5 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.44102126899998, 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85923808499999998, 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154090 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979134+0000", 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979133+0000", 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979137+0000", 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.332e-06 2023-10-09T10:15:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979144+0000", 2023-10-09T10:15:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0809999999999996e-06 2023-10-09T10:15:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979149+0000", 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5770000000000001e-06 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979256+0000", 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106482 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.979263+0000", 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2509999999999997e-06 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838364+0000", 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85910122099999997 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.838372+0000", 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7430000000000002e-06 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154096 7.8 7.af393d68 (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.42549437299999, 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84222455699999998, 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154096 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994661+0000", 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994659+0000", 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994668+0000", 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0710000000000001e-06 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994692+0000", 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3887999999999999e-05 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994702+0000", 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0111e-05 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994765+0000", 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3428000000000003e-05 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.994779+0000", 2023-10-09T10:15:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3946e-05 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836878+0000", 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84209816000000004 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836885+0000", 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7600000000000002e-06 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154097 7.e 7.c78fc5fe (undecoded) ondisk+write+known_if_redirected e221)", 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 539.42471605799994, 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84114410299999998, 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154097 2023-10-09T10:15:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995439+0000", 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995438+0000", 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995444+0000", 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2299999999999999e-06 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995459+0000", 2023-10-09T10:15:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5116999999999999e-05 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995467+0000", 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5329999999999994e-06 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995527+0000", 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0043000000000001e-05 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:01.995535+0000", 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9219999999999997e-06 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836576+0000", 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.84104053400000001 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.836583+0000", 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4970000000000004e-06 2023-10-09T10:15:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:160622 7.e e227/222 7:7d366766:::benchmark_data_smithi012_221426_object52844:head v 227'9935, mlcod=196'3292)", 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 523.73954130200002, 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069261389000000007, 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680614+0000", 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680613+0000", 2023-10-09T10:15:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680622+0000", 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2410000000000001e-06 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680638+0000", 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5410999999999999e-05 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680646+0000", 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0849999999999997e-06 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680752+0000", 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105919 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.680779+0000", 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7885000000000002e-05 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749829+0000", 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069049252000000005 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:17.749875+0000", 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6548000000000002e-05 2023-10-09T10:15:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.35157.0:189857 7.e e240/235 7:72010e1c:::benchmark_data_smithi012_221426_object82079:head v 240'11708, mlcod=0'0)", 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:09.473016+0000", 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 471.94713916000001, 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067498027000000002, 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473016+0000", 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473016+0000", 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473015+0000", 2023-10-09T10:15:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473021+0000", 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3619999999999997e-06 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473029+0000", 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1899999999999995e-06 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.473034+0000", 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7029999999999997e-06 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.492979+0000", 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019945086000000001 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.492994+0000", 2023-10-09T10:15:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5645999999999999e-05 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.540489+0000", 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.047494579000000002 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:09.540514+0000", 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5024999999999998e-05 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGRecoveryDeleteReply(7.3 e251,250 7:c51775e5:::benchmark_data_smithi012_221426_object58922:head,230'10428)", 2023-10-09T10:15:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:43.116592+0000", 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 438.30356343900002, 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.083055424000000003, 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116592+0000", 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116592+0000", 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116591+0000", 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116597+0000", 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8200000000000002e-06 2023-10-09T10:15:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116609+0000", 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2516e-05 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116616+0000", 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8900000000000001e-06 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116664+0000", 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7862999999999999e-05 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199647+0000", 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082982967000000005 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(7.5 251/234 [PushOp(7:ad2a751e:::benchmark_data_smithi012_221426_object89411:head, version: 206'5575, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:ad2a751e:::benchmark_data_smithi012_221426_object89411:head@206'5575, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:43.116655+0000", 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 438.30349975500002, 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082982908999999994, 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116655+0000", 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116655+0000", 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116655+0000", 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116682+0000", 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.747e-05 2023-10-09T10:15:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116696+0000", 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3750999999999999e-05 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116711+0000", 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4603000000000001e-05 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199489+0000", 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082778400000000002 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199497+0000", 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.028e-06 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199638+0000", 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014106700000000001 2023-10-09T10:15:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "pg_backfill(progress 7.5 e 251/251 lb 7:ad27a7d6:::benchmark_data_smithi012_221426_object93576:head)", 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:07:43.116729+0000", 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 438.30342658500001, 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082978932000000005, 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116729+0000", 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116729+0000", 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116728+0000", 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.116735+0000", 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.528e-06 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199466+0000", 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082731616999999993 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199477+0000", 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0193999999999999e-05 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199664+0000", 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018752100000000001 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199676+0000", 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2228999999999999e-05 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:07:43.199707+0000", 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1142000000000001e-05 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:209942 7.3 7:c41ee6b6:::benchmark_data_smithi012_221426_object102164:head [delete] snapc 0=[] ondisk+write+known_if_redirected e261)", 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:08:18.370550+0000", 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 403.049605375, 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.082343928999999996, 2023-10-09T10:15:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 209942 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370550+0000", 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370550+0000", 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370549+0000", 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370553+0000", 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.534e-06 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370575+0000", 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2643999999999999e-05 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.370593+0000", 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7706e-05 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.393648+0000", 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023055218999999998 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.393804+0000", 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000155613 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.393940+0000", 2023-10-09T10:15:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013610599999999999 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.407833+0000", 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013893476 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.407842+0000", 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.867e-06 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.452613+0000", 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044770694 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.452668+0000", 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4614999999999998e-05 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:08:18.452894+0000", 2023-10-09T10:15:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00022602500000000001 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:95854 8.f e290/288 8:f5fc2dfd:::benchmark_data_smithi012_271002_object95853:head v 290'6024, mlcod=276'276)", 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:02.773123+0000", 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 298.64703216100003, 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066014849, 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773123+0000", 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773123+0000", 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773122+0000", 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773149+0000", 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7163999999999999e-05 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773163+0000", 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.362e-05 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.773170+0000", 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0040000000000002e-06 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.781520+0000", 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.008349825 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.781557+0000", 2023-10-09T10:15:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7011999999999997e-05 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.839085+0000", 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057528982999999999 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:02.839138+0000", 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2321000000000003e-05 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:108330 8.f e292/288 8:fca24286:::benchmark_data_smithi012_271002_object108329:head v 292'6808, mlcod=277'654)", 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 280.34783315200002, 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068658946999999998, 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:15:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072351+0000", 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9468e-05 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072361+0000", 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5270000000000004e-06 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072368+0000", 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0310000000000001e-06 2023-10-09T10:15:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.081510+0000", 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091419859999999995 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.081536+0000", 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6675000000000001e-05 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.140944+0000", 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059407616000000003 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.140981+0000", 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7005999999999998e-05 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:110180 8.4 e293/288 8:2cd3d405:::benchmark_data_smithi012_271002_object1497:head v 293'6943, mlcod=0'0)", 2023-10-09T10:15:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 277.54663395300003, 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068331423000000002, 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873525+0000", 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5710000000000002e-06 2023-10-09T10:15:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873529+0000", 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4320000000000001e-06 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873533+0000", 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6629999999999999e-06 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873743+0000", 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000210688 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873766+0000", 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2617e-05 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.941812+0000", 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068045934000000002 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.941853+0000", 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0711999999999999e-05 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:136718 8.4 e297/288 8:2f021f19:::benchmark_data_smithi012_271002_object28035:head v 297'8596, mlcod=0'0)", 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 236.88408278700001, 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069669067000000001, 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536071+0000", 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536082+0000", 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1034e-05 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536091+0000", 2023-10-09T10:15:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1370000000000008e-06 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536097+0000", 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4620000000000004e-06 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.544788+0000", 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0086914880000000007 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.544818+0000", 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0079000000000001e-05 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.605693+0000", 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.06087447 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.605741+0000", 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8838000000000002e-05 2023-10-09T10:15:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:02.034 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:15:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:02.035 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:02.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:02.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:02.219 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:02.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:02 smithi012 ceph-mon[147926]: 9.7 scrub ok 2023-10-09T10:15:02.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:02 smithi012 ceph-mon[147926]: 9.8 deep-scrub starts 2023-10-09T10:15:02.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:02 smithi012 ceph-mon[147926]: 9.4 scrub starts 2023-10-09T10:15:02.309 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 85 16 108916 108900 79.9833 57.875 0.00956065 0.0124987 2023-10-09T10:15:02.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:02 smithi012 ceph-mon[152263]: 9.7 scrub ok 2023-10-09T10:15:02.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:02 smithi012 ceph-mon[152263]: 9.8 deep-scrub starts 2023-10-09T10:15:02.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:02 smithi012 ceph-mon[152263]: 9.4 scrub starts 2023-10-09T10:15:02.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:02 smithi110 ceph-mon[136032]: 9.7 scrub ok 2023-10-09T10:15:02.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:02 smithi110 ceph-mon[136032]: 9.8 deep-scrub starts 2023-10-09T10:15:02.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:02 smithi110 ceph-mon[136032]: 9.4 scrub starts 2023-10-09T10:15:02.730 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 5, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 3, 4] 2023-10-09T10:15:02.730 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:15:02.730 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [1, 3, 6, 7, 4, 0, 5, 2] 2023-10-09T10:15:02.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T10:15:03.186 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:03.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:03.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:03.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:15:03.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:03.310 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 86 16 109808 109792 79.7017 55.75 0.0258615 0.0125413 2023-10-09T10:15:03.333 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:03.362 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:03 smithi012 ceph-mon[152263]: pgmap v958: 99 pgs: 2 peering, 9 unknown, 88 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 655 op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:15:03.362 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:03 smithi012 ceph-mon[152263]: 9.0 scrub starts 2023-10-09T10:15:03.363 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:03 smithi012 ceph-mon[147926]: pgmap v958: 99 pgs: 2 peering, 9 unknown, 88 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 655 op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:15:03.363 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:03 smithi012 ceph-mon[147926]: 9.0 scrub starts 2023-10-09T10:15:03.381 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:03.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:03.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:03.488 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:03.519 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:03.561 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:03.572 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:03.574 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:03 smithi110 ceph-mon[136032]: pgmap v958: 99 pgs: 2 peering, 9 unknown, 88 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 655 op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:15:03.574 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:03 smithi110 ceph-mon[136032]: 9.0 scrub starts 2023-10-09T10:15:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:03.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:03.676 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:03.715 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:03.757 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:03.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:03.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:03.865 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:03.867 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:03.902 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:03.942 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:03.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:04.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:15:04.136 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:04.310 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 87 16 110610 110594 79.3621 50.125 0.0179638 0.0125964 2023-10-09T10:15:04.356 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:15:04.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:04 smithi012 ceph-mon[147926]: 9.0 scrub ok 2023-10-09T10:15:04.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3507192268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:04.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:04 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:04.391 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:04 smithi012 ceph-mon[152263]: 9.0 scrub ok 2023-10-09T10:15:04.391 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3507192268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:04.391 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:04 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:04.496 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:15:04.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:04.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:15:04.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:04.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:04 smithi110 ceph-mon[136032]: 9.0 scrub ok 2023-10-09T10:15:04.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3507192268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:04.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:04 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:04.653 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:04.710 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:04.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:04.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:04.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:04.872 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:04.940 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:04.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:05.049 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:05.090 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:05.138 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:05.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:05.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:05.208 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:05.242 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:05.277 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:05.312 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 88 16 111566 111550 79.1372 59.75 0.0230014 0.0126308 2023-10-09T10:15:05.333 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:05.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:05.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[152263]: pgmap v959: 99 pgs: 1 active+clean+scrubbing+deep, 2 peering, 2 unknown, 94 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 29 MiB/s wr, 469 op/s; 18 KiB/s, 0 objects/s recovering 2023-10-09T10:15:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[152263]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:15:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[152263]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T10:15:05.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[152263]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[152263]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[152263]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[147926]: pgmap v959: 99 pgs: 1 active+clean+scrubbing+deep, 2 peering, 2 unknown, 94 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 29 MiB/s wr, 469 op/s; 18 KiB/s, 0 objects/s recovering 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[147926]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[147926]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[147926]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[147926]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:05 smithi012 ceph-mon[147926]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:15:05.518 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:05.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:05 smithi110 ceph-mon[136032]: pgmap v959: 99 pgs: 1 active+clean+scrubbing+deep, 2 peering, 2 unknown, 94 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 29 MiB/s wr, 469 op/s; 18 KiB/s, 0 objects/s recovering 2023-10-09T10:15:05.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:05 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:05.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:05 smithi110 ceph-mon[136032]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:05 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:15:05.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:05 smithi110 ceph-mon[136032]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T10:15:05.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:05 smithi110 ceph-mon[136032]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:05 smithi110 ceph-mon[136032]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:05 smithi110 ceph-mon[136032]: failed to encode map e340 with expected crc 2023-10-09T10:15:05.795 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:15:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:05.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:15:05.906 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:05.941 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:05.986 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:05.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:06.090 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:06.123 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:06.161 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:06.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:06.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:06.255 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:15:06.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:06.293 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:15:06.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 89 16 112427 112411 78.8529 53.8125 0.0146022 0.0126766 2023-10-09T10:15:06.326 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:15:06.336 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:06.374 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.5. 2023-10-09T10:15:06.395 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:15:06.422 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:15:06.432 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:15:06.450 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:15:06.469 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:15:06.470 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:15:06.470 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:15:06.470 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:15:06.470 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:15:06.471 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:15:06.471 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:15:06.471 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:15:06.471 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:15:06.497 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:15:06.194+0000 7f85a99d3700 -1 mon.a@0(leader).osd e340 definitely_dead 0 2023-10-09T10:15:06.498 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:15:06.498 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:15:06.498 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2903572819' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:15:06.498 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:15:06.498 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[147926]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T10:15:06.498 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1173092767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:06.498 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:15:06.498 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:15:06.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2903572819' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:15:06.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:15:06.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[152263]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T10:15:06.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1173092767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:06 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:15:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:06 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:15:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2903572819' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:15:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:06 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T10:15:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:06 smithi110 ceph-mon[136032]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T10:15:06.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1173092767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:06.725 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:06.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:06.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:06.830 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:06.857 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:06.891 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:06.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:06.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:06.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:15:07.025 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:07.068 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:15:07.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:07.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:15:07.174 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:07.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:07.235 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:07.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:07.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:15:07.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 90 16 112472 112456 78.0088 2.8125 0.00317125 0.0126787 2023-10-09T10:15:07.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:07.401 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:07.443 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:07.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:07.539 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:15:07.539 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:15:07.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[147926]: pgmap v961: 99 pgs: 1 active+clean+scrubbing+deep, 98 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s, 0 objects/s recovering 2023-10-09T10:15:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[147926]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T10:15:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T10:15:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[147926]: osdmap e341: 8 total, 8 up, 7 in 2023-10-09T10:15:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[147926]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[147926]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[152263]: pgmap v961: 99 pgs: 1 active+clean+scrubbing+deep, 98 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s, 0 objects/s recovering 2023-10-09T10:15:07.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[152263]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T10:15:07.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T10:15:07.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[152263]: osdmap e341: 8 total, 8 up, 7 in 2023-10-09T10:15:07.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[152263]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-mon[152263]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.576 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:07 smithi110 ceph-mon[136032]: pgmap v961: 99 pgs: 1 active+clean+scrubbing+deep, 98 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s, 0 objects/s recovering 2023-10-09T10:15:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:07 smithi110 ceph-mon[136032]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:07 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T10:15:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T10:15:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:07 smithi110 ceph-mon[136032]: osdmap e341: 8 total, 8 up, 7 in 2023-10-09T10:15:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:07 smithi110 ceph-mon[136032]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:07 smithi110 ceph-mon[136032]: failed to encode map e341 with expected crc 2023-10-09T10:15:07.872 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:07.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:07.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:07.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:15:07.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:08.002 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:08.018 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:08.034 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:08.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:08.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:08.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:15:08.164 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:15:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:15:07] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T10:15:08.171 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:08.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 91 16 112472 112456 77.1524 0 - 0.0126787 2023-10-09T10:15:08.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[152263]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[152263]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[152263]: osdmap e342: 8 total, 8 up, 7 in 2023-10-09T10:15:08.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[152263]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[152263]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[152263]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[152263]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[152263]: osdmap e343: 8 total, 8 up, 7 in 2023-10-09T10:15:08.438 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[147926]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.438 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[147926]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.438 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[147926]: osdmap e342: 8 total, 8 up, 7 in 2023-10-09T10:15:08.438 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[147926]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.438 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[147926]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.439 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[147926]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.439 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[147926]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.439 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:08 smithi012 ceph-mon[147926]: osdmap e343: 8 total, 8 up, 7 in 2023-10-09T10:15:08.616 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:15:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:08.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:15:08.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:08 smithi110 ceph-mon[136032]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:08 smithi110 ceph-mon[136032]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:08 smithi110 ceph-mon[136032]: osdmap e342: 8 total, 8 up, 7 in 2023-10-09T10:15:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:08 smithi110 ceph-mon[136032]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:08 smithi110 ceph-mon[136032]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:08 smithi110 ceph-mon[136032]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:08 smithi110 ceph-mon[136032]: failed to encode map e342 with expected crc 2023-10-09T10:15:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:08 smithi110 ceph-mon[136032]: osdmap e343: 8 total, 8 up, 7 in 2023-10-09T10:15:08.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:15:08.912 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:08.981 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:15:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:08.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:15:09.086 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:15:09.087 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:09.119 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:09.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total time run: 91.7368 2023-10-09T10:15:09.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total writes made: 112472 2023-10-09T10:15:09.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Write size: 65536 2023-10-09T10:15:09.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object size: 65536 2023-10-09T10:15:09.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Bandwidth (MB/sec): 76.6268 2023-10-09T10:15:09.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Bandwidth: 34.8827 2023-10-09T10:15:09.313 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max bandwidth (MB/sec): 193.5 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min bandwidth (MB/sec): 0 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average IOPS: 1226 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev IOPS: 558.123 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max IOPS: 3096 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min IOPS: 0 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average Latency(s): 0.0130311 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Latency(s): 0.033062 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max latency(s): 2.56896 2023-10-09T10:15:09.314 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min latency(s): 0.00139647 2023-10-09T10:15:09.324 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:15:09.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[147926]: pgmap v964: 99 pgs: 2 unknown, 18 peering, 79 active+clean; 6.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:15:09.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[147926]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[147926]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[147926]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 340'7095 2023-10-09T10:15:09.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:15:09.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[147926]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[147926]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[147926]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[152263]: pgmap v964: 99 pgs: 2 unknown, 18 peering, 79 active+clean; 6.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[152263]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[152263]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[152263]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 340'7095 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[152263]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[152263]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:09 smithi012 ceph-mon[152263]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:09 smithi110 ceph-mon[136032]: pgmap v964: 99 pgs: 2 unknown, 18 peering, 79 active+clean; 6.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:15:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:09 smithi110 ceph-mon[136032]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:09 smithi110 ceph-mon[136032]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:09 smithi110 ceph-mon[136032]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 340'7095 2023-10-09T10:15:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:09 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:15:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:09 smithi110 ceph-mon[136032]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:09 smithi110 ceph-mon[136032]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:09 smithi110 ceph-mon[136032]: failed to encode map e343 with expected crc 2023-10-09T10:15:09.784 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:09.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:09.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:09.898 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:09.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:09.967 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:09.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:10.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:10.157 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:10.245 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:10.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:10.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:10.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:10.462 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:10.588 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:10.647 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:10.658 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:10.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:10.767 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:10.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:10 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:10.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:10 smithi110 ceph-mon[136032]: pgmap v966: 99 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 unknown, 18 peering, 73 active+clean; 6.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 654 op/s; 24309/337545 objects degraded (7.202%); 14003/337545 objects misplaced (4.148%); 124 KiB/s, 2 objects/s recovering 2023-10-09T10:15:10.795 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:10.833 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:10.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:10.917 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:10 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:10.917 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:10 smithi012 ceph-mon[152263]: pgmap v966: 99 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 unknown, 18 peering, 73 active+clean; 6.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 654 op/s; 24309/337545 objects degraded (7.202%); 14003/337545 objects misplaced (4.148%); 124 KiB/s, 2 objects/s recovering 2023-10-09T10:15:10.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:10 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:10.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:10 smithi012 ceph-mon[147926]: pgmap v966: 99 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 unknown, 18 peering, 73 active+clean; 6.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 654 op/s; 24309/337545 objects degraded (7.202%); 14003/337545 objects misplaced (4.148%); 124 KiB/s, 2 objects/s recovering 2023-10-09T10:15:10.937 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:15:11.077 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:11.411 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:15:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:11.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:15:11.476 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:11 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 24309/337545 objects degraded (7.202%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:11.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:11.647 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:11.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:11 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 24309/337545 objects degraded (7.202%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:11.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:11 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 24309/337545 objects degraded (7.202%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:12.037 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:12.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:12.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:12.052 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2] out_osds: [5] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 3, 4] 2023-10-09T10:15:12.052 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:15:12.052 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 1, 5, 6, 7, 2, 3, 4] 2023-10-09T10:15:12.053 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T10:15:12.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T10:15:12.144 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:12.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:12.209 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:12.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:12.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:12.342 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:12 smithi012 systemd[1]: Stopping Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:15:12.437 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:12.550 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:15:12.586 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:12.591 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:12 smithi012 ceph-mon[147926]: pgmap v967: 99 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 unknown, 18 peering, 73 active+clean; 6.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 33 MiB/s wr, 526 op/s; 24309/337662 objects degraded (7.199%); 14003/337662 objects misplaced (4.147%); 107 KiB/s, 1 objects/s recovering 2023-10-09T10:15:12.591 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2307272706' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:12.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:12 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:12.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:12.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:12.593 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:12 smithi012 ceph-mon[152263]: pgmap v967: 99 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 unknown, 18 peering, 73 active+clean; 6.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 33 MiB/s wr, 526 op/s; 24309/337662 objects degraded (7.199%); 14003/337662 objects misplaced (4.147%); 107 KiB/s, 1 objects/s recovering 2023-10-09T10:15:12.593 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2307272706' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:12.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:12 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:12.694 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:12.723 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:12.761 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:12.763 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:12 smithi110 ceph-mon[136032]: pgmap v967: 99 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 unknown, 18 peering, 73 active+clean; 6.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 33 MiB/s wr, 526 op/s; 24309/337662 objects degraded (7.199%); 14003/337662 objects misplaced (4.147%); 107 KiB/s, 1 objects/s recovering 2023-10-09T10:15:12.764 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2307272706' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:12.764 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:12 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:12.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:12.865 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:12.894 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:12.933 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:12.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:13.036 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:13.135 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:15:13.135 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:15:13.174 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:13.429 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[256974]: 2023-10-09T10:15:13.393+0000 7fe118337700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:15:13.430 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[256974]: 2023-10-09T10:15:13.393+0000 7fe118337700 -1 osd.3 344 *** Got signal Terminated *** 2023-10-09T10:15:13.430 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[256974]: 2023-10-09T10:15:13.393+0000 7fe118337700 -1 osd.3 344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:15:13.472 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:13.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:13.517 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:15:13.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:13.584 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:15:13.612 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:13.621 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:15:13.651 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:13.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:13.692 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:15:13.724 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:15:13.755 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:13.762 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:15:13.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:13 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:13.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:13 smithi110 ceph-mon[136032]: failed to encode map e344 with expected crc 2023-10-09T10:15:13.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:13 smithi110 ceph-mon[136032]: failed to encode map e344 with expected crc 2023-10-09T10:15:13.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:13.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:13 smithi110 ceph-mon[136032]: osdmap e344: 8 total, 8 up, 7 in 2023-10-09T10:15:13.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:13 smithi110 ceph-mon[136032]: osd.3 marked itself down and dead 2023-10-09T10:15:13.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2911609083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:13.784 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:13.813 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:15:13.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:13.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:13.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:15:13.871 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:15:13.871 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:15:13.871 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:15:13.871 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:15:13.871 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:15:13.871 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:15:13.872 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:15:13.872 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:15:13.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:13.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[147926]: failed to encode map e344 with expected crc 2023-10-09T10:15:13.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[147926]: failed to encode map e344 with expected crc 2023-10-09T10:15:13.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[147926]: osdmap e344: 8 total, 8 up, 7 in 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[147926]: osd.3 marked itself down and dead 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2911609083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[152263]: failed to encode map e344 with expected crc 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[152263]: failed to encode map e344 with expected crc 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[152263]: osdmap e344: 8 total, 8 up, 7 in 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[152263]: osd.3 marked itself down and dead 2023-10-09T10:15:13.890 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2911609083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:13.928 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:13.956 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:13.962 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:15:13.996 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:13.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:14.099 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:14.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:14.173 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:14.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:14.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:14.277 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:14.302 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:14.335 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:14.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:14.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:14.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:14.469 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:14.508 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:14.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:14.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:14.616 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:14.643 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:14.675 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:14.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:14.779 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:14.799 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:14.805 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:14.838 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:14.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:14.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:14.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:14.976 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:15.024 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:15.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:15.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:15.154 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:15.193 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:15.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:15.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:14 smithi110 ceph-mon[136032]: pgmap v969: 99 pgs: 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 12 MiB/s wr, 294 op/s; 56075/335721 objects degraded (16.703%); 17480/335721 objects misplaced (5.207%); 1.7 MiB/s, 28 objects/s recovering 2023-10-09T10:15:15.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:15.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:15.202 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:14 smithi110 ceph-mon[136032]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:15:15.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:14 smithi110 ceph-mon[136032]: failed to encode map e345 with expected crc 2023-10-09T10:15:15.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:14 smithi110 ceph-mon[136032]: osdmap e345: 8 total, 7 up, 7 in 2023-10-09T10:15:15.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:14 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:15:15.237 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:15 smithi012 bash[327357]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T10:15:15.237 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[147926]: pgmap v969: 99 pgs: 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 12 MiB/s wr, 294 op/s; 56075/335721 objects degraded (16.703%); 17480/335721 objects misplaced (5.207%); 1.7 MiB/s, 28 objects/s recovering 2023-10-09T10:15:15.238 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:15.238 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:15.238 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[147926]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:15:15.238 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[147926]: failed to encode map e345 with expected crc 2023-10-09T10:15:15.238 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[147926]: osdmap e345: 8 total, 7 up, 7 in 2023-10-09T10:15:15.238 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:15:15.245 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[152263]: pgmap v969: 99 pgs: 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 12 MiB/s wr, 294 op/s; 56075/335721 objects degraded (16.703%); 17480/335721 objects misplaced (5.207%); 1.7 MiB/s, 28 objects/s recovering 2023-10-09T10:15:15.246 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:15.246 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:15.246 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[152263]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:15:15.246 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[152263]: failed to encode map e345 with expected crc 2023-10-09T10:15:15.246 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[152263]: osdmap e345: 8 total, 7 up, 7 in 2023-10-09T10:15:15.246 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:14 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:15:15.303 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:15.329 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:15.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:15:15.362 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:15.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:15.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:15.492 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:15.523 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:15.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:15.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:15.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:15.666 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:15.857 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:15.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:15.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:15.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:15.987 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:16.020 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:16.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:16.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:16.124 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:16.149 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:16.182 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:16.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:16.286 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:16.311 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:16 smithi110 ceph-mon[136032]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.312 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:16 smithi110 ceph-mon[136032]: osdmap e346: 8 total, 7 up, 7 in 2023-10-09T10:15:16.312 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:16 smithi110 ceph-mon[136032]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.312 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:16 smithi110 ceph-mon[136032]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.312 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:16 smithi110 ceph-mon[136032]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.312 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:16 smithi110 ceph-mon[136032]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.312 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:16 smithi110 ceph-mon[136032]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:16 smithi110 ceph-mon[136032]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.317 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:16.329 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:15:16.356 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:16.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:16.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:16.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:16.489 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:16.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[147926]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[147926]: osdmap e346: 8 total, 7 up, 7 in 2023-10-09T10:15:16.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[147926]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[147926]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[147926]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[147926]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[147926]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[147926]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[152263]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[152263]: osdmap e346: 8 total, 7 up, 7 in 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[152263]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[152263]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[152263]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[152263]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[152263]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.525 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:16 smithi012 ceph-mon[152263]: failed to encode map e346 with expected crc 2023-10-09T10:15:16.525 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:16.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:16.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:16.707 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:17.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:17 smithi110 ceph-mon[136032]: pgmap v972: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 17 stale+active+clean, 6 active+undersized+degraded, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized, 1 active+recovering+undersized+remapped, 54 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 757 KiB/s wr, 345 op/s; 62766/331749 objects degraded (18.920%); 16998/331749 objects misplaced (5.124%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:15:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:17 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:15:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:17 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 62766/331749 objects degraded (18.920%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:17 smithi110 ceph-mon[136032]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:17 smithi110 ceph-mon[136032]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:17 smithi110 ceph-mon[136032]: osdmap e347: 8 total, 7 up, 7 in 2023-10-09T10:15:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:17 smithi110 ceph-mon[136032]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:17 smithi110 ceph-mon[136032]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[147926]: pgmap v972: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 17 stale+active+clean, 6 active+undersized+degraded, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized, 1 active+recovering+undersized+remapped, 54 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 757 KiB/s wr, 345 op/s; 62766/331749 objects degraded (18.920%); 16998/331749 objects misplaced (5.124%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:15:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:15:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 62766/331749 objects degraded (18.920%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[147926]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[147926]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[147926]: osdmap e347: 8 total, 7 up, 7 in 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[147926]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[147926]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[152263]: pgmap v972: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 17 stale+active+clean, 6 active+undersized+degraded, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized, 1 active+recovering+undersized+remapped, 54 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 757 KiB/s wr, 345 op/s; 62766/331749 objects degraded (18.920%); 16998/331749 objects misplaced (5.124%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 62766/331749 objects degraded (18.920%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[152263]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[152263]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[152263]: osdmap e347: 8 total, 7 up, 7 in 2023-10-09T10:15:17.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[152263]: failed to encode map e347 with expected crc 2023-10-09T10:15:17.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-mon[152263]: failed to encode map e347 with expected crc 2023-10-09T10:15:18.166 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:18.171 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:15:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:15:17] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T10:15:18.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:18.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:18.273 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:18.301 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:18.335 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:18.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:18.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:18.474 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:19.078 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:19.127 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:19.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:19.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:19.231 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:19.261 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:19.296 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:19 smithi012 ceph-mon[147926]: pgmap v974: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 17 stale+active+clean, 6 active+undersized+degraded, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized, 1 active+recovering+undersized+remapped, 54 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 204 B/s rd, 204 B/s wr, 264 op/s; 62766/331749 objects degraded (18.920%); 16998/331749 objects misplaced (5.124%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T10:15:19.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:19 smithi012 ceph-mon[152263]: pgmap v974: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 17 stale+active+clean, 6 active+undersized+degraded, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized, 1 active+recovering+undersized+remapped, 54 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 204 B/s rd, 204 B/s wr, 264 op/s; 62766/331749 objects degraded (18.920%); 16998/331749 objects misplaced (5.124%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T10:15:19.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:19 smithi110 ceph-mon[136032]: pgmap v974: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 17 stale+active+clean, 6 active+undersized+degraded, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized, 1 active+recovering+undersized+remapped, 54 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 204 B/s rd, 204 B/s wr, 264 op/s; 62766/331749 objects degraded (18.920%); 16998/331749 objects misplaced (5.124%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T10:15:20.203 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:20.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:20.274 INFO:teuthology.orchestra.run.smithi012.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2023-10-09T10:15:20.310 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:20.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:20.369 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:20.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:20.472 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:20.499 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:20.532 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:20.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:20.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:20.538 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:20 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:20.538 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:20 smithi012 ceph-mon[147926]: pgmap v975: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+remapped+backfill_wait, 11 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized, 2 active+recovery_wait+degraded+remapped, 46 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 170 B/s wr, 270 op/s; 79503/330855 objects degraded (24.030%); 27220/330855 objects misplaced (8.227%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T10:15:20.539 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:20 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:20.540 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:20 smithi012 ceph-mon[152263]: pgmap v975: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+remapped+backfill_wait, 11 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized, 2 active+recovery_wait+degraded+remapped, 46 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 170 B/s wr, 270 op/s; 79503/330855 objects degraded (24.030%); 27220/330855 objects misplaced (8.227%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T10:15:20.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:20.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:20 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:20.660 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:20 smithi110 ceph-mon[136032]: pgmap v975: 99 pgs: 2 remapped+peering, 3 peering, 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+remapped+backfill_wait, 11 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized, 2 active+recovery_wait+degraded+remapped, 46 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 170 B/s wr, 270 op/s; 79503/330855 objects degraded (24.030%); 27220/330855 objects misplaced (8.227%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T10:15:20.662 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:20.697 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:20.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:20.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:20.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:20.839 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:21.263 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:21.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:21.305 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:15:21.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:15:21.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:21.396 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:21.428 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:21.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:21.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:21.532 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:21.559 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:21.595 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:21.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:21.617 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:21 smithi012 ceph-mon[152263]: Health check failed: 1/109935 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:21.617 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:21 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 79451/329805 objects degraded (24.090%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:21.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:21 smithi012 ceph-mon[147926]: Health check failed: 1/109935 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:21.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:21 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 79451/329805 objects degraded (24.090%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:21.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:21 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T10:15:21.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:21 smithi110 ceph-mon[136032]: Health check failed: 1/109935 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:21.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:21 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 79451/329805 objects degraded (24.090%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:21.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:21 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T10:15:21.699 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:21.734 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:21.775 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:21.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:21.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:21.879 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:21.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:21 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T10:15:21.906 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:21.941 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:21.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:22.045 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:22.078 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:22.112 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:22.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:22.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:22.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:22.287 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:22.367 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:22.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:22.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:22.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:22.510 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:22.546 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:22.637 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:15:22.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:22 smithi012 ceph-mon[147926]: pgmap v976: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 79451/329805 objects degraded (24.090%); 46654/329805 objects misplaced (14.146%); 1/109935 objects unfound (0.001%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:15:22.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/927836762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:15:22.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:15:22.792 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:22 smithi012 ceph-mon[152263]: pgmap v976: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 79451/329805 objects degraded (24.090%); 46654/329805 objects misplaced (14.146%); 1/109935 objects unfound (0.001%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:15:22.792 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/927836762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:15:22.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:15:22.820 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:22.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:22.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:22.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:22 smithi110 ceph-mon[136032]: pgmap v976: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 79451/329805 objects degraded (24.090%); 46654/329805 objects misplaced (14.146%); 1/109935 objects unfound (0.001%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:15:22.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/927836762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:15:22.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:15:22.975 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:23.092 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:23.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:23.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:23.209 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:23.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:23.281 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:23.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:23.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:15:23.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:23.387 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:15:23.411 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:15:23.413 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:23.440 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:15:23.451 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:23.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:23.466 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:15:23.494 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:15:23.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:15:23.555 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:23.581 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:23.586 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T10:15:23.617 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:23.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:23.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:23.646 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:15:23.646 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T10:15:23.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:23.749 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:23.749 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[147926]: failed to encode map e348 with expected crc 2023-10-09T10:15:23.749 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[147926]: failed to encode map e348 with expected crc 2023-10-09T10:15:23.749 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:15:23.749 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[147926]: osdmap e348: 8 total, 7 up, 7 in 2023-10-09T10:15:23.749 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2631694256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:23.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:23.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[152263]: failed to encode map e348 with expected crc 2023-10-09T10:15:23.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[152263]: failed to encode map e348 with expected crc 2023-10-09T10:15:23.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:15:23.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[152263]: osdmap e348: 8 total, 7 up, 7 in 2023-10-09T10:15:23.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:23 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2631694256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:23.753 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:23.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:23 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:23.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:23 smithi110 ceph-mon[136032]: failed to encode map e348 with expected crc 2023-10-09T10:15:23.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:23 smithi110 ceph-mon[136032]: failed to encode map e348 with expected crc 2023-10-09T10:15:23.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:15:23.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:23 smithi110 ceph-mon[136032]: osdmap e348: 8 total, 7 up, 7 in 2023-10-09T10:15:23.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:23 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2631694256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:24.116 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:24.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:24.158 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:15:24.159 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:15:24.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:24.276 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:24.320 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:24.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:24.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:24.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:24.451 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:24.488 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:24.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:24.592 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:24.620 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:24.653 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:24.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:24.676 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:15:24.676 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:15:24.757 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:24.784 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:24.822 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:24.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:24.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:24.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:24.972 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:25.024 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:25.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:25.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:25.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:15:25.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:15:25.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:25.155 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:25.188 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:25.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:25.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:25.338 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:25 smithi012 ceph-mon[152263]: pgmap v978: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 79406/329670 objects degraded (24.087%); 46654/329670 objects misplaced (14.152%); 1/109890 objects unfound (0.001%); 12 MiB/s, 194 objects/s recovering 2023-10-09T10:15:25.345 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:25 smithi012 ceph-mon[147926]: pgmap v978: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 79406/329670 objects degraded (24.087%); 46654/329670 objects misplaced (14.152%); 1/109890 objects unfound (0.001%); 12 MiB/s, 194 objects/s recovering 2023-10-09T10:15:25.347 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:25.402 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:25.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:25.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:25 smithi110 ceph-mon[136032]: pgmap v978: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 79406/329670 objects degraded (24.087%); 46654/329670 objects misplaced (14.152%); 1/109890 objects unfound (0.001%); 12 MiB/s, 194 objects/s recovering 2023-10-09T10:15:25.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:25.536 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:25.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:25.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:25.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:25.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:25.708 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:25.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:25.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:25.777 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:25.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:25.923 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:25.948 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:25.991 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:25.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:25.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:26.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:26.128 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:26.160 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:15:26.170 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:26.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:26.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:26.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:26.337 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:26.389 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:26.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:26.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:26.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:26.531 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:26.581 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:26.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:26.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:26.712 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:26.751 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:26.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:26.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:26.856 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:26.882 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:26.918 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:26.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:26.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:27.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:27.050 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:27.094 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:27.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:27.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:27.225 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:27.261 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:27.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:27.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:27.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:27.408 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:27.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:27 smithi110 ceph-mon[136032]: pgmap v979: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 78316/327306 objects degraded (23.927%); 46795/327306 objects misplaced (14.297%); 1/109102 objects unfound (0.001%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:15:27.528 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:27.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:27.536 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:27.539 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:27 smithi012 ceph-mon[147926]: pgmap v979: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 78316/327306 objects degraded (23.927%); 46795/327306 objects misplaced (14.297%); 1/109102 objects unfound (0.001%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:15:27.539 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:27 smithi012 ceph-mon[152263]: pgmap v979: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 78316/327306 objects degraded (23.927%); 46795/327306 objects misplaced (14.297%); 1/109102 objects unfound (0.001%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:15:27.637 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:27.666 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:27.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:27.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:27.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:27.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:27.839 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:27.875 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:27.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:27.968 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:15:27.968 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:15:27.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:28.039 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:28.154 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:15:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:15:27] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T10:15:28.204 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:28.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:28.241 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:15:28.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:28.339 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:28.377 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:28.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:28.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1460271528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:28 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:28 smithi012 ceph-mon[147926]: Health check update: 1/109102 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:28.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 78316/327306 objects degraded (23.927%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:28.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1460271528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:28.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:28 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:28.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:28 smithi012 ceph-mon[152263]: Health check update: 1/109102 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:28.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 78316/327306 objects degraded (23.927%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:28.481 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:28.505 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1460271528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:28.505 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:28 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:28.505 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:28 smithi110 ceph-mon[136032]: Health check update: 1/109102 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:28.506 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 78316/327306 objects degraded (23.927%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:28.508 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:28.540 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:28.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:28.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:28.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:28.647 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2] out_osds: [5] dead_osds: [3] live_osds: [0, 1, 5, 6, 7, 2, 4] 2023-10-09T10:15:28.647 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:15:28.647 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2023-10-09T10:15:28.647 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2023-10-09T10:15:28.648 DEBUG:teuthology.orchestra.run.smithi012:osd.3> sudo journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T10:15:28.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl start ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T10:15:28.678 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:28.692 INFO:journalctl@ceph.osd.3.smithi012.stdout:-- Logs begin at Mon 2023-10-09 09:26:33 UTC. -- 2023-10-09T10:15:28.927 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:28 smithi012 systemd[1]: Starting Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:15:28.976 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:28.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:28.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:28.992 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:15:29.085 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:29.112 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:29.150 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:29.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:29.254 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:29.281 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:29.315 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:29.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:29.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[147926]: pgmap v980: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 80972/325938 objects degraded (24.843%); 43764/325938 objects misplaced (13.427%); 1/108646 objects unfound (0.001%); 11 MiB/s, 189 objects/s recovering 2023-10-09T10:15:29.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:15:29.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[147926]: failed to encode map e349 with expected crc 2023-10-09T10:15:29.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[147926]: failed to encode map e349 with expected crc 2023-10-09T10:15:29.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:29.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[147926]: osdmap e349: 8 total, 7 up, 7 in 2023-10-09T10:15:29.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:29.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:29.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[152263]: pgmap v980: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 80972/325938 objects degraded (24.843%); 43764/325938 objects misplaced (13.427%); 1/108646 objects unfound (0.001%); 11 MiB/s, 189 objects/s recovering 2023-10-09T10:15:29.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:15:29.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[152263]: failed to encode map e349 with expected crc 2023-10-09T10:15:29.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[152263]: failed to encode map e349 with expected crc 2023-10-09T10:15:29.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:29.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[152263]: osdmap e349: 8 total, 7 up, 7 in 2023-10-09T10:15:29.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:29.373 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:29.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:29 smithi110 ceph-mon[136032]: pgmap v980: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 80972/325938 objects degraded (24.843%); 43764/325938 objects misplaced (13.427%); 1/108646 objects unfound (0.001%); 11 MiB/s, 189 objects/s recovering 2023-10-09T10:15:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:29 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:15:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:29 smithi110 ceph-mon[136032]: failed to encode map e349 with expected crc 2023-10-09T10:15:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:29 smithi110 ceph-mon[136032]: failed to encode map e349 with expected crc 2023-10-09T10:15:29.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:29.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:29 smithi110 ceph-mon[136032]: osdmap e349: 8 total, 7 up, 7 in 2023-10-09T10:15:29.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:29.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:29.451 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:29.487 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:29.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:29.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:29.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:29.648 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:29.655 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:15:29.693 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:15:29.724 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:29.726 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:15:29.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:29.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:29.756 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:15:29.783 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:15:29.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:15:29.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:29.838 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:15:29.839 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:15:29.839 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:15:29.839 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:15:29.839 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:15:29.839 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:15:29.839 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:15:29.839 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:15:29.840 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:15:29.872 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:29.937 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:29.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:29.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:30.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:30.072 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:30.209 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1178647416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:30.209 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1178647416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:30.217 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:15:30.218 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:15:30.232 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:30.232 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:30.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:30.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:30.336 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:30.391 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:30.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1178647416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:31.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:15:31.074 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:30 smithi012 bash[329806]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T10:15:31.074 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:30 smithi012 bash[329806]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T10:15:31.074 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:30 smithi012 bash[329806]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T10:15:31.074 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:30 smithi012 bash[329806]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T10:15:31.074 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:30 smithi012 bash[329806]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T10:15:31.074 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:30 smithi012 bash[329806]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T10:15:31.074 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:30 smithi012 bash[329806]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T10:15:31.075 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:30 smithi012 bash[329806]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T10:15:31.332 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:31.333 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:31 smithi012 ceph-mon[152263]: pgmap v982: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 80460/323298 objects degraded (24.887%); 43451/323298 objects misplaced (13.440%); 1/107766 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:15:31.333 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:31 smithi012 ceph-mon[147926]: pgmap v982: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 80460/323298 objects degraded (24.887%); 43451/323298 objects misplaced (13.440%); 1/107766 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:15:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:31.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:31.435 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:31.462 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:31 smithi110 ceph-mon[136032]: pgmap v982: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 80460/323298 objects degraded (24.887%); 43451/323298 objects misplaced (13.440%); 1/107766 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:15:31.465 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:31.499 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:31.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:31.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:31.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:31.652 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:31.754 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:31.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:31.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:31.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:31.932 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:32.044 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:32.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:32.155 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:32.182 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:32.215 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:32.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:32.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:15:32.319 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:32.347 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:32.383 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:32.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:32.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:32.418 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:15:32.418 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154067 7.e 7:729cdabe:::benchmark_data_smithi012_221426_object46289:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 569.57920618100002, 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.711856684, 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.419 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154067 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833146+0000", 2023-10-09T10:15:32.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833144+0000", 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833150+0000", 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0079999999999999e-06 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833167+0000", 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6198999999999999e-05 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833176+0000", 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4490000000000005e-06 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833926+0000", 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00074946799999999999 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op must wait for map", 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833934+0000", 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1589999999999993e-06 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841253+0000", 2023-10-09T10:15:32.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073188610000000003 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841264+0000", 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1069e-05 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497295+0000", 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6560310870000001 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.497435+0000", 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013998400000000001 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.544884+0000", 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.047449005000000002 2023-10-09T10:15:32.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.545002+0000", 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000118678 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154074 7.e 7:7a8c6f84:::benchmark_data_smithi012_221426_object46296:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:15:32.426 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 569.57847250600003, 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7164543729999999, 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154074 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833879+0000", 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:15:32.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833884+0000", 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1059999999999999e-06 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833895+0000", 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1639e-05 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833906+0000", 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1104000000000001e-05 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833999+0000", 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2387999999999997e-05 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:15:32.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841286+0000", 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072812739999999999 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841293+0000", 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7290000000000003e-06 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500331+0000", 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.659038185 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500464+0000", 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013265799999999999 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550212+0000", 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049747711999999999 2023-10-09T10:15:32.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.550334+0000", 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012227200000000001 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.433 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154083 7.e 7:757b33cc:::benchmark_data_smithi012_221426_object46305:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:15:32.434 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:15:32.434 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 569.57840810699997, 2023-10-09T10:15:32.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7033852390000002, 2023-10-09T10:15:32.434 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.434 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.434 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.434 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154083 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833944+0000", 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833943+0000", 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833946+0000", 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0000000000000001e-06 2023-10-09T10:15:32.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833952+0000", 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9000000000000003e-06 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833957+0000", 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9629999999999997e-06 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834051+0000", 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4141999999999994e-05 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834060+0000", 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8079999999999994e-06 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841319+0000", 2023-10-09T10:15:32.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072592009999999998 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841327+0000", 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.571e-06 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500500+0000", 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6591724889999999 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.500595+0000", 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5142000000000005e-05 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.501295+0000", 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00069987000000000001 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510695+0000", 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094008289999999994 2023-10-09T10:15:32.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520701+0000", 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010005623 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534431+0000", 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013730086000000001 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.534503+0000", 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2304000000000004e-05 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537329+0000", 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028256510000000002 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.442 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.35157.0:154097 7.e 7:7fa3f1e3:::benchmark_data_smithi012_221426_object46319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e222)", 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 569.57835386600004, 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.706036967, 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.35157", 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 154097 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.833998+0000", 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834000+0000", 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.384e-06 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834005+0000", 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2499999999999997e-06 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834010+0000", 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8359999999999999e-06 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834090+0000", 2023-10-09T10:15:32.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9957999999999996e-05 2023-10-09T10:15:32.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:15:32.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.834096+0000", 2023-10-09T10:15:32.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3350000000000003e-06 2023-10-09T10:15:32.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841347+0000", 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072508479999999998 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:02.841354+0000", 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7159999999999997e-06 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510724+0000", 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.669369753 2023-10-09T10:15:32.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.510903+0000", 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017934200000000001 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.511077+0000", 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017371899999999999 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.520762+0000", 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0096847530000000008 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.525146+0000", 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043842450000000002 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537388+0000", 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012242494 2023-10-09T10:15:32.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.537446+0000", 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7565999999999997e-05 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:06:04.540035+0000", 2023-10-09T10:15:32.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025890919999999999 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114948 9.18 9:1ed3790b:::benchmark_data_smithi012_313365_object2473:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 18.188850285000001, 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9028626509999995, 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.452 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114948 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223501+0000", 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223504+0000", 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.932e-06 2023-10-09T10:15:32.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223510+0000", 2023-10-09T10:15:32.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6890000000000001e-06 2023-10-09T10:15:32.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223515+0000", 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7820000000000001e-06 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223563+0000", 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8705000000000002e-05 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223573+0000", 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3149999999999998e-06 2023-10-09T10:15:32.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.185664+0000", 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96209155300000004 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.185676+0000", 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2165e-05 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.092908+0000", 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9072312829999998 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093028+0000", 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120679 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093121+0000", 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2170999999999996e-05 2023-10-09T10:15:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.105712+0000", 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012591150000000001 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124350+0000", 2023-10-09T10:15:32.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018638268 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124362+0000", 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2362e-05 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124426+0000", 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3637000000000001e-05 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.126364+0000", 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019383549999999999 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114976 9.8 9:131c68e0:::benchmark_data_smithi012_313365_object2501:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 17.106471127999999, 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.83748516, 2023-10-09T10:15:32.461 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114976 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305879+0000", 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305887+0000", 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7619999999999995e-06 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305900+0000", 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3369e-05 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305912+0000", 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2313000000000001e-05 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.309645+0000", 2023-10-09T10:15:32.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003732815 2023-10-09T10:15:32.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.309659+0000", 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3890000000000001e-05 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.063120+0000", 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7534609549999995 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.073887+0000", 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010766901000000001 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.106337+0000", 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032450537000000002 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.110335+0000", 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003997212 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.118546+0000", 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0082116299999999993 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.127135+0000", 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085889409999999992 2023-10-09T10:15:32.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.128467+0000", 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001332176 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.130828+0000", 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00236046 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.133144+0000", 2023-10-09T10:15:32.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023158419999999998 2023-10-09T10:15:32.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.143366+0000", 2023-10-09T10:15:32.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010222337 2023-10-09T10:15:32.470 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.470 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114990 9.8 9:10d5a285:::benchmark_data_smithi012_313365_object2515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 16.149872226999999, 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0309713049999996, 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.471 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114990 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262478+0000", 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262492+0000", 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4115e-05 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.267636+0000", 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051444480000000002 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.267653+0000", 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6095000000000001e-05 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.144210+0000", 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8765578109999996 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155494+0000", 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011283985999999999 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.201082+0000", 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.045587640999999998 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.205016+0000", 2023-10-09T10:15:32.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039338280000000003 2023-10-09T10:15:32.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.214090+0000", 2023-10-09T10:15:32.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090739989999999993 2023-10-09T10:15:32.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.243654+0000", 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029564373000000001 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.265457+0000", 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021802591 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.267777+0000", 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023206949999999998 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.271653+0000", 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038760219999999998 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.293451+0000", 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021797512000000002 2023-10-09T10:15:32.478 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115028 9.18 9:1ec11435:::benchmark_data_smithi012_313365_object2553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 16.008069903999999, 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7778839790000003, 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.479 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115028 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404280+0000", 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404289+0000", 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5809999999999997e-06 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404316+0000", 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6720000000000002e-05 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404326+0000", 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0453e-05 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155549+0000", 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7512224199999995 2023-10-09T10:15:32.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155657+0000", 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108597 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155763+0000", 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106253 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.167754+0000", 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011990404 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179961+0000", 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012206995999999999 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179968+0000", 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9029999999999998e-06 2023-10-09T10:15:32.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.180006+0000", 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8136000000000002e-05 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.182166+0000", 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021601250000000002 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.485 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.486 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115046 9.8 9:10ac3fd1:::benchmark_data_smithi012_313365_object2571:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.486 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:15:32.486 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.953821135, 2023-10-09T10:15:32.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8549557910000001, 2023-10-09T10:15:32.486 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115046 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458530+0000", 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458533+0000", 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.79e-06 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458539+0000", 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4199999999999998e-06 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458543+0000", 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8060000000000002e-06 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.168371+0000", 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.709827089 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179894+0000", 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011523197000000001 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.244426+0000", 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064532795000000004 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.249276+0000", 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004849969 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.257311+0000", 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0080343040000000008 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.295944+0000", 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.038633611999999998 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.297677+0000", 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001732803 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.298328+0000", 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00065045599999999999 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.301789+0000", 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034609559999999998 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313487+0000", 2023-10-09T10:15:32.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011697982000000001 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115063 9.8 9:12e13310:::benchmark_data_smithi012_313365_object2588:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.890778096, 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9951984300000003, 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.494 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115063 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521572+0000", 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521582+0000", 2023-10-09T10:15:32.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3870000000000001e-06 2023-10-09T10:15:32.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521600+0000", 2023-10-09T10:15:32.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8029e-05 2023-10-09T10:15:32.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521609+0000", 2023-10-09T10:15:32.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7890000000000006e-06 2023-10-09T10:15:32.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.314437+0000", 2023-10-09T10:15:32.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7928272840000004 2023-10-09T10:15:32.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:32.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330107+0000", 2023-10-09T10:15:32.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015670738 2023-10-09T10:15:32.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.392854+0000", 2023-10-09T10:15:32.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062746280000000001 2023-10-09T10:15:32.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.396715+0000", 2023-10-09T10:15:32.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038610379999999998 2023-10-09T10:15:32.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.405209+0000", 2023-10-09T10:15:32.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084946369999999993 2023-10-09T10:15:32.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.468006+0000", 2023-10-09T10:15:32.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062796789000000006 2023-10-09T10:15:32.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.492245+0000", 2023-10-09T10:15:32.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024238563000000001 2023-10-09T10:15:32.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.495737+0000", 2023-10-09T10:15:32.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034923879999999999 2023-10-09T10:15:32.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.499136+0000", 2023-10-09T10:15:32.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033991400000000001 2023-10-09T10:15:32.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.516772+0000", 2023-10-09T10:15:32.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017636013999999998 2023-10-09T10:15:32.512 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.512 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.512 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.513 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115064 9.18 9:1d8cd058:::benchmark_data_smithi012_313365_object2589:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.513 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:15:32.513 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.886981001000001, 2023-10-09T10:15:32.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8195911320000002, 2023-10-09T10:15:32.514 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.514 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.514 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.515 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115064 2023-10-09T10:15:32.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.515 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:15:32.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:15:32.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525369+0000", 2023-10-09T10:15:32.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:15:32.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525377+0000", 2023-10-09T10:15:32.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4749999999999993e-06 2023-10-09T10:15:32.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525393+0000", 2023-10-09T10:15:32.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6319000000000001e-05 2023-10-09T10:15:32.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525405+0000", 2023-10-09T10:15:32.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1484000000000001e-05 2023-10-09T10:15:32.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313537+0000", 2023-10-09T10:15:32.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7881318369999999 2023-10-09T10:15:32.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313674+0000", 2023-10-09T10:15:32.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137074 2023-10-09T10:15:32.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313794+0000", 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120281 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330174+0000", 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016380212000000002 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330198+0000", 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3269e-05 2023-10-09T10:15:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342558+0000", 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012360884000000001 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342624+0000", 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5277000000000005e-05 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.344962+0000", 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023383079999999999 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 347/342 [PushOp(9:0c166afc:::benchmark_data_smithi012_313365_object2594:head, version: 320'166, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0c166afc:::benchmark_data_smithi012_313365_object2594:head@320'166, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.865518217, 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4620482260000003, 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.532 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546828+0000", 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999948 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546890+0000", 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1514999999999996e-05 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546920+0000", 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0239e-05 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546947+0000", 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7192e-05 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.007036+0000", 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4600890619999998 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.007054+0000", 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.785e-05 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.008882+0000", 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001827841 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115082 9.18 9:1d0726ed:::benchmark_data_smithi012_313365_object2607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.782786251999999, 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8115987789999997, 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115082 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629565+0000", 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629569+0000", 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.377e-06 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629577+0000", 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0830000000000004e-06 2023-10-09T10:15:32.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629584+0000", 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315693+0000", 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6861087850000001 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315831+0000", 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013798599999999999 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315971+0000", 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000139887 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330217+0000", 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014246325000000001 2023-10-09T10:15:32.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330237+0000", 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9962e-05 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.438747+0000", 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10850990200000001 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.438804+0000", 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6765000000000003e-05 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.441164+0000", 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023603669999999999 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.543 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115093 9.8 9:161687e0:::benchmark_data_smithi012_313365_object2618:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.764588701999999, 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8841582529999998, 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115093 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647762+0000", 2023-10-09T10:15:32.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647769+0000", 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2479999999999999e-06 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647781+0000", 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2565000000000001e-05 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647791+0000", 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3360000000000001e-06 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330722+0000", 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6829310240000002 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342528+0000", 2023-10-09T10:15:32.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011806542999999999 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.439337+0000", 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.096808401000000002 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.445587+0000", 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062502110000000003 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.458972+0000", 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013385052 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.493763+0000", 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.034790897000000001 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.517445+0000", 2023-10-09T10:15:32.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023681722999999998 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.519054+0000", 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016091409999999999 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.521336+0000", 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022823409999999998 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.531921+0000", 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010585331 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115107 9.8 9:160e4e76:::benchmark_data_smithi012_313365_object2632:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.736823269, 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9200908160000001, 2023-10-09T10:15:32.551 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115107 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675527+0000", 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675535+0000", 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.695e-06 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675546+0000", 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0796e-05 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675553+0000", 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0149999999999998e-06 2023-10-09T10:15:32.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.343412+0000", 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6678597110000002 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.360622+0000", 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017209067000000002 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532751+0000", 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.172129479 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.539246+0000", 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0064948999999999996 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.549540+0000", 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010293656999999999 2023-10-09T10:15:32.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.577693+0000", 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028153504999999999 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578635+0000", 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094198399999999999 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.581004+0000", 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023692980000000002 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.584057+0000", 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003052768 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595619+0000", 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011562329999999999 2023-10-09T10:15:32.558 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 347/342 [PushOp(9:0b40069f:::benchmark_data_smithi012_313365_object2631:head, version: 320'167, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0b40069f:::benchmark_data_smithi012_313365_object2631:head@320'167, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.726881389000001, 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3934248150000004, 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685469+0000", 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685554+0000", 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5025999999999995e-05 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685577+0000", 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.319e-05 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685596+0000", 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8238000000000001e-05 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.077623+0000", 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3920269019999996 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.077631+0000", 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0590000000000003e-06 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.078895+0000", 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001264668 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115110 9.18 9:189e6899:::benchmark_data_smithi012_313365_object2635:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.563 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.724132167, 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.870415833, 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115110 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688219+0000", 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688222+0000", 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2199999999999999e-06 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688226+0000", 2023-10-09T10:15:32.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9859999999999996e-06 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688229+0000", 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2629999999999999e-06 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.531968+0000", 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8437393259999997 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532080+0000", 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111876 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532180+0000", 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9580999999999994e-05 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.536128+0000", 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039479809999999997 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.536166+0000", 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8197e-05 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556671+0000", 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020505332000000001 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556708+0000", 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6332999999999999e-05 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.558636+0000", 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019280180000000001 2023-10-09T10:15:32.570 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115117 9.18 9:1bb15d0c:::benchmark_data_smithi012_313365_object2642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.707552142000001, 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8692873749999999, 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.571 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115117 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:15:32.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704799+0000", 2023-10-09T10:15:32.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704805+0000", 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0100000000000001e-06 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704817+0000", 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2054e-05 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.574 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704823+0000", 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9900000000000002e-06 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.533734+0000", 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.828911078 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.533880+0000", 2023-10-09T10:15:32.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014652400000000001 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.534039+0000", 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015923399999999999 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556665+0000", 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022625698 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.558646+0000", 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019810460000000002 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571804+0000", 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013158174 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571859+0000", 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5074000000000002e-05 2023-10-09T10:15:32.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.574087+0000", 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002227672 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115136 9.8 9:102eeaef:::benchmark_data_smithi012_313365_object2661:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.578 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.631307022, 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9345335690000001, 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115136 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781043+0000", 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781051+0000", 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2010000000000002e-06 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781062+0000", 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1688000000000001e-05 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781071+0000", 2023-10-09T10:15:32.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3089999999999996e-06 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.557513+0000", 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7764423310000002 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571744+0000", 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014230893 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.636487+0000", 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064743060000000005 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.642690+0000", 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062035450000000004 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.660819+0000", 2023-10-09T10:15:32.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018128339 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.700703+0000", 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.039884233999999998 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.702344+0000", 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016405899999999999 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.702817+0000", 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000473578 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.705696+0000", 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028788009999999998 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.715578+0000", 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0098824919999999997 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115142 9.18 9:1de397b1:::benchmark_data_smithi012_313365_object2667:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.601807975, 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7874058789999996, 2023-10-09T10:15:32.586 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115142 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810542+0000", 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810549+0000", 2023-10-09T10:15:32.588 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7070000000000003e-06 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810562+0000", 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2543e-05 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810570+0000", 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6279999999999994e-06 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.589 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578683+0000", 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7681127830000003 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578869+0000", 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018629700000000001 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.579038+0000", 2023-10-09T10:15:32.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000168355 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.582715+0000", 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0036773040000000002 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.582723+0000", 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0139999999999998e-06 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595701+0000", 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012977700999999999 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595774+0000", 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2996000000000004e-05 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.597950+0000", 2023-10-09T10:15:32.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021762019999999999 2023-10-09T10:15:32.593 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.593 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.593 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.593 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:32.593 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:32.593 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:15:32.593 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:32.594 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:32.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:32.652 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:32.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:32.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:32.778 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:32 smithi012 podman[330523]: 2023-10-09T10:15:32.786 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:32.820 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:32.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:32.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:15:32.923 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:32.951 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:32.990 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:32.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:33.095 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:33.124 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:33.144 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:32 smithi012 bash[330523]: d0e1dbeaa729f2034dcc7fece8411b89428d8b39391ee628913439c105a298a7 2023-10-09T10:15:33.144 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:32 smithi012 systemd[1]: Started Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T10:15:33.158 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:33.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:33.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:33.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:33.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:15:33.300 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:33.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:33 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[330575]: 2023-10-09T10:15:33.291+0000 7f0f08898640 -1 Falling back to public interface 2023-10-09T10:15:33.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:33 smithi012 ceph-mon[147926]: pgmap v983: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 80171/321216 objects degraded (24.959%); 43154/321216 objects misplaced (13.435%); 1/107072 objects unfound (0.001%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:15:33.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:33 smithi012 ceph-mon[147926]: Health check update: 1/107072 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:33.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 80171/321216 objects degraded (24.959%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:33.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:33 smithi012 ceph-mon[152263]: pgmap v983: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 80171/321216 objects degraded (24.959%); 43154/321216 objects misplaced (13.435%); 1/107072 objects unfound (0.001%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:15:33.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:33 smithi012 ceph-mon[152263]: Health check update: 1/107072 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:33.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 80171/321216 objects degraded (24.959%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:33.504 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:33.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:33.574 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:33.621 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:33.661 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:33.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:33 smithi110 ceph-mon[136032]: pgmap v983: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 80171/321216 objects degraded (24.959%); 43154/321216 objects misplaced (13.435%); 1/107072 objects unfound (0.001%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:15:33.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:33 smithi110 ceph-mon[136032]: Health check update: 1/107072 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:33.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 80171/321216 objects degraded (24.959%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:33.798 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:33.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:33.906 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:33.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:33.966 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:33.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:34.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:34.083 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:34.149 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:34.368 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:34.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:34.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:34.486 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:34.512 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:34.524 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:34.551 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:34.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:34.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:34.706 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:34.858 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:34.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:34.968 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:34.998 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:35.031 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:35.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:35.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:35.135 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:35.199 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:35.241 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:15:35.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:35 smithi012 ceph-mon[152263]: pgmap v984: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 80082/320949 objects degraded (24.952%); 43154/320949 objects misplaced (13.446%); 1/106983 objects unfound (0.001%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T10:15:35.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/347976466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:35.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:35.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:35 smithi012 ceph-mon[147926]: pgmap v984: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 80082/320949 objects degraded (24.952%); 43154/320949 objects misplaced (13.446%); 1/106983 objects unfound (0.001%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T10:15:35.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/347976466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:35.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:35.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:35 smithi110 ceph-mon[136032]: pgmap v984: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 80082/320949 objects degraded (24.952%); 43154/320949 objects misplaced (13.446%); 1/106983 objects unfound (0.001%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T10:15:35.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/347976466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:35.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:15:36.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:36.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[147926]: failed to encode map e350 with expected crc 2023-10-09T10:15:36.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:15:36.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[147926]: osdmap e350: 8 total, 7 up, 7 in 2023-10-09T10:15:36.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[147926]: failed to encode map e350 with expected crc 2023-10-09T10:15:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[152263]: failed to encode map e350 with expected crc 2023-10-09T10:15:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:15:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[152263]: osdmap e350: 8 total, 7 up, 7 in 2023-10-09T10:15:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:36 smithi012 ceph-mon[152263]: failed to encode map e350 with expected crc 2023-10-09T10:15:36.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:36 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:36.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:36 smithi110 ceph-mon[136032]: failed to encode map e350 with expected crc 2023-10-09T10:15:36.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:15:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:36 smithi110 ceph-mon[136032]: osdmap e350: 8 total, 7 up, 7 in 2023-10-09T10:15:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:36 smithi110 ceph-mon[136032]: failed to encode map e350 with expected crc 2023-10-09T10:15:36.781 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:36.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:36.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:36.942 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:37.067 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:37.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:37.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:37.268 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:37.297 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:15:37.297 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:15:37.297 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:15:37.297 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:15:37.338 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:15:37.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:15:37.384 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:15:37.384 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:15:37.384 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:15:37.384 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:15:37.384 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T10:15:37.384 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:15:37.384 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:15:37.384 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:15:37.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:15:37.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:37 smithi012 ceph-mon[147926]: pgmap v986: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 345 op/s; 79396/317631 objects degraded (24.996%); 42731/317631 objects misplaced (13.453%); 1/105877 objects unfound (0.001%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T10:15:37.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3283780456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:37.399 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:37 smithi012 ceph-mon[152263]: pgmap v986: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 345 op/s; 79396/317631 objects degraded (24.996%); 42731/317631 objects misplaced (13.453%); 1/105877 objects unfound (0.001%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T10:15:37.399 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3283780456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:37.423 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:37.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:37.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:37.530 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:37.554 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:37 smithi110 ceph-mon[136032]: pgmap v986: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 345 op/s; 79396/317631 objects degraded (24.996%); 42731/317631 objects misplaced (13.453%); 1/105877 objects unfound (0.001%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T10:15:37.554 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3283780456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:37.556 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:37.590 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:37.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:37.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:37.693 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:37.721 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:37.758 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:37.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:37.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:37.888 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:37.917 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:15:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:15:37] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.33.4" 2023-10-09T10:15:37.923 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:37.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:37.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:37.973 INFO:teuthology.orchestra.run.smithi012.stderr:no valid command found; 10 closest matches: 2023-10-09T10:15:37.973 INFO:teuthology.orchestra.run.smithi012.stderr:0 2023-10-09T10:15:37.973 INFO:teuthology.orchestra.run.smithi012.stderr:1 2023-10-09T10:15:37.973 INFO:teuthology.orchestra.run.smithi012.stderr:2 2023-10-09T10:15:37.973 INFO:teuthology.orchestra.run.smithi012.stderr:abort 2023-10-09T10:15:37.974 INFO:teuthology.orchestra.run.smithi012.stderr:assert 2023-10-09T10:15:37.974 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs debug_inject_read_zeros 2023-10-09T10:15:37.974 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs files list 2023-10-09T10:15:37.974 INFO:teuthology.orchestra.run.smithi012.stderr:bluefs stats 2023-10-09T10:15:37.974 INFO:teuthology.orchestra.run.smithi012.stderr:bluestore allocator dump block 2023-10-09T10:15:37.974 INFO:teuthology.orchestra.run.smithi012.stderr:bluestore allocator fragmentation block 2023-10-09T10:15:37.974 INFO:teuthology.orchestra.run.smithi012.stderr:admin_socket: invalid command 2023-10-09T10:15:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:15:38.004 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:15:38.028 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:38.056 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:38.091 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:38.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:38.198 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:38.229 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:38.268 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:38.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:38.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:38.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:38.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:38.435 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:38.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:38.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:38 smithi012 ceph-mon[147926]: Health check update: 1/105877 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:38.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 79396/317631 objects degraded (24.996%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:38.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:38 smithi012 ceph-mon[152263]: Health check update: 1/105877 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:38.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 79396/317631 objects degraded (24.996%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:38.516 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:15:38.516 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:15:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:15:38.517 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2023-10-09T10:15:38.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:38.627 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:38.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:38 smithi110 ceph-mon[136032]: Health check update: 1/105877 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 79396/317631 objects degraded (24.996%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:38.886 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:38.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:38.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:15:38.997 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:39.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:39.062 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:39.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:39.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:39.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:39.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:15:39.201 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:39.238 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:39.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:39.342 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:39.367 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:39 smithi110 ceph-mon[136032]: pgmap v987: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 79325/316470 objects degraded (25.066%); 42529/316470 objects misplaced (13.439%); 1/105490 objects unfound (0.001%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T10:15:39.369 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:39.404 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:39.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:39.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:39.534 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:39.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:39.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:39.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:39.597 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:39 smithi012 ceph-mon[147926]: pgmap v987: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 79325/316470 objects degraded (25.066%); 42529/316470 objects misplaced (13.439%); 1/105490 objects unfound (0.001%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T10:15:39.597 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[330575]: 2023-10-09T10:15:39.590+0000 7f0f08898640 -1 osd.3 344 log_to_monitors true 2023-10-09T10:15:39.598 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:39 smithi012 ceph-mon[152263]: pgmap v987: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 79325/316470 objects degraded (25.066%); 42529/316470 objects misplaced (13.439%); 1/105490 objects unfound (0.001%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T10:15:39.678 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:39.704 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:39.738 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:39.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:39.775 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:39.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:39.889 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:39.931 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:39.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:40.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:15:40.017 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:40.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:40.114 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:40.174 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:40.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:40.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:40.346 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:40.426 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:40.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:15:40 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[330575]: 2023-10-09T10:15:40.339+0000 7f0ef64fd700 -1 osd.3 344 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:15:40.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:40 smithi012 ceph-mon[147926]: from='osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:15:40.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:40 smithi012 ceph-mon[147926]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:15:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:40 smithi012 ceph-mon[152263]: from='osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:15:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:40 smithi012 ceph-mon[152263]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:15:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:40.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:40.534 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:40.560 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:40 smithi110 ceph-mon[136032]: from='osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:15:40.560 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:40 smithi110 ceph-mon[136032]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:15:40.562 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:40.603 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:40.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:40.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:40.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:40.758 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:41.074 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:41.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:41.180 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:41.207 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:41.244 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:41.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:41.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:41.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[147926]: pgmap v988: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 78797/313548 objects degraded (25.131%); 42145/313548 objects misplaced (13.441%); 1/104516 objects unfound (0.001%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:15:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[147926]: failed to encode map e351 with expected crc 2023-10-09T10:15:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[147926]: failed to encode map e351 with expected crc 2023-10-09T10:15:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[147926]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:15:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[147926]: osdmap e351: 8 total, 7 up, 7 in 2023-10-09T10:15:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[147926]: from='osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:15:41.382 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[147926]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:15:41.383 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[152263]: pgmap v988: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 78797/313548 objects degraded (25.131%); 42145/313548 objects misplaced (13.441%); 1/104516 objects unfound (0.001%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:15:41.383 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[152263]: failed to encode map e351 with expected crc 2023-10-09T10:15:41.383 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[152263]: failed to encode map e351 with expected crc 2023-10-09T10:15:41.383 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[152263]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:15:41.383 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[152263]: osdmap e351: 8 total, 7 up, 7 in 2023-10-09T10:15:41.383 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[152263]: from='osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:15:41.383 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:41 smithi012 ceph-mon[152263]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:15:41.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:41.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:41.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:41.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:41 smithi110 ceph-mon[136032]: pgmap v988: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 78797/313548 objects degraded (25.131%); 42145/313548 objects misplaced (13.441%); 1/104516 objects unfound (0.001%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:15:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:41 smithi110 ceph-mon[136032]: failed to encode map e351 with expected crc 2023-10-09T10:15:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:41 smithi110 ceph-mon[136032]: failed to encode map e351 with expected crc 2023-10-09T10:15:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:41 smithi110 ceph-mon[136032]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:15:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:41 smithi110 ceph-mon[136032]: osdmap e351: 8 total, 7 up, 7 in 2023-10-09T10:15:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:41 smithi110 ceph-mon[136032]: from='osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:15:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:41 smithi110 ceph-mon[136032]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2023-10-09T10:15:41.718 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:41.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:41.741 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:15:41.741 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:15:41.775 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:41.990 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:41.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:41.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:42.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:42.120 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:42.153 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:42.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:42.257 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:42.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:42.307 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:15:42.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:42.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:42.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:42.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:15:42.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: failed to encode map e352 with expected crc 2023-10-09T10:15:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407] boot 2023-10-09T10:15:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: osdmap e352: 8 total, 8 up, 7 in 2023-10-09T10:15:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:15:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3211515280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3211515280' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:42.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:42 smithi110 ceph-mon[136032]: osdmap e353: 8 total, 8 up, 7 in 2023-10-09T10:15:42.473 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:42.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:15:42.552 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: failed to encode map e352 with expected crc 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407] boot 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: osdmap e352: 8 total, 8 up, 7 in 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3211515280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3211515280' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[147926]: osdmap e353: 8 total, 8 up, 7 in 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: failed to encode map e352 with expected crc 2023-10-09T10:15:42.553 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: osd.3 [v2:172.21.15.12:6826/1499112407,v1:172.21.15.12:6827/1499112407] boot 2023-10-09T10:15:42.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: osdmap e352: 8 total, 8 up, 7 in 2023-10-09T10:15:42.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:15:42.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3211515280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:15:42.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:15:42.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3211515280' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:15:42.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:42 smithi012 ceph-mon[152263]: osdmap e353: 8 total, 8 up, 7 in 2023-10-09T10:15:43.518 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:15:43.585 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: pgmap v991: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 78417/311163 objects degraded (25.201%); 41854/311163 objects misplaced (13.451%); 1/103721 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: Health check update: 1/103721 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 78417/311163 objects degraded (25.201%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:43.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[147926]: osdmap e354: 8 total, 8 up, 7 in 2023-10-09T10:15:43.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: pgmap v991: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 78417/311163 objects degraded (25.201%); 41854/311163 objects misplaced (13.451%); 1/103721 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:15:43.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:15:43.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: Health check update: 1/103721 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:43.588 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 78417/311163 objects degraded (25.201%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:43.588 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:43.588 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:43.588 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:43 smithi012 ceph-mon[152263]: osdmap e354: 8 total, 8 up, 7 in 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: pgmap v991: 99 pgs: 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 78417/311163 objects degraded (25.201%); 41854/311163 objects misplaced (13.451%); 1/103721 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: failed to encode map e353 with expected crc 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: Health check update: 1/103721 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 78417/311163 objects degraded (25.201%), 28 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:43.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:43 smithi110 ceph-mon[136032]: osdmap e354: 8 total, 8 up, 7 in 2023-10-09T10:15:44.310 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[147926]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.310 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[147926]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.310 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[147926]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.310 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[147926]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.310 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.310 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:44.310 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.310 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.626 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:44.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[152263]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[152263]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[152263]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[152263]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:44.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:44.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:44.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:15:44.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:44 smithi110 ceph-mon[136032]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:44 smithi110 ceph-mon[136032]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:44 smithi110 ceph-mon[136032]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:44 smithi110 ceph-mon[136032]: failed to encode map e354 with expected crc 2023-10-09T10:15:44.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:44.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:44.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:44.766 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:45.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:45 smithi012 ceph-mon[152263]: pgmap v994: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 13 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 71877/310902 objects degraded (23.119%); 41854/310902 objects misplaced (13.462%); 1/103634 objects unfound (0.001%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:15:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:45 smithi012 ceph-mon[147926]: pgmap v994: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 13 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 71877/310902 objects degraded (23.119%); 41854/310902 objects misplaced (13.462%); 1/103634 objects unfound (0.001%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:15:45.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:45 smithi110 ceph-mon[136032]: pgmap v994: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 13 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 30 active+undersized, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 40 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 71877/310902 objects degraded (23.119%); 41854/310902 objects misplaced (13.462%); 1/103634 objects unfound (0.001%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:15:47.162 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:47.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:47.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:47.267 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:47.303 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:47.338 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:47.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:47.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:47.372 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:47.372 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:47.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:47 smithi012 ceph-mon[147926]: pgmap v995: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 51603/309408 objects degraded (16.678%); 25588/309408 objects misplaced (8.270%); 1/103136 objects unfound (0.001%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T10:15:47.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:47 smithi012 ceph-mon[152263]: pgmap v995: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 51603/309408 objects degraded (16.678%); 25588/309408 objects misplaced (8.270%); 1/103136 objects unfound (0.001%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T10:15:47.442 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:47.485 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:47.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:47 smithi110 ceph-mon[136032]: pgmap v995: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 51603/309408 objects degraded (16.678%); 25588/309408 objects misplaced (8.270%); 1/103136 objects unfound (0.001%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T10:15:47.774 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:47.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:47.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:47.864 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:15:47.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:47.894 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:15:47.915 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:47.919 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:15:47.953 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:15:47.964 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:47.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:47.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:47.976 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:15:48.001 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:15:48.028 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:15:48.044 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:15:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:15:47] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-09T10:15:48.054 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:15:48.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:15:48.058 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:15:48.059 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:15:48.059 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:15:48.059 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:15:48.059 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:15:48.059 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:15:48.059 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:15:48.069 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:48.118 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:48.152 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:48.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:48.256 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:48.283 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:48.319 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:48.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:48.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:48.423 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:48.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1587939618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:48.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:48 smithi110 ceph-mon[136032]: Health check update: 1/103136 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:48.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 51603/309408 objects degraded (16.678%), 22 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:48.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:48.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:48 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:48.449 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:48.489 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:48.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:48.594 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:48.622 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:48.643 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1587939618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:48.644 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[147926]: Health check update: 1/103136 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:48.644 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 51603/309408 objects degraded (16.678%), 22 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:48.644 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:48.644 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:48.644 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1587939618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:15:48.644 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[152263]: Health check update: 1/103136 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:15:48.644 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 51603/309408 objects degraded (16.678%), 22 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:48.644 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:48.645 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:48 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:48.656 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:48.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:48.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:48.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:15:48.760 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:15:48.787 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:48.820 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:15:48.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:48.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:15:48.923 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:48.950 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:48.984 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:48.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:48.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:49.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:49.119 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:49 smithi012 ceph-mon[147926]: pgmap v996: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 51603/309408 objects degraded (16.678%); 25588/309408 objects misplaced (8.270%); 1/103136 objects unfound (0.001%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:15:49.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:49 smithi012 ceph-mon[152263]: pgmap v996: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 51603/309408 objects degraded (16.678%); 25588/309408 objects misplaced (8.270%); 1/103136 objects unfound (0.001%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:15:49.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:49 smithi110 ceph-mon[136032]: pgmap v996: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 51603/309408 objects degraded (16.678%); 25588/309408 objects misplaced (8.270%); 1/103136 objects unfound (0.001%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:15:50.361 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:50.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:50.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:50.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:50.501 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:50.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:50 smithi012 ceph-mon[147926]: Health check cleared: OBJECT_UNFOUND (was: 1/103136 objects unfound (0.001%)) 2023-10-09T10:15:50.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:50 smithi012 ceph-mon[152263]: Health check cleared: OBJECT_UNFOUND (was: 1/103136 objects unfound (0.001%)) 2023-10-09T10:15:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:50 smithi110 ceph-mon[136032]: Health check cleared: OBJECT_UNFOUND (was: 1/103136 objects unfound (0.001%)) 2023-10-09T10:15:51.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:51 smithi012 ceph-mon[147926]: pgmap v997: 99 pgs: 2 activating+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 50026/309348 objects degraded (16.171%); 21567/309348 objects misplaced (6.972%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T10:15:51.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:51 smithi012 ceph-mon[152263]: pgmap v997: 99 pgs: 2 activating+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 50026/309348 objects degraded (16.171%); 21567/309348 objects misplaced (6.972%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T10:15:51.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:51 smithi110 ceph-mon[136032]: pgmap v997: 99 pgs: 2 activating+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 50026/309348 objects degraded (16.171%); 21567/309348 objects misplaced (6.972%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T10:15:53.479 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:53.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:53.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:53.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:53 smithi012 ceph-mon[147926]: pgmap v998: 99 pgs: 2 activating+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 50026/309273 objects degraded (16.175%); 20924/309273 objects misplaced (6.766%); 12 MiB/s, 186 objects/s recovering 2023-10-09T10:15:53.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:53 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 50026/309273 objects degraded (16.175%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:53 smithi012 ceph-mon[152263]: pgmap v998: 99 pgs: 2 activating+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 50026/309273 objects degraded (16.175%); 20924/309273 objects misplaced (6.766%); 12 MiB/s, 186 objects/s recovering 2023-10-09T10:15:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:53 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 50026/309273 objects degraded (16.175%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:53.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:15:53.621 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:53.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:53 smithi110 ceph-mon[136032]: pgmap v998: 99 pgs: 2 activating+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 50026/309273 objects degraded (16.175%); 20924/309273 objects misplaced (6.766%); 12 MiB/s, 186 objects/s recovering 2023-10-09T10:15:53.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:53 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 50026/309273 objects degraded (16.175%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:54.683 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:15:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:54.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:15:54.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:54.870 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:55.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:15:55.623 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:55.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:55.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:55.640 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:55 smithi012 ceph-mon[147926]: pgmap v999: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 52534/308127 objects degraded (17.049%); 18299/308127 objects misplaced (5.939%); 11 MiB/s, 179 objects/s recovering 2023-10-09T10:15:55.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:55 smithi012 ceph-mon[152263]: pgmap v999: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 52534/308127 objects degraded (17.049%); 18299/308127 objects misplaced (5.939%); 11 MiB/s, 179 objects/s recovering 2023-10-09T10:15:55.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:55 smithi110 ceph-mon[136032]: pgmap v999: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 52534/308127 objects degraded (17.049%); 18299/308127 objects misplaced (5.939%); 11 MiB/s, 179 objects/s recovering 2023-10-09T10:15:55.733 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:55.775 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:55.843 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:55.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:55.948 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:55.977 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:56.012 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:56.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:56.117 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:15:56.144 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:56.179 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:15:56.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:56.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:15:56.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:56.311 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:56.350 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:56.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:56.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:56.454 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:56.481 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:56.518 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:56.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:56.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:56.658 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:56.812 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:56.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:56.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:56.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:15:57.001 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:57.109 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:15:57.718 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:57 smithi110 ceph-mon[136032]: pgmap v1000: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 162 op/s; 52449/305046 objects degraded (17.194%); 17552/305046 objects misplaced (5.754%); 13 MiB/s, 208 objects/s recovering 2023-10-09T10:15:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:57.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:15:57.721 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:15:57.721 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:15:57.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:15:57.721 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:15:57.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.721 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114935 9.f 9:f4378248:::benchmark_data_smithi012_313365_object2460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:15:57.721 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:15:57.721 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 43.01176882, 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2871666070000001, 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114935 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162479+0000", 2023-10-09T10:15:57.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162485+0000", 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3759999999999996e-06 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162497+0000", 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162503+0000", 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9770000000000003e-06 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174344+0000", 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01184062 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174510+0000", 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000166026 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174655+0000", 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000144563 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.181694+0000", 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007039694 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.214204+0000", 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.03250981 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.222717+0000", 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085130050000000006 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.222730+0000", 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3229e-05 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.728 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196525+0000", 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97379490400000002 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196531+0000", 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.879e-06 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.428687+0000", 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.232155509 2023-10-09T10:15:57.729 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.428882+0000", 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019557999999999999 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.449518+0000", 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020635767999999999 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.730 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.449647+0000", 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000129014 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114936 9.a 9:58a6faa8:::benchmark_data_smithi012_313365_object2461:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 43.011738108000003, 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.092944401, 2023-10-09T10:15:57.731 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114936 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.732 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.733 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:15:57.733 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.733 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.733 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.733 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:15:57.733 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162513+0000", 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1610000000000001e-06 2023-10-09T10:15:57.734 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162518+0000", 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7310000000000001e-06 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162522+0000", 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9920000000000001e-06 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164493+0000", 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019717620000000002 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164686+0000", 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019211 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:15:57.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164874+0000", 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018879699999999999 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.168656+0000", 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037811670000000002 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.169482+0000", 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082636199999999997 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.737 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.212142+0000", 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042660245999999999 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.212158+0000", 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5977999999999998e-05 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.738 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.195199+0000", 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98304075999999996 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.195213+0000", 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4025e-05 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236218+0000", 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0410054230000001 2023-10-09T10:15:57.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236372+0000", 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015330899999999999 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255344+0000", 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018972025 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255456+0000", 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111928 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114940 9.f 9:f2b30a6a:::benchmark_data_smithi012_313365_object2465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 43.000633258000001, 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2912956169999998, 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.741 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114940 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.742 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173614+0000", 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173632+0000", 2023-10-09T10:15:57.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8349999999999999e-05 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173651+0000", 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8372e-05 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173660+0000", 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0429999999999996e-06 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.177952+0000", 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042923120000000004 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.178136+0000", 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018356399999999999 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.178304+0000", 2023-10-09T10:15:57.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016803400000000001 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.181873+0000", 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035693090000000001 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.214244+0000", 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032370566000000003 2023-10-09T10:15:57.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223012+0000", 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087685690000000004 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223019+0000", 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196547+0000", 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97352829299999999 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196551+0000", 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9949999999999999e-06 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.748 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.434365+0000", 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2378136799999999 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.434545+0000", 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000180044 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.464846+0000", 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030301600000000001 2023-10-09T10:15:57.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.464912+0000", 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5380000000000001e-05 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114944 9.2 9:4465be14:::benchmark_data_smithi012_313365_object2469:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:15:57.750 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.213860+0000", 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 42.960389483, 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0296730190000001, 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114944 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213860+0000", 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213860+0000", 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213858+0000", 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:15:57.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213866+0000", 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7150000000000004e-06 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213879+0000", 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2781e-05 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213888+0000", 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6619999999999994e-06 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.220743+0000", 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0068547720000000003 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.754 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.220753+0000", 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0428000000000001e-05 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.211156+0000", 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99040253599999994 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.211283+0000", 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000127074 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2", 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.211705+0000", 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000422533 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.214439+0000", 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027343419999999998 2023-10-09T10:15:57.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.243377+0000", 2023-10-09T10:15:57.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028937577999999999 2023-10-09T10:15:57.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.243433+0000", 2023-10-09T10:15:57.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5665999999999999e-05 2023-10-09T10:15:57.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.243533+0000", 2023-10-09T10:15:57.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100103 2023-10-09T10:15:57.759 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.759 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.759 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114947 9.12 9:49290503:::benchmark_data_smithi012_313365_object2472:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.213894+0000", 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 42.960354823000003, 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0099479920000001, 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114947 2023-10-09T10:15:57.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213894+0000", 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213894+0000", 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213894+0000", 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213897+0000", 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4789999999999999e-06 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.762 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213902+0000", 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6710000000000004e-06 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.213907+0000", 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5639999999999998e-06 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.220773+0000", 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006865978 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.220779+0000", 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9150000000000001e-06 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.209829+0000", 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98905031300000001 2023-10-09T10:15:57.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.209958+0000", 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012907000000000001 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2", 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.210247+0000", 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00028939100000000001 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.214433+0000", 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041858959999999997 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.223678+0000", 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092444099999999998 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.223742+0000", 2023-10-09T10:15:57.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4579999999999995e-05 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.223842+0000", 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010016 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114950 9.f 9:f115a5aa:::benchmark_data_smithi012_313365_object2475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:15:57.767 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 42.950621967000004, 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2705686389999999, 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114950 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:15:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223625+0000", 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223632+0000", 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8639999999999998e-06 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223644+0000", 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1824000000000001e-05 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223651+0000", 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5499999999999997e-06 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223694+0000", 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2765999999999997e-05 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223699+0000", 2023-10-09T10:15:57.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2569999999999998e-06 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196570+0000", 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97287036000000005 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196576+0000", 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5710000000000003e-06 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.439509+0000", 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.242932758 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.440273+0000", 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00076375 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474247+0000", 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033973640999999999 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474363+0000", 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116765 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474458+0000", 2023-10-09T10:15:57.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4829999999999998e-05 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.479012+0000", 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045542890000000004 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.481102+0000", 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020900659999999998 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.489625+0000", 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085221180000000004 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.489680+0000", 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5405999999999998e-05 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.494196+0000", 2023-10-09T10:15:57.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045159930000000003 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114963 9.a 9:5c4833c5:::benchmark_data_smithi012_313365_object2488:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:15.225359+0000", 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 41.948890048999999, 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0302989380000001, 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114963 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225359+0000", 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225359+0000", 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225357+0000", 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225368+0000", 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0895e-05 2023-10-09T10:15:57.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225382+0000", 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3910000000000001e-05 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225392+0000", 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0088999999999999e-05 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225439+0000", 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7324999999999998e-05 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225450+0000", 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0961999999999999e-05 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236629+0000", 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.011179286 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236768+0000", 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013870000000000001 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236922+0000", 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015346899999999999 2023-10-09T10:15:57.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.247521+0000", 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010599450999999999 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.247541+0000", 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9745000000000002e-05 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255494+0000", 2023-10-09T10:15:57.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0079536539999999992 2023-10-09T10:15:57.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255540+0000", 2023-10-09T10:15:57.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6007000000000002e-05 2023-10-09T10:15:57.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255658+0000", 2023-10-09T10:15:57.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000117834 2023-10-09T10:15:57.786 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.786 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.786 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114965 9.a 9:5bf480de:::benchmark_data_smithi012_313365_object2490:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:15.225410+0000", 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 41.948839730000003, 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0385700630000001, 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.787 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.788 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.788 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114965 2023-10-09T10:15:57.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.788 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225410+0000", 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225410+0000", 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225409+0000", 2023-10-09T10:15:57.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225412+0000", 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7889999999999999e-06 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225418+0000", 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7309999999999998e-06 2023-10-09T10:15:57.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225423+0000", 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8160000000000001e-06 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225470+0000", 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7450000000000001e-05 2023-10-09T10:15:57.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.225476+0000", 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.054e-06 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.240296+0000", 2023-10-09T10:15:57.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0148203200000001 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.240444+0000", 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000147918 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.240596+0000", 2023-10-09T10:15:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000151636 2023-10-09T10:15:57.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.247550+0000", 2023-10-09T10:15:57.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0069544250000000002 2023-10-09T10:15:57.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.247594+0000", 2023-10-09T10:15:57.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3278999999999997e-05 2023-10-09T10:15:57.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.263688+0000", 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016094549 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.263779+0000", 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0760000000000005e-05 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.263980+0000", 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020071299999999999 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114970 9.f 9:f5977bb9:::benchmark_data_smithi012_313365_object2495:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:15.262520+0000", 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 41.911729025, 2023-10-09T10:15:57.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.340960916, 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114970 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.262520+0000", 2023-10-09T10:15:57.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.262520+0000", 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.262518+0000", 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.262529+0000", 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0678e-05 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.262546+0000", 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7399999999999999e-05 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.262560+0000", 2023-10-09T10:15:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4499e-05 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.284983+0000", 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022422630999999998 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.284993+0000", 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0293e-05 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.477520+0000", 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1925261970000001 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.478209+0000", 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00068909799999999997 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.503481+0000", 2023-10-09T10:15:57.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025272745999999999 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.503574+0000", 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2161000000000001e-05 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.503661+0000", 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7738000000000006e-05 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.520563+0000", 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016901879000000002 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.598467+0000", 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.077904344 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.598514+0000", 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6767999999999997e-05 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.598574+0000", 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.982e-05 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.603481+0000", 2023-10-09T10:15:57.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049071150000000001 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115069 9.10 9:0c166afc:::benchmark_data_smithi012_313365_object2594:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 40.628602143999998, 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5496961840000001, 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.807 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115069 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545646+0000", 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545653+0000", 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5399999999999998e-06 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545670+0000", 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7144000000000001e-05 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545679+0000", 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5830000000000007e-06 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545730+0000", 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1313000000000002e-05 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546723+0000", 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099291799999999997 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079001+0000", 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5322773119999997 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079100+0000", 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9010000000000002e-05 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079372+0000", 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000272782 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.080850+0000", 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014771509999999999 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.082687+0000", 2023-10-09T10:15:57.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001837336 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.082730+0000", 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3612999999999997e-05 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093025+0000", 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010295011 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093094+0000", 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8725000000000004e-05 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095343+0000", 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022491519999999999 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115106 9.10 9:0b40069f:::benchmark_data_smithi012_313365_object2631:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 40.490417438999998, 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4133900090000004, 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.816 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115106 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683831+0000", 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683836+0000", 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0440000000000003e-06 2023-10-09T10:15:57.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683846+0000", 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9250000000000007e-06 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683854+0000", 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7959999999999996e-06 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683942+0000", 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7602000000000006e-05 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.684891+0000", 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094994899999999995 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089254+0000", 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4043629649999998 2023-10-09T10:15:57.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089396+0000", 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141886 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089758+0000", 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036181000000000001 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.091841+0000", 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020825930000000002 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095413+0000", 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035718400000000002 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095477+0000", 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4685999999999997e-05 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095503+0000", 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5575000000000002e-05 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095538+0000", 2023-10-09T10:15:57.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5116000000000001e-05 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.097222+0000", 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001683941 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122337 9.10 9:0bd6554e:::benchmark_data_smithi012_313365_object9862:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.824 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 6.277270004, 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6698403750000002, 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122337 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896978+0000", 2023-10-09T10:15:57.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896984+0000", 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7479999999999999e-06 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896996+0000", 2023-10-09T10:15:57.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.199e-05 2023-10-09T10:15:57.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897001+0000", 2023-10-09T10:15:57.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8189999999999997e-06 2023-10-09T10:15:57.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897056+0000", 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4818999999999998e-05 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897145+0000", 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8943000000000005e-05 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.517542+0000", 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6203965519999999 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.517672+0000", 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013004499999999999 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.518031+0000", 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035930199999999998 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.536908+0000", 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018877416000000001 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.536918+0000", 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9369999999999999e-06 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.537422+0000", 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00050323499999999999 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564538+0000", 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027116735999999999 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564585+0000", 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6592999999999999e-05 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.566820+0000", 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022347140000000001 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122499 9.10 9:087dc532:::benchmark_data_smithi012_313365_object10024:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.834 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.333924+0000", 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 5.8403254530000002, 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.26869386, 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122499 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.333924+0000", 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.333924+0000", 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.333922+0000", 2023-10-09T10:15:57.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.333932+0000", 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6460000000000005e-06 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.333946+0000", 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4459e-05 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.333956+0000", 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6399999999999992e-06 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.334011+0000", 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5374000000000002e-05 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.334086+0000", 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4317e-05 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.543939+0000", 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.209853042 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.544069+0000", 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000130269 2023-10-09T10:15:57.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.544462+0000", 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00039287399999999998 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564455+0000", 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019993405999999998 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564500+0000", 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4399e-05 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.571105+0000", 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066057420000000004 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.600294+0000", 2023-10-09T10:15:57.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.02918857 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.600363+0000", 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8717e-05 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.602618+0000", 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002255033 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.842 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122523 9.10 9:0d3d12ba:::benchmark_data_smithi012_313365_object10048:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 5.731920712, 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7874849670000001, 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.843 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122523 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442327+0000", 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442334+0000", 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.221e-06 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442347+0000", 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2809000000000001e-05 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442355+0000", 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0760000000000003e-06 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442400+0000", 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4570000000000002e-05 2023-10-09T10:15:57.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442467+0000", 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7063999999999994e-05 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194401+0000", 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7519345159999999 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194518+0000", 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116872 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194929+0000", 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041051599999999998 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.211495+0000", 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016566498999999998 2023-10-09T10:15:57.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.211544+0000", 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9190999999999997e-05 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.214081+0000", 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025366910000000002 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.227445+0000", 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013363808 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.227497+0000", 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1860000000000002e-05 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.850 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.229814+0000", 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023168160000000002 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122561 9.10 9:092dbf94:::benchmark_data_smithi012_313365_object10086:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:15:57.851 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 5.632811641, 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1569061710000001, 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122561 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:15:57.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541437+0000", 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541441+0000", 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1919999999999998e-06 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541449+0000", 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.746e-06 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541454+0000", 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4639999999999997e-06 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541494+0000", 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9505999999999997e-05 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541543+0000", 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9872e-05 2023-10-09T10:15:57.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.660487+0000", 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1189437330000001 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.660622+0000", 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135024 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.661025+0000", 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040268299999999998 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.668459+0000", 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074342699999999998 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.668490+0000", 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1279999999999999e-05 2023-10-09T10:15:57.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683619+0000", 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015128694999999999 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.696066+0000", 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012446472 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.696109+0000", 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3887999999999997e-05 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698344+0000", 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022343480000000001 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122689 9.10 9:0a3555ea:::benchmark_data_smithi012_313365_object10214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 5.2193941300000004, 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7453082640000002, 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122689 2023-10-09T10:15:57.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954853+0000", 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954862+0000", 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2959999999999999e-06 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954874+0000", 2023-10-09T10:15:57.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2072000000000001e-05 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954882+0000", 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1030000000000002e-06 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954938+0000", 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6165999999999997e-05 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.955019+0000", 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1001000000000006e-05 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.666823+0000", 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7118038929999999 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.666922+0000", 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8803999999999998e-05 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.667247+0000", 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032501299999999997 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683704+0000", 2023-10-09T10:15:57.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016457423999999998 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683804+0000", 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100046 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683814+0000", 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0083e-05 2023-10-09T10:15:57.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698412+0000", 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014598266 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698458+0000", 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5503999999999997e-05 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.700163+0000", 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001705441 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122759 9.10 9:0b678b9b:::benchmark_data_smithi012_313365_object10284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 5.0497422900000002, 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.605785577, 2023-10-09T10:15:57.868 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122759 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:15:57.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124506+0000", 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:57.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124513+0000", 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124528+0000", 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4893e-05 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124536+0000", 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1319999999999994e-06 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124601+0000", 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5038999999999999e-05 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.872 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124689+0000", 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7894999999999998e-05 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704065+0000", 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5793759000000001 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704190+0000", 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012584799999999999 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704594+0000", 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040309699999999999 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.727961+0000", 2023-10-09T10:15:57.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023367024 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728012+0000", 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1174000000000002e-05 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728024+0000", 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1966e-05 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728058+0000", 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3911000000000001e-05 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728098+0000", 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0286e-05 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.730293+0000", 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021946410000000002 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122850 9.10 9:0ca7b949:::benchmark_data_smithi012_313365_object10375:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:52.368637+0000", 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4.805612408, 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.402450548, 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.877 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122850 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368637+0000", 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368637+0000", 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368636+0000", 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.879 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368644+0000", 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0160000000000008e-06 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368661+0000", 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7278999999999998e-05 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368670+0000", 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6900000000000004e-06 2023-10-09T10:15:57.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368716+0000", 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5191999999999997e-05 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368770+0000", 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4789e-05 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.726225+0000", 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3574544230000001 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.726304+0000", 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9400000000000006e-05 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.726690+0000", 2023-10-09T10:15:57.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000386024 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.736145+0000", 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094551549999999998 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.736206+0000", 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0946999999999998e-05 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.744450+0000", 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0082431880000000003 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.769110+0000", 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024660049 2023-10-09T10:15:57.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.769187+0000", 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7342999999999995e-05 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.771087+0000", 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001900451 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.885 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122894 9.10 9:0bdcfdfe:::benchmark_data_smithi012_313365_object10419:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:52.523941+0000", 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4.6503081880000003, 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2493680540000001, 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.886 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122894 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.523941+0000", 2023-10-09T10:15:57.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.523941+0000", 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.523940+0000", 2023-10-09T10:15:57.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.523946+0000", 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4700000000000001e-06 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.523960+0000", 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3967e-05 2023-10-09T10:15:57.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.523966+0000", 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8399999999999997e-06 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.537454+0000", 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013487383 2023-10-09T10:15:57.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.539742+0000", 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022884120000000001 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.734109+0000", 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1943662319999999 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.734216+0000", 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010767 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.734585+0000", 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036902400000000002 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.746567+0000", 2023-10-09T10:15:57.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011982188 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.746602+0000", 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4456999999999999e-05 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.762733+0000", 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016131389999999999 2023-10-09T10:15:57.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.771151+0000", 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084174820000000004 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.771206+0000", 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5507000000000002e-05 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.773309+0000", 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021029519999999999 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122990 9.10 9:08b7387e:::benchmark_data_smithi012_313365_object10515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:52.726844+0000", 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4.4474052689999999, 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.048764539, 2023-10-09T10:15:57.895 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122990 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.726844+0000", 2023-10-09T10:15:57.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.726844+0000", 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.726844+0000", 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.726846+0000", 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4600000000000002e-06 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.726856+0000", 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0132999999999999e-05 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.726863+0000", 2023-10-09T10:15:57.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.002e-06 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.726917+0000", 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3698e-05 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.727007+0000", 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9610000000000004e-05 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.742736+0000", 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0157296200000001 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.742828+0000", 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2247e-05 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.743184+0000", 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035564199999999998 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.762778+0000", 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019593826000000002 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.762787+0000", 2023-10-09T10:15:57.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9230000000000008e-06 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.762826+0000", 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9598000000000003e-05 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.773424+0000", 2023-10-09T10:15:57.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010597942000000001 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.773482+0000", 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7151000000000002e-05 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.775609+0000", 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021270690000000001 2023-10-09T10:15:57.903 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.904 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.904 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.904 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:15:57.904 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:15:57.904 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:15:57.904 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:57.905 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:57 smithi012 ceph-mon[147926]: pgmap v1000: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 162 op/s; 52449/305046 objects degraded (17.194%); 17552/305046 objects misplaced (5.754%); 13 MiB/s, 208 objects/s recovering 2023-10-09T10:15:57.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:57 smithi012 ceph-mon[152263]: pgmap v1000: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 162 op/s; 52449/305046 objects degraded (17.194%); 17552/305046 objects misplaced (5.754%); 13 MiB/s, 208 objects/s recovering 2023-10-09T10:15:57.905 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:15:57.905 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:15:57.905 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:15:57.906 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:15:57.906 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:15:57.906 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:15:57.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:15:57.943 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:57.977 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:15:57.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:57.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:15:58.084 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:15:58.112 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:58.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:15:58.158 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:15:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:15:57] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-09T10:15:58.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:58.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:15:58.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:58.330 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:58.420 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:58.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:58.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:58.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:58.584 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:58.641 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:15:58.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 52449/305046 objects degraded (17.194%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:58.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 52449/305046 objects degraded (17.194%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:58.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 52449/305046 objects degraded (17.194%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:15:59.366 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:15:59.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:59.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:15:59.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:15:59.588 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:59.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:59 smithi012 ceph-mon[147926]: pgmap v1001: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 52449/305046 objects degraded (17.194%); 17552/305046 objects misplaced (5.754%); 11 MiB/s, 169 objects/s recovering 2023-10-09T10:15:59.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:59.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:15:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:59.630 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:59 smithi012 ceph-mon[152263]: pgmap v1001: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 52449/305046 objects degraded (17.194%); 17552/305046 objects misplaced (5.754%); 11 MiB/s, 169 objects/s recovering 2023-10-09T10:15:59.630 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:59.630 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:15:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:59.684 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:15:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:59 smithi110 ceph-mon[136032]: pgmap v1001: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 52449/305046 objects degraded (17.194%); 17552/305046 objects misplaced (5.754%); 11 MiB/s, 169 objects/s recovering 2023-10-09T10:15:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:15:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:15:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:15:59.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:15:59.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:15:59.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:15:59.914 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:15:59.941 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:15:59.941 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:15:59.941 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:15:59.942 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:15:59.998 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:00.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:00.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:00.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:00.175 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:00.201 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:16:00.250 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:00.255 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:16:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:00.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:00.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:16:00.355 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:16:00.371 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:00.397 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:16:00.433 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:16:00.467 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:16:00.478 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:16:00.495 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:00.519 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:16:00.520 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:00 smithi012 ceph-mon[152263]: pgmap v1002: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 52251/301008 objects degraded (17.359%); 17303/301008 objects misplaced (5.748%); 12 MiB/s, 190 objects/s recovering 2023-10-09T10:16:00.520 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3609764851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:00.520 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/78921913' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:00.520 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:00 smithi012 ceph-mon[147926]: pgmap v1002: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 52251/301008 objects degraded (17.359%); 17303/301008 objects misplaced (5.748%); 12 MiB/s, 190 objects/s recovering 2023-10-09T10:16:00.521 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3609764851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:00.521 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/78921913' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:00.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:16:00.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:16:00.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:16:00.543 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:16:00.544 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:16:00.544 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:16:00.544 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:16:00.544 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:16:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:00 smithi110 ceph-mon[136032]: pgmap v1002: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 52251/301008 objects degraded (17.359%); 17303/301008 objects misplaced (5.748%); 12 MiB/s, 190 objects/s recovering 2023-10-09T10:16:00.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3609764851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:00.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/78921913' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/78921913' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:16:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: osdmap e355: 8 total, 8 up, 7 in 2023-10-09T10:16:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:01.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:01.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/78921913' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:16:01.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: osdmap e355: 8 total, 8 up, 7 in 2023-10-09T10:16:01.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:01.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:01.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/78921913' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:16:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: osdmap e355: 8 total, 8 up, 7 in 2023-10-09T10:16:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: failed to encode map e355 with expected crc 2023-10-09T10:16:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:01.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:02.777 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:02 smithi110 ceph-mon[136032]: pgmap v1004: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 52251/299991 objects degraded (17.418%); 16778/299991 objects misplaced (5.593%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T10:16:02.778 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:16:02.778 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:16:02.778 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:02.784 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:02.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:02.802 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:16:02.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:02.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:02 smithi012 ceph-mon[152263]: pgmap v1004: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 52251/299991 objects degraded (17.418%); 16778/299991 objects misplaced (5.593%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T10:16:02.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:16:02.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:16:02.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:02.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:02 smithi012 ceph-mon[147926]: pgmap v1004: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 52251/299991 objects degraded (17.418%); 16778/299991 objects misplaced (5.593%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T10:16:02.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:16:02.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:16:02.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:02.962 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:03.012 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:03.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:03.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:16:03.117 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:03.148 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:03.187 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:03.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:03.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:03.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:03.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:16:03.381 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:03.388 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2] out_osds: [5] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:16:03.388 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:16:03.388 INFO:tasks.thrashosds.thrasher:Reweighting osd 3 to 0.39554993270737426 2023-10-09T10:16:03.388 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 0.39554993270737426 2023-10-09T10:16:03.478 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:03.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:03.585 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:03.614 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:03.655 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:03.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:03.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 52251/299991 objects degraded (17.418%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:16:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 52251/299991 objects degraded (17.418%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:16:03.761 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:03.788 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 52251/299991 objects degraded (17.418%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:16:03.790 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:03.828 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:03.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:03.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:03.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:03.965 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:03.983 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:04.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:04.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:04.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:04.108 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:04.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:04.151 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:04.247 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:04.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:04.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:04.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:04.387 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:04.489 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:04.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:04 smithi110 ceph-mon[136032]: pgmap v1005: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 52145/298311 objects degraded (17.480%); 16778/298311 objects misplaced (5.624%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T10:16:04.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:04 smithi012 ceph-mon[152263]: pgmap v1005: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 52145/298311 objects degraded (17.480%); 16778/298311 objects misplaced (5.624%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T10:16:04.916 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:04 smithi012 ceph-mon[147926]: pgmap v1005: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 52145/298311 objects degraded (17.480%); 16778/298311 objects misplaced (5.624%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T10:16:04.942 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:05.101 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:05.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:05.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:05.207 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:05.237 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:05.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:05.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:05.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:05.478 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:05.562 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:05.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:05.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:05.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:05.729 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:05.799 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:05.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:05.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:05.940 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:16:05.968 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:05.985 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:16:06.021 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:16:06.024 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:06.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:06.051 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:16:06.081 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:16:06.112 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:16:06.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:06.153 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:16:06.160 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:06.193 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:16:06.196 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:16:06.196 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:16:06.196 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:16:06.196 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:16:06.196 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:16:06.196 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:16:06.196 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:16:06.197 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:16:06.201 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:06.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:06.308 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:06.340 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:06.381 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:06.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:06.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:06.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:06.581 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:06.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:06 smithi012 ceph-mon[147926]: pgmap v1006: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 52047/294810 objects degraded (17.654%); 15874/294810 objects misplaced (5.384%); 10 MiB/s, 165 objects/s recovering 2023-10-09T10:16:06.627 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1565054469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:06.730 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:06.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:06.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:06.770 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:06 smithi110 ceph-mon[136032]: pgmap v1006: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 52047/294810 objects degraded (17.654%); 15874/294810 objects misplaced (5.384%); 10 MiB/s, 165 objects/s recovering 2023-10-09T10:16:06.771 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1565054469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:06.849 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:06.881 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:06.895 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:06 smithi012 ceph-mon[152263]: pgmap v1006: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 52047/294810 objects degraded (17.654%); 15874/294810 objects misplaced (5.384%); 10 MiB/s, 165 objects/s recovering 2023-10-09T10:16:06.895 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1565054469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:06.923 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:06.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:06.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:07.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:07.108 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:07.204 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:16:07.204 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:16:07.204 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:16:07.204 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:16:07.205 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:16:07.265 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:07.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:07.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:07.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:07.411 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:07.550 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:16:07.554 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:07.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:07.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:07.705 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.3 to 0.39555 (6542) 2023-10-09T10:16:07.727 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:07.825 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:07.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:07.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:07.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1753255842' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.39554993270737426}]: dispatch 2023-10-09T10:16:07.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1753255842' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.39554993270737426}]: dispatch 2023-10-09T10:16:07.930 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:07.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1753255842' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.39554993270737426}]: dispatch 2023-10-09T10:16:07.960 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:07.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:07.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:08.097 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:08.126 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:16:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:16:07] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-10-09T10:16:08.131 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:08.184 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:08.221 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:08.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:08.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:08.240 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:16:08.329 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:08.404 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:08.494 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:08.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:08.605 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:08.634 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: pgmap v1007: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 52047/294810 objects degraded (17.654%); 15874/294810 objects misplaced (5.384%); 10 MiB/s, 165 objects/s recovering 2023-10-09T10:16:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1753255842' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.39554993270737426}]': finished 2023-10-09T10:16:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: osdmap e356: 8 total, 8 up, 7 in 2023-10-09T10:16:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/312951188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 52047/294810 objects degraded (17.654%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: osdmap e357: 8 total, 8 up, 7 in 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[152263]: failed to encode map e357 with expected crc 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: pgmap v1007: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 52047/294810 objects degraded (17.654%); 15874/294810 objects misplaced (5.384%); 10 MiB/s, 165 objects/s recovering 2023-10-09T10:16:08.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1753255842' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.39554993270737426}]': finished 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: osdmap e356: 8 total, 8 up, 7 in 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/312951188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 52047/294810 objects degraded (17.654%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: osdmap e357: 8 total, 8 up, 7 in 2023-10-09T10:16:08.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:08 smithi012 ceph-mon[147926]: failed to encode map e357 with expected crc 2023-10-09T10:16:08.682 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:08.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:08.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:08.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:16:08.786 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:08.815 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:08.853 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:08.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:08.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: pgmap v1007: 99 pgs: 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 52047/294810 objects degraded (17.654%); 15874/294810 objects misplaced (5.384%); 10 MiB/s, 165 objects/s recovering 2023-10-09T10:16:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1753255842' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.39554993270737426}]': finished 2023-10-09T10:16:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: osdmap e356: 8 total, 8 up, 7 in 2023-10-09T10:16:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: failed to encode map e356 with expected crc 2023-10-09T10:16:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/312951188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 52047/294810 objects degraded (17.654%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T10:16:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:16:08.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:08.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: osdmap e357: 8 total, 8 up, 7 in 2023-10-09T10:16:08.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:08 smithi110 ceph-mon[136032]: failed to encode map e357 with expected crc 2023-10-09T10:16:08.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:08.994 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:09.051 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:09.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:09.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:09.158 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:09.188 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:09.273 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:09.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:09.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:09.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:09.468 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:09.533 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:09.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:09.562 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:16:09.562 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:16:09.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:16:09.562 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:16:09.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.562 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189822 7.e 7:75c53490:::benchmark_data_smithi012_221426_object82044:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e235)", 2023-10-09T10:16:09.562 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.352313+0000", 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 547.20642843099995, 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1682787279999998, 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189822 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352313+0000", 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352313+0000", 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352311+0000", 2023-10-09T10:16:09.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1600000000000001e-05 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352322+0000", 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6101000000000001e-05 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352338+0000", 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3510000000000008e-06 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352348+0000", 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000208784 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352556+0000", 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.38e-05 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352570+0000", 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028903000000000002 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.355460+0000", 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4120000000000007e-06 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.355470+0000", 2023-10-09T10:16:09.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0820199669999999 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.437490+0000", 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033238400000000002 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.440814+0000", 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079646506000000006 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.520460+0000", 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013110199999999999 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.520591+0000", 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1682787279999998 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189857 7.e 7:72010e1c:::benchmark_data_smithi012_221426_object82079:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e235)", 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.352444+0000", 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 547.20629732800001, 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2263163940000004, 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.570 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189857 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352444+0000", 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352444+0000", 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352442+0000", 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6990000000000003e-06 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352450+0000", 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5135e-05 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352465+0000", 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1009999999999992e-06 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352473+0000", 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017449699999999999 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352648+0000", 2023-10-09T10:16:09.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0151e-05 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.352658+0000", 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028324719999999999 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.355490+0000", 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5820000000000001e-06 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.355496+0000", 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1057662949999996 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.461262+0000", 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037577500000000001 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.461638+0000", 2023-10-09T10:16:09.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010975377 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.472614+0000", 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7334e-05 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.472711+0000", 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015850099999999999 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.472869+0000", 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054263319999999999 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478296+0000", 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.656e-05 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478362+0000", 2023-10-09T10:16:09.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.099068411999999995 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.577431+0000", 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1085000000000003e-05 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.577482+0000", 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012783359999999999 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578760+0000", 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2263163940000004 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189895 7.e 7:706a3b6c:::benchmark_data_smithi012_221426_object82117:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.472620+0000", 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 547.08612093800002, 2023-10-09T10:16:09.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1068740220000004, 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189895 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472620+0000", 2023-10-09T10:16:09.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472620+0000", 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472618+0000", 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6149999999999999e-06 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472626+0000", 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3054000000000001e-05 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472639+0000", 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3929999999999996e-06 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472645+0000", 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0260999999999998e-05 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472706+0000", 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1963999999999999e-05 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.472717+0000", 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9890024180000001 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.461720+0000", 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029004900000000001 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462010+0000", 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016384402999999999 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478394+0000", 2023-10-09T10:16:09.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100259 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478495+0000", 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016704099999999999 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.478662+0000", 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096639559999999996 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.488326+0000", 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017766220000000001 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.490102+0000", 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.088771355999999996 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578874+0000", 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8349000000000002e-05 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578912+0000", 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058215199999999995 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579494+0000", 2023-10-09T10:16:09.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1068740220000004 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189935 7.e 7:7b9491a5:::benchmark_data_smithi012_221426_object82157:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.525002+0000", 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 547.03373943099996, 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0551137519999996, 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.588 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189935 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525002+0000", 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525002+0000", 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525000+0000", 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0190999999999999e-05 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525010+0000", 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1472999999999999e-05 2023-10-09T10:16:09.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525031+0000", 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.191e-06 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525039+0000", 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4003000000000004e-05 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525103+0000", 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2624e-05 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.525116+0000", 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9369911579999997 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462107+0000", 2023-10-09T10:16:09.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000335633 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462443+0000", 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021951134000000001 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.484394+0000", 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000110223 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.484504+0000", 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000190457 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.484695+0000", 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010366334 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.495061+0000", 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1199999999999996e-06 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.495068+0000", 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084530545999999998 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579599+0000", 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2638000000000002e-05 2023-10-09T10:16:09.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579631+0000", 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048402200000000002 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580115+0000", 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0551137519999996 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.596 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189951 7.e 7:72370faa:::benchmark_data_smithi012_221426_object82173:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.550258+0000", 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 547.00848273400004, 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0304842059999997, 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189951 2023-10-09T10:16:09.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550258+0000", 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550258+0000", 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550256+0000", 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0445000000000001e-05 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550267+0000", 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0763000000000001e-05 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550288+0000", 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0390000000000005e-06 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550296+0000", 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1124999999999995e-05 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550357+0000", 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7599999999999997e-06 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.550366+0000", 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9122201030000001 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462587+0000", 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032054199999999999 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462907+0000", 2023-10-09T10:16:09.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.027249079999999998 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.490156+0000", 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118588 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.490275+0000", 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019666 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.490471+0000", 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016806707000000001 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.507278+0000", 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011562615 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.518841+0000", 2023-10-09T10:16:09.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.061361996000000002 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580203+0000", 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2552999999999998e-05 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580235+0000", 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050716400000000005 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580742+0000", 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0304842059999997 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189992 7.e 7:7e96e1e7:::benchmark_data_smithi012_221426_object82214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.627928+0000", 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.93081251800004, 2023-10-09T10:16:09.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9534130049999998, 2023-10-09T10:16:09.606 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.606 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.606 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.606 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.606 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.606 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189992 2023-10-09T10:16:09.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.607 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627928+0000", 2023-10-09T10:16:09.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627928+0000", 2023-10-09T10:16:09.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:09.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627927+0000", 2023-10-09T10:16:09.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1340000000000004e-06 2023-10-09T10:16:09.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627935+0000", 2023-10-09T10:16:09.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6446000000000001e-05 2023-10-09T10:16:09.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627951+0000", 2023-10-09T10:16:09.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0209999999999996e-06 2023-10-09T10:16:09.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.627957+0000", 2023-10-09T10:16:09.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.333e-05 2023-10-09T10:16:09.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.628001+0000", 2023-10-09T10:16:09.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9800000000000004e-06 2023-10-09T10:16:09.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.628010+0000", 2023-10-09T10:16:09.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.834941336 2023-10-09T10:16:09.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.462951+0000", 2023-10-09T10:16:09.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023041399999999999 2023-10-09T10:16:09.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463181+0000", 2023-10-09T10:16:09.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.034042462000000002 2023-10-09T10:16:09.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.497224+0000", 2023-10-09T10:16:09.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1834000000000002e-05 2023-10-09T10:16:09.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.497316+0000", 2023-10-09T10:16:09.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017792799999999999 2023-10-09T10:16:09.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.497494+0000", 2023-10-09T10:16:09.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015801953000000001 2023-10-09T10:16:09.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.513296+0000", 2023-10-09T10:16:09.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013098742 2023-10-09T10:16:09.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526394+0000", 2023-10-09T10:16:09.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054431805999999999 2023-10-09T10:16:09.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580826+0000", 2023-10-09T10:16:09.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1927000000000001e-05 2023-10-09T10:16:09.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580858+0000", 2023-10-09T10:16:09.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048334099999999998 2023-10-09T10:16:09.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581341+0000", 2023-10-09T10:16:09.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9534130049999998 2023-10-09T10:16:09.622 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.622 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.622 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.623 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:189997 7.e 7:7601f845:::benchmark_data_smithi012_221426_object82219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.623 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.637282+0000", 2023-10-09T10:16:09.623 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.92145904500001, 2023-10-09T10:16:09.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9446827339999997, 2023-10-09T10:16:09.623 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.623 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.624 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.624 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.624 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.624 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 189997 2023-10-09T10:16:09.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.625 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637282+0000", 2023-10-09T10:16:09.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637282+0000", 2023-10-09T10:16:09.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:09.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637280+0000", 2023-10-09T10:16:09.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6240000000000002e-06 2023-10-09T10:16:09.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637288+0000", 2023-10-09T10:16:09.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4859e-05 2023-10-09T10:16:09.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637303+0000", 2023-10-09T10:16:09.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4529999999999999e-06 2023-10-09T10:16:09.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637309+0000", 2023-10-09T10:16:09.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9356e-05 2023-10-09T10:16:09.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637359+0000", 2023-10-09T10:16:09.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1171e-05 2023-10-09T10:16:09.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.637370+0000", 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8259225219999999 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463292+0000", 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023791400000000001 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463530+0000", 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.039758138999999998 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.503288+0000", 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108609 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.503397+0000", 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000200063 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.503597+0000", 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015166198000000001 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.518763+0000", 2023-10-09T10:16:09.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076912940000000004 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526454+0000", 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054970941000000002 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581425+0000", 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3358000000000001e-05 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581469+0000", 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00049589800000000004 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581965+0000", 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9446827339999997 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190020 7.e 7:726633f4:::benchmark_data_smithi012_221426_object82242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.665505+0000", 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.893235839, 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.917070517, 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.636 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190020 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665505+0000", 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665505+0000", 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:09.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665505+0000", 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.492e-06 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665507+0000", 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0849000000000001e-05 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665518+0000", 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.233e-06 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665522+0000", 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6986e-05 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665569+0000", 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0998999999999999e-05 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.665580+0000", 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7979873040000003 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463568+0000", 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026484300000000001 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463832+0000", 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.045393021999999998 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.509225+0000", 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010867800000000001 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.509334+0000", 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018993699999999999 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.509524+0000", 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010873582 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.520398+0000", 2023-10-09T10:16:09.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061112220000000004 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526509+0000", 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055545221999999998 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582054+0000", 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2280000000000003e-05 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582086+0000", 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048930100000000004 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582576+0000", 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.917070517 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190037 7.e 7:7898f990:::benchmark_data_smithi012_221426_object82259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.690696+0000", 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.86804542300001, 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8928224829999998, 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190037 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.645 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690696+0000", 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690696+0000", 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690695+0000", 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7029999999999997e-06 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690700+0000", 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0607e-05 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690710+0000", 2023-10-09T10:16:09.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5809999999999998e-06 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690715+0000", 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8108000000000001e-05 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690763+0000", 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5320000000000001e-06 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690770+0000", 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7731051459999998 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.463876+0000", 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000234665 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464110+0000", 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051168810000000002 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.515279+0000", 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8494999999999998e-05 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.515378+0000", 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017001200000000001 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.515548+0000", 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061315839999999998 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.521679+0000", 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048848420000000004 2023-10-09T10:16:09.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526564+0000", 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056062085999999997 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582626+0000", 2023-10-09T10:16:09.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6119999999999998e-05 2023-10-09T10:16:09.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582682+0000", 2023-10-09T10:16:09.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00083589099999999996 2023-10-09T10:16:09.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.583518+0000", 2023-10-09T10:16:09.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8928224829999998 2023-10-09T10:16:09.654 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.654 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.655 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.655 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190038 7.e 7:7015ad23:::benchmark_data_smithi012_221426_object82260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.655 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.690725+0000", 2023-10-09T10:16:09.655 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.86801618300001, 2023-10-09T10:16:09.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8935087460000002, 2023-10-09T10:16:09.656 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.656 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.656 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.656 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190038 2023-10-09T10:16:09.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.657 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690725+0000", 2023-10-09T10:16:09.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690725+0000", 2023-10-09T10:16:09.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:09.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690724+0000", 2023-10-09T10:16:09.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2280000000000001e-06 2023-10-09T10:16:09.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690727+0000", 2023-10-09T10:16:09.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6109999999999998e-06 2023-10-09T10:16:09.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690730+0000", 2023-10-09T10:16:09.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4559999999999999e-06 2023-10-09T10:16:09.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690733+0000", 2023-10-09T10:16:09.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7816999999999999e-05 2023-10-09T10:16:09.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690791+0000", 2023-10-09T10:16:09.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1880000000000004e-06 2023-10-09T10:16:09.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.690795+0000", 2023-10-09T10:16:09.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7733637140000003 2023-10-09T10:16:09.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464158+0000", 2023-10-09T10:16:09.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042128800000000002 2023-10-09T10:16:09.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464580+0000", 2023-10-09T10:16:09.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057145987000000002 2023-10-09T10:16:09.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.521726+0000", 2023-10-09T10:16:09.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7033999999999994e-05 2023-10-09T10:16:09.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.521813+0000", 2023-10-09T10:16:09.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015166100000000001 2023-10-09T10:16:09.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.521964+0000", 2023-10-09T10:16:09.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046447090000000003 2023-10-09T10:16:09.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526609+0000", 2023-10-09T10:16:09.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9210000000000003e-06 2023-10-09T10:16:09.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.526616+0000", 2023-10-09T10:16:09.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056952488000000002 2023-10-09T10:16:09.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.583569+0000", 2023-10-09T10:16:09.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4563e-05 2023-10-09T10:16:09.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.583613+0000", 2023-10-09T10:16:09.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062047100000000004 2023-10-09T10:16:09.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.584234+0000", 2023-10-09T10:16:09.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8935087460000002 2023-10-09T10:16:09.671 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.671 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.671 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.672 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190119 7.e 7:73998f4a:::benchmark_data_smithi012_221426_object82341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.672 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.813673+0000", 2023-10-09T10:16:09.672 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.74506827599998, 2023-10-09T10:16:09.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7712925779999997, 2023-10-09T10:16:09.673 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.673 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.673 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.673 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.673 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.673 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190119 2023-10-09T10:16:09.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.674 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813673+0000", 2023-10-09T10:16:09.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813673+0000", 2023-10-09T10:16:09.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:09.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813671+0000", 2023-10-09T10:16:09.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4869999999999999e-06 2023-10-09T10:16:09.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813678+0000", 2023-10-09T10:16:09.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0417999999999998e-05 2023-10-09T10:16:09.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813718+0000", 2023-10-09T10:16:09.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7849999999999999e-06 2023-10-09T10:16:09.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813723+0000", 2023-10-09T10:16:09.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6807999999999997e-05 2023-10-09T10:16:09.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813760+0000", 2023-10-09T10:16:09.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9610000000000002e-06 2023-10-09T10:16:09.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.813766+0000", 2023-10-09T10:16:09.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6508705859999999 2023-10-09T10:16:09.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464636+0000", 2023-10-09T10:16:09.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034053700000000002 2023-10-09T10:16:09.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.464977+0000", 2023-10-09T10:16:09.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.077599679000000005 2023-10-09T10:16:09.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.542577+0000", 2023-10-09T10:16:09.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8681999999999998e-05 2023-10-09T10:16:09.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.542635+0000", 2023-10-09T10:16:09.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3194000000000003e-05 2023-10-09T10:16:09.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.542719+0000", 2023-10-09T10:16:09.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014864018 2023-10-09T10:16:09.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.557583+0000", 2023-10-09T10:16:09.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021250937000000001 2023-10-09T10:16:09.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578834+0000", 2023-10-09T10:16:09.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054931679999999997 2023-10-09T10:16:09.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.584327+0000", 2023-10-09T10:16:09.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4540999999999998e-05 2023-10-09T10:16:09.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.584381+0000", 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058408199999999998 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.584965+0000", 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7712925779999997 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190129 7.e 7:77d17918:::benchmark_data_smithi012_221426_object82351:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.831462+0000", 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.72727935600005, 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7543196769999998, 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.688 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190129 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831462+0000", 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831462+0000", 2023-10-09T10:16:09.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831460+0000", 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1391e-05 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831471+0000", 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1126e-05 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831482+0000", 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3470000000000004e-06 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831489+0000", 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7751000000000003e-05 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831547+0000", 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1390000000000001e-06 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.831556+0000", 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6334554919999995 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465011+0000", 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036681399999999997 2023-10-09T10:16:09.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465378+0000", 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082331029 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.547709+0000", 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4887000000000001e-05 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.547804+0000", 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015043500000000001 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.547954+0000", 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016570558999999999 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.564525+0000", 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015033075999999999 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579558+0000", 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054733459999999996 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585031+0000", 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1279999999999997e-05 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585083+0000", 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00069872300000000005 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585781+0000", 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7543196769999998 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.696 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190148 7.e 7:73a672b0:::benchmark_data_smithi012_221426_object82370:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.862419+0000", 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.696322055, 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7241475929999996, 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190148 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862419+0000", 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862419+0000", 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862417+0000", 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6969999999999999e-06 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862426+0000", 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6313000000000001e-05 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862442+0000", 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.319e-06 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862448+0000", 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9616000000000001e-05 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862498+0000", 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0275000000000001e-05 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.862508+0000", 2023-10-09T10:16:09.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6029239119999996 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465432+0000", 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039124599999999999 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465823+0000", 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.087867027 2023-10-09T10:16:09.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.553690+0000", 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6755999999999994e-05 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.553777+0000", 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000137775 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.553915+0000", 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017295237000000001 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.571210+0000", 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0089565500000000006 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580167+0000", 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056861580000000002 2023-10-09T10:16:09.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585853+0000", 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2290000000000002e-05 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.585915+0000", 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00065133699999999999 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.586567+0000", 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7241475929999996 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190161 7.e 7:78ca3d82:::benchmark_data_smithi012_221426_object82383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.885594+0000", 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.67314694499998, 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7015902470000004, 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.705 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190161 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885594+0000", 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885594+0000", 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885593+0000", 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4420000000000001e-06 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885600+0000", 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.325e-05 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885613+0000", 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2089999999999999e-06 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885621+0000", 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6032999999999998e-05 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885667+0000", 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0569999999999998e-06 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.885673+0000", 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5802007260000002 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.465873+0000", 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000347401 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466221+0000", 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.093410134000000006 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.559631+0000", 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119565 2023-10-09T10:16:09.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.559750+0000", 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018952699999999999 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.559940+0000", 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017418316 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.577358+0000", 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003434739 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.580793+0000", 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058168040000000001 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.586610+0000", 2023-10-09T10:16:09.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3302000000000001e-05 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.586643+0000", 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00054114400000000002 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587184+0000", 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7015902470000004 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.713 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190171 7.e 7:7bc672db:::benchmark_data_smithi012_221426_object82393:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.906118+0000", 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.65262277700003, 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6816790670000001, 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.714 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190171 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906118+0000", 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906118+0000", 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906117+0000", 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3710000000000002e-06 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906122+0000", 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0206e-05 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906132+0000", 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6589999999999998e-06 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906137+0000", 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0462e-05 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906177+0000", 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9599999999999997e-06 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.906183+0000", 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5600940379999999 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466277+0000", 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000401865 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466679+0000", 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.099618462000000005 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.566297+0000", 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000115087 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.566413+0000", 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020621100000000001 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.566619+0000", 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012227658000000001 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.578846+0000", 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025430679999999999 2023-10-09T10:16:09.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.581390+0000", 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058345560000000003 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587224+0000", 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2561999999999998e-05 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587257+0000", 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00054065999999999999 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587797+0000", 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6816790670000001 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190180 7.e 7:709efe90:::benchmark_data_smithi012_221426_object82402:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:02.945202+0000", 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 546.61353934500005, 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6432413459999999, 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190180 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945202+0000", 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945202+0000", 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945201+0000", 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4689999999999999e-06 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945205+0000", 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0219999999999993e-06 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945214+0000", 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0990000000000001e-06 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945218+0000", 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6171999999999999e-05 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945255+0000", 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4559999999999999e-06 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:02.945260+0000", 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5214660279999999 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466726+0000", 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025202900000000002 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.466978+0000", 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10637545399999999 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.573354+0000", 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6915000000000003e-05 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.573451+0000", 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000172957 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,2", 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.573623+0000", 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059456320000000002 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.579569+0000", 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024467289999999999 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.582016+0000", 2023-10-09T10:16:09.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058284809999999999 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587844+0000", 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4780000000000002e-05 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.587889+0000", 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00055391200000000005 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:09.588443+0000", 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6432413459999999 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:85019 8.4 8:26bd7172:::benchmark_data_smithi012_271002_object85018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:46.061719+0000", 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 383.49702162300002, 2023-10-09T10:16:09.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.398916023, 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 85019 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061719+0000", 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061719+0000", 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061718+0000", 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5325000000000002e-05 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061783+0000", 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7779999999999999e-05 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061801+0000", 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8170000000000001e-06 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061808+0000", 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7277999999999997e-05 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061855+0000", 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.832e-06 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061862+0000", 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93984965899999995 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001712+0000", 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7869999999999996e-06 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001722+0000", 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4215940549999999 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.423316+0000", 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011345000000000001 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.423429+0000", 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000177066 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.423606+0000", 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010226247000000001 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433832+0000", 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3725000000000004e-05 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433896+0000", 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022457444 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.456354+0000", 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5872000000000003e-05 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.456409+0000", 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042259159999999997 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.460635+0000", 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.398916023 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.739 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:85022 8.4 8:259e8b82:::benchmark_data_smithi012_271002_object85021:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:46.064695+0000", 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 383.49404614999997, 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.39940138, 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:16:09.740 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 85022 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064695+0000", 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064695+0000", 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064693+0000", 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6102999999999999e-05 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064779+0000", 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3779999999999999e-05 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064803+0000", 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.562e-06 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064810+0000", 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4813000000000004e-05 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064875+0000", 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432999999999999e-05 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064887+0000", 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.936856048 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001743+0000", 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8660000000000006e-06 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001751+0000", 2023-10-09T10:16:09.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.424300347 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.426051+0000", 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105982 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.426157+0000", 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013454 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.426292+0000", 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075499429999999999 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433842+0000", 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7359000000000001e-05 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433939+0000", 2023-10-09T10:16:09.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026765071000000001 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.460704+0000", 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3587000000000001e-05 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.460748+0000", 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033485020000000002 2023-10-09T10:16:09.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.464096+0000", 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.39940138 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:85023 8.4 8:25543529:::benchmark_data_smithi012_271002_object85022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:46.064828+0000", 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 383.49391317800001, 2023-10-09T10:16:09.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.401756979, 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 85023 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064828+0000", 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064828+0000", 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064827+0000", 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7450000000000002e-05 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064865+0000", 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6919999999999997e-06 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064874+0000", 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5569999999999997e-06 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064879+0000", 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5850999999999999e-05 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064925+0000", 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4500000000000004e-06 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064933+0000", 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93683761200000004 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001771+0000", 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2289999999999997e-06 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001778+0000", 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.426578484 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.428357+0000", 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132087 2023-10-09T10:16:09.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.428489+0000", 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000149829 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.428638+0000", 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052126619999999999 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433851+0000", 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013186900000000001 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433983+0000", 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.03017481 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.464158+0000", 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0639000000000002e-05 2023-10-09T10:16:09.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.464198+0000", 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023863640000000002 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.466585+0000", 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.401756979 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.756 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:85024 8.4 8:2abed64e:::benchmark_data_smithi012_271002_object85023:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:46.064887+0000", 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 383.49385421599999, 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4040639640000001, 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:16:09.757 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 85024 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064887+0000", 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064887+0000", 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:09.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064886+0000", 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4586000000000001e-05 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064911+0000", 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.844e-06 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064918+0000", 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9160000000000003e-06 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064922+0000", 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9326999999999999e-05 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064951+0000", 2023-10-09T10:16:09.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9810000000000003e-06 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064956+0000", 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93684371 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001800+0000", 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2970000000000004e-06 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001808+0000", 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4321932479999999 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.434001+0000", 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012312900000000001 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:09.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.434124+0000", 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000124043 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.434248+0000", 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070481770000000001 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.441296+0000", 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014025800000000001 2023-10-09T10:16:09.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.441437+0000", 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025203030000000001 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.466640+0000", 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8245999999999999e-05 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.466678+0000", 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022727250000000002 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.468951+0000", 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4040639640000001 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:09.765 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:09.766 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:16:09.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:09.767 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[152263]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[152263]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[152263]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[152263]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[152263]: osdmap e358: 8 total, 8 up, 7 in 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[152263]: failed to encode map e358 with expected crc 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[152263]: failed to encode map e358 with expected crc 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[147926]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.798 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[147926]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[147926]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[147926]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[147926]: osdmap e358: 8 total, 8 up, 7 in 2023-10-09T10:16:09.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[147926]: failed to encode map e358 with expected crc 2023-10-09T10:16:09.799 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:09 smithi012 ceph-mon[147926]: failed to encode map e358 with expected crc 2023-10-09T10:16:09.802 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:09.846 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:09.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:09.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:09 smithi110 ceph-mon[136032]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:09 smithi110 ceph-mon[136032]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:09 smithi110 ceph-mon[136032]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:09 smithi110 ceph-mon[136032]: failed to encode map e357 with expected crc 2023-10-09T10:16:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:09 smithi110 ceph-mon[136032]: osdmap e358: 8 total, 8 up, 7 in 2023-10-09T10:16:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:09 smithi110 ceph-mon[136032]: failed to encode map e358 with expected crc 2023-10-09T10:16:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:09 smithi110 ceph-mon[136032]: failed to encode map e358 with expected crc 2023-10-09T10:16:09.949 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:09.991 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:10.015 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:16:10.032 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:10.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:10.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:10.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:10.165 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:10.213 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:10.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:10.318 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:10.345 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:10.381 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:10.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:10.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:10.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:10.519 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:10.561 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:10.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:10.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:16:10.662 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:10.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:10.722 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:10.771 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:10.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 355'8065 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 355'8044 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 355'7757 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: pgmap v1011: 99 pgs: 7 remapped+peering, 6 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 69 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 30078/291594 objects degraded (10.315%); 15765/291594 objects misplaced (5.406%); 11 MiB/s, 195 objects/s recovering 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: failed to encode map e359 with expected crc 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: osdmap e359: 8 total, 8 up, 7 in 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[147926]: failed to encode map e359 with expected crc 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.774 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 355'8065 2023-10-09T10:16:10.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 355'8044 2023-10-09T10:16:10.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 355'7757 2023-10-09T10:16:10.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: pgmap v1011: 99 pgs: 7 remapped+peering, 6 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 69 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 30078/291594 objects degraded (10.315%); 15765/291594 objects misplaced (5.406%); 11 MiB/s, 195 objects/s recovering 2023-10-09T10:16:10.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: failed to encode map e359 with expected crc 2023-10-09T10:16:10.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: osdmap e359: 8 total, 8 up, 7 in 2023-10-09T10:16:10.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:10 smithi012 ceph-mon[152263]: failed to encode map e359 with expected crc 2023-10-09T10:16:10.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:10.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:10.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:10.923 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.923 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.923 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 355'8065 2023-10-09T10:16:10.923 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: failed to encode map e358 with expected crc 2023-10-09T10:16:10.923 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 355'8044 2023-10-09T10:16:10.924 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 355'7757 2023-10-09T10:16:10.924 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: pgmap v1011: 99 pgs: 7 remapped+peering, 6 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 69 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 30078/291594 objects degraded (10.315%); 15765/291594 objects misplaced (5.406%); 11 MiB/s, 195 objects/s recovering 2023-10-09T10:16:10.924 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: failed to encode map e359 with expected crc 2023-10-09T10:16:10.924 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: osdmap e359: 8 total, 8 up, 7 in 2023-10-09T10:16:10.924 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:10 smithi110 ceph-mon[136032]: failed to encode map e359 with expected crc 2023-10-09T10:16:10.927 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:11.077 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:11.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:11.170 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:16:11.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:11.197 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:16:11.219 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:11.231 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:16:11.263 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:16:11.288 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:16:11.310 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:16:11.324 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:11.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:11.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:11.336 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:16:11.359 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:16:11.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:16:11.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:16:11.360 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:16:11.361 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:16:11.361 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:16:11.361 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:16:11.361 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:16:11.361 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:16:11.371 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:11.429 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:11.458 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:11.494 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:11.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:11.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:11.598 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:11.626 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:11.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:11 smithi012 ceph-mon[147926]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:11 smithi012 ceph-mon[147926]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:11 smithi012 ceph-mon[147926]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/538521418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:11.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:11 smithi012 ceph-mon[152263]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:11 smithi012 ceph-mon[152263]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:11 smithi012 ceph-mon[152263]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/538521418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:11.765 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:11.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:11.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:11.802 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:11.802 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:11.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:11.915 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:11 smithi110 ceph-mon[136032]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:11 smithi110 ceph-mon[136032]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:11 smithi110 ceph-mon[136032]: failed to encode map e359 with expected crc 2023-10-09T10:16:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/538521418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:11.957 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:11.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:11.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:12.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:12.090 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:12.208 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:12.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:12.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:12.299 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:16:12.313 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:12.371 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:12.407 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:12.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:12.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:12.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:12.547 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:12.659 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:12.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:12.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:12 smithi110 ceph-mon[136032]: pgmap v1013: 99 pgs: 1 activating, 7 remapped+peering, 6 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 69 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 39712/291072 objects degraded (13.643%); 18227/291072 objects misplaced (6.262%); 10 MiB/s, 257 objects/s recovering 2023-10-09T10:16:12.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:12 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T10:16:12.756 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:12 smithi012 ceph-mon[147926]: pgmap v1013: 99 pgs: 1 activating, 7 remapped+peering, 6 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 69 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 39712/291072 objects degraded (13.643%); 18227/291072 objects misplaced (6.262%); 10 MiB/s, 257 objects/s recovering 2023-10-09T10:16:12.756 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:12 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T10:16:12.756 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:12 smithi012 ceph-mon[152263]: pgmap v1013: 99 pgs: 1 activating, 7 remapped+peering, 6 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 69 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 39712/291072 objects degraded (13.643%); 18227/291072 objects misplaced (6.262%); 10 MiB/s, 257 objects/s recovering 2023-10-09T10:16:12.756 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:12 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T10:16:12.764 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:12.792 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:12.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:12.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:12.922 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:12.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:12.970 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:13.012 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:13.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:13.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:13.117 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:13.145 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:13.181 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:13.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:13.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:13.286 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:13.313 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2] out_osds: [5] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:16:13.313 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:16:13.313 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T10:16:13.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T10:16:13.316 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:13.403 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:13.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:13.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:13.508 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:13.535 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:13.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 39712/291072 objects degraded (13.643%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:13.666 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4008567132' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:16:13.796 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:13.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:13.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:16:13.901 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:13.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 39712/291072 objects degraded (13.643%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:13.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4008567132' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:16:13.935 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 39712/291072 objects degraded (13.643%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:13.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4008567132' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:16:13.989 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:13.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:14.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:14.000 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:14.101 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:14.135 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:14.199 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:14.199 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:14.218 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:16:14.648 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:16:15.272 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:15.354 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:15.358 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: pgmap v1014: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 2 remapped+peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 73 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 64116/291006 objects degraded (22.033%); 20168/291006 objects misplaced (6.930%); 1/97002 objects unfound (0.001%); 11 MiB/s, 243 objects/s recovering 2023-10-09T10:16:15.358 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:15.358 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:15.358 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: Health check failed: 1/97002 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:15.358 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:15.358 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.358 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4008567132' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: osdmap e360: 8 total, 8 up, 7 in 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[152263]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: pgmap v1014: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 2 remapped+peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 73 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 64116/291006 objects degraded (22.033%); 20168/291006 objects misplaced (6.930%); 1/97002 objects unfound (0.001%); 11 MiB/s, 243 objects/s recovering 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: Health check failed: 1/97002 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:15.359 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4008567132' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:16:15.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: osdmap e360: 8 total, 8 up, 7 in 2023-10-09T10:16:15.360 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:14 smithi012 ceph-mon[147926]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:15.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: pgmap v1014: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 2 remapped+peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 73 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 64116/291006 objects degraded (22.033%); 20168/291006 objects misplaced (6.930%); 1/97002 objects unfound (0.001%); 11 MiB/s, 243 objects/s recovering 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: Health check failed: 1/97002 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4008567132' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: osdmap e360: 8 total, 8 up, 7 in 2023-10-09T10:16:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:14 smithi110 ceph-mon[136032]: failed to encode map e360 with expected crc 2023-10-09T10:16:15.462 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:15.494 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:15.641 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:15.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:15.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:15.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:15.779 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:15.975 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:15.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:15.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:16.081 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:16.109 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:15 smithi110 ceph-mon[136032]: failed to encode map e360 with expected crc 2023-10-09T10:16:16.113 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:16.163 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:16.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:16.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:16.290 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:15 smithi012 ceph-mon[147926]: failed to encode map e360 with expected crc 2023-10-09T10:16:16.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:15 smithi012 ceph-mon[152263]: failed to encode map e360 with expected crc 2023-10-09T10:16:16.303 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:16.449 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:16.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:16.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:16.487 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:16.487 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:16.553 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:16.583 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:16.688 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:16.841 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:16.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:16.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:16.895 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:16:16.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:16.981 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:17.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:16 smithi012 ceph-mon[147926]: pgmap v1016: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+degraded, 74 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 76608/290898 objects degraded (26.335%); 25356/290898 objects misplaced (8.716%); 1/96966 objects unfound (0.001%); 13 MiB/s, 258 objects/s recovering 2023-10-09T10:16:17.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:16 smithi012 ceph-mon[152263]: pgmap v1016: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+degraded, 74 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 76608/290898 objects degraded (26.335%); 25356/290898 objects misplaced (8.716%); 1/96966 objects unfound (0.001%); 13 MiB/s, 258 objects/s recovering 2023-10-09T10:16:17.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:16 smithi110 ceph-mon[136032]: pgmap v1016: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+degraded, 74 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 76608/290898 objects degraded (26.335%); 25356/290898 objects misplaced (8.716%); 1/96966 objects unfound (0.001%); 13 MiB/s, 258 objects/s recovering 2023-10-09T10:16:17.223 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:17.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:17.328 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:17.356 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:17.399 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:17.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:17.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:17.518 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:17.532 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:17.650 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:17.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:17.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:17.791 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:16:17.513+0000 7f85a99d3700 -1 mon.a@0(leader).osd e360 definitely_dead 0 2023-10-09T10:16:17.799 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:17.866 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:17.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:17.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:17.971 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:17.996 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.5. 2023-10-09T10:16:18.043 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:18.059 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:17 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2864566352' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:16:18.059 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:16:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:16:17] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.33.4" 2023-10-09T10:16:18.060 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:17 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2864566352' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:16:18.140 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:18.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:18.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:18.305 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:17 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2864566352' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T10:16:18.310 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:18.384 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:18.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:18.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:18.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:16:18.495 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:18.529 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:18.558 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T10:16:18.622 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:18.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:18.726 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:18.758 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:18.807 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:18.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:18.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:18.912 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:18.919 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:18.920 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:18.944 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:19.033 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:19.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:19.088 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:19.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:19.168 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: pgmap v1017: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+degraded, 74 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 76608/290898 objects degraded (26.335%); 25356/290898 objects misplaced (8.716%); 1/96966 objects unfound (0.001%); 9.9 MiB/s, 204 objects/s recovering 2023-10-09T10:16:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e361 with expected crc 2023-10-09T10:16:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2864566352' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T10:16:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T10:16:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e361 with expected crc 2023-10-09T10:16:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 76608/290898 objects degraded (26.335%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1272282471' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:19.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: pgmap v1017: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+degraded, 74 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 76608/290898 objects degraded (26.335%); 25356/290898 objects misplaced (8.716%); 1/96966 objects unfound (0.001%); 9.9 MiB/s, 204 objects/s recovering 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e361 with expected crc 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2864566352' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e361 with expected crc 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 76608/290898 objects degraded (26.335%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1272282471' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: pgmap v1017: 99 pgs: 2 activating+undersized+degraded+remapped, 2 activating, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+degraded, 74 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 76608/290898 objects degraded (26.335%); 25356/290898 objects misplaced (8.716%); 1/96966 objects unfound (0.001%); 9.9 MiB/s, 204 objects/s recovering 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e361 with expected crc 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2864566352' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e361 with expected crc 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 76608/290898 objects degraded (26.335%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T10:16:19.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: failed to encode map e362 with expected crc 2023-10-09T10:16:19.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1272282471' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:19.256 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:19.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:19.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:19.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:16:19.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:19.404 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:19.452 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:19.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:19.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:19.589 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:19.923 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:19.956 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:19.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:20.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:20.097 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:20.139 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:20.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:20.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:20.221 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[147926]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.221 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[147926]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.221 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[147926]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T10:16:20.221 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[147926]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.221 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[147926]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[152263]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[152263]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[152263]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T10:16:20.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[152263]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:20 smithi012 ceph-mon[152263]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.245 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:20.275 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:20.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:20.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:20.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:20 smithi110 ceph-mon[136032]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:20 smithi110 ceph-mon[136032]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:20 smithi110 ceph-mon[136032]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T10:16:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:20 smithi110 ceph-mon[136032]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:20 smithi110 ceph-mon[136032]: failed to encode map e363 with expected crc 2023-10-09T10:16:20.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:20.528 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:20.703 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:20.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:20.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:20.808 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:20.840 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:20.904 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:20.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:21.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:21.044 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:21.087 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:21.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:21.194 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:21.203 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:21.204 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:21.220 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:16:21.228 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:21.323 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:21.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:21.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:21.354 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[147926]: pgmap v1021: 99 pgs: 2 remapped+peering, 13 peering, 2 activating+undersized+degraded+remapped, 2 activating, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 62 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 58610/290487 objects degraded (20.176%); 13358/290487 objects misplaced (4.598%); 1/96829 objects unfound (0.001%); 5.4 MiB/s, 140 objects/s recovering 2023-10-09T10:16:21.354 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[147926]: Health check update: 1/96829 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:21.354 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[147926]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[147926]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[147926]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[147926]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/54242921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[152263]: pgmap v1021: 99 pgs: 2 remapped+peering, 13 peering, 2 activating+undersized+degraded+remapped, 2 activating, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 62 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 58610/290487 objects degraded (20.176%); 13358/290487 objects misplaced (4.598%); 1/96829 objects unfound (0.001%); 5.4 MiB/s, 140 objects/s recovering 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[152263]: Health check update: 1/96829 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[152263]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[152263]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[152263]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T10:16:21.355 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[152263]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.356 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/54242921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:21.427 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:21.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:21.582 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:21.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:21 smithi110 ceph-mon[136032]: pgmap v1021: 99 pgs: 2 remapped+peering, 13 peering, 2 activating+undersized+degraded+remapped, 2 activating, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 62 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 58610/290487 objects degraded (20.176%); 13358/290487 objects misplaced (4.598%); 1/96829 objects unfound (0.001%); 5.4 MiB/s, 140 objects/s recovering 2023-10-09T10:16:21.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:21 smithi110 ceph-mon[136032]: Health check update: 1/96829 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:21.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:21 smithi110 ceph-mon[136032]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.585 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:21 smithi110 ceph-mon[136032]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.585 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:21 smithi110 ceph-mon[136032]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T10:16:21.585 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:21 smithi110 ceph-mon[136032]: failed to encode map e364 with expected crc 2023-10-09T10:16:21.585 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/54242921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:21.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:21.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:21.652 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:16:21.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:21.720 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:22.240 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:22.240 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:16:22.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:22.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:22.264 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:16:22.286 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:16:22.294 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:22.311 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:16:22.335 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:16:22.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:22.357 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:16:22.379 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:22.380 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:16:22.407 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:16:22.409 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:16:22.409 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:16:22.409 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:16:22.409 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:16:22.409 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:16:22.410 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:16:22.410 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:16:22.410 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:16:22.423 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:22.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:22.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:22.502 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:22 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:22.502 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/54242921' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:22.502 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:22 smithi110 ceph-mon[136032]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.503 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:22 smithi110 ceph-mon[136032]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T10:16:22.503 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:22 smithi110 ceph-mon[136032]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.503 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:22 smithi110 ceph-mon[136032]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.503 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:22 smithi110 ceph-mon[136032]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.503 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:22 smithi110 ceph-mon[136032]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:22.560 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:22.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:22.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/54242921' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:22.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[147926]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[147926]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T10:16:22.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[147926]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[147926]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[147926]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[147926]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/54242921' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[152263]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[152263]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[152263]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[152263]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[152263]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.574 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:22 smithi012 ceph-mon[152263]: failed to encode map e365 with expected crc 2023-10-09T10:16:22.598 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:22.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:22.702 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:22.732 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:22.775 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:22.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:22.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:22.910 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:22.949 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:22.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:23.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:23.090 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:23.189 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:23.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:23.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:23.326 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:23.366 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:23.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:23.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:23 smithi110 ceph-mon[136032]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 13 peering, 2 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 63 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 47 op/s; 52390/290358 objects degraded (18.043%); 13577/290358 objects misplaced (4.676%); 1/96786 objects unfound (0.001%); 8.0 MiB/s, 213 objects/s recovering 2023-10-09T10:16:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:23 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T10:16:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:23 smithi110 ceph-mon[136032]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:23 smithi110 ceph-mon[136032]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:23 smithi110 ceph-mon[136032]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T10:16:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:23 smithi110 ceph-mon[136032]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:23 smithi110 ceph-mon[136032]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 52390/290358 objects degraded (18.043%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:23.471 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:23.505 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190193 7.8 7:1a8121c5:::benchmark_data_smithi012_221426_object82415:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e240)", 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.10401679300003, 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.051354506, 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.506 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190193 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397657+0000", 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397656+0000", 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7679999999999997e-06 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397662+0000", 2023-10-09T10:16:23.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0439999999999996e-06 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397670+0000", 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0260000000000004e-06 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397676+0000", 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00094881100000000003 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398625+0000", 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2380000000000003e-06 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398634+0000", 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0429888860000001 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441623+0000", 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011069399999999999 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441733+0000", 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4486000000000003e-05 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.441828+0000", 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033881190000000002 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445216+0000", 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036284030000000001 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.448844+0000", 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.923e-05 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.448904+0000", 2023-10-09T10:16:23.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010786 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449012+0000", 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.051354506 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.513 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190203 7.8 7:1d4612d0:::benchmark_data_smithi012_221426_object82425:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:16:23.514 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:16:23.514 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.10399371300002, 2023-10-09T10:16:23.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0571886909999999, 2023-10-09T10:16:23.514 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.514 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.514 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.514 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:23.514 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190203 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397680+0000", 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0150000000000002e-06 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.516 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397682+0000", 2023-10-09T10:16:23.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.647e-06 2023-10-09T10:16:23.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397685+0000", 2023-10-09T10:16:23.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7410000000000001e-06 2023-10-09T10:16:23.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397688+0000", 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099381899999999995 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398682+0000", 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5570000000000003e-06 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398688+0000", 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.046572745 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445260+0000", 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012410700000000001 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445384+0000", 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000154804 2023-10-09T10:16:23.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.445539+0000", 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058752240000000001 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451414+0000", 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033171709999999998 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454732+0000", 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5908000000000001e-05 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454788+0000", 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1254999999999994e-05 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454869+0000", 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0571886909999999 2023-10-09T10:16:23.521 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190210 7.8 7:1b3a4cee:::benchmark_data_smithi012_221426_object82432:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.10398243199995, 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0644897120000001, 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190210 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397691+0000", 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7489999999999999e-06 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397693+0000", 2023-10-09T10:16:23.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0630000000000002e-06 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397696+0000", 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2269999999999996e-06 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397703+0000", 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001020782 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398724+0000", 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3509999999999996e-06 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398731+0000", 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0503301599999999 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449062+0000", 2023-10-09T10:16:23.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013289700000000001 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449194+0000", 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018222 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.449377+0000", 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012586909 2023-10-09T10:16:23.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.461964+0000", 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4403e-05 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462028+0000", 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3278e-05 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462081+0000", 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9912000000000002e-05 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462181+0000", 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0644897120000001 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190212 7.8 7:1e06d44a:::benchmark_data_smithi012_221426_object82434:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:16:23.529 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.103968098, 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.071536394, 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190212 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:16:23.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397706+0000", 2023-10-09T10:16:23.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:23.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397705+0000", 2023-10-09T10:16:23.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7659999999999999e-06 2023-10-09T10:16:23.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397707+0000", 2023-10-09T10:16:23.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8750000000000002e-06 2023-10-09T10:16:23.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397711+0000", 2023-10-09T10:16:23.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4600000000000002e-06 2023-10-09T10:16:23.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397714+0000", 2023-10-09T10:16:23.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010809909999999999 2023-10-09T10:16:23.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398795+0000", 2023-10-09T10:16:23.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5300000000000004e-06 2023-10-09T10:16:23.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398800+0000", 2023-10-09T10:16:23.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0526496919999999 2023-10-09T10:16:23.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451450+0000", 2023-10-09T10:16:23.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013340300000000001 2023-10-09T10:16:23.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451583+0000", 2023-10-09T10:16:23.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017290599999999999 2023-10-09T10:16:23.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:16:23.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.451756+0000", 2023-10-09T10:16:23.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010213186000000001 2023-10-09T10:16:23.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.539 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.539 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.461969+0000", 2023-10-09T10:16:23.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071399239999999997 2023-10-09T10:16:23.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469109+0000", 2023-10-09T10:16:23.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9132000000000003e-05 2023-10-09T10:16:23.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469158+0000", 2023-10-09T10:16:23.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3792e-05 2023-10-09T10:16:23.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469242+0000", 2023-10-09T10:16:23.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.071536394 2023-10-09T10:16:23.542 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.542 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.542 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.543 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190219 7.8 7:155dd6d1:::benchmark_data_smithi012_221426_object82441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:16:23.543 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:16:23.543 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.10395710099999, 2023-10-09T10:16:23.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0716708749999999, 2023-10-09T10:16:23.543 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.544 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.544 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.544 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:23.544 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:23.544 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190219 2023-10-09T10:16:23.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.545 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:16:23.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397717+0000", 2023-10-09T10:16:23.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:23.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397716+0000", 2023-10-09T10:16:23.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6100000000000002e-06 2023-10-09T10:16:23.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.548 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397723+0000", 2023-10-09T10:16:23.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5250000000000001e-06 2023-10-09T10:16:23.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.548 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397727+0000", 2023-10-09T10:16:23.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.711e-06 2023-10-09T10:16:23.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397729+0000", 2023-10-09T10:16:23.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001098705 2023-10-09T10:16:23.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398828+0000", 2023-10-09T10:16:23.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5500000000000002e-06 2023-10-09T10:16:23.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398834+0000", 2023-10-09T10:16:23.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0560779309999999 2023-10-09T10:16:23.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.454912+0000", 2023-10-09T10:16:23.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012238900000000001 2023-10-09T10:16:23.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.455034+0000", 2023-10-09T10:16:23.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015768999999999999 2023-10-09T10:16:23.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:16:23.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.455192+0000", 2023-10-09T10:16:23.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010377422000000001 2023-10-09T10:16:23.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465569+0000", 2023-10-09T10:16:23.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037032409999999999 2023-10-09T10:16:23.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469272+0000", 2023-10-09T10:16:23.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1624e-05 2023-10-09T10:16:23.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469314+0000", 2023-10-09T10:16:23.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3724000000000001e-05 2023-10-09T10:16:23.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469388+0000", 2023-10-09T10:16:23.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0716708749999999 2023-10-09T10:16:23.556 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.557 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.557 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.557 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190227 7.8 7:129c86af:::benchmark_data_smithi012_221426_object82449:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:16:23.557 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:16:23.557 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.10394214400003, 2023-10-09T10:16:23.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0717717900000001, 2023-10-09T10:16:23.558 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.558 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.558 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.558 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:23.558 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:23.559 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190227 2023-10-09T10:16:23.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.559 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:16:23.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397732+0000", 2023-10-09T10:16:23.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:23.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397731+0000", 2023-10-09T10:16:23.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5820000000000001e-06 2023-10-09T10:16:23.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397733+0000", 2023-10-09T10:16:23.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8680000000000001e-06 2023-10-09T10:16:23.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397737+0000", 2023-10-09T10:16:23.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4420000000000001e-06 2023-10-09T10:16:23.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397739+0000", 2023-10-09T10:16:23.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011216900000000001 2023-10-09T10:16:23.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398861+0000", 2023-10-09T10:16:23.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4060000000000004e-06 2023-10-09T10:16:23.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398866+0000", 2023-10-09T10:16:23.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0599802039999999 2023-10-09T10:16:23.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.458847+0000", 2023-10-09T10:16:23.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9543000000000006e-05 2023-10-09T10:16:23.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.458946+0000", 2023-10-09T10:16:23.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5651999999999999e-05 2023-10-09T10:16:23.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:16:23.567 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.459032+0000", 2023-10-09T10:16:23.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010030136 2023-10-09T10:16:23.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.567 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.567 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469062+0000", 2023-10-09T10:16:23.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036131999999999999 2023-10-09T10:16:23.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469423+0000", 2023-10-09T10:16:23.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0482000000000001e-05 2023-10-09T10:16:23.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469454+0000", 2023-10-09T10:16:23.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9725000000000002e-05 2023-10-09T10:16:23.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469503+0000", 2023-10-09T10:16:23.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0717717900000001 2023-10-09T10:16:23.570 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.570 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.571 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.571 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190233 7.8 7:13f66138:::benchmark_data_smithi012_221426_object82455:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:16:23.571 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:16:23.571 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.10393235900005, 2023-10-09T10:16:23.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.076501538, 2023-10-09T10:16:23.572 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.572 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.572 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.572 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:23.572 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:23.572 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190233 2023-10-09T10:16:23.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.572 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397741+0000", 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4050000000000001e-06 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397743+0000", 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.63e-06 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397746+0000", 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.255e-06 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397749+0000", 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011455700000000001 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398894+0000", 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3720000000000004e-06 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398899+0000", 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0633114340000001 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462211+0000", 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8008000000000001e-05 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.576 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462309+0000", 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013975599999999999 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.462449+0000", 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068008950000000004 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469250+0000", 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048359240000000001 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474086+0000", 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7553999999999998e-05 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474143+0000", 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9965999999999998e-05 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.474243+0000", 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.076501538 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.35157.0:190240 7.8 7:1bb20cb0:::benchmark_data_smithi012_221426_object82462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 553.10392336999996, 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0788701549999999, 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.579 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.35157", 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1623964932", 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 190240 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397750+0000", 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.508e-06 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397752+0000", 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5329999999999999e-06 2023-10-09T10:16:23.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397755+0000", 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1100000000000001e-06 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.397757+0000", 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001159413 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398917+0000", 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4650000000000002e-06 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:10.398922+0000", 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.066675222 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465597+0000", 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5288000000000001e-05 2023-10-09T10:16:23.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465683+0000", 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107876 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2", 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.465791+0000", 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036041200000000002 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.469395+0000", 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071257839999999996 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476521+0000", 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2398999999999998e-05 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476563+0000", 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7669e-05 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:07:11.476621+0000", 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0788701549999999 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65159 9.3 9:cf47ceeb:::benchmark_data_smithi012_313365_object65158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 128.493408174, 2023-10-09T10:16:23.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19414071499999999, 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65159 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008264+0000", 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8745000000000001e-05 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008323+0000", 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1172000000000001e-05 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008334+0000", 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4099999999999996e-06 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008341+0000", 2023-10-09T10:16:23.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6387999999999998e-05 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008377+0000", 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015405 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008531+0000", 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016722800000000001 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008698+0000", 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023079260000000001 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.011006+0000", 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19127018400000001 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202277+0000", 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9379999999999999e-05 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202316+0000", 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0695999999999997e-05 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202367+0000", 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9728000000000003e-05 2023-10-09T10:16:23.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202406+0000", 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19414071499999999 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:65165 9.d e328/317 9:bf9ec36c:::benchmark_data_smithi012_313365_object65164:head v 328'4139, mlcod=328'4137)", 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:15.015534+0000", 2023-10-09T10:16:23.593 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 128.48613949400001, 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.191150345, 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015534+0000", 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015534+0000", 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015534+0000", 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6866000000000006e-05 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015611+0000", 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8059999999999995e-06 2023-10-09T10:16:23.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015619+0000", 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4700000000000001e-06 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.015624+0000", 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.18184143899999999 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197465+0000", 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8736999999999999e-05 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.197494+0000", 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0091424269999999998 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.206637+0000", 2023-10-09T10:16:23.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8059000000000002e-05 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.206685+0000", 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.191150345 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114942 9.11 9:8d57c7a6:::benchmark_data_smithi012_313365_object2467:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:16:23.598 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 69.287730354999994, 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0078972580000001, 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114942 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999967 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213940+0000", 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.791e-05 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213958+0000", 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5698999999999998e-05 2023-10-09T10:16:23.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213994+0000", 2023-10-09T10:16:23.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.436e-05 2023-10-09T10:16:23.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214008+0000", 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122045 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214130+0000", 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3575999999999999e-05 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214154+0000", 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99465683800000004 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.208810+0000", 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042772410000000002 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213088+0000", 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031397499999999997 2023-10-09T10:16:23.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213402+0000", 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061402180000000002 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.219542+0000", 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021434869999999999 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221685+0000", 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7170999999999999e-05 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221743+0000", 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8197999999999997e-05 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221841+0000", 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0078972580000001 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114943 9.7 9:e38f5304:::benchmark_data_smithi012_313365_object2468:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 69.287640913000004, 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1890109880000002, 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.606 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114943 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214032+0000", 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9030000000000004e-06 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214040+0000", 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.774e-05 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214057+0000", 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6160000000000008e-06 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214067+0000", 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013272499999999999 2023-10-09T10:16:23.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214200+0000", 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2798000000000004e-05 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214262+0000", 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98051140199999998 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194774+0000", 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8810000000000008e-06 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194784+0000", 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1642592760000001 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359043+0000", 2023-10-09T10:16:23.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000316631 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359360+0000", 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026861052 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386221+0000", 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8226999999999994e-05 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386309+0000", 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030679600000000002 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386616+0000", 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060189639999999999 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392635+0000", 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1379999999999999e-05 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392646+0000", 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095982680000000001 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402244+0000", 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8916999999999999e-05 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402303+0000", 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00074067200000000005 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.403044+0000", 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1890109880000002 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.615 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114945 9.7 9:e1cd6305:::benchmark_data_smithi012_313365_object2470:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 69.287579910999995, 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.171928715, 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114945 2023-10-09T10:16:23.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:23.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214093+0000", 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3230000000000003e-06 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214099+0000", 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6458999999999999e-05 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214116+0000", 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1797000000000001e-05 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214128+0000", 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015993 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214287+0000", 2023-10-09T10:16:23.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214295+0000", 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98052440100000005 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194820+0000", 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0089999999999998e-06 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194825+0000", 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1645754820000001 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359400+0000", 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035076999999999998 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359751+0000", 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015525427 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375276+0000", 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0332000000000004e-05 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375347+0000", 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032258599999999998 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375669+0000", 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095005880000000008 2023-10-09T10:16:23.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385170+0000", 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0859999999999997e-05 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385231+0000", 2023-10-09T10:16:23.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5485999999999999e-05 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385266+0000", 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9339999999999999e-05 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385306+0000", 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00071704500000000003 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386023+0000", 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.171928715 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114953 9.1 9:85379d0d:::benchmark_data_smithi012_313365_object2478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 69.267135640000006, 2023-10-09T10:16:23.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98747086900000003, 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114953 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:16:23.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234537+0000", 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234543+0000", 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3334e-05 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234557+0000", 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1240000000000004e-06 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234562+0000", 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0877000000000002e-05 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234603+0000", 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5950000000000001e-06 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234610+0000", 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98059611300000005 2023-10-09T10:16:23.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215206+0000", 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012311700000000001 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215329+0000", 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038846200000000001 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-10-09T10:16:23.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215718+0000", 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038298339999999998 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.219548+0000", 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023326010000000001 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221880+0000", 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4744999999999999e-05 2023-10-09T10:16:23.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221925+0000", 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3997000000000002e-05 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.222009+0000", 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98747086900000003 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114959 9.13 9:cf007434:::benchmark_data_smithi012_313365_object2484:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 69.225967780999994, 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96965389000000002, 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114959 2023-10-09T10:16:23.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275705+0000", 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1540000000000002e-06 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275713+0000", 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4321000000000001e-05 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275727+0000", 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4679999999999997e-06 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275733+0000", 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8029999999999999e-05 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275782+0000", 2023-10-09T10:16:23.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0797e-05 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275792+0000", 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93510872899999997 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.210901+0000", 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013108099999999999 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.211032+0000", 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035745900000000002 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1", 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.211390+0000", 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052776840000000004 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216667+0000", 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028541474000000001 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245209+0000", 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6895000000000003e-05 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245266+0000", 2023-10-09T10:16:23.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4310999999999999e-05 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245360+0000", 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96965389000000002 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114960 9.17 9:ee9c2079:::benchmark_data_smithi012_313365_object2485:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:16:23.641 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 69.223475669999999, 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1241943320000001, 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114960 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278197+0000", 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6739999999999996e-06 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278202+0000", 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1857e-05 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278214+0000", 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4840000000000003e-06 2023-10-09T10:16:23.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278219+0000", 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1851000000000002e-05 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278271+0000", 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6990000000000003e-06 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278279+0000", 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91653466100000003 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194814+0000", 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7139999999999996e-06 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194820+0000", 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1641976199999999 2023-10-09T10:16:23.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359018+0000", 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000340787 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359359+0000", 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026806301000000001 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386165+0000", 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1144999999999996e-05 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386246+0000", 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000216963 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386463+0000", 2023-10-09T10:16:23.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025360859999999999 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.388999+0000", 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003603794 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392603+0000", 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077717259999999996 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400375+0000", 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0828999999999999e-05 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400406+0000", 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019869430000000001 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402393+0000", 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1241943320000001 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114968 9.7 9:e6edcf82:::benchmark_data_smithi012_313365_object2493:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 68.250117035000002, 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.148768977, 2023-10-09T10:16:23.651 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114968 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251556+0000", 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1169999999999998e-06 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251563+0000", 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1269e-05 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251574+0000", 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5419999999999996e-06 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251579+0000", 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049893740000000004 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.256569+0000", 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0169e-05 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.256579+0000", 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.103208593 2023-10-09T10:16:23.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359788+0000", 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028333999999999998 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.360071+0000", 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018386280000000001 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378457+0000", 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0075999999999998e-05 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378527+0000", 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023643399999999999 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:16:23.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378764+0000", 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064134869999999998 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385177+0000", 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090589800000000003 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386083+0000", 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013396399 2023-10-09T10:16:23.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.399480+0000", 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0691999999999993e-05 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.399550+0000", 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00077554999999999996 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400326+0000", 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.148768977 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128018 9.17 9:e84c3436:::benchmark_data_smithi012_313365_object15543:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.848385327000001, 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0015004270000001, 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.660 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128018 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653287+0000", 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8730000000000004e-06 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653293+0000", 2023-10-09T10:16:23.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6543999999999999e-05 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653310+0000", 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3600000000000001e-06 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653316+0000", 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0634999999999999e-05 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653367+0000", 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000173043 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653540+0000", 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036610800000000003 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:16:23.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653906+0000", 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023299039999999998 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.656236+0000", 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042009109999999999 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.660437+0000", 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1856e-05 2023-10-09T10:16:23.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.660448+0000", 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97237176999999997 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.632820+0000", 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013645000000000001 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.632957+0000", 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021710798 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.654667+0000", 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121478 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.654789+0000", 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0015004270000001 2023-10-09T10:16:23.667 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128022 9.13 9:cae3607a:::benchmark_data_smithi012_313365_object15547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.707089606, 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.71981764000000004, 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.668 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128022 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:16:23.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794581+0000", 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7827999999999999e-05 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794599+0000", 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0599000000000003e-05 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794630+0000", 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2262000000000001e-05 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794642+0000", 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7308e-05 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794709+0000", 2023-10-09T10:16:23.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1711e-05 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794721+0000", 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.712568863 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507290+0000", 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013420899999999999 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507424+0000", 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036085199999999999 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507785+0000", 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003524667 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.511310+0000", 2023-10-09T10:16:23.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00080993999999999999 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512119+0000", 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00066952899999999998 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512789+0000", 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0370999999999997e-05 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512829+0000", 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015724840000000001 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.514402+0000", 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.71981764000000004 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128023 9.17 9:e92c6749:::benchmark_data_smithi012_313365_object15548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.696741091, 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86638305000000004, 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128023 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.676 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:23.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804931+0000", 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3567000000000001e-05 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804945+0000", 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9168e-05 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804964+0000", 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2880000000000004e-06 2023-10-09T10:16:23.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804971+0000", 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1679000000000004e-05 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.805033+0000", 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3217999999999999e-05 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.805046+0000", 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.85093918000000002 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.655985+0000", 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130085 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.656115+0000", 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031436700000000002 2023-10-09T10:16:23.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.656430+0000", 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065475230000000004 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.662977+0000", 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4881000000000004e-05 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.663052+0000", 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058596330000000004 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.668912+0000", 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9499999999999998e-05 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.668951+0000", 2023-10-09T10:16:23.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023647130000000001 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.671316+0000", 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86638305000000004 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:23.683 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:16:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:23.684 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:23.685 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:23.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:23.685 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:16:23.685 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:16:23.685 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:16:23.685 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-10-09T10:16:23.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T10:16:23.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[147926]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 13 peering, 2 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 63 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 47 op/s; 52390/290358 objects degraded (18.043%); 13577/290358 objects misplaced (4.676%); 1/96786 objects unfound (0.001%); 8.0 MiB/s, 213 objects/s recovering 2023-10-09T10:16:23.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T10:16:23.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[147926]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[147926]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[147926]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[147926]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[147926]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 52390/290358 objects degraded (18.043%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[152263]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 13 peering, 2 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 63 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 47 op/s; 52390/290358 objects degraded (18.043%); 13577/290358 objects misplaced (4.676%); 1/96786 objects unfound (0.001%); 8.0 MiB/s, 213 objects/s recovering 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[152263]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[152263]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[152263]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T10:16:23.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[152263]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.688 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[152263]: failed to encode map e366 with expected crc 2023-10-09T10:16:23.688 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 52390/290358 objects degraded (18.043%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:16:23.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:23.818 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:23.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:16:23.865 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:23.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:23.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:24.000 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:24.039 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:24.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:24.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:24.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:24.174 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:24.212 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:24.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:24.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:24.317 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:24.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:16:24.346 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:24.369 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:24.382 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:24.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:24.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:24.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:24.520 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:24.621 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:24.671 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:24.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:24.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:24.777 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:24.804 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:24.825 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:16:24.841 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:24.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:24.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:24.946 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:24.974 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:25.010 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:25.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:25.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:25.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:25.153 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:25.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:25 smithi012 ceph-mon[147926]: pgmap v1026: 99 pgs: 1 active+remapped+backfilling, 1 activating+degraded, 1 active+recovering+undersized+remapped, 3 remapped+peering, 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 235 B/s wr, 29 op/s; 27230/290130 objects degraded (9.385%); 9797/290130 objects misplaced (3.377%); 1/96710 objects unfound (0.001%); 40 MiB/s, 772 objects/s recovering 2023-10-09T10:16:25.392 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:25 smithi012 ceph-mon[152263]: pgmap v1026: 99 pgs: 1 active+remapped+backfilling, 1 activating+degraded, 1 active+recovering+undersized+remapped, 3 remapped+peering, 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 235 B/s wr, 29 op/s; 27230/290130 objects degraded (9.385%); 9797/290130 objects misplaced (3.377%); 1/96710 objects unfound (0.001%); 40 MiB/s, 772 objects/s recovering 2023-10-09T10:16:25.445 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:25.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:25.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:25.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:25.585 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:25.621 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:25.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:25 smithi110 ceph-mon[136032]: pgmap v1026: 99 pgs: 1 active+remapped+backfilling, 1 activating+degraded, 1 active+recovering+undersized+remapped, 3 remapped+peering, 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 235 B/s wr, 29 op/s; 27230/290130 objects degraded (9.385%); 9797/290130 objects misplaced (3.377%); 1/96710 objects unfound (0.001%); 40 MiB/s, 772 objects/s recovering 2023-10-09T10:16:26.209 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:26.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:26.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:26.358 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:26.399 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:26.458 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:26.463 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:26 smithi012 ceph-mon[147926]: Health check update: 1/96695 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:26.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:26.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:26.464 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:26 smithi012 ceph-mon[152263]: Health check update: 1/96695 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:26.565 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:26.592 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:26 smithi110 ceph-mon[136032]: Health check update: 1/96695 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-10-09T10:16:26.594 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:26.632 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:26.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:26.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:26.699 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:16:26.699 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:16:26.699 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:16:26.700 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:16:26.736 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:26.764 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:26.801 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:26.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:26.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:26.906 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:26.942 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:26.985 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:26.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:26.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:27.091 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:27.119 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:27.159 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:27.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:27.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:27.264 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:27.290 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:27 smithi012 ceph-mon[147926]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 activating+degraded, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s wr, 23 op/s; 34455/290085 objects degraded (11.878%); 9140/290085 objects misplaced (3.151%); 1/96695 objects unfound (0.001%); 36 MiB/s, 692 objects/s recovering 2023-10-09T10:16:27.290 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/650079193' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:16:27.291 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:27 smithi012 ceph-mon[152263]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 activating+degraded, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s wr, 23 op/s; 34455/290085 objects degraded (11.878%); 9140/290085 objects misplaced (3.151%); 1/96695 objects unfound (0.001%); 36 MiB/s, 692 objects/s recovering 2023-10-09T10:16:27.291 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/650079193' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:16:27.295 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:27.303 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T10:16:27.333 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:27.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:27.437 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:27 smithi110 ceph-mon[136032]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 activating+degraded, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s wr, 23 op/s; 34455/290085 objects degraded (11.878%); 9140/290085 objects misplaced (3.151%); 1/96695 objects unfound (0.001%); 36 MiB/s, 692 objects/s recovering 2023-10-09T10:16:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/650079193' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:16:27.473 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:27.517 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:27.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:27.624 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:27.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:27.917 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:16:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:16:27] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.33.4" 2023-10-09T10:16:28.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[147926]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/650079193' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:16:28.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[147926]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[147926]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T10:16:28.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[147926]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 34455/290085 objects degraded (11.878%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:28.673 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[152263]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.673 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/650079193' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:16:28.673 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[152263]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.673 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[152263]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T10:16:28.673 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[152263]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.673 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 34455/290085 objects degraded (11.878%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:28.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:28 smithi110 ceph-mon[136032]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/650079193' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:16:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:28 smithi110 ceph-mon[136032]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:28 smithi110 ceph-mon[136032]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T10:16:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:28 smithi110 ceph-mon[136032]: failed to encode map e367 with expected crc 2023-10-09T10:16:28.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 34455/290085 objects degraded (11.878%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:28.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:16:29.069 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:29.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:29.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:29.174 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:29.204 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:29.247 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:29.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:29.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:29 smithi012 ceph-mon[147926]: pgmap v1029: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 161 B/s wr, 18 op/s; 34512/290031 objects degraded (11.899%); 12186/290031 objects misplaced (4.202%); 30 MiB/s, 571 objects/s recovering 2023-10-09T10:16:29.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:29 smithi012 ceph-mon[147926]: Health check cleared: OBJECT_UNFOUND (was: 1/96695 objects unfound (0.001%)) 2023-10-09T10:16:29.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:29.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:29.328 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:29 smithi012 ceph-mon[152263]: pgmap v1029: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 161 B/s wr, 18 op/s; 34512/290031 objects degraded (11.899%); 12186/290031 objects misplaced (4.202%); 30 MiB/s, 571 objects/s recovering 2023-10-09T10:16:29.328 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:29 smithi012 ceph-mon[152263]: Health check cleared: OBJECT_UNFOUND (was: 1/96695 objects unfound (0.001%)) 2023-10-09T10:16:29.328 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:29.328 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:29.351 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:29.382 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:29.425 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:29.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:29.530 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:29.580 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:29.638 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:29.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:29 smithi110 ceph-mon[136032]: pgmap v1029: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 161 B/s wr, 18 op/s; 34512/290031 objects degraded (11.899%); 12186/290031 objects misplaced (4.202%); 30 MiB/s, 571 objects/s recovering 2023-10-09T10:16:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:29 smithi110 ceph-mon[136032]: Health check cleared: OBJECT_UNFOUND (was: 1/96695 objects unfound (0.001%)) 2023-10-09T10:16:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:29.745 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:29.800 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:29.856 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:29.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:29.865 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:29.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:29.989 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:30.030 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:30.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:30.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:30.134 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:30.165 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:30.204 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:30.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:30.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:30.331 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:16:30.343 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:16:30.394 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:30.415 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:16:30.457 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:16:30.468 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:30.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:30.492 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/752477769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:30.492 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:30.492 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2390827551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:30.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/752477769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:30.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:30.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2390827551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:30.502 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:16:30.535 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:16:30.574 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:16:30.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:30.608 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:16:30.649 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:16:30.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:16:30.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:16:30.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:16:30.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:16:30.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:16:30.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:16:30.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:16:30.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:16:30.681 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:30.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/752477769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:30.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:16:30.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2390827551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:31.264 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:31.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:31.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:31.412 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:31.461 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:31.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:31.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:31.616 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[147926]: pgmap v1030: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s wr, 60 op/s; 34439/288930 objects degraded (11.919%); 9291/288930 objects misplaced (3.216%); 46 MiB/s, 819 objects/s recovering 2023-10-09T10:16:31.616 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:31.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:16:31.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[147926]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T10:16:31.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[147926]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[147926]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[147926]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[147926]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.617 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[152263]: pgmap v1030: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s wr, 60 op/s; 34439/288930 objects degraded (11.919%); 9291/288930 objects misplaced (3.216%); 46 MiB/s, 819 objects/s recovering 2023-10-09T10:16:31.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:31.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:16:31.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[152263]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T10:16:31.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[152263]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[152263]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[152263]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.618 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:31 smithi012 ceph-mon[152263]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.639 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:31 smithi110 ceph-mon[136032]: pgmap v1030: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s wr, 60 op/s; 34439/288930 objects degraded (11.919%); 9291/288930 objects misplaced (3.216%); 46 MiB/s, 819 objects/s recovering 2023-10-09T10:16:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:31 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:31 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:16:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:31 smithi110 ceph-mon[136032]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T10:16:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:31 smithi110 ceph-mon[136032]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:31 smithi110 ceph-mon[136032]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:31 smithi110 ceph-mon[136032]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:31 smithi110 ceph-mon[136032]: failed to encode map e368 with expected crc 2023-10-09T10:16:31.698 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:31.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:31.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:31.805 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:31.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:31.894 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:31.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:31.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:32.000 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:32.062 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:32.134 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:32.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:32.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:32.284 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:32.284 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:32.313 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:32.708 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:32.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:32.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:32.739 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:16:32.816 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:32.853 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:32.905 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:32.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:32.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:33.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:33.120 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:33.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:16:33.198 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:33.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:33.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:33.316 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:33.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:16:33.353 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:33.392 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:33 smithi012 ceph-mon[152263]: pgmap v1032: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 36326/288294 objects degraded (12.600%); 7305/288294 objects misplaced (2.534%); 30 MiB/s, 504 objects/s recovering 2023-10-09T10:16:33.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 36326/288294 objects degraded (12.600%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:33.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:33 smithi012 ceph-mon[152263]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T10:16:33.395 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:33 smithi012 ceph-mon[147926]: pgmap v1032: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 36326/288294 objects degraded (12.600%); 7305/288294 objects misplaced (2.534%); 30 MiB/s, 504 objects/s recovering 2023-10-09T10:16:33.395 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 36326/288294 objects degraded (12.600%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:33.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:33 smithi012 ceph-mon[147926]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T10:16:33.396 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:33.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:33.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:33.501 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:33.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:33.565 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:33.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:33.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:33.602 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:33.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:33.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:33 smithi110 ceph-mon[136032]: pgmap v1032: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 36326/288294 objects degraded (12.600%); 7305/288294 objects misplaced (2.534%); 30 MiB/s, 504 objects/s recovering 2023-10-09T10:16:33.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 36326/288294 objects degraded (12.600%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:33.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:33 smithi110 ceph-mon[136032]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T10:16:33.712 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:33.803 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:33.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:33.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:33.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:33.935 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:34.159 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:34.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:34.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:34.250 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:16:34.250 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:16:34.250 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:16:34.251 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.950184 2023-10-09T10:16:34.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0.9501837256655521 2023-10-09T10:16:34.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:34.389 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[147926]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[147926]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[147926]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[147926]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[147926]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T10:16:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[152263]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[152263]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[152263]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[152263]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:34 smithi012 ceph-mon[152263]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T10:16:34.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:34 smithi110 ceph-mon[136032]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:34 smithi110 ceph-mon[136032]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:34 smithi110 ceph-mon[136032]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:34 smithi110 ceph-mon[136032]: failed to encode map e369 with expected crc 2023-10-09T10:16:34.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:34 smithi110 ceph-mon[136032]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T10:16:34.810 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:34.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:34.922 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:34.956 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:35.011 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:35.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:35.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:35.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:35.132 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:35.220 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:35.339 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:35 smithi110 ceph-mon[136032]: pgmap v1034: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 36324/285783 objects degraded (12.710%); 5850/285783 objects misplaced (2.047%); 47 MiB/s, 744 objects/s recovering 2023-10-09T10:16:35.339 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:35 smithi110 ceph-mon[136032]: failed to encode map e370 with expected crc 2023-10-09T10:16:35.339 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:35 smithi110 ceph-mon[136032]: failed to encode map e370 with expected crc 2023-10-09T10:16:35.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:35 smithi012 ceph-mon[147926]: pgmap v1034: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 36324/285783 objects degraded (12.710%); 5850/285783 objects misplaced (2.047%); 47 MiB/s, 744 objects/s recovering 2023-10-09T10:16:35.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:35 smithi012 ceph-mon[147926]: failed to encode map e370 with expected crc 2023-10-09T10:16:35.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:35 smithi012 ceph-mon[147926]: failed to encode map e370 with expected crc 2023-10-09T10:16:35.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:35 smithi012 ceph-mon[152263]: pgmap v1034: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 36324/285783 objects degraded (12.710%); 5850/285783 objects misplaced (2.047%); 47 MiB/s, 744 objects/s recovering 2023-10-09T10:16:35.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:35 smithi012 ceph-mon[152263]: failed to encode map e370 with expected crc 2023-10-09T10:16:35.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:35 smithi012 ceph-mon[152263]: failed to encode map e370 with expected crc 2023-10-09T10:16:35.631 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:35.647 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:35.654 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:35.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:35.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:35.766 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:35.804 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:35.847 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:35.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:35.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:36.032 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:36.420 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:36.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:36.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:36.433 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:36.433 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:36.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:36.611 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:36 smithi012 ceph-mon[147926]: 9.1 scrub starts 2023-10-09T10:16:36.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:36 smithi012 ceph-mon[152263]: 9.1 scrub starts 2023-10-09T10:16:36.620 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:36.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:36 smithi110 ceph-mon[136032]: 9.1 scrub starts 2023-10-09T10:16:37.237 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:16:37.434 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:37.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:37.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:37.544 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:37.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:37 smithi012 ceph-mon[147926]: pgmap v1036: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 36004/280140 objects degraded (12.852%); 5777/280140 objects misplaced (2.062%); 21 MiB/s, 330 objects/s recovering 2023-10-09T10:16:37.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:37 smithi012 ceph-mon[152263]: pgmap v1036: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 36004/280140 objects degraded (12.852%); 5777/280140 objects misplaced (2.062%); 21 MiB/s, 330 objects/s recovering 2023-10-09T10:16:37.636 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:37 smithi110 ceph-mon[136032]: pgmap v1036: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 36004/280140 objects degraded (12.852%); 5777/280140 objects misplaced (2.062%); 21 MiB/s, 330 objects/s recovering 2023-10-09T10:16:37.917 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:16:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:16:37] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-10-09T10:16:37.945 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:38.068 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:38.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:38.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:38.222 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:38.270 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:38.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:38.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:38.378 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:38.403 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:16:38.438 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:38.509 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1589387583' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:38.510 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 36004/280140 objects degraded (12.852%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:38.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1589387583' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:38.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 36004/280140 objects degraded (12.852%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:38.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1589387583' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:38.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 36004/280140 objects degraded (12.852%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:38.865 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:38.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:38.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:39.016 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:39.074 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:39.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:39.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:39.178 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:39.232 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:39.278 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:39.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:39.332 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:39.332 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:39.384 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:39.385 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[147926]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 36004/277551 objects degraded (12.972%); 5460/277551 objects misplaced (1.967%); 17 MiB/s, 276 objects/s recovering 2023-10-09T10:16:39.385 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:16:39.385 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[147926]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1589387583' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[147926]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[147926]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[147926]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[147926]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[152263]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 36004/277551 objects degraded (12.972%); 5460/277551 objects misplaced (1.967%); 17 MiB/s, 276 objects/s recovering 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[152263]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.386 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1589387583' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:39.387 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[152263]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T10:16:39.387 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[152263]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.387 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[152263]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.387 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:39 smithi012 ceph-mon[152263]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.430 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:39.476 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:16:39.519 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:16:39.568 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:16:39.605 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:16:39.635 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:16:39.671 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:39.680 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:16:39.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:39.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:39.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:39 smithi110 ceph-mon[136032]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 36004/277551 objects degraded (12.972%); 5460/277551 objects misplaced (1.967%); 17 MiB/s, 276 objects/s recovering 2023-10-09T10:16:39.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:39 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:16:39.685 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:39 smithi110 ceph-mon[136032]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.685 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1589387583' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:39.685 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:39 smithi110 ceph-mon[136032]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T10:16:39.685 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:39 smithi110 ceph-mon[136032]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.685 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:39 smithi110 ceph-mon[136032]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.685 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:39 smithi110 ceph-mon[136032]: failed to encode map e371 with expected crc 2023-10-09T10:16:39.713 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:16:39.719 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:16:39.750 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:16:39.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:16:39.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:16:39.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:16:39.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:16:39.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:16:39.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:16:39.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:16:39.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:16:39.785 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:39.848 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:40.164 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:40.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:40.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:40.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:40.326 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:40.381 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:40.394 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:40.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:40.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:40.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/74931250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:40.413 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2341624790' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]: dispatch 2023-10-09T10:16:40.413 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]: dispatch 2023-10-09T10:16:40.414 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.5 primary-affinity to 0.950184 (8622712) 2023-10-09T10:16:40.503 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:40.561 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:40.625 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:40.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:40.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:40.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/74931250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:40.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2341624790' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]: dispatch 2023-10-09T10:16:40.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]: dispatch 2023-10-09T10:16:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/74931250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:40.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2341624790' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]: dispatch 2023-10-09T10:16:40.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]: dispatch 2023-10-09T10:16:40.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:40.799 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:41.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:16:41.415 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[152263]: pgmap v1039: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 35579/272442 objects degraded (13.059%); 5361/272442 objects misplaced (1.968%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T10:16:41.415 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[152263]: failed to encode map e372 with expected crc 2023-10-09T10:16:41.415 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[147926]: pgmap v1039: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 35579/272442 objects degraded (13.059%); 5361/272442 objects misplaced (1.968%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T10:16:41.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[147926]: failed to encode map e372 with expected crc 2023-10-09T10:16:41.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]': finished 2023-10-09T10:16:41.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[147926]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T10:16:41.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[147926]: 9.11 scrub starts 2023-10-09T10:16:41.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]': finished 2023-10-09T10:16:41.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[152263]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T10:16:41.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:41 smithi012 ceph-mon[152263]: 9.11 scrub starts 2023-10-09T10:16:41.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:41 smithi110 ceph-mon[136032]: pgmap v1039: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 35579/272442 objects degraded (13.059%); 5361/272442 objects misplaced (1.968%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T10:16:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:41 smithi110 ceph-mon[136032]: failed to encode map e372 with expected crc 2023-10-09T10:16:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:41 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.9501837256655521}]': finished 2023-10-09T10:16:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:41 smithi110 ceph-mon[136032]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T10:16:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:41 smithi110 ceph-mon[136032]: 9.11 scrub starts 2023-10-09T10:16:42.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:16:42.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:42 smithi110 ceph-mon[136032]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T10:16:42.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:42 smithi110 ceph-mon[136032]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:42 smithi110 ceph-mon[136032]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:42 smithi110 ceph-mon[136032]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:42 smithi110 ceph-mon[136032]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:42 smithi110 ceph-mon[136032]: pgmap v1042: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 35579/270441 objects degraded (13.156%); 5361/270441 objects misplaced (1.982%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T10:16:42.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[147926]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T10:16:42.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[147926]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[147926]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[147926]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[147926]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[147926]: pgmap v1042: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 35579/270441 objects degraded (13.156%); 5361/270441 objects misplaced (1.982%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T10:16:42.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[152263]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T10:16:42.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[152263]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[152263]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[152263]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[152263]: failed to encode map e373 with expected crc 2023-10-09T10:16:42.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:42 smithi012 ceph-mon[152263]: pgmap v1042: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 35579/270441 objects degraded (13.156%); 5361/270441 objects misplaced (1.982%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T10:16:43.266 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:43.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:43.324 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:16:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:16:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:16:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:16:43.325 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:16:43.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:43.462 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:43.526 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:43.532 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:43.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:43.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:43.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:43 smithi012 ceph-mon[152263]: 9.1 scrub starts 2023-10-09T10:16:43.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 35579/270441 objects degraded (13.156%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:43.537 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:43 smithi012 ceph-mon[147926]: 9.1 scrub starts 2023-10-09T10:16:43.537 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 35579/270441 objects degraded (13.156%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:43.608 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:43.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:43.682 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:43 smithi110 ceph-mon[136032]: 9.1 scrub starts 2023-10-09T10:16:43.682 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 35579/270441 objects degraded (13.156%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:43.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:16:43.703 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:44.163 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:44.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:44.273 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:44.330 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:44.458 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:16:44.501 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:44.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:44.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:44.708 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:45.061 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:44 smithi110 ceph-mon[136032]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 35579/267204 objects degraded (13.315%); 5048/267204 objects misplaced (1.889%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T10:16:45.061 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:45.061 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:45.228 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:44 smithi012 ceph-mon[147926]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 35579/267204 objects degraded (13.315%); 5048/267204 objects misplaced (1.889%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T10:16:45.228 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:45.228 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:45.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:44 smithi012 ceph-mon[152263]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 35579/267204 objects degraded (13.315%); 5048/267204 objects misplaced (1.889%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T10:16:45.228 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:45.229 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:45.298 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:45.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:45.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:45.491 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:45.606 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:45.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:45.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:45.793 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:45.874 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:16:45.874 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:16:45.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:16:45.874 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:16:45.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.874 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:149449 8.4 e300/288 8:22b86a6f:::benchmark_data_smithi012_271002_object40766:head v 300'9437, mlcod=0'0)", 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:11:25.725204+0000", 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 320.14587213800002, 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023525069999999999, 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725204+0000", 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725204+0000", 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725204+0000", 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4489999999999998e-06 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725209+0000", 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1371e-05 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725221+0000", 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.95e-06 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725226+0000", 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0683000000000003e-05 2023-10-09T10:16:45.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725266+0000", 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5409e-05 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.725292+0000", 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023398059999999998 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.748690+0000", 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9641999999999998e-05 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:11:25.748729+0000", 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023525069999999999 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:184836 8.5 e306/271 8:a910306a:::benchmark_data_smithi012_271002_object76153:head v 306'11700, mlcod=306'11698)", 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:24.261702+0000", 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 261.60937461200001, 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022325889000000002, 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261702+0000", 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261702+0000", 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261701+0000", 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1691e-05 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261713+0000", 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4887e-05 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261727+0000", 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9949999999999999e-06 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261734+0000", 2023-10-09T10:16:45.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1493999999999999e-05 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261776+0000", 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9335999999999999e-05 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.261795+0000", 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022185633 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.283981+0000", 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6918e-05 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:24.284028+0000", 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022325889000000002 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:195316 8.4 e308/288 8:2c727ca6:::benchmark_data_smithi012_271002_object86633:head v 308'12322, mlcod=0'0)", 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:40.818673+0000", 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 245.052403105, 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023273819000000001, 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818673+0000", 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818673+0000", 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818673+0000", 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7129999999999994e-06 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818681+0000", 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7350000000000002e-05 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818699+0000", 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.64e-06 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818705+0000", 2023-10-09T10:16:45.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6796999999999998e-05 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818762+0000", 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2611000000000001e-05 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.818785+0000", 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023134630999999999 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.841919+0000", 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7971e-05 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.841947+0000", 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023273819000000001 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:195367 8.e e309/271 8:722ced42:::benchmark_data_smithi012_271002_object86684:head v 309'12167, mlcod=308'12163)", 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:40.903616+0000", 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 244.96746084899999, 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022218882999999998, 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903616+0000", 2023-10-09T10:16:45.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903616+0000", 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903615+0000", 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.73e-06 2023-10-09T10:16:45.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903624+0000", 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4894e-05 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903639+0000", 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2600000000000002e-06 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903646+0000", 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4731000000000002e-05 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903700+0000", 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5386999999999999e-05 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.903726+0000", 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022070419000000001 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.925796+0000", 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8512e-05 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:40.925835+0000", 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022218882999999998 2023-10-09T10:16:45.894 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:196465 8.5 e309/271 8:a839e85e:::benchmark_data_smithi012_271002_object87782:head v 309'12423, mlcod=309'12421)", 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:42.428599+0000", 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 243.442477975, 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022158028, 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428599+0000", 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428599+0000", 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428597+0000", 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1518999999999999e-05 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428609+0000", 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1316000000000001e-05 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428630+0000", 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1689999999999992e-06 2023-10-09T10:16:45.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428638+0000", 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5626999999999999e-05 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428714+0000", 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1158e-05 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.428735+0000", 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021970321000000001 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.450705+0000", 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1100000000000002e-05 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:42.450757+0000", 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022158028 2023-10-09T10:16:45.899 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:204399 8.4 e312/288 8:24e25026:::benchmark_data_smithi012_271002_object95716:head v 312'12896, mlcod=293'7264)", 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:55.621795+0000", 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 230.24928139900001, 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024465166, 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621795+0000", 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621795+0000", 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621794+0000", 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.52e-06 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621802+0000", 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3852000000000001e-05 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621816+0000", 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4289999999999998e-06 2023-10-09T10:16:45.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.621822+0000", 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059956109999999996 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.627818+0000", 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5525999999999999e-05 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.627843+0000", 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018155171000000001 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.645998+0000", 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026192700000000001 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:55.646260+0000", 2023-10-09T10:16:45.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024465166 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:206140 8.2 e313/271 8:4aaddc3b:::benchmark_data_smithi012_271002_object97457:head v 313'12708, mlcod=313'12705)", 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:58.507955+0000", 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 227.363121855, 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02358936, 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.905 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507955+0000", 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507955+0000", 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:45.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507953+0000", 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1479e-05 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507965+0000", 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5814000000000001e-05 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507981+0000", 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2949999999999996e-06 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.507988+0000", 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00056878400000000002 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.508557+0000", 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2687000000000001e-05 2023-10-09T10:16:45.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.508579+0000", 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022922205000000001 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.531502+0000", 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2348000000000002e-05 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:58.531544+0000", 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.02358936 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:206825 8.2 e313/271 8:4fa78ce4:::benchmark_data_smithi012_271002_object98142:head v 313'12745, mlcod=313'12741)", 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:12:59.563962+0000", 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 226.30711405599999, 2023-10-09T10:16:45.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023624422999999999, 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563962+0000", 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563962+0000", 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563962+0000", 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.186e-06 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563967+0000", 2023-10-09T10:16:45.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6150000000000002e-06 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563974+0000", 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7530000000000001e-06 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.563979+0000", 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010570637000000001 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.574549+0000", 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0530000000000002e-05 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.574570+0000", 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012928932000000001 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.587499+0000", 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8201999999999993e-05 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:12:59.587587+0000", 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023624422999999999 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.915 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45518.0:212994 8.e e314/271 8:73cb14b3:::benchmark_data_smithi012_271002_object104311:head v 314'13240, mlcod=314'13237)", 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:13:09.062601+0000", 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 216.80847577700001, 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023475941, 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062601+0000", 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062601+0000", 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062600+0000", 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5499999999999996e-06 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062605+0000", 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4129999999999997e-06 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062610+0000", 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4860000000000002e-06 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.062614+0000", 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0072164890000000004 2023-10-09T10:16:45.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.069830+0000", 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0945e-05 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.069851+0000", 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016199209999999999 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.919 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.086051+0000", 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6112999999999999e-05 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:13:09.086077+0000", 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.023475941 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:98787 9.e e336/317 9:7e4e1e7c:::benchmark_data_smithi012_313365_object98786:head v 336'6128, mlcod=336'6123)", 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:49.985821+0000", 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 115.885255657, 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022871335999999999, 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:49.985821+0000", 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:49.985821+0000", 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:49.985820+0000", 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010737284999999999 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:49.996558+0000", 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1985999999999999e-05 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:49.996579+0000", 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3459999999999994e-06 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:49.996588+0000", 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023655 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:49.996824+0000", 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6574e-05 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:49.996851+0000", 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01179097 2023-10-09T10:16:45.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:50.008642+0000", 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0287000000000002e-05 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:50.008692+0000", 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022871335999999999 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:106538 9.e e339/317 9:74995394:::benchmark_data_smithi012_313365_object106537:head v 339'6623, mlcod=339'6621)", 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:00.077525+0000", 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 105.793551292, 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022255535, 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.077525+0000", 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.077525+0000", 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.077524+0000", 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011286299999999999 2023-10-09T10:16:45.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.077637+0000", 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5576000000000001e-05 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.077653+0000", 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9450000000000004e-06 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.077660+0000", 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020686200000000001 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.077867+0000", 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3983999999999999e-05 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.077891+0000", 2023-10-09T10:16:45.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021847907 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.099739+0000", 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2200000000000003e-05 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:00.099781+0000", 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022255535 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.930 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114937 9.c 9:3de29534:::benchmark_data_smithi012_313365_object2462:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.655861740000006, 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0079670510000001, 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.931 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114937 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:16:45.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215214+0000", 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9450000000000005e-06 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215224+0000", 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4849999999999992e-06 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215232+0000", 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3200000000000002e-06 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215239+0000", 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8306000000000002e-05 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215278+0000", 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3389999999999998e-06 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215284+0000", 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98804788700000001 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.203332+0000", 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00200256 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.205335+0000", 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015448999999999999 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.205489+0000", 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013383071999999999 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218872+0000", 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041514890000000004 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223024+0000", 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7812999999999997e-05 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223081+0000", 2023-10-09T10:16:45.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100445 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223182+0000", 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0079670510000001 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114938 9.c 9:317dc30b:::benchmark_data_smithi012_313365_object2463:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:16:45.938 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.655828424000006, 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0080620979999999, 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114938 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:16:45.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.979e-06 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215250+0000", 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5199999999999997e-06 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215258+0000", 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1250000000000001e-06 2023-10-09T10:16:45.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215261+0000", 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8232000000000001e-05 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215320+0000", 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1710000000000002e-06 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.942 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215328+0000", 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99764452400000003 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.212972+0000", 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010863 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213081+0000", 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012440199999999999 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213205+0000", 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056800169999999999 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218885+0000", 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043305829999999998 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223216+0000", 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1077000000000001e-05 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223247+0000", 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3231000000000005e-05 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223310+0000", 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0080620979999999 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114946 9.c 9:3c5b83c8:::benchmark_data_smithi012_313365_object2471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.655795224000002, 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0081399559999999, 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.946 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114946 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6449999999999999e-06 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215283+0000", 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7349999999999998e-06 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215286+0000", 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7470000000000001e-06 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215289+0000", 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2694000000000001e-05 2023-10-09T10:16:45.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215342+0000", 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2049999999999998e-06 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215345+0000", 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.000627545 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215973+0000", 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011626899999999999 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216089+0000", 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132232 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216221+0000", 2023-10-09T10:16:45.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027664209999999998 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218988+0000", 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043527119999999999 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223340+0000", 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9572000000000001e-05 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223370+0000", 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1471000000000002e-05 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223421+0000", 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0081399559999999 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.953 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128021 9.c 9:3ed78aba:::benchmark_data_smithi012_313365_object15546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 38.167853473999998, 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999, 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.954 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128021 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:16:45.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703222+0000", 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7700000000000004e-06 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703229+0000", 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1178e-05 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703240+0000", 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0509999999999996e-06 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703245+0000", 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9240000000000002e-05 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703304+0000", 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.429e-05 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703318+0000", 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.498177753 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201496+0000", 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2643e-05 2023-10-09T10:16:45.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201519+0000", 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1325606320000001 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334079+0000", 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016077600000000001 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334240+0000", 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016287699999999999 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334403+0000", 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050087500000000002 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339412+0000", 2023-10-09T10:16:45.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1050000000000007e-05 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339473+0000", 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013735 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339610+0000", 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041248470000000001 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343735+0000", 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9188999999999999e-05 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343784+0000", 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111032 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343895+0000", 2023-10-09T10:16:45.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128030 9.c 9:39ef457f:::benchmark_data_smithi012_313365_object15555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 38.027842397999997, 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999, 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.963 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128030 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843233+0000", 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843241+0000", 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3162e-05 2023-10-09T10:16:45.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843254+0000", 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5490000000000001e-06 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843261+0000", 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2747999999999997e-05 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843314+0000", 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7970000000000006e-06 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843323+0000", 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35848033200000001 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201803+0000", 2023-10-09T10:16:45.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6119999999999999e-06 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201812+0000", 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1353121740000001 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337124+0000", 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8503000000000003e-05 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337212+0000", 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0490999999999998e-05 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337303+0000", 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002260742 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:45.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339564+0000", 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1705999999999994e-05 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339635+0000", 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002741145 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.342376+0000", 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001560173 2023-10-09T10:16:45.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343937+0000", 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6062000000000001e-05 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343973+0000", 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4776000000000002e-05 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.344068+0000", 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128782 9.c 9:3e7b84df:::benchmark_data_smithi012_313365_object16307:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:28.004865+0000", 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 17.866211145000001, 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.071993426999999999, 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.972 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128782 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004865+0000", 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004865+0000", 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004864+0000", 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0590000000000003e-06 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004872+0000", 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5293000000000001e-05 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004887+0000", 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7459999999999997e-06 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004893+0000", 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8399000000000006e-05 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004971+0000", 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109824 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.005081+0000", 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000124888 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:16:45.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.005206+0000", 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025613400000000001 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.007767+0000", 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3368000000000005e-05 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.007851+0000", 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00040009399999999998 2023-10-09T10:16:45.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.008251+0000", 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.068476722000000004 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.076728+0000", 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9867999999999998e-05 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.076778+0000", 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1296000000000002e-05 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.076859+0000", 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.071993426999999999 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.979 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:129554 9.c 9:3ee6beeb:::benchmark_data_smithi012_313365_object17079:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:30.306417+0000", 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.564659644000001, 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038617190000000003, 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 129554 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306417+0000", 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306417+0000", 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306415+0000", 2023-10-09T10:16:45.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7190000000000007e-06 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306424+0000", 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8678e-05 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306442+0000", 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1040000000000001e-06 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306449+0000", 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4561000000000007e-05 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306524+0000", 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000189667 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306714+0000", 2023-10-09T10:16:45.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017995500000000001 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306894+0000", 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036508880000000001 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.310544+0000", 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002046572 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.312591+0000", 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024919819999999998 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.315083+0000", 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029807935000000001 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.344891+0000", 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4691999999999997e-05 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.344946+0000", 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8405000000000005e-05 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.345034+0000", 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038617190000000003 2023-10-09T10:16:45.986 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:129562 9.c 9:39b51c53:::benchmark_data_smithi012_313365_object17087:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:30.333070+0000", 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.538006580999999, 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031552197999999997, 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.987 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 129562 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333070+0000", 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333070+0000", 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333068+0000", 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1009000000000001e-05 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333079+0000", 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0988999999999998e-05 2023-10-09T10:16:45.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333100+0000", 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9189999999999999e-06 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333108+0000", 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0591000000000002e-05 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333178+0000", 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0890000000000006e-06 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333187+0000", 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011867011 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.345054+0000", 2023-10-09T10:16:45.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103452 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.345158+0000", 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103909 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.345262+0000", 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086899980000000009 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.353952+0000", 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1491e-05 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.353993+0000", 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029029170000000001 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.356896+0000", 2023-10-09T10:16:45.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075623560000000001 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364459+0000", 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3745999999999996e-05 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364522+0000", 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9827999999999994e-05 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364622+0000", 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031552197999999997 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:129563 9.c 9:3efd6b8d:::benchmark_data_smithi012_313365_object17088:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:30.333122+0000", 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.537954286, 2023-10-09T10:16:45.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031645813000000002, 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 129563 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333122+0000", 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:16:45.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333122+0000", 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333122+0000", 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0860000000000002e-06 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333125+0000", 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7100000000000004e-06 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333131+0000", 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.805e-06 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333134+0000", 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102803 2023-10-09T10:16:45.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333237+0000", 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7409999999999992e-06 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333245+0000", 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014888682 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:45.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.348134+0000", 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013137199999999999 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.348265+0000", 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100295 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.348365+0000", 2023-10-09T10:16:46.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056663199999999999 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.354032+0000", 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1625999999999998e-05 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.354073+0000", 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003101683 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:46.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.357175+0000", 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074756780000000004 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364651+0000", 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3782000000000003e-05 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364684+0000", 2023-10-09T10:16:46.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3739999999999994e-05 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364768+0000", 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031645813000000002 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:16:46.003 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:16:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:46.005 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:46.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:46.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:46.187 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:46.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:16:46.274 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:46.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:46.305 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:16:46.350 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:16:46.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:46.402 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:16:46.438 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:16:46.483 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:46.487 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:16:46.522 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:16:46.569 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:16:46.584 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:46.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:46.634 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:16:46.641 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:16:46.642 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:16:46.642 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:16:46.642 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:16:46.642 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:16:46.642 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:16:46.642 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:16:46.642 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:16:46.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:46.788 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:47.019 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:16:47.051 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:46 smithi012 ceph-mon[147926]: pgmap v1044: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 35109/259776 objects degraded (13.515%); 4936/259776 objects misplaced (1.900%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T10:16:47.052 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:46 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1483279020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:47.052 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:46 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1291189187' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:16:47.052 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:46 smithi012 ceph-mon[152263]: pgmap v1044: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 35109/259776 objects degraded (13.515%); 4936/259776 objects misplaced (1.900%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T10:16:47.052 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:46 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1483279020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:47.052 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:46 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1291189187' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:16:47.093 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:47.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:47.198 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:47.243 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:46 smithi110 ceph-mon[136032]: pgmap v1044: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 35109/259776 objects degraded (13.515%); 4936/259776 objects misplaced (1.900%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T10:16:47.243 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:46 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1483279020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:47.243 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:46 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1291189187' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:16:47.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:47.310 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:47.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:47.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:47.473 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:47.601 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:47.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:47.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:47.632 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:47.709 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:47.766 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:47.824 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:47.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:47.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:47.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:48.027 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:48.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:48.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1291189187' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:16:48.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[147926]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T10:16:48.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[147926]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[147926]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[147926]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.165 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:16:47] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.33.4" 2023-10-09T10:16:48.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:48.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1291189187' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:16:48.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[152263]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T10:16:48.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[152263]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[152263]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:47 smithi012 ceph-mon[152263]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:47 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:48.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1291189187' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:16:48.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:47 smithi110 ceph-mon[136032]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T10:16:48.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:47 smithi110 ceph-mon[136032]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:47 smithi110 ceph-mon[136032]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:47 smithi110 ceph-mon[136032]: failed to encode map e374 with expected crc 2023-10-09T10:16:48.331 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:16:48.332 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:16:48.332 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T10:16:48.332 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:16:48.532 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:48.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:48.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:48.745 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:48.868 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:48.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:48.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:48.979 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:49.034 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:49.086 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:49.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:49.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:49.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:16:49.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:48 smithi012 ceph-mon[147926]: pgmap v1046: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 35109/256407 objects degraded (13.693%); 4643/256407 objects misplaced (1.811%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:16:49.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 35109/256407 objects degraded (13.693%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:49.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:48 smithi012 ceph-mon[152263]: pgmap v1046: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 35109/256407 objects degraded (13.693%); 4643/256407 objects misplaced (1.811%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:16:49.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 35109/256407 objects degraded (13.693%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:49.182 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:16:49.182 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:16:49.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:49.271 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:49.287 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:49.386 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:49.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:49.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:16:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:48 smithi110 ceph-mon[136032]: pgmap v1046: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 35109/256407 objects degraded (13.693%); 4643/256407 objects misplaced (1.811%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:16:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 35109/256407 objects degraded (13.693%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:49.494 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:49.548 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:49.603 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:16:49.615 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:49.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:49.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:49.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:49.774 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:49.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:49.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:49.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:50.001 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:50.580 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:50.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:50.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:50.755 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:51 smithi012 ceph-mon[147926]: pgmap v1047: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 34615/250701 objects degraded (13.807%); 4500/250701 objects misplaced (1.795%); 10 MiB/s, 161 objects/s recovering 2023-10-09T10:16:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:51 smithi012 ceph-mon[152263]: pgmap v1047: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 34615/250701 objects degraded (13.807%); 4500/250701 objects misplaced (1.795%); 10 MiB/s, 161 objects/s recovering 2023-10-09T10:16:51.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:51 smithi110 ceph-mon[136032]: pgmap v1047: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 34615/250701 objects degraded (13.807%); 4500/250701 objects misplaced (1.795%); 10 MiB/s, 161 objects/s recovering 2023-10-09T10:16:51.611 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:51.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:51.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:51.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:51.802 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:52.622 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:52.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:52.728 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:52.734 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:16:52.802 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:53.017 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:53.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:53.125 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:53.169 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:53.214 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:53.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:53.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:53.244 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:53.319 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:53.334 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:53.373 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:53.373 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:53 smithi110 ceph-mon[136032]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 34615/248583 objects degraded (13.925%); 4500/248583 objects misplaced (1.810%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T10:16:53.376 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:53 smithi012 ceph-mon[147926]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 34615/248583 objects degraded (13.925%); 4500/248583 objects misplaced (1.810%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T10:16:53.378 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:53 smithi012 ceph-mon[152263]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 34615/248583 objects degraded (13.925%); 4500/248583 objects misplaced (1.810%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T10:16:53.436 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:53.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:53.545 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:53.589 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:53.632 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:53.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:53.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:53.738 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:16:53.777 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:53.828 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:16:53.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:53.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:16:53.934 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:53.970 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:54.031 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:54.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:54.079 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:54 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 34615/248583 objects degraded (13.925%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:54.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:54 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 34615/248583 objects degraded (13.925%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:54.135 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:54.171 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:54.176 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:16:54.187 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:16:54.217 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:54.221 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:54 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 34615/248583 objects degraded (13.925%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:54.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:54.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:54.227 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:54.323 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:54.372 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:54.426 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:54.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:54.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:54.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:16:54.623 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:54.954 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:16:54.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:54.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:16:55.066 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:16:55.098 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:55.099 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1049,"stamp":"2023-10-09T10:16:53.570949+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5349611448,"num_objects":81645,"num_object_clones":0,"num_object_copies":244935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34615,"num_objects_misplaced":4175,"num_objects_unfound":0,"num_objects_dirty":81645,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":136979,"num_write_kb":6772078,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21014,"num_bytes_recovered":1273179866,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":190230,"ondisk_log_size":190230,"up":297,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":340,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21676996,"kb_used_data":17317124,"kb_used_omap":172,"kb_used_meta":4359571,"kb_avail":728284220,"statfs":{"total":767960285184,"available":745763041280,"internally_reserved":0,"allocated":17732734976,"data_stored":17724505096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":176164,"internal_metadata":4464201692},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-486473728,"num_objects":-7423,"num_object_clones":0,"num_object_copies":-22269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-964,"num_objects_misplaced":-873,"num_objects_unfound":0,"num_objects_dirty":-7423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7423,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1328,"num_bytes_recovered":87031808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006282"},"pg_stats":[{"pgid":"9.10","version":"374'7610","reported_seq":11293,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:49.179328+0000","last_change":"2023-10-09T10:16:08.277548+0000","last_active":"2023-10-09T10:16:49.179328+0000","last_peered":"2023-10-09T10:16:49.179328+0000","last_clean":"2023-10-09T10:15:06.274037+0000","last_became_active":"2023-10-09T10:15:08.875469+0000","last_became_peered":"2023-10-09T10:15:08.875469+0000","last_unstale":"2023-10-09T10:16:49.179328+0000","last_undegraded":"2023-10-09T10:16:08.267271+0000","last_fullsized":"2023-10-09T10:15:08.357982+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7610,"log_dups_size":0,"ondisk_log_size":7610,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167018496,"num_objects":2549,"num_object_clones":0,"num_object_copies":7647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5050,"num_objects_misplaced":2525,"num_objects_unfound":0,"num_objects_dirty":2549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4347,"num_write_kb":220640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":900,"num_bytes_recovered":58982400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[1,0],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"0,1","objects":24},{"shards":"1,2,6","objects":2525}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":218,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.451759+0000","last_change":"2023-10-09T10:08:35.095619+0000","last_active":"2023-10-09T10:16:41.451759+0000","last_peered":"2023-10-09T10:16:41.451759+0000","last_clean":"2023-10-09T10:16:41.451759+0000","last_became_active":"2023-10-09T10:08:35.094996+0000","last_became_peered":"2023-10-09T10:08:35.094996+0000","last_unstale":"2023-10-09T10:16:41.451759+0000","last_undegraded":"2023-10-09T10:16:41.451759+0000","last_fullsized":"2023-10-09T10:16:41.451759+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:12.129727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":298,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.445513+0000","last_change":"2023-10-09T10:16:08.230865+0000","last_active":"2023-10-09T10:16:41.445513+0000","last_peered":"2023-10-09T10:16:41.445513+0000","last_clean":"2023-10-09T10:16:41.445513+0000","last_became_active":"2023-10-09T10:16:08.230559+0000","last_became_peered":"2023-10-09T10:16:08.230559+0000","last_unstale":"2023-10-09T10:16:41.445513+0000","last_undegraded":"2023-10-09T10:16:41.445513+0000","last_fullsized":"2023-10-09T10:16:41.445513+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:02.674870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.13","version":"374'7674","reported_seq":7781,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:50.195913+0000","last_change":"2023-10-09T10:16:18.493982+0000","last_active":"2023-10-09T10:16:50.195913+0000","last_peered":"2023-10-09T10:16:50.195913+0000","last_clean":"2023-10-09T10:15:14.025151+0000","last_became_active":"2023-10-09T10:16:18.481725+0000","last_became_peered":"2023-10-09T10:16:18.481725+0000","last_unstale":"2023-10-09T10:16:50.195913+0000","last_undegraded":"2023-10-09T10:16:18.277288+0000","last_fullsized":"2023-10-09T10:16:18.237757+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7674,"log_dups_size":0,"ondisk_log_size":7674,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168919040,"num_objects":2578,"num_object_clones":0,"num_object_copies":7734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3459,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4408,"num_write_kb":223520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,3,7","objects":2544},{"shards":"1,7","objects":-423}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":394,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.444674+0000","last_change":"2023-10-09T10:15:42.314309+0000","last_active":"2023-10-09T10:16:41.444674+0000","last_peered":"2023-10-09T10:16:41.444674+0000","last_clean":"2023-10-09T10:16:41.444674+0000","last_became_active":"2023-10-09T10:15:42.313553+0000","last_became_peered":"2023-10-09T10:15:42.313553+0000","last_unstale":"2023-10-09T10:16:41.444674+0000","last_undegraded":"2023-10-09T10:16:41.444674+0000","last_fullsized":"2023-10-09T10:16:41.444674+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:00.421233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":267,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.440732+0000","last_change":"2023-10-09T10:16:08.309799+0000","last_active":"2023-10-09T10:16:41.440732+0000","last_peered":"2023-10-09T10:16:41.440732+0000","last_clean":"2023-10-09T10:16:41.440732+0000","last_became_active":"2023-10-09T10:16:08.309448+0000","last_became_peered":"2023-10-09T10:16:08.309448+0000","last_unstale":"2023-10-09T10:16:41.440732+0000","last_undegraded":"2023-10-09T10:16:41.440732+0000","last_fullsized":"2023-10-09T10:16:41.440732+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:58.976233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"374'7620","reported_seq":7712,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:48.955183+0000","last_change":"2023-10-09T10:16:08.571296+0000","last_active":"2023-10-09T10:16:48.955183+0000","last_peered":"2023-10-09T10:16:48.955183+0000","last_clean":"2023-10-09T10:15:14.060602+0000","last_became_active":"2023-10-09T10:16:08.554288+0000","last_became_peered":"2023-10-09T10:16:08.554288+0000","last_unstale":"2023-10-09T10:16:48.955183+0000","last_undegraded":"2023-10-09T10:16:08.301014+0000","last_fullsized":"2023-10-09T10:16:08.200940+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7620,"log_dups_size":0,"ondisk_log_size":7620,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172949504,"num_objects":2639,"num_object_clones":0,"num_object_copies":7917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3475,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4351,"num_write_kb":223680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-394},{"shards":"1,2,3","objects":2619}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":334,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.450705+0000","last_change":"2023-10-09T10:16:16.789610+0000","last_active":"2023-10-09T10:16:41.450705+0000","last_peered":"2023-10-09T10:16:41.450705+0000","last_clean":"2023-10-09T10:16:41.450705+0000","last_became_active":"2023-10-09T10:16:13.361832+0000","last_became_peered":"2023-10-09T10:16:13.361832+0000","last_unstale":"2023-10-09T10:16:41.450705+0000","last_undegraded":"2023-10-09T10:16:41.450705+0000","last_fullsized":"2023-10-09T10:16:41.450705+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:46.197713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":166,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.478033+0000","last_change":"2023-10-09T10:16:41.473093+0000","last_active":"2023-10-09T10:16:41.478033+0000","last_peered":"2023-10-09T10:16:41.478033+0000","last_clean":"2023-10-09T10:16:41.478033+0000","last_became_active":"2023-10-09T10:16:41.472774+0000","last_became_peered":"2023-10-09T10:16:41.472774+0000","last_unstale":"2023-10-09T10:16:41.478033+0000","last_undegraded":"2023-10-09T10:16:41.478033+0000","last_fullsized":"2023-10-09T10:16:41.478033+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:07.117932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":301,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.441258+0000","last_change":"2023-10-09T10:16:18.273288+0000","last_active":"2023-10-09T10:16:41.441258+0000","last_peered":"2023-10-09T10:16:41.441258+0000","last_clean":"2023-10-09T10:16:41.441258+0000","last_became_active":"2023-10-09T10:16:18.272936+0000","last_became_peered":"2023-10-09T10:16:18.272936+0000","last_unstale":"2023-10-09T10:16:41.441258+0000","last_undegraded":"2023-10-09T10:16:41.441258+0000","last_fullsized":"2023-10-09T10:16:41.441258+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:39:34.847781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":319,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.446480+0000","last_change":"2023-10-09T10:16:08.619899+0000","last_active":"2023-10-09T10:16:41.446480+0000","last_peered":"2023-10-09T10:16:41.446480+0000","last_clean":"2023-10-09T10:16:41.446480+0000","last_became_active":"2023-10-09T10:16:08.619549+0000","last_became_peered":"2023-10-09T10:16:08.619549+0000","last_unstale":"2023-10-09T10:16:41.446480+0000","last_undegraded":"2023-10-09T10:16:41.446480+0000","last_fullsized":"2023-10-09T10:16:41.446480+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:07.870610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.455599+0000","last_change":"2023-10-09T10:16:18.280500+0000","last_active":"2023-10-09T10:16:41.455599+0000","last_peered":"2023-10-09T10:16:41.455599+0000","last_clean":"2023-10-09T10:16:41.455599+0000","last_became_active":"2023-10-09T10:16:18.275348+0000","last_became_peered":"2023-10-09T10:16:18.275348+0000","last_unstale":"2023-10-09T10:16:41.455599+0000","last_undegraded":"2023-10-09T10:16:41.455599+0000","last_fullsized":"2023-10-09T10:16:41.455599+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":282,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.430698+0000","last_change":"2023-10-09T10:15:42.396546+0000","last_active":"2023-10-09T10:16:41.430698+0000","last_peered":"2023-10-09T10:16:41.430698+0000","last_clean":"2023-10-09T10:16:41.430698+0000","last_became_active":"2023-10-09T10:15:42.367055+0000","last_became_peered":"2023-10-09T10:15:42.367055+0000","last_unstale":"2023-10-09T10:16:41.430698+0000","last_undegraded":"2023-10-09T10:16:41.430698+0000","last_fullsized":"2023-10-09T10:16:41.430698+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:57.600046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":427,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.443316+0000","last_change":"2023-10-09T10:15:42.342644+0000","last_active":"2023-10-09T10:16:41.443316+0000","last_peered":"2023-10-09T10:16:41.443316+0000","last_clean":"2023-10-09T10:16:41.443316+0000","last_became_active":"2023-10-09T10:15:42.342300+0000","last_became_peered":"2023-10-09T10:15:42.342300+0000","last_unstale":"2023-10-09T10:16:41.443316+0000","last_undegraded":"2023-10-09T10:16:41.443316+0000","last_fullsized":"2023-10-09T10:16:41.443316+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":371,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.451570+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:16:41.451570+0000","last_peered":"2023-10-09T10:16:41.451570+0000","last_clean":"2023-10-09T10:16:41.451570+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:16:41.451570+0000","last_undegraded":"2023-10-09T10:16:41.451570+0000","last_fullsized":"2023-10-09T10:16:41.451570+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":261,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.443080+0000","last_change":"2023-10-09T10:08:35.086519+0000","last_active":"2023-10-09T10:16:41.443080+0000","last_peered":"2023-10-09T10:16:41.443080+0000","last_clean":"2023-10-09T10:16:41.443080+0000","last_became_active":"2023-10-09T10:08:35.086165+0000","last_became_peered":"2023-10-09T10:08:35.086165+0000","last_unstale":"2023-10-09T10:16:41.443080+0000","last_undegraded":"2023-10-09T10:16:41.443080+0000","last_fullsized":"2023-10-09T10:16:41.443080+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":243,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.445128+0000","last_change":"2023-10-09T10:08:35.094871+0000","last_active":"2023-10-09T10:16:41.445128+0000","last_peered":"2023-10-09T10:16:41.445128+0000","last_clean":"2023-10-09T10:16:41.445128+0000","last_became_active":"2023-10-09T10:08:35.092795+0000","last_became_peered":"2023-10-09T10:08:35.092795+0000","last_unstale":"2023-10-09T10:16:41.445128+0000","last_undegraded":"2023-10-09T10:16:41.445128+0000","last_fullsized":"2023-10-09T10:16:41.445128+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:16.153583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.19","version":"374'8029","reported_seq":8148,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:52.201637+0000","last_change":"2023-10-09T10:16:18.393528+0000","last_active":"2023-10-09T10:16:52.201637+0000","last_peered":"2023-10-09T10:16:52.201637+0000","last_clean":"2023-10-09T10:15:06.355378+0000","last_became_active":"2023-10-09T10:16:18.369005+0000","last_became_peered":"2023-10-09T10:16:18.369005+0000","last_unstale":"2023-10-09T10:16:52.201637+0000","last_undegraded":"2023-10-09T10:16:18.226963+0000","last_fullsized":"2023-10-09T10:16:18.205099+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8029,"log_dups_size":0,"ondisk_log_size":8029,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169050112,"num_objects":2580,"num_object_clones":0,"num_object_copies":7740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4646,"num_write_kb":231200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2580}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":445,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.447083+0000","last_change":"2023-10-09T10:08:35.094200+0000","last_active":"2023-10-09T10:16:41.447083+0000","last_peered":"2023-10-09T10:16:41.447083+0000","last_clean":"2023-10-09T10:16:41.447083+0000","last_became_active":"2023-10-09T10:08:35.093656+0000","last_became_peered":"2023-10-09T10:08:35.093656+0000","last_unstale":"2023-10-09T10:16:41.447083+0000","last_undegraded":"2023-10-09T10:16:41.447083+0000","last_fullsized":"2023-10-09T10:16:41.447083+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":326,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.431128+0000","last_change":"2023-10-09T10:15:42.406082+0000","last_active":"2023-10-09T10:16:41.431128+0000","last_peered":"2023-10-09T10:16:41.431128+0000","last_clean":"2023-10-09T10:16:41.431128+0000","last_became_active":"2023-10-09T10:15:42.405487+0000","last_became_peered":"2023-10-09T10:15:42.405487+0000","last_unstale":"2023-10-09T10:16:41.431128+0000","last_undegraded":"2023-10-09T10:16:41.431128+0000","last_fullsized":"2023-10-09T10:16:41.431128+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:58.713810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.18","version":"374'8048","reported_seq":9134,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:52.284208+0000","last_change":"2023-10-09T10:16:27.077109+0000","last_active":"2023-10-09T10:16:52.284208+0000","last_peered":"2023-10-09T10:16:52.284208+0000","last_clean":"2023-10-09T10:15:06.273595+0000","last_became_active":"2023-10-09T10:16:27.068683+0000","last_became_peered":"2023-10-09T10:16:27.068683+0000","last_unstale":"2023-10-09T10:16:52.284208+0000","last_undegraded":"2023-10-09T10:16:26.827722+0000","last_fullsized":"2023-10-09T10:16:22.231298+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8048,"log_dups_size":0,"ondisk_log_size":8048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167378944,"num_objects":2554,"num_object_clones":0,"num_object_copies":7662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3454,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4654,"num_write_kb":230656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":483,"num_bytes_recovered":31653888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0,3,5","objects":2584},{"shards":"0,3,7","objects":2},{"shards":"0,5","objects":-552}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":266,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.440760+0000","last_change":"2023-10-09T10:16:08.455134+0000","last_active":"2023-10-09T10:16:41.440760+0000","last_peered":"2023-10-09T10:16:41.440760+0000","last_clean":"2023-10-09T10:16:41.440760+0000","last_became_active":"2023-10-09T10:16:08.454793+0000","last_became_peered":"2023-10-09T10:16:08.454793+0000","last_unstale":"2023-10-09T10:16:41.440760+0000","last_undegraded":"2023-10-09T10:16:41.440760+0000","last_fullsized":"2023-10-09T10:16:41.440760+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:25:33.883382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.455227+0000","last_change":"2023-10-09T10:16:18.255609+0000","last_active":"2023-10-09T10:16:41.455227+0000","last_peered":"2023-10-09T10:16:41.455227+0000","last_clean":"2023-10-09T10:16:41.455227+0000","last_became_active":"2023-10-09T10:16:18.244618+0000","last_became_peered":"2023-10-09T10:16:18.244618+0000","last_unstale":"2023-10-09T10:16:41.455227+0000","last_undegraded":"2023-10-09T10:16:41.455227+0000","last_fullsized":"2023-10-09T10:16:41.455227+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":242,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.442974+0000","last_change":"2023-10-09T10:08:35.098102+0000","last_active":"2023-10-09T10:16:41.442974+0000","last_peered":"2023-10-09T10:16:41.442974+0000","last_clean":"2023-10-09T10:16:41.442974+0000","last_became_active":"2023-10-09T10:08:35.097497+0000","last_became_peered":"2023-10-09T10:08:35.097497+0000","last_unstale":"2023-10-09T10:16:41.442974+0000","last_undegraded":"2023-10-09T10:16:41.442974+0000","last_fullsized":"2023-10-09T10:16:41.442974+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":265,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.440819+0000","last_change":"2023-10-09T10:16:08.232247+0000","last_active":"2023-10-09T10:16:41.440819+0000","last_peered":"2023-10-09T10:16:41.440819+0000","last_clean":"2023-10-09T10:16:41.440819+0000","last_became_active":"2023-10-09T10:16:08.231927+0000","last_became_peered":"2023-10-09T10:16:08.231927+0000","last_unstale":"2023-10-09T10:16:41.440819+0000","last_undegraded":"2023-10-09T10:16:41.440819+0000","last_fullsized":"2023-10-09T10:16:41.440819+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:20.408880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":311,"reported_epoch":373,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:16:41.450854+0000","last_change":"2023-10-09T10:16:26.743265+0000","last_active":"2023-10-09T10:16:41.450854+0000","last_peered":"2023-10-09T10:16:41.450854+0000","last_clean":"2023-10-09T10:16:00.577412+0000","last_became_active":"2023-10-09T10:16:22.206357+0000","last_became_peered":"2023-10-09T10:16:22.206357+0000","last_unstale":"2023-10-09T10:16:41.450854+0000","last_undegraded":"2023-10-09T10:16:18.203819+0000","last_fullsized":"2023-10-09T10:16:41.450854+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:47.851069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":319,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.457723+0000","last_change":"2023-10-09T10:16:08.618851+0000","last_active":"2023-10-09T10:16:41.457723+0000","last_peered":"2023-10-09T10:16:41.457723+0000","last_clean":"2023-10-09T10:16:41.457723+0000","last_became_active":"2023-10-09T10:16:08.618501+0000","last_became_peered":"2023-10-09T10:16:08.618501+0000","last_unstale":"2023-10-09T10:16:41.457723+0000","last_undegraded":"2023-10-09T10:16:41.457723+0000","last_fullsized":"2023-10-09T10:16:41.457723+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:45.133486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"374'7815","reported_seq":9199,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:52.310172+0000","last_change":"2023-10-09T10:16:18.358604+0000","last_active":"2023-10-09T10:16:52.310172+0000","last_peered":"2023-10-09T10:16:52.310172+0000","last_clean":"2023-10-09T10:15:06.284821+0000","last_became_active":"2023-10-09T10:16:18.344039+0000","last_became_peered":"2023-10-09T10:16:18.344039+0000","last_unstale":"2023-10-09T10:16:52.310172+0000","last_undegraded":"2023-10-09T10:16:18.231303+0000","last_fullsized":"2023-10-09T10:16:18.204454+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7815,"log_dups_size":0,"ondisk_log_size":7815,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162725888,"num_objects":2483,"num_object_clones":0,"num_object_copies":7449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4513,"num_write_kb":223872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":644,"num_bytes_recovered":42205184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2483}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":406,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.443062+0000","last_change":"2023-10-09T10:08:35.088362+0000","last_active":"2023-10-09T10:16:41.443062+0000","last_peered":"2023-10-09T10:16:41.443062+0000","last_clean":"2023-10-09T10:16:41.443062+0000","last_became_active":"2023-10-09T10:08:35.088009+0000","last_became_peered":"2023-10-09T10:08:35.088009+0000","last_unstale":"2023-10-09T10:16:41.443062+0000","last_undegraded":"2023-10-09T10:16:41.443062+0000","last_fullsized":"2023-10-09T10:16:41.443062+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.454644+0000","last_change":"2023-10-09T10:16:18.237380+0000","last_active":"2023-10-09T10:16:41.454644+0000","last_peered":"2023-10-09T10:16:41.454644+0000","last_clean":"2023-10-09T10:16:41.454644+0000","last_became_active":"2023-10-09T10:16:18.236585+0000","last_became_peered":"2023-10-09T10:16:18.236585+0000","last_unstale":"2023-10-09T10:16:41.454644+0000","last_undegraded":"2023-10-09T10:16:41.454644+0000","last_fullsized":"2023-10-09T10:16:41.454644+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"374'7709","reported_seq":8243,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:49.005611+0000","last_change":"2023-10-09T10:16:20.239479+0000","last_active":"2023-10-09T10:16:49.005611+0000","last_peered":"2023-10-09T10:16:49.005611+0000","last_clean":"2023-10-09T10:16:49.005611+0000","last_became_active":"2023-10-09T10:16:20.231381+0000","last_became_peered":"2023-10-09T10:16:20.231381+0000","last_unstale":"2023-10-09T10:16:49.005611+0000","last_undegraded":"2023-10-09T10:16:49.005611+0000","last_fullsized":"2023-10-09T10:16:49.005611+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7709,"log_dups_size":0,"ondisk_log_size":7709,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164364288,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4450,"num_write_kb":222656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":215,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":394,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.443250+0000","last_change":"2023-10-09T10:08:35.091559+0000","last_active":"2023-10-09T10:16:41.443250+0000","last_peered":"2023-10-09T10:16:41.443250+0000","last_clean":"2023-10-09T10:16:41.443250+0000","last_became_active":"2023-10-09T10:08:35.091260+0000","last_became_peered":"2023-10-09T10:08:35.091260+0000","last_unstale":"2023-10-09T10:16:41.443250+0000","last_undegraded":"2023-10-09T10:16:41.443250+0000","last_fullsized":"2023-10-09T10:16:41.443250+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":311,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.444573+0000","last_change":"2023-10-09T10:16:08.229304+0000","last_active":"2023-10-09T10:16:41.444573+0000","last_peered":"2023-10-09T10:16:41.444573+0000","last_clean":"2023-10-09T10:16:41.444573+0000","last_became_active":"2023-10-09T10:16:08.228997+0000","last_became_peered":"2023-10-09T10:16:08.228997+0000","last_unstale":"2023-10-09T10:16:41.444573+0000","last_undegraded":"2023-10-09T10:16:41.444573+0000","last_fullsized":"2023-10-09T10:16:41.444573+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:11.084115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"374'7731","reported_seq":11628,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:50.243684+0000","last_change":"2023-10-09T10:16:27.084024+0000","last_active":"2023-10-09T10:16:50.243684+0000","last_peered":"2023-10-09T10:16:50.243684+0000","last_clean":"2023-10-09T10:15:06.266980+0000","last_became_active":"2023-10-09T10:16:18.443378+0000","last_became_peered":"2023-10-09T10:16:18.443378+0000","last_unstale":"2023-10-09T10:16:50.243684+0000","last_undegraded":"2023-10-09T10:16:27.083274+0000","last_fullsized":"2023-10-09T10:16:18.208408+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"339'6595","last_scrub_stamp":"2023-10-09T10:15:01.144444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:15:01.144444+0000","objects_scrubbed":0,"log_size":7731,"log_dups_size":0,"ondisk_log_size":7731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166068224,"num_objects":2534,"num_object_clones":0,"num_object_copies":7602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1887,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1164,"num_write_kb":10560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1848,"num_bytes_recovered":116129792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[7,2],"avail_no_missing":["7"],"object_location_counts":[{"shards":"2,7","objects":849},{"shards":"3,5,7","objects":1685}],"blocked_by":[],"up_primary":5,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":26,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.454929+0000","last_change":"2023-10-09T10:16:18.362151+0000","last_active":"2023-10-09T10:16:41.454929+0000","last_peered":"2023-10-09T10:16:41.454929+0000","last_clean":"2023-10-09T10:16:41.454929+0000","last_became_active":"2023-10-09T10:16:18.323076+0000","last_became_peered":"2023-10-09T10:16:18.323076+0000","last_unstale":"2023-10-09T10:16:41.454929+0000","last_undegraded":"2023-10-09T10:16:41.454929+0000","last_fullsized":"2023-10-09T10:16:41.454929+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":217,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.469241+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:16:41.469241+0000","last_peered":"2023-10-09T10:16:41.469241+0000","last_clean":"2023-10-09T10:16:41.469241+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:16:41.469241+0000","last_undegraded":"2023-10-09T10:16:41.469241+0000","last_fullsized":"2023-10-09T10:16:41.469241+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"374'7786","reported_seq":8954,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:49.029251+0000","last_change":"2023-10-09T10:16:18.313517+0000","last_active":"2023-10-09T10:16:49.029251+0000","last_peered":"2023-10-09T10:16:49.029251+0000","last_clean":"2023-10-09T10:15:06.224242+0000","last_became_active":"2023-10-09T10:16:18.305795+0000","last_became_peered":"2023-10-09T10:16:18.305795+0000","last_unstale":"2023-10-09T10:16:49.029251+0000","last_undegraded":"2023-10-09T10:16:18.214537+0000","last_fullsized":"2023-10-09T10:16:18.207078+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7786,"log_dups_size":0,"ondisk_log_size":7786,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166821888,"num_objects":2545,"num_object_clones":0,"num_object_copies":7635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":500,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4469,"num_write_kb":224480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":536,"num_bytes_recovered":35127296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2544},{"shards":"1,2,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.455137+0000","last_change":"2023-10-09T10:16:18.237242+0000","last_active":"2023-10-09T10:16:41.455137+0000","last_peered":"2023-10-09T10:16:41.455137+0000","last_clean":"2023-10-09T10:16:41.455137+0000","last_became_active":"2023-10-09T10:16:18.236617+0000","last_became_peered":"2023-10-09T10:16:18.236617+0000","last_unstale":"2023-10-09T10:16:41.455137+0000","last_undegraded":"2023-10-09T10:16:41.455137+0000","last_fullsized":"2023-10-09T10:16:41.455137+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":243,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.442379+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:16:41.442379+0000","last_peered":"2023-10-09T10:16:41.442379+0000","last_clean":"2023-10-09T10:16:41.442379+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:16:41.442379+0000","last_undegraded":"2023-10-09T10:16:41.442379+0000","last_fullsized":"2023-10-09T10:16:41.442379+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"374'7953","reported_seq":8047,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:50.274492+0000","last_change":"2023-10-09T10:16:42.443549+0000","last_active":"2023-10-09T10:16:50.274492+0000","last_peered":"2023-10-09T10:16:50.274492+0000","last_clean":"2023-10-09T10:16:50.274492+0000","last_became_active":"2023-10-09T10:16:10.230777+0000","last_became_peered":"2023-10-09T10:16:10.230777+0000","last_unstale":"2023-10-09T10:16:50.274492+0000","last_undegraded":"2023-10-09T10:16:50.274492+0000","last_fullsized":"2023-10-09T10:16:50.274492+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7953,"log_dups_size":0,"ondisk_log_size":7953,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167247872,"num_objects":2552,"num_object_clones":0,"num_object_copies":7656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4596,"num_write_kb":228736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":228,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":245,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.442306+0000","last_change":"2023-10-09T10:16:08.363203+0000","last_active":"2023-10-09T10:16:41.442306+0000","last_peered":"2023-10-09T10:16:41.442306+0000","last_clean":"2023-10-09T10:16:41.442306+0000","last_became_active":"2023-10-09T10:16:08.248248+0000","last_became_peered":"2023-10-09T10:16:08.248248+0000","last_unstale":"2023-10-09T10:16:41.442306+0000","last_undegraded":"2023-10-09T10:16:41.442306+0000","last_fullsized":"2023-10-09T10:16:41.442306+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:48.937824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":207,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.430990+0000","last_change":"2023-10-09T10:15:42.389995+0000","last_active":"2023-10-09T10:16:41.430990+0000","last_peered":"2023-10-09T10:16:41.430990+0000","last_clean":"2023-10-09T10:16:41.430990+0000","last_became_active":"2023-10-09T10:15:42.389652+0000","last_became_peered":"2023-10-09T10:15:42.389652+0000","last_unstale":"2023-10-09T10:16:41.430990+0000","last_undegraded":"2023-10-09T10:16:41.430990+0000","last_fullsized":"2023-10-09T10:16:41.430990+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:19:40.955859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"374'7705","reported_seq":7861,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:49.075677+0000","last_change":"2023-10-09T10:15:03.504099+0000","last_active":"2023-10-09T10:16:49.075677+0000","last_peered":"2023-10-09T10:16:49.075677+0000","last_clean":"2023-10-09T10:16:49.075677+0000","last_became_active":"2023-10-09T10:14:59.337388+0000","last_became_peered":"2023-10-09T10:14:59.337388+0000","last_unstale":"2023-10-09T10:16:49.075677+0000","last_undegraded":"2023-10-09T10:16:49.075677+0000","last_fullsized":"2023-10-09T10:16:49.075677+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"339'6671","last_scrub_stamp":"2023-10-09T10:15:03.503975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:15:03.503975+0000","objects_scrubbed":3418,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:10:40.212861+0000","scrub_duration":0.90657236399999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169082880,"num_objects":2580,"num_object_clones":0,"num_object_copies":7740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1046,"num_write_kb":6656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":301,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.451492+0000","last_change":"2023-10-09T10:08:35.095576+0000","last_active":"2023-10-09T10:16:41.451492+0000","last_peered":"2023-10-09T10:16:41.451492+0000","last_clean":"2023-10-09T10:16:41.451492+0000","last_became_active":"2023-10-09T10:08:35.094995+0000","last_became_peered":"2023-10-09T10:08:35.094995+0000","last_unstale":"2023-10-09T10:16:41.451492+0000","last_undegraded":"2023-10-09T10:16:41.451492+0000","last_fullsized":"2023-10-09T10:16:41.451492+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:58.433210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":307,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.445428+0000","last_change":"2023-10-09T10:09:47.007132+0000","last_active":"2023-10-09T10:16:41.445428+0000","last_peered":"2023-10-09T10:16:41.445428+0000","last_clean":"2023-10-09T10:16:41.445428+0000","last_became_active":"2023-10-09T10:09:47.006714+0000","last_became_peered":"2023-10-09T10:09:47.006714+0000","last_unstale":"2023-10-09T10:16:41.445428+0000","last_undegraded":"2023-10-09T10:16:41.445428+0000","last_fullsized":"2023-10-09T10:16:41.445428+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:46.659049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"374'7717","reported_seq":7820,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:50.196461+0000","last_change":"2023-10-09T10:16:18.493990+0000","last_active":"2023-10-09T10:16:50.196461+0000","last_peered":"2023-10-09T10:16:50.196461+0000","last_clean":"2023-10-09T10:15:14.172820+0000","last_became_active":"2023-10-09T10:16:18.481842+0000","last_became_peered":"2023-10-09T10:16:18.481842+0000","last_unstale":"2023-10-09T10:16:50.196461+0000","last_undegraded":"2023-10-09T10:16:18.235307+0000","last_fullsized":"2023-10-09T10:16:18.204778+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165052416,"num_objects":2518,"num_object_clones":0,"num_object_copies":7554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4450,"num_write_kb":223008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,3,7","objects":2552},{"shards":"1,7","objects":-491}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":268,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.442898+0000","last_change":"2023-10-09T10:08:35.097276+0000","last_active":"2023-10-09T10:16:41.442898+0000","last_peered":"2023-10-09T10:16:41.442898+0000","last_clean":"2023-10-09T10:16:41.442898+0000","last_became_active":"2023-10-09T10:08:35.097036+0000","last_became_peered":"2023-10-09T10:08:35.097036+0000","last_unstale":"2023-10-09T10:16:41.442898+0000","last_undegraded":"2023-10-09T10:16:41.442898+0000","last_fullsized":"2023-10-09T10:16:41.442898+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.445595+0000","last_change":"2023-10-09T10:08:35.095746+0000","last_active":"2023-10-09T10:16:41.445595+0000","last_peered":"2023-10-09T10:16:41.445595+0000","last_clean":"2023-10-09T10:16:41.445595+0000","last_became_active":"2023-10-09T10:08:35.095397+0000","last_became_peered":"2023-10-09T10:08:35.095397+0000","last_unstale":"2023-10-09T10:16:41.445595+0000","last_undegraded":"2023-10-09T10:16:41.445595+0000","last_fullsized":"2023-10-09T10:16:41.445595+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:58.639252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"374'7679","reported_seq":7778,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:49.126336+0000","last_change":"2023-10-09T10:16:27.105001+0000","last_active":"2023-10-09T10:16:49.126336+0000","last_peered":"2023-10-09T10:16:49.126336+0000","last_clean":"2023-10-09T10:15:14.051568+0000","last_became_active":"2023-10-09T10:16:08.554375+0000","last_became_peered":"2023-10-09T10:16:08.554375+0000","last_unstale":"2023-10-09T10:16:49.126336+0000","last_undegraded":"2023-10-09T10:16:08.315559+0000","last_fullsized":"2023-10-09T10:16:08.202043+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7679,"log_dups_size":0,"ondisk_log_size":7679,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166068224,"num_objects":2534,"num_object_clones":0,"num_object_copies":7602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3490,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4410,"num_write_kb":222208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-475},{"shards":"1,2,3","objects":2554}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":321,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.446152+0000","last_change":"2023-10-09T10:16:08.267092+0000","last_active":"2023-10-09T10:16:41.446152+0000","last_peered":"2023-10-09T10:16:41.446152+0000","last_clean":"2023-10-09T10:16:41.446152+0000","last_became_active":"2023-10-09T10:16:08.244668+0000","last_became_peered":"2023-10-09T10:16:08.244668+0000","last_unstale":"2023-10-09T10:16:41.446152+0000","last_undegraded":"2023-10-09T10:16:41.446152+0000","last_fullsized":"2023-10-09T10:16:41.446152+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:35:59.836060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":394,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.457675+0000","last_change":"2023-10-09T10:15:42.317788+0000","last_active":"2023-10-09T10:16:41.457675+0000","last_peered":"2023-10-09T10:16:41.457675+0000","last_clean":"2023-10-09T10:16:41.457675+0000","last_became_active":"2023-10-09T10:15:42.317524+0000","last_became_peered":"2023-10-09T10:15:42.317524+0000","last_unstale":"2023-10-09T10:16:41.457675+0000","last_undegraded":"2023-10-09T10:16:41.457675+0000","last_fullsized":"2023-10-09T10:16:41.457675+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:54.507373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"374'8961","reported_seq":12458,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:49.186411+0000","last_change":"2023-10-09T10:16:19.493844+0000","last_active":"2023-10-09T10:16:49.186411+0000","last_peered":"2023-10-09T10:16:49.186411+0000","last_clean":"2023-10-09T10:15:06.273272+0000","last_became_active":"2023-10-09T10:16:19.421047+0000","last_became_peered":"2023-10-09T10:16:19.421047+0000","last_unstale":"2023-10-09T10:16:49.186411+0000","last_undegraded":"2023-10-09T10:16:19.351119+0000","last_fullsized":"2023-10-09T10:16:19.212441+0000","mapping_epoch":362,"log_start":"330'4399","ondisk_log_start":"330'4399","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":4562,"log_dups_size":0,"ondisk_log_size":4562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:15.198806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344129536,"num_objects":5251,"num_object_clones":0,"num_object_copies":15753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5913,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8961,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1697,"num_bytes_recovered":111083520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[1,2],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,5","objects":5251}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":379,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.446155+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:16:41.446155+0000","last_peered":"2023-10-09T10:16:41.446155+0000","last_clean":"2023-10-09T10:16:41.446155+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:16:41.446155+0000","last_undegraded":"2023-10-09T10:16:41.446155+0000","last_fullsized":"2023-10-09T10:16:41.446155+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.455019+0000","last_change":"2023-10-09T10:16:18.323515+0000","last_active":"2023-10-09T10:16:41.455019+0000","last_peered":"2023-10-09T10:16:41.455019+0000","last_clean":"2023-10-09T10:16:41.455019+0000","last_became_active":"2023-10-09T10:16:18.321171+0000","last_became_peered":"2023-10-09T10:16:18.321171+0000","last_unstale":"2023-10-09T10:16:41.455019+0000","last_undegraded":"2023-10-09T10:16:41.455019+0000","last_fullsized":"2023-10-09T10:16:41.455019+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":353,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.451832+0000","last_change":"2023-10-09T10:16:18.261462+0000","last_active":"2023-10-09T10:16:41.451832+0000","last_peered":"2023-10-09T10:16:41.451832+0000","last_clean":"2023-10-09T10:16:41.451832+0000","last_became_active":"2023-10-09T10:16:18.261023+0000","last_became_peered":"2023-10-09T10:16:18.261023+0000","last_unstale":"2023-10-09T10:16:41.451832+0000","last_undegraded":"2023-10-09T10:16:41.451832+0000","last_fullsized":"2023-10-09T10:16:41.451832+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:43.268197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"374'8065","reported_seq":8182,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:52.255891+0000","last_change":"2023-10-09T10:16:18.400385+0000","last_active":"2023-10-09T10:16:52.255891+0000","last_peered":"2023-10-09T10:16:52.255891+0000","last_clean":"2023-10-09T10:15:06.364805+0000","last_became_active":"2023-10-09T10:16:18.392713+0000","last_became_peered":"2023-10-09T10:16:18.392713+0000","last_unstale":"2023-10-09T10:16:52.255891+0000","last_undegraded":"2023-10-09T10:16:18.227890+0000","last_fullsized":"2023-10-09T10:16:18.203511+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8065,"log_dups_size":0,"ondisk_log_size":8065,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165511168,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4681,"num_write_kb":230624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2525}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.454858+0000","last_change":"2023-10-09T10:16:18.255781+0000","last_active":"2023-10-09T10:16:41.454858+0000","last_peered":"2023-10-09T10:16:41.454858+0000","last_clean":"2023-10-09T10:16:41.454858+0000","last_became_active":"2023-10-09T10:16:18.238531+0000","last_became_peered":"2023-10-09T10:16:18.238531+0000","last_unstale":"2023-10-09T10:16:41.454858+0000","last_undegraded":"2023-10-09T10:16:41.454858+0000","last_fullsized":"2023-10-09T10:16:41.454858+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":193,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:42.324549+0000","last_change":"2023-10-09T10:08:35.084417+0000","last_active":"2023-10-09T10:16:42.324549+0000","last_peered":"2023-10-09T10:16:42.324549+0000","last_clean":"2023-10-09T10:16:42.324549+0000","last_became_active":"2023-10-09T10:08:35.084075+0000","last_became_peered":"2023-10-09T10:08:35.084075+0000","last_unstale":"2023-10-09T10:16:42.324549+0000","last_undegraded":"2023-10-09T10:16:42.324549+0000","last_fullsized":"2023-10-09T10:16:42.324549+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:53.471399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":426,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.446080+0000","last_change":"2023-10-09T10:16:08.230567+0000","last_active":"2023-10-09T10:16:41.446080+0000","last_peered":"2023-10-09T10:16:41.446080+0000","last_clean":"2023-10-09T10:16:41.446080+0000","last_became_active":"2023-10-09T10:16:08.229677+0000","last_became_peered":"2023-10-09T10:16:08.229677+0000","last_unstale":"2023-10-09T10:16:41.446080+0000","last_undegraded":"2023-10-09T10:16:41.446080+0000","last_fullsized":"2023-10-09T10:16:41.446080+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:52.866334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"374'8601","reported_seq":8741,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:49.116295+0000","last_change":"2023-10-09T10:16:19.311764+0000","last_active":"2023-10-09T10:16:49.116295+0000","last_peered":"2023-10-09T10:16:49.116295+0000","last_clean":"2023-10-09T10:15:06.345974+0000","last_became_active":"2023-10-09T10:16:19.245712+0000","last_became_peered":"2023-10-09T10:16:19.245712+0000","last_unstale":"2023-10-09T10:16:49.116295+0000","last_undegraded":"2023-10-09T10:16:19.223036+0000","last_fullsized":"2023-10-09T10:16:19.216151+0000","mapping_epoch":362,"log_start":"328'4012","ondisk_log_start":"328'4012","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":4589,"log_dups_size":0,"ondisk_log_size":4589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:58:11.000412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330629120,"num_objects":5045,"num_object_clones":0,"num_object_copies":15135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":550,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8601,"num_write_kb":436672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[1,7],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,3,7","objects":1},{"shards":"1,7","objects":5044}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":283,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.440982+0000","last_change":"2023-10-09T10:16:08.269361+0000","last_active":"2023-10-09T10:16:41.440982+0000","last_peered":"2023-10-09T10:16:41.440982+0000","last_clean":"2023-10-09T10:16:41.440982+0000","last_became_active":"2023-10-09T10:16:08.266453+0000","last_became_peered":"2023-10-09T10:16:08.266453+0000","last_unstale":"2023-10-09T10:16:41.440982+0000","last_undegraded":"2023-10-09T10:16:41.440982+0000","last_fullsized":"2023-10-09T10:16:41.440982+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:07.633138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":372,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:47.004705+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:16:47.004705+0000","last_peered":"2023-10-09T10:16:47.004705+0000","last_clean":"2023-10-09T10:16:47.004705+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:16:47.004705+0000","last_undegraded":"2023-10-09T10:16:47.004705+0000","last_fullsized":"2023-10-09T10:16:47.004705+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":410,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.457628+0000","last_change":"2023-10-09T10:16:18.266234+0000","last_active":"2023-10-09T10:16:41.457628+0000","last_peered":"2023-10-09T10:16:41.457628+0000","last_clean":"2023-10-09T10:16:41.457628+0000","last_became_active":"2023-10-09T10:16:18.265721+0000","last_became_peered":"2023-10-09T10:16:18.265721+0000","last_unstale":"2023-10-09T10:16:41.457628+0000","last_undegraded":"2023-10-09T10:16:41.457628+0000","last_fullsized":"2023-10-09T10:16:41.457628+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:59.391242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"374'8944","reported_seq":12848,"reported_epoch":374,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:16:52.336045+0000","last_change":"2023-10-09T10:16:30.394073+0000","last_active":"2023-10-09T10:16:52.336045+0000","last_peered":"2023-10-09T10:16:52.336045+0000","last_clean":"2023-10-09T10:15:06.308775+0000","last_became_active":"2023-10-09T10:16:26.751379+0000","last_became_peered":"2023-10-09T10:16:26.751379+0000","last_unstale":"2023-10-09T10:16:52.336045+0000","last_undegraded":"2023-10-09T10:16:52.336045+0000","last_fullsized":"2023-10-09T10:16:18.203512+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":1211,"log_size":8944,"log_dups_size":0,"ondisk_log_size":8944,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167575552,"num_objects":2557,"num_object_clones":0,"num_object_copies":7671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1650,"num_objects_unfound":0,"num_objects_dirty":2557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4629,"num_write_kb":229952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1846,"num_bytes_recovered":120717312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,7],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0,3,5","objects":1650},{"shards":"0,7","objects":907}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.454888+0000","last_change":"2023-10-09T10:16:18.477748+0000","last_active":"2023-10-09T10:16:41.454888+0000","last_peered":"2023-10-09T10:16:41.454888+0000","last_clean":"2023-10-09T10:16:41.454888+0000","last_became_active":"2023-10-09T10:16:18.477398+0000","last_became_peered":"2023-10-09T10:16:18.477398+0000","last_unstale":"2023-10-09T10:16:41.454888+0000","last_undegraded":"2023-10-09T10:16:41.454888+0000","last_fullsized":"2023-10-09T10:16:41.454888+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":620,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.444527+0000","last_change":"2023-10-09T10:02:43.741043+0000","last_active":"2023-10-09T10:16:41.444527+0000","last_peered":"2023-10-09T10:16:41.444527+0000","last_clean":"2023-10-09T10:16:41.444527+0000","last_became_active":"2023-10-09T10:02:43.740603+0000","last_became_peered":"2023-10-09T10:02:43.740603+0000","last_unstale":"2023-10-09T10:16:41.444527+0000","last_undegraded":"2023-10-09T10:16:41.444527+0000","last_fullsized":"2023-10-09T10:16:41.444527+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.455015+0000","last_change":"2023-10-09T10:16:18.255461+0000","last_active":"2023-10-09T10:16:41.455015+0000","last_peered":"2023-10-09T10:16:41.455015+0000","last_clean":"2023-10-09T10:16:41.455015+0000","last_became_active":"2023-10-09T10:16:18.247752+0000","last_became_peered":"2023-10-09T10:16:18.247752+0000","last_unstale":"2023-10-09T10:16:41.455015+0000","last_undegraded":"2023-10-09T10:16:41.455015+0000","last_fullsized":"2023-10-09T10:16:41.455015+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":289,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.443349+0000","last_change":"2023-10-09T10:08:35.089592+0000","last_active":"2023-10-09T10:16:41.443349+0000","last_peered":"2023-10-09T10:16:41.443349+0000","last_clean":"2023-10-09T10:16:41.443349+0000","last_became_active":"2023-10-09T10:08:35.089101+0000","last_became_peered":"2023-10-09T10:08:35.089101+0000","last_unstale":"2023-10-09T10:16:41.443349+0000","last_undegraded":"2023-10-09T10:16:41.443349+0000","last_fullsized":"2023-10-09T10:16:41.443349+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"374'8991","reported_seq":9121,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:52.186483+0000","last_change":"2023-10-09T10:16:18.471067+0000","last_active":"2023-10-09T10:16:52.186483+0000","last_peered":"2023-10-09T10:16:52.186483+0000","last_clean":"2023-10-09T10:15:06.321287+0000","last_became_active":"2023-10-09T10:16:18.457037+0000","last_became_peered":"2023-10-09T10:16:18.457037+0000","last_unstale":"2023-10-09T10:16:52.186483+0000","last_undegraded":"2023-10-09T10:16:18.237892+0000","last_fullsized":"2023-10-09T10:16:18.218927+0000","mapping_epoch":361,"log_start":"323'1635","ondisk_log_start":"323'1635","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7356,"log_dups_size":0,"ondisk_log_size":7356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:31:55.519719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330104832,"num_objects":5037,"num_object_clones":0,"num_object_copies":15111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8991,"num_write_kb":448896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":1441792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":5037}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":352,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.446550+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:16:41.446550+0000","last_peered":"2023-10-09T10:16:41.446550+0000","last_clean":"2023-10-09T10:16:41.446550+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:16:41.446550+0000","last_undegraded":"2023-10-09T10:16:41.446550+0000","last_fullsized":"2023-10-09T10:16:41.446550+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":286,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.441071+0000","last_change":"2023-10-09T10:16:18.393880+0000","last_active":"2023-10-09T10:16:41.441071+0000","last_peered":"2023-10-09T10:16:41.441071+0000","last_clean":"2023-10-09T10:16:41.441071+0000","last_became_active":"2023-10-09T10:16:18.375766+0000","last_became_peered":"2023-10-09T10:16:18.375766+0000","last_unstale":"2023-10-09T10:16:41.441071+0000","last_undegraded":"2023-10-09T10:16:41.441071+0000","last_fullsized":"2023-10-09T10:16:41.441071+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:56:50.927077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":290,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.440654+0000","last_change":"2023-10-09T10:15:07.378501+0000","last_active":"2023-10-09T10:16:41.440654+0000","last_peered":"2023-10-09T10:16:41.440654+0000","last_clean":"2023-10-09T10:16:41.440654+0000","last_became_active":"2023-10-09T10:15:07.378146+0000","last_became_peered":"2023-10-09T10:15:07.378146+0000","last_unstale":"2023-10-09T10:16:41.440654+0000","last_undegraded":"2023-10-09T10:16:41.440654+0000","last_fullsized":"2023-10-09T10:16:41.440654+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:56.827491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"374'8820","reported_seq":8887,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:49.180406+0000","last_change":"2023-10-09T10:13:32.576814+0000","last_active":"2023-10-09T10:16:49.180406+0000","last_peered":"2023-10-09T10:16:49.180406+0000","last_clean":"2023-10-09T10:16:49.180406+0000","last_became_active":"2023-10-09T10:13:32.576567+0000","last_became_peered":"2023-10-09T10:13:32.576567+0000","last_unstale":"2023-10-09T10:16:49.180406+0000","last_undegraded":"2023-10-09T10:16:49.180406+0000","last_fullsized":"2023-10-09T10:16:49.180406+0000","mapping_epoch":317,"log_start":"329'4190","ondisk_log_start":"329'4190","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":4630,"log_dups_size":0,"ondisk_log_size":4630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:37.032579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338558976,"num_objects":5166,"num_object_clones":0,"num_object_copies":15498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8820,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":393,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.443274+0000","last_change":"2023-10-09T10:08:35.089414+0000","last_active":"2023-10-09T10:16:41.443274+0000","last_peered":"2023-10-09T10:16:41.443274+0000","last_clean":"2023-10-09T10:16:41.443274+0000","last_became_active":"2023-10-09T10:08:35.088710+0000","last_became_peered":"2023-10-09T10:08:35.088710+0000","last_unstale":"2023-10-09T10:16:41.443274+0000","last_undegraded":"2023-10-09T10:16:41.443274+0000","last_fullsized":"2023-10-09T10:16:41.443274+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":426,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.446236+0000","last_change":"2023-10-09T10:16:18.261069+0000","last_active":"2023-10-09T10:16:41.446236+0000","last_peered":"2023-10-09T10:16:41.446236+0000","last_clean":"2023-10-09T10:16:41.446236+0000","last_became_active":"2023-10-09T10:16:18.260736+0000","last_became_peered":"2023-10-09T10:16:18.260736+0000","last_unstale":"2023-10-09T10:16:41.446236+0000","last_undegraded":"2023-10-09T10:16:41.446236+0000","last_fullsized":"2023-10-09T10:16:41.446236+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:18.217603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":263,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.442259+0000","last_change":"2023-10-09T10:16:18.477779+0000","last_active":"2023-10-09T10:16:41.442259+0000","last_peered":"2023-10-09T10:16:41.442259+0000","last_clean":"2023-10-09T10:16:41.442259+0000","last_became_active":"2023-10-09T10:16:18.477390+0000","last_became_peered":"2023-10-09T10:16:18.477390+0000","last_unstale":"2023-10-09T10:16:41.442259+0000","last_undegraded":"2023-10-09T10:16:41.442259+0000","last_fullsized":"2023-10-09T10:16:41.442259+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:06.576807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"374'9011","reported_seq":9095,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:49.162821+0000","last_change":"2023-10-09T10:16:18.335820+0000","last_active":"2023-10-09T10:16:49.162821+0000","last_peered":"2023-10-09T10:16:49.162821+0000","last_clean":"2023-10-09T10:16:49.162821+0000","last_became_active":"2023-10-09T10:16:18.323451+0000","last_became_peered":"2023-10-09T10:16:18.323451+0000","last_unstale":"2023-10-09T10:16:49.162821+0000","last_undegraded":"2023-10-09T10:16:49.162821+0000","last_fullsized":"2023-10-09T10:16:49.162821+0000","mapping_epoch":361,"log_start":"330'4426","ondisk_log_start":"330'4426","created":317,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":4585,"log_dups_size":0,"ondisk_log_size":4585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:19:20.069884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339476504,"num_objects":5181,"num_object_clones":0,"num_object_copies":15543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5181,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9011,"num_write_kb":454081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":332,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.431087+0000","last_change":"2023-10-09T10:15:42.391115+0000","last_active":"2023-10-09T10:16:41.431087+0000","last_peered":"2023-10-09T10:16:41.431087+0000","last_clean":"2023-10-09T10:16:41.431087+0000","last_became_active":"2023-10-09T10:15:42.390765+0000","last_became_peered":"2023-10-09T10:15:42.390765+0000","last_unstale":"2023-10-09T10:16:41.431087+0000","last_undegraded":"2023-10-09T10:16:41.431087+0000","last_fullsized":"2023-10-09T10:16:41.431087+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:03.271932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":353,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.445674+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:16:41.445674+0000","last_peered":"2023-10-09T10:16:41.445674+0000","last_clean":"2023-10-09T10:16:41.445674+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:16:41.445674+0000","last_undegraded":"2023-10-09T10:16:41.445674+0000","last_fullsized":"2023-10-09T10:16:41.445674+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":320,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.444610+0000","last_change":"2023-10-09T10:16:16.785913+0000","last_active":"2023-10-09T10:16:41.444610+0000","last_peered":"2023-10-09T10:16:41.444610+0000","last_clean":"2023-10-09T10:16:41.444610+0000","last_became_active":"2023-10-09T10:16:16.785530+0000","last_became_peered":"2023-10-09T10:16:16.785530+0000","last_unstale":"2023-10-09T10:16:41.444610+0000","last_undegraded":"2023-10-09T10:16:41.444610+0000","last_fullsized":"2023-10-09T10:16:41.444610+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"374'9130","reported_seq":22172,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:52.301505+0000","last_change":"2023-10-09T10:16:34.253888+0000","last_active":"2023-10-09T10:16:52.301505+0000","last_peered":"2023-10-09T10:16:52.301505+0000","last_clean":"2023-10-09T10:16:52.301505+0000","last_became_active":"2023-10-09T10:16:34.237728+0000","last_became_peered":"2023-10-09T10:16:34.237728+0000","last_unstale":"2023-10-09T10:16:52.301505+0000","last_undegraded":"2023-10-09T10:16:52.301505+0000","last_fullsized":"2023-10-09T10:16:52.301505+0000","mapping_epoch":369,"log_start":"323'1792","ondisk_log_start":"323'1792","created":317,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7338,"log_dups_size":0,"ondisk_log_size":7338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:06.101348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327155712,"num_objects":4992,"num_object_clones":0,"num_object_copies":14976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9130,"num_write_kb":451904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6469,"num_bytes_recovered":393936896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":282,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.469631+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:16:41.469631+0000","last_peered":"2023-10-09T10:16:41.469631+0000","last_clean":"2023-10-09T10:16:41.469631+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:16:41.469631+0000","last_undegraded":"2023-10-09T10:16:41.469631+0000","last_fullsized":"2023-10-09T10:16:41.469631+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":247,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.440570+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:16:41.440570+0000","last_peered":"2023-10-09T10:16:41.440570+0000","last_clean":"2023-10-09T10:16:41.440570+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:16:41.440570+0000","last_undegraded":"2023-10-09T10:16:41.440570+0000","last_fullsized":"2023-10-09T10:16:41.440570+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":22,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.454698+0000","last_change":"2023-10-09T10:16:18.244156+0000","last_active":"2023-10-09T10:16:41.454698+0000","last_peered":"2023-10-09T10:16:41.454698+0000","last_clean":"2023-10-09T10:16:41.454698+0000","last_became_active":"2023-10-09T10:16:18.243857+0000","last_became_peered":"2023-10-09T10:16:18.243857+0000","last_unstale":"2023-10-09T10:16:41.454698+0000","last_undegraded":"2023-10-09T10:16:41.454698+0000","last_fullsized":"2023-10-09T10:16:41.454698+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"374'7938","reported_seq":8030,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:50.269560+0000","last_change":"2023-10-09T10:16:40.412556+0000","last_active":"2023-10-09T10:16:50.269560+0000","last_peered":"2023-10-09T10:16:50.269560+0000","last_clean":"2023-10-09T10:16:50.269560+0000","last_became_active":"2023-10-09T10:16:22.227875+0000","last_became_peered":"2023-10-09T10:16:22.227875+0000","last_unstale":"2023-10-09T10:16:50.269560+0000","last_undegraded":"2023-10-09T10:16:50.269560+0000","last_fullsized":"2023-10-09T10:16:50.269560+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7938,"log_dups_size":0,"ondisk_log_size":7938,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167575552,"num_objects":2557,"num_object_clones":0,"num_object_copies":7671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4581,"num_write_kb":228416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":233,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":241,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.442158+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:16:41.442158+0000","last_peered":"2023-10-09T10:16:41.442158+0000","last_clean":"2023-10-09T10:16:41.442158+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:16:41.442158+0000","last_undegraded":"2023-10-09T10:16:41.442158+0000","last_fullsized":"2023-10-09T10:16:41.442158+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":338,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.446310+0000","last_change":"2023-10-09T10:16:08.231228+0000","last_active":"2023-10-09T10:16:41.446310+0000","last_peered":"2023-10-09T10:16:41.446310+0000","last_clean":"2023-10-09T10:16:41.446310+0000","last_became_active":"2023-10-09T10:16:08.230940+0000","last_became_peered":"2023-10-09T10:16:08.230940+0000","last_unstale":"2023-10-09T10:16:41.446310+0000","last_undegraded":"2023-10-09T10:16:41.446310+0000","last_fullsized":"2023-10-09T10:16:41.446310+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:52.611954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.16","version":"374'7766","reported_seq":8238,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:49.088908+0000","last_change":"2023-10-09T10:16:18.352731+0000","last_active":"2023-10-09T10:16:49.088908+0000","last_peered":"2023-10-09T10:16:49.088908+0000","last_clean":"2023-10-09T10:15:06.314173+0000","last_became_active":"2023-10-09T10:16:18.345487+0000","last_became_peered":"2023-10-09T10:16:18.345487+0000","last_unstale":"2023-10-09T10:16:49.088908+0000","last_undegraded":"2023-10-09T10:16:18.224255+0000","last_fullsized":"2023-10-09T10:16:18.206382+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7766,"log_dups_size":0,"ondisk_log_size":7766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169443328,"num_objects":2586,"num_object_clones":0,"num_object_copies":7758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":508,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4450,"num_write_kb":225120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":190,"num_bytes_recovered":12451840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2586}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":26,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.455069+0000","last_change":"2023-10-09T10:16:18.299039+0000","last_active":"2023-10-09T10:16:41.455069+0000","last_peered":"2023-10-09T10:16:41.455069+0000","last_clean":"2023-10-09T10:16:41.455069+0000","last_became_active":"2023-10-09T10:16:18.228601+0000","last_became_peered":"2023-10-09T10:16:18.228601+0000","last_unstale":"2023-10-09T10:16:41.455069+0000","last_undegraded":"2023-10-09T10:16:41.455069+0000","last_fullsized":"2023-10-09T10:16:41.455069+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":221,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.430467+0000","last_change":"2023-10-09T10:15:42.385888+0000","last_active":"2023-10-09T10:16:41.430467+0000","last_peered":"2023-10-09T10:16:41.430467+0000","last_clean":"2023-10-09T10:16:41.430467+0000","last_became_active":"2023-10-09T10:15:42.385230+0000","last_became_peered":"2023-10-09T10:15:42.385230+0000","last_unstale":"2023-10-09T10:16:41.430467+0000","last_undegraded":"2023-10-09T10:16:41.430467+0000","last_fullsized":"2023-10-09T10:16:41.430467+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:45.160761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.17","version":"374'7685","reported_seq":15963,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:50.274751+0000","last_change":"2023-10-09T10:16:18.442217+0000","last_active":"2023-10-09T10:16:50.274751+0000","last_peered":"2023-10-09T10:16:50.274751+0000","last_clean":"2023-10-09T10:15:06.303688+0000","last_became_active":"2023-10-09T10:16:18.348754+0000","last_became_peered":"2023-10-09T10:16:18.348754+0000","last_unstale":"2023-10-09T10:16:50.274751+0000","last_undegraded":"2023-10-09T10:16:18.207994+0000","last_fullsized":"2023-10-09T10:16:18.204785+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7685,"log_dups_size":0,"ondisk_log_size":7685,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168361984,"num_objects":2569,"num_object_clones":0,"num_object_copies":7707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":264,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4419,"num_write_kb":223616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3878,"num_bytes_recovered":248971264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,3,7","objects":1},{"shards":"2,7","objects":2568}],"blocked_by":[],"up_primary":5,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":26,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.454883+0000","last_change":"2023-10-09T10:16:18.351145+0000","last_active":"2023-10-09T10:16:41.454883+0000","last_peered":"2023-10-09T10:16:41.454883+0000","last_clean":"2023-10-09T10:16:41.454883+0000","last_became_active":"2023-10-09T10:16:18.321691+0000","last_became_peered":"2023-10-09T10:16:18.321691+0000","last_unstale":"2023-10-09T10:16:41.454883+0000","last_undegraded":"2023-10-09T10:16:41.454883+0000","last_fullsized":"2023-10-09T10:16:41.454883+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":230,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.430259+0000","last_change":"2023-10-09T10:15:42.393203+0000","last_active":"2023-10-09T10:16:41.430259+0000","last_peered":"2023-10-09T10:16:41.430259+0000","last_clean":"2023-10-09T10:16:41.430259+0000","last_became_active":"2023-10-09T10:15:42.381511+0000","last_became_peered":"2023-10-09T10:15:42.381511+0000","last_unstale":"2023-10-09T10:16:41.430259+0000","last_undegraded":"2023-10-09T10:16:41.430259+0000","last_fullsized":"2023-10-09T10:16:41.430259+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:23.196321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"374'7662","reported_seq":8190,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T10:16:49.052046+0000","last_change":"2023-10-09T10:16:10.248177+0000","last_active":"2023-10-09T10:16:49.052046+0000","last_peered":"2023-10-09T10:16:49.052046+0000","last_clean":"2023-10-09T10:16:49.052046+0000","last_became_active":"2023-10-09T10:16:10.240509+0000","last_became_peered":"2023-10-09T10:16:10.240509+0000","last_unstale":"2023-10-09T10:16:49.052046+0000","last_undegraded":"2023-10-09T10:16:49.052046+0000","last_fullsized":"2023-10-09T10:16:49.052046+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7662,"log_dups_size":0,"ondisk_log_size":7662,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164167680,"num_objects":2505,"num_object_clones":0,"num_object_copies":7515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4403,"num_write_kb":221056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":28,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.444850+0000","last_change":"2023-10-09T10:16:08.240555+0000","last_active":"2023-10-09T10:16:41.444850+0000","last_peered":"2023-10-09T10:16:41.444850+0000","last_clean":"2023-10-09T10:16:41.444850+0000","last_became_active":"2023-10-09T10:16:08.239692+0000","last_became_peered":"2023-10-09T10:16:08.239692+0000","last_unstale":"2023-10-09T10:16:41.444850+0000","last_undegraded":"2023-10-09T10:16:41.444850+0000","last_fullsized":"2023-10-09T10:16:41.444850+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:39.501717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":413,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.440896+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:16:41.440896+0000","last_peered":"2023-10-09T10:16:41.440896+0000","last_clean":"2023-10-09T10:16:41.440896+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:16:41.440896+0000","last_undegraded":"2023-10-09T10:16:41.440896+0000","last_fullsized":"2023-10-09T10:16:41.440896+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.15","version":"374'7840","reported_seq":10963,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:16:52.292478+0000","last_change":"2023-10-09T10:16:18.463184+0000","last_active":"2023-10-09T10:16:52.292478+0000","last_peered":"2023-10-09T10:16:52.292478+0000","last_clean":"2023-10-09T10:15:06.324825+0000","last_became_active":"2023-10-09T10:16:18.336894+0000","last_became_peered":"2023-10-09T10:16:18.336894+0000","last_unstale":"2023-10-09T10:16:52.292478+0000","last_undegraded":"2023-10-09T10:16:18.463001+0000","last_fullsized":"2023-10-09T10:16:18.205452+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7840,"log_dups_size":0,"ondisk_log_size":7840,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163577856,"num_objects":2496,"num_object_clones":0,"num_object_copies":7488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":489,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4538,"num_write_kb":225088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1511,"num_bytes_recovered":98041856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2496}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":353,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.441057+0000","last_change":"2023-10-09T10:16:18.354987+0000","last_active":"2023-10-09T10:16:41.441057+0000","last_peered":"2023-10-09T10:16:41.441057+0000","last_clean":"2023-10-09T10:16:41.441057+0000","last_became_active":"2023-10-09T10:16:18.272788+0000","last_became_peered":"2023-10-09T10:16:18.272788+0000","last_unstale":"2023-10-09T10:16:41.441057+0000","last_undegraded":"2023-10-09T10:16:41.441057+0000","last_fullsized":"2023-10-09T10:16:41.441057+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:36.355149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":243,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T10:16:41.446051+0000","last_change":"2023-10-09T10:08:35.094235+0000","last_active":"2023-10-09T10:16:41.446051+0000","last_peered":"2023-10-09T10:16:41.446051+0000","last_clean":"2023-10-09T10:16:41.446051+0000","last_became_active":"2023-10-09T10:08:35.093870+0000","last_became_peered":"2023-10-09T10:08:35.093870+0000","last_unstale":"2023-10-09T10:16:41.446051+0000","last_undegraded":"2023-10-09T10:16:41.446051+0000","last_fullsized":"2023-10-09T10:16:41.446051+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.322250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":10906,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":5349015576,"num_objects":81621,"num_object_clones":0,"num_object_copies":244863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34611,"num_objects_misplaced":4175,"num_objects_unfound":0,"num_objects_dirty":81621,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136719,"num_write_kb":6769633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20934,"num_bytes_recovered":1271988224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17655869440,"data_stored":17655857224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":190036,"ondisk_log_size":190036,"up":78,"acting":60,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430566,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3899684,"kb_used_data":2995420,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":89845468,"statfs":{"total":95995035648,"available":92001759232,"internally_reserved":0,"allocated":3067310080,"data_stored":3066631572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:15:54 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":2.1190000000000002,"15min":2.1469999999999998},"min":{"1min":0.23599999999999999,"5min":0.223,"15min":0.20599999999999999},"max":{"1min":5.0730000000000004,"5min":11.164,"15min":50.591999999999999},"last":3.202},{"interface":"front","average":{"1min":0.92400000000000004,"5min":2.0720000000000001,"15min":2.1070000000000002},"min":{"1min":0.26100000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.3140000000000001,"5min":15.974,"15min":50.747},"last":3.294}]},{"osd":1,"last update":"Mon Oct 9 10:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":2.169,"15min":2.0960000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22500000000000001},"max":{"1min":9.657,"5min":11.92,"15min":25.338000000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.1850000000000001,"5min":2.2669999999999999,"15min":1.9550000000000001},"min":{"1min":0.31,"5min":0.249,"15min":0.249},"max":{"1min":5.2599999999999998,"5min":14.5,"15min":18.27},"last":7.407}]},{"osd":2,"last update":"Mon Oct 9 10:16:10 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":2.387,"15min":2.226},"min":{"1min":0.315,"5min":0.29499999999999998,"15min":0.246},"max":{"1min":5.6500000000000004,"5min":13.775,"15min":13.775},"last":4.8940000000000001},{"interface":"front","average":{"1min":2.3450000000000002,"5min":2.508,"15min":2.1949999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":12.845000000000001,"5min":12.845000000000001,"15min":12.845000000000001},"last":0.97299999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":7.444},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.359,"5min":7.359,"15min":7.359},"last":8.0779999999999994}]},{"osd":4,"last update":"Mon Oct 9 10:16:49 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":1.45,"15min":1.1359999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":7.8890000000000002,"5min":12.699999999999999,"15min":12.699999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.125,"15min":1.0029999999999999},"min":{"1min":0.315,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.3609999999999998,"5min":12.003,"15min":12.003},"last":0.315}]},{"osd":5,"last update":"Mon Oct 9 10:16:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.647,"15min":1.2909999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":1.7270000000000001,"5min":12.048,"15min":25.277999999999999},"last":0.374},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.502,"15min":1.2549999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.22600000000000001},"max":{"1min":0.877,"5min":18.547999999999998,"15min":25.754000000000001},"last":0.38200000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:16:10 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.2829999999999999,"15min":1.099},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26100000000000001},"max":{"1min":0.64200000000000002,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":1.3420000000000001,"15min":1.089},"min":{"1min":0.307,"5min":0.307,"15min":0.27200000000000002},"max":{"1min":0.59699999999999998,"5min":10.625,"15min":10.625},"last":0.36299999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626794,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1652308,"kb_used_data":1050316,"kb_used_omap":0,"kb_used_meta":601984,"kb_avail":92092844,"statfs":{"total":95995035648,"available":94303072256,"internally_reserved":0,"allocated":1075523584,"data_stored":1074862390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616431616},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:16:00 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.831,"15min":1.746},"min":{"1min":0.248,"5min":0.23100000000000001,"15min":0.22600000000000001},"max":{"1min":2.887,"5min":9.6940000000000008,"15min":15.435},"last":0.96199999999999997},{"interface":"front","average":{"1min":2.0329999999999999,"5min":1.873,"15min":1.8080000000000001},"min":{"1min":0.26700000000000002,"5min":0.22,"15min":0.22},"max":{"1min":10.539999999999999,"5min":13.385,"15min":15.308999999999999},"last":8.0999999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:16:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":2.1739999999999999,"15min":2.1720000000000002},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":17.925000000000001,"5min":17.925000000000001,"15min":35.609999999999999},"last":7.9800000000000004},{"interface":"front","average":{"1min":2.0920000000000001,"5min":2.149,"15min":2.2250000000000001},"min":{"1min":0.311,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.090999999999999,"5min":10.58,"15min":32.396000000000001},"last":3.0019999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.9830000000000001,"15min":1.762},"min":{"1min":0.33900000000000002,"5min":0.316,"15min":0.26200000000000001},"max":{"1min":6.9569999999999999,"5min":7.7359999999999998,"15min":12.898},"last":10.435},{"interface":"front","average":{"1min":1.887,"5min":2.3500000000000001,"15min":2.6589999999999998},"min":{"1min":0.33600000000000002,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":6.1929999999999996,"5min":14.542999999999999,"15min":14.542999999999999},"last":8.0630000000000006}]},{"osd":3,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":1.6739999999999999,"15min":1.6739999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":8.0229999999999997,"5min":8.0229999999999997,"15min":8.0229999999999997},"last":8.0229999999999997},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":5.2400000000000002,"5min":5.2400000000000002,"15min":5.2400000000000002},"last":3.7480000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:15:50 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":1.056,"15min":0.66700000000000004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.1739999999999999,"5min":10.113,"15min":10.113},"last":1.1850000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.93899999999999995,"15min":0.68000000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":2.5960000000000001,"5min":9.218,"15min":9.218},"last":2.2280000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.0549999999999999,"15min":0.89800000000000002},"min":{"1min":0.34999999999999998,"5min":0.29099999999999998,"15min":0.23699999999999999},"max":{"1min":2.9100000000000001,"5min":6.3170000000000002,"15min":7.9260000000000002},"last":1.3460000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":1.0589999999999999,"15min":0.97499999999999998},"min":{"1min":0.36899999999999999,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":1.008,"5min":14.465999999999999,"15min":14.465999999999999},"last":1.008}]},{"osd":7,"last update":"Mon Oct 9 10:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.167,"15min":0.95499999999999996},"min":{"1min":0.376,"5min":0.31900000000000001,"15min":0.252},"max":{"1min":2.4350000000000001,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":2.2589999999999999},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.206,"15min":0.997},"min":{"1min":0.41599999999999998,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":2.464,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":2.2799999999999998}]}]},{"osd":1,"up_from":84,"seq":360777253107,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3730620,"kb_used_data":3110516,"kb_used_omap":9,"kb_used_meta":620086,"kb_avail":90014532,"statfs":{"total":95995035648,"available":92174880768,"internally_reserved":0,"allocated":3185168384,"data_stored":3183818914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":634969065},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:16:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":2.2480000000000002,"15min":2.5750000000000002},"min":{"1min":0.216,"5min":0.20000000000000001,"15min":0.183},"max":{"1min":3.1080000000000001,"5min":31.007000000000001,"15min":31.007000000000001},"last":8.1370000000000005},{"interface":"front","average":{"1min":1.048,"5min":2.2349999999999999,"15min":2.762},"min":{"1min":0.247,"5min":0.216,"15min":0.192},"max":{"1min":6.1769999999999996,"5min":10.577999999999999,"15min":20.501999999999999},"last":0.51700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:15:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":2.895,"15min":3.3809999999999998},"min":{"1min":0.24399999999999999,"5min":0.22800000000000001,"15min":0.217},"max":{"1min":10.211,"5min":23.105,"15min":23.105},"last":1.716},{"interface":"front","average":{"1min":1.29,"5min":2.5569999999999999,"15min":3.2200000000000002},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":7.4939999999999998,"5min":28.024999999999999,"15min":28.024999999999999},"last":2.6970000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:16:44 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":1.177,"15min":1.177},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.673,"5min":6.673,"15min":6.673},"last":2.7440000000000002},{"interface":"front","average":{"1min":1.714,"5min":1.714,"15min":1.714},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.2160000000000002,"5min":7.2160000000000002,"15min":7.2160000000000002},"last":0.314}]},{"osd":4,"last update":"Mon Oct 9 10:15:48 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.5269999999999999,"15min":1.4179999999999999},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.28699999999999998},"max":{"1min":7.3789999999999996,"5min":21.620000000000001,"15min":21.620000000000001},"last":2.7839999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":2.3210000000000002,"15min":1.7629999999999999},"min":{"1min":0.314,"5min":0.308,"15min":0.308},"max":{"1min":1.7769999999999999,"5min":12.904,"15min":12.904},"last":0.629}]},{"osd":5,"last update":"Mon Oct 9 10:16:30 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":3.2959999999999998,"15min":2.613},"min":{"1min":0.33900000000000002,"5min":0.33300000000000002,"15min":0.29099999999999998},"max":{"1min":4.0179999999999998,"5min":92.177999999999997,"15min":92.177999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.085,"5min":3.7730000000000001,"15min":2.9039999999999999},"min":{"1min":0.374,"5min":0.36299999999999999,"15min":0.29099999999999998},"max":{"1min":7.7350000000000003,"5min":91.869,"15min":91.869},"last":1.647}]},{"osd":6,"last update":"Mon Oct 9 10:16:30 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.849,"15min":2.0129999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.307},"max":{"1min":4.0199999999999996,"5min":21.670999999999999,"15min":21.670999999999999},"last":2.8450000000000002},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.929,"15min":2.1869999999999998},"min":{"1min":0.42599999999999999,"5min":0.36899999999999999,"15min":0.35099999999999998},"max":{"1min":3.988,"5min":9.9440000000000008,"15min":15.481999999999999},"last":0.59899999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:16:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":2.2919999999999998,"15min":2.403},"min":{"1min":0.39300000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.8010000000000002,"5min":10.723000000000001,"15min":16.331},"last":1.6890000000000001},{"interface":"front","average":{"1min":0.70899999999999996,"5min":1.825,"15min":2.1019999999999999},"min":{"1min":0.38500000000000001,"5min":0.371,"15min":0.371},"max":{"1min":3.0579999999999998,"5min":21.593,"15min":21.593},"last":2.8149999999999999}]}]},{"osd":0,"up_from":79,"seq":339302416632,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2128752,"kb_used_data":1629928,"kb_used_omap":28,"kb_used_meta":498787,"kb_avail":91616400,"statfs":{"total":95995035648,"available":93815193600,"internally_reserved":0,"allocated":1669046272,"data_stored":1667706239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":510758736},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:16:26 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":2.7309999999999999,"15min":2.214},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.16600000000000001},"max":{"1min":6.7169999999999996,"5min":24.399999999999999,"15min":24.399999999999999},"last":2.1280000000000001},{"interface":"front","average":{"1min":0.84399999999999997,"5min":3.4350000000000001,"15min":3.3900000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.21299999999999999},"max":{"1min":4.9260000000000002,"5min":31.925999999999998,"15min":31.925999999999998},"last":4.0609999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:16:04 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":2.7959999999999998,"15min":2.3170000000000002},"min":{"1min":0.23699999999999999,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":12.202999999999999,"5min":24.43,"15min":24.43},"last":2.2410000000000001},{"interface":"front","average":{"1min":1.2090000000000001,"5min":2.464,"15min":2.7109999999999999},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.23300000000000001},"max":{"1min":6.5519999999999996,"5min":24.459,"15min":24.459},"last":2.1589999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:16:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.4059999999999997,"5min":6.4059999999999997,"15min":6.4059999999999997},"last":3.9430000000000001},{"interface":"front","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.367,"5min":3.367,"15min":3.367},"last":2.6720000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:15:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":2.6309999999999998,"15min":1.978},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.875,"5min":20.494,"15min":20.494},"last":4.1420000000000003},{"interface":"front","average":{"1min":1.401,"5min":2.5270000000000001,"15min":2.1699999999999999},"min":{"1min":0.40999999999999998,"5min":0.34599999999999997,"15min":0.34499999999999997},"max":{"1min":7.4109999999999996,"5min":15.907,"15min":15.907},"last":3.8159999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:15:58 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":5.218,"15min":3.3700000000000001},"min":{"1min":0.35299999999999998,"5min":0.35099999999999998,"15min":0.28000000000000003},"max":{"1min":7.4539999999999997,"5min":229.589,"15min":229.589},"last":2.2509999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":4.7699999999999996,"15min":2.5739999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27000000000000002},"max":{"1min":1.762,"5min":228.88900000000001,"15min":228.88900000000001},"last":2.1819999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:15:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":2.6819999999999999,"15min":1.984},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.31900000000000001},"max":{"1min":1.73,"5min":24.341999999999999,"15min":24.341999999999999},"last":8.7959999999999994},{"interface":"front","average":{"1min":1.1919999999999999,"5min":3.0779999999999998,"15min":2.8519999999999999},"min":{"1min":0.38500000000000001,"5min":0.378,"15min":0.35699999999999998},"max":{"1min":4.9610000000000003,"5min":23.106999999999999,"15min":23.106999999999999},"last":3.984}]},{"osd":7,"last update":"Mon Oct 9 10:15:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":2.879,"15min":2.7280000000000002},"min":{"1min":0.436,"5min":0.35499999999999998,"15min":0.34300000000000003},"max":{"1min":7.4829999999999997,"5min":18.904,"15min":26.978000000000002},"last":1.954},{"interface":"front","average":{"1min":0.70399999999999996,"5min":2.6349999999999998,"15min":2.1219999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":1.726,"5min":24.372,"15min":26.768000000000001},"last":2.2040000000000002}]}]},{"osd":2,"up_from":217,"seq":932007903367,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1628984,"kb_used_data":1263664,"kb_used_omap":83,"kb_used_meta":365228,"kb_avail":92116168,"statfs":{"total":95995035648,"available":94326956032,"internally_reserved":0,"allocated":1293991936,"data_stored":1292706287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":373993967},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:16:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":2.5030000000000001,"15min":2.0009999999999999},"min":{"1min":0.20200000000000001,"5min":0.185,"15min":0.17899999999999999},"max":{"1min":2.4740000000000002,"5min":17.946000000000002,"15min":19.372},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.835,"5min":3.2410000000000001,"15min":3.056},"min":{"1min":0.23799999999999999,"5min":0.23599999999999999,"15min":0.23000000000000001},"max":{"1min":10.929,"5min":20.143000000000001,"15min":20.143000000000001},"last":2.4420000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:16:00 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":3.0499999999999998,"15min":2.665},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":5.8490000000000002,"5min":19.283999999999999,"15min":19.283999999999999},"last":3.4830000000000001},{"interface":"front","average":{"1min":2.1469999999999998,"5min":3.0899999999999999,"15min":2.7690000000000001},"min":{"1min":0.30099999999999999,"5min":0.254,"15min":0.254},"max":{"1min":11.041,"5min":19.027000000000001,"15min":19.027000000000001},"last":2.4820000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:16:41 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.872,"15min":0.872},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.5670000000000002,"5min":3.5670000000000002,"15min":3.5670000000000002},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.4620000000000002,"5min":3.4620000000000002,"15min":3.4620000000000002},"last":7.8099999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:15:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":2.3769999999999998,"15min":1.3100000000000001},"min":{"1min":0.34100000000000003,"5min":0.28999999999999998,"15min":0.26600000000000001},"max":{"1min":10.996,"5min":18.873999999999999,"15min":18.873999999999999},"last":0.622},{"interface":"front","average":{"1min":1.593,"5min":2.0910000000000002,"15min":1.365},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.314},"max":{"1min":6.5419999999999998,"5min":15.276,"15min":15.276},"last":0.55500000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":2.8170000000000002,"15min":1.994},"min":{"1min":0.32100000000000001,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":6.0369999999999999,"5min":57.298000000000002,"15min":57.298000000000002},"last":0.502},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.7829999999999999,"15min":2.028},"min":{"1min":0.35399999999999998,"5min":0.315,"15min":0.315},"max":{"1min":5.6429999999999998,"5min":57.828000000000003,"15min":57.828000000000003},"last":0.71099999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.7280000000000002,"15min":1.6899999999999999},"min":{"1min":0.377,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.962999999999999,"5min":18.943999999999999,"15min":18.943999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.157,"5min":2.5019999999999998,"15min":1.994},"min":{"1min":0.40200000000000002,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":5.5700000000000003,"5min":15.378,"15min":15.378},"last":0.59699999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":2.8740000000000001,"15min":1.7589999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":11.022,"5min":18.972999999999999,"15min":18.972999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.27,"5min":3.0219999999999998,"15min":2.1469999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.9039999999999999,"5min":15.353,"15min":15.353},"last":0.64300000000000002}]}]},{"osd":3,"up_from":352,"seq":1511828488207,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3222276,"kb_used_data":3081148,"kb_used_omap":15,"kb_used_meta":141104,"kb_avail":90522876,"statfs":{"total":95995035648,"available":92695425024,"internally_reserved":0,"allocated":3155095552,"data_stored":3153506186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":144490950},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.125,"15min":1.125},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.024,"5min":6.024,"15min":6.024},"last":3.3010000000000002},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":5.0170000000000003,"5min":5.0170000000000003,"15min":5.0170000000000003},"last":6.3220000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.111,"15min":1.111},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.1319999999999997,"5min":4.1319999999999997,"15min":4.1319999999999997},"last":15.970000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.679,"5min":1.679,"15min":1.679},"last":3.4260000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":2.1880000000000002,"5min":2.1880000000000002,"15min":2.1880000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.7970000000000006,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":2.4670000000000001},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.3099999999999996,"5min":7.3099999999999996,"15min":7.3099999999999996},"last":16.018000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.73,"5min":1.73,"15min":1.73},"last":3.1179999999999999},{"interface":"front","average":{"1min":1.278,"5min":1.278,"15min":1.278},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.988,"5min":3.988,"15min":3.988},"last":2.383}]},{"osd":5,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.097,"5min":2.097,"15min":2.097},"last":3.2400000000000002},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.7050000000000001,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":3.1970000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.0629999999999997,"5min":4.0629999999999997,"15min":4.0629999999999997},"last":2.444},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.3570000000000002,"5min":7.3570000000000002,"15min":7.3570000000000002},"last":3.2709999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:16:45 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.751,"15min":0.751},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":3.1659999999999999},{"interface":"front","average":{"1min":1.347,"5min":1.347,"15min":1.347},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":4.0339999999999998,"5min":4.0339999999999998,"15min":4.0339999999999998},"last":2.419}]}]},{"osd":4,"up_from":266,"seq":1142461300837,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1274648,"kb_used_data":947792,"kb_used_omap":35,"kb_used_meta":326812,"kb_avail":92470504,"statfs":{"total":95995035648,"available":94689796096,"internally_reserved":0,"allocated":970539008,"data_stored":969872150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":334655726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":1.2190000000000001,"15min":0.99099999999999999},"min":{"1min":0.26900000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.286,"5min":8.1319999999999997,"15min":10.881},"last":1.6379999999999999},{"interface":"front","average":{"1min":0.90400000000000003,"5min":2.0680000000000001,"15min":1.393},"min":{"1min":0.248,"5min":0.224,"15min":0.224},"max":{"1min":2.6869999999999998,"5min":12.246,"15min":12.246},"last":2.7290000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.448,"15min":1.478},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25},"max":{"1min":5.8680000000000003,"5min":18.303000000000001,"15min":18.303000000000001},"last":2.7559999999999998},{"interface":"front","average":{"1min":0.997,"5min":1.9370000000000001,"15min":1.5860000000000001},"min":{"1min":0.28399999999999997,"5min":0.254,"15min":0.254},"max":{"1min":4.7489999999999997,"5min":18.036000000000001,"15min":18.036000000000001},"last":2.6299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:15:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.8999999999999999,"15min":1.375},"min":{"1min":0.30599999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6669999999999998,"5min":9.2469999999999999,"15min":9.2469999999999999},"last":2.605},{"interface":"front","average":{"1min":2.0129999999999999,"5min":2.1619999999999999,"15min":1.3700000000000001},"min":{"1min":0.28199999999999997,"5min":0.28100000000000003,"15min":0.27700000000000002},"max":{"1min":6.3029999999999999,"5min":11.778,"15min":11.778},"last":2.6840000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:16:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.6779999999999999,"15min":1.6779999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":9.6170000000000009,"5min":9.6170000000000009,"15min":9.6170000000000009},"last":2.6680000000000001},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.3119999999999998,"5min":3.3119999999999998,"15min":3.3119999999999998},"last":2.5739999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":2.2189999999999999,"15min":1.25},"min":{"1min":0.26100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.6040000000000001,"5min":99.585999999999999,"15min":99.585999999999999},"last":2.79},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.944,"15min":1.0649999999999999},"min":{"1min":0.28899999999999998,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":2.0350000000000001,"5min":99.384,"15min":99.384},"last":1.6699999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.0900000000000001,"15min":0.73999999999999999},"min":{"1min":0.33300000000000002,"5min":0.313,"15min":0.311},"max":{"1min":2.4199999999999999,"5min":6.851,"15min":6.851},"last":1.6970000000000001},{"interface":"front","average":{"1min":0.77300000000000002,"5min":1.2809999999999999,"15min":0.82499999999999996},"min":{"1min":0.38600000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.2619999999999996,"5min":12.255000000000001,"15min":12.255000000000001},"last":3.383}]},{"osd":7,"last update":"Mon Oct 9 10:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.4650000000000001,"15min":0.89900000000000002},"min":{"1min":0.32900000000000001,"5min":0.312,"15min":0.312},"max":{"1min":2.3570000000000002,"5min":12.237,"15min":12.237},"last":2.698},{"interface":"front","average":{"1min":0.74399999999999999,"5min":1.357,"15min":1.0029999999999999},"min":{"1min":0.33900000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.641,"5min":14.964,"15min":14.964},"last":3.5369999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823022,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4139724,"kb_used_data":3238340,"kb_used_omap":0,"kb_used_meta":901312,"kb_avail":89605428,"statfs":{"total":95995035648,"available":91755958272,"internally_reserved":0,"allocated":3316060160,"data_stored":3315401358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922943488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:16:46 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.782,"15min":1.786},"min":{"1min":0.23499999999999999,"5min":0.219,"15min":0.21099999999999999},"max":{"1min":3.4889999999999999,"5min":16.079000000000001,"15min":16.079000000000001},"last":5.1050000000000004},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.504,"15min":1.764},"min":{"1min":0.253,"5min":0.253,"15min":0.22900000000000001},"max":{"1min":3.302,"5min":15.907,"15min":15.907},"last":1.5700000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:16:24 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":2.1309999999999998,"15min":1.8640000000000001},"min":{"1min":0.254,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":3.3559999999999999,"5min":16.056999999999999,"15min":16.056999999999999},"last":8.0890000000000004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.7869999999999999,"15min":1.927},"min":{"1min":0.29599999999999999,"5min":0.252,"15min":0.252},"max":{"1min":2.371,"5min":23.099,"15min":23.099},"last":1.548}]},{"osd":2,"last update":"Mon Oct 9 10:16:03 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.2949999999999999,"15min":2.2450000000000001},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":5.2039999999999997,"5min":15.973000000000001,"15min":15.973000000000001},"last":5.0750000000000002},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.514,"15min":1.8979999999999999},"min":{"1min":0.36399999999999999,"5min":0.32200000000000001,"15min":0.313},"max":{"1min":2.4740000000000002,"5min":15.929,"15min":15.929},"last":1.5900000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:16:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":6.1120000000000001,"5min":6.1120000000000001,"15min":6.1120000000000001},"last":0.375},{"interface":"front","average":{"1min":1.47,"5min":1.47,"15min":1.47},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":4.7450000000000001,"5min":4.7450000000000001,"15min":4.7450000000000001},"last":1.512}]},{"osd":4,"last update":"Mon Oct 9 10:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":1.0509999999999999,"15min":0.82099999999999995},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.24299999999999999},"max":{"1min":3.7599999999999998,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.72299999999999998,"15min":0.72399999999999998},"min":{"1min":0.35399999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":4.7110000000000003,"5min":8.5559999999999992,"15min":8.5559999999999992},"last":4.6029999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:16:46 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.274,"15min":1.05},"min":{"1min":0.33700000000000002,"5min":0.28599999999999998,"15min":0.25},"max":{"1min":4.6420000000000003,"5min":10.222,"15min":13.483000000000001},"last":1.5029999999999999},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.3440000000000001,"15min":1.109},"min":{"1min":0.38,"5min":0.26700000000000002,"15min":0.22600000000000001},"max":{"1min":9.4160000000000004,"5min":15.997,"15min":18.600999999999999},"last":5.04}]},{"osd":7,"last update":"Mon Oct 9 10:16:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":1.2290000000000001,"15min":1.0569999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.25600000000000001},"max":{"1min":0.60099999999999998,"5min":16.030999999999999,"15min":16.030999999999999},"last":5.1710000000000003},{"interface":"front","average":{"1min":0.438,"5min":0.68799999999999994,"15min":0.82799999999999996},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26800000000000002},"max":{"1min":0.58999999999999997,"5min":8.5220000000000002,"15min":9.6430000000000007},"last":5.3239999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1653080064,"data_stored":1653080064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3176665088,"data_stored":3176661016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1284636672,"data_stored":1284636672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3143827456,"data_stored":3143827456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":966524928,"data_stored":966524928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3306164224,"data_stored":3306160152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1067778048,"data_stored":1067778048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3057192960,"data_stored":3057188888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:16:55.106 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:16:55.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 36 2023-10-09T10:16:55.138 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:16:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:55.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:16:55.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:16:55.306 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:55 smithi110 ceph-mon[136032]: pgmap v1049: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 34615/244935 objects degraded (14.132%); 4175/244935 objects misplaced (1.705%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:16:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:55 smithi110 ceph-mon[136032]: from='client.56314 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:16:55.518 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:55 smithi012 ceph-mon[147926]: pgmap v1049: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 34615/244935 objects degraded (14.132%); 4175/244935 objects misplaced (1.705%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:16:55.518 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:55 smithi012 ceph-mon[147926]: from='client.56314 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:16:55.519 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:55 smithi012 ceph-mon[152263]: pgmap v1049: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 34615/244935 objects degraded (14.132%); 4175/244935 objects misplaced (1.705%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:16:55.519 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:55 smithi012 ceph-mon[152263]: from='client.56314 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:16:55.762 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:16:55.775 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:16:55.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:55.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:16:55.847 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:16:55.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:16:55.892 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:16:55.932 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:16:55.940 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:55.964 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:16:55.991 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:16:55.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:55.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:16:56.026 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:16:56.075 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:16:56.100 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:56.115 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:16:56.134 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:16:56.134 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:16:56.134 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:16:56.134 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:16:56.134 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:16:56.134 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:16:56.134 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:16:56.134 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:16:56.205 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:56.587 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2967318376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:56.589 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2967318376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:56.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2967318376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:16:56.713 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:56.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:56.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:16:56.827 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:16:56.899 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:56.969 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:16:56.970 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:16:56.970 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:16:56.970 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:16:56.970 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:16:57.267 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:16:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:57.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:16:57.301 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:16:57.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:16:57.429 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:57.479 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:16:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:57.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:16:57.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:16:57.651 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:57 smithi012 ceph-mon[147926]: pgmap v1050: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 34168/237120 objects degraded (14.410%); 4063/237120 objects misplaced (1.713%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T10:16:57.651 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1447271810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:57.651 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:57 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:57.651 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:57 smithi012 ceph-mon[152263]: pgmap v1050: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 34168/237120 objects degraded (14.410%); 4063/237120 objects misplaced (1.713%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T10:16:57.651 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1447271810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:57.652 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:57 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:57.660 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:16:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:57 smithi110 ceph-mon[136032]: pgmap v1050: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 34168/237120 objects degraded (14.410%); 4063/237120 objects misplaced (1.713%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T10:16:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1447271810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:57.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:57 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:16:58.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:16:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:16:57] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.33.4" 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 34168/237120 objects degraded (14.410%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:58.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:58.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:58.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T10:16:58.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 34168/237120 objects degraded (14.410%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: failed to encode map e375 with expected crc 2023-10-09T10:16:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 34168/237120 objects degraded (14.410%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:16:58.769 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:16:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:59 smithi012 ceph-mon[152263]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 34168/233817 objects degraded (14.613%); 3683/233817 objects misplaced (1.575%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:16:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:59.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:16:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:59 smithi012 ceph-mon[147926]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 34168/233817 objects degraded (14.613%); 3683/233817 objects misplaced (1.575%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:16:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:16:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:59 smithi110 ceph-mon[136032]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 34168/233817 objects degraded (14.613%); 3683/233817 objects misplaced (1.575%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:16:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:16:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:16:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:16:59.951 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:16:59.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:16:59.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:16:59.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:00.065 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:00.099 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:00.142 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:00.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:00.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:00.334 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:01 smithi012 ceph-mon[147926]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 33709/228435 objects degraded (14.756%); 3567/228435 objects misplaced (1.561%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T10:17:01.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:01 smithi012 ceph-mon[152263]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 33709/228435 objects degraded (14.756%); 3567/228435 objects misplaced (1.561%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T10:17:01.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:01 smithi110 ceph-mon[136032]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 33709/228435 objects degraded (14.756%); 3567/228435 objects misplaced (1.561%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T10:17:03.009 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:03.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:03.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:03.185 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:03.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:03 smithi012 ceph-mon[152263]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 33709/226452 objects degraded (14.886%); 3567/226452 objects misplaced (1.575%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T10:17:03.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:03 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:17:03.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 33709/226452 objects degraded (14.886%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:17:03.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:03 smithi012 ceph-mon[147926]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 33709/226452 objects degraded (14.886%); 3567/226452 objects misplaced (1.575%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T10:17:03.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:03 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:17:03.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 33709/226452 objects degraded (14.886%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:17:03.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:03 smithi110 ceph-mon[136032]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 33709/226452 objects degraded (14.886%); 3567/226452 objects misplaced (1.575%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T10:17:03.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:03 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:17:03.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 33709/226452 objects degraded (14.886%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:17:03.757 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:03.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:03.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:03.864 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:03.911 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:03.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:17:04.428 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:04.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:04.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:04.540 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:04.623 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:04.635 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:04.650 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:04.886 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:04.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:04.991 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:05.048 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:05.118 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:05.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:05.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:05.225 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:05.289 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:05.348 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:05.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:05.455 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:05.541 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:05.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:05 smithi012 ceph-mon[147926]: pgmap v1055: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 33709/222972 objects degraded (15.118%); 3129/222972 objects misplaced (1.403%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:17:05.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:05 smithi012 ceph-mon[152263]: pgmap v1055: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 33709/222972 objects degraded (15.118%); 3129/222972 objects misplaced (1.403%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:17:05.704 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:05 smithi110 ceph-mon[136032]: pgmap v1055: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 33709/222972 objects degraded (15.118%); 3129/222972 objects misplaced (1.403%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:17:05.996 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:06.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:06.105 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:06.157 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:06.219 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:06.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:06.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:06.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:06.358 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:06.399 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:06.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:06.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:06.548 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:06.594 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:06.606 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:06.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:06.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:06.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:06.802 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:06.915 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:06.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:06.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:07.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:07.084 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:07.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:07 smithi012 ceph-mon[147926]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 33276/215619 objects degraded (15.433%); 3037/215619 objects misplaced (1.409%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:17:07.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3072061229' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:17:07.369 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:17:07.369 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:07 smithi012 ceph-mon[152263]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 33276/215619 objects degraded (15.433%); 3037/215619 objects misplaced (1.409%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:17:07.369 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3072061229' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:17:07.369 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:17:07.394 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 9 pg_num to 36 2023-10-09T10:17:07.488 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:07.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:07.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:07.685 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:07.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:07 smithi110 ceph-mon[136032]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 33276/215619 objects degraded (15.433%); 3037/215619 objects misplaced (1.409%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:17:07.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3072061229' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:17:07.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:17:07.795 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:07.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:07.902 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:07.920 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:17:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:17:07] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-10-09T10:17:07.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:08.047 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:08.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:08.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:08.220 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:08.466 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-10-09T10:17:08.467 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T10:17:08.467 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.467 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.467 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.468 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.468 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.468 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:17:08.468 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:17:08.468 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 33276/215619 objects degraded (15.433%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:17:08.469 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-10-09T10:17:08.469 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T10:17:08.469 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.469 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.469 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.470 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.470 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.470 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:17:08.470 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:17:08.470 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 33276/215619 objects degraded (15.433%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:17:08.566 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:08.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:08.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:08.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: failed to encode map e376 with expected crc 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:17:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 33276/215619 objects degraded (15.433%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:17:08.751 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:08.900 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:08.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:08.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:09.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:09.040 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:09.231 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:09.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:09.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:09.385 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121890 9.1 9.4309a8a1 (undecoded) ondisk+retry+write+known_if_redirected e352)", 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 87.869769622000007, 2023-10-09T10:17:09.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99073293299999998, 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121890 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:17:09.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300149+0000", 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:09.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300155+0000", 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2280000000000003e-06 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300163+0000", 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8429999999999993e-06 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300170+0000", 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5989999999999998e-06 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300211+0000", 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1511000000000003e-05 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300219+0000", 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6190000000000002e-06 2023-10-09T10:17:09.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.290871+0000", 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99065166100000002 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.290884+0000", 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2998e-05 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.458 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121893 9.11 9.c051ab51 (undecoded) ondisk+write+known_if_redirected e352)", 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 87.863860977000002, 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98772148100000001, 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.459 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121893 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306058+0000", 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306064+0000", 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1700000000000002e-06 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306074+0000", 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0094e-05 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306084+0000", 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6579999999999997e-06 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306174+0000", 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0655000000000005e-05 2023-10-09T10:17:09.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306187+0000", 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2376e-05 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.293771+0000", 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98758401100000004 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.293781+0000", 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0017000000000001e-05 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121898 9.c 9.187a12cc (undecoded) ondisk+write+known_if_redirected e352)", 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 87.836622783999999, 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94351869099999996, 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.464 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121898 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333295+0000", 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333306+0000", 2023-10-09T10:17:09.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0361e-05 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333346+0000", 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0145999999999998e-05 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333357+0000", 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1377e-05 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333428+0000", 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0835999999999997e-05 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333441+0000", 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2449000000000001e-05 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.276804+0000", 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94336297999999996 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.276816+0000", 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2670999999999999e-05 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.469 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:123521 9.2 e354/353 9:46c69193:::benchmark_data_smithi012_313365_object11046:head v 354'7084, mlcod=344'6827)", 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:54.203163+0000", 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 74.966757408000007, 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023058531, 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203163+0000", 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203163+0000", 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203162+0000", 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203167+0000", 2023-10-09T10:17:09.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7600000000000002e-06 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203174+0000", 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1180000000000003e-06 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203180+0000", 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5310000000000001e-06 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.206390+0000", 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032101149999999999 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.206421+0000", 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1452999999999999e-05 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.226183+0000", 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019761772 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.226221+0000", 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8231e-05 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:123709 9.a e354/353 9:5c5183a2:::benchmark_data_smithi012_313365_object11234:head v 354'7475, mlcod=0'0)", 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:54.758440+0000", 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 74.411480193000003, 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025809789999999999, 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758440+0000", 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758440+0000", 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758440+0000", 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758444+0000", 2023-10-09T10:17:09.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5940000000000004e-06 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758451+0000", 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7009999999999999e-06 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758456+0000", 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9150000000000004e-06 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.766498+0000", 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0080422059999999997 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.766526+0000", 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8189000000000001e-05 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.784204+0000", 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017677531999999999 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.784250+0000", 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5983999999999999e-05 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:125277 9.a e354/353 9:5960c75e:::benchmark_data_smithi012_313365_object12802:head v 354'7584, mlcod=0'0)", 2023-10-09T10:17:09.479 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:59.479531+0000", 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 69.690389206999996, 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023230032000000001, 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479531+0000", 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479531+0000", 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479530+0000", 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479538+0000", 2023-10-09T10:17:09.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1099999999999997e-06 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479548+0000", 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0301e-05 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479555+0000", 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2389999999999996e-06 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.483886+0000", 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043309020000000002 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.483915+0000", 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8530999999999998e-05 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.502627+0000", 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018712876 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.502761+0000", 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000133719 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:125279 9.2 e354/353 9:40d3f81e:::benchmark_data_smithi012_313365_object12804:head v 354'7141, mlcod=344'6827)", 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:59.482769+0000", 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 69.687151387, 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029954650999999999, 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482769+0000", 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482769+0000", 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482768+0000", 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482777+0000", 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5490000000000006e-06 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482790+0000", 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2818e-05 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482798+0000", 2023-10-09T10:17:09.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3820000000000004e-06 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.491998+0000", 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092004579999999999 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.492024+0000", 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5789e-05 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512700+0000", 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020676132999999999 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512724+0000", 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3291000000000001e-05 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:125278 9.a e354/353 9:59d42cbe:::benchmark_data_smithi012_313365_object12803:head v 354'7585, mlcod=0'0)", 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:59.482806+0000", 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 69.687114205, 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.02989023, 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482806+0000", 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482806+0000", 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482806+0000", 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482810+0000", 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0949999999999998e-06 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482815+0000", 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4489999999999998e-06 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482819+0000", 2023-10-09T10:17:09.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0759999999999996e-06 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.494191+0000", 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011372044 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.494219+0000", 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7787e-05 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512667+0000", 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018447519999999999 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512696+0000", 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9553e-05 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:125284 9.2 e354/353 9:476a4c30:::benchmark_data_smithi012_313365_object12809:head v 354'7142, mlcod=344'6827)", 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:59.488043+0000", 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 69.681877760000006, 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024718561999999999, 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488043+0000", 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488043+0000", 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488042+0000", 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488051+0000", 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7359999999999996e-06 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488066+0000", 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4744e-05 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488075+0000", 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.234e-06 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.504341+0000", 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016265601000000001 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.504364+0000", 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3397999999999999e-05 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.498 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512726+0000", 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0083614750000000002 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512761+0000", 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5423999999999998e-05 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.499 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127796 9.a e355/353 9:5b549510:::benchmark_data_smithi012_313365_object15321:head v 355'7734, mlcod=0'0)", 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.929123+0000", 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.240797121999996, 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029092829000000001, 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929123+0000", 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929123+0000", 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929123+0000", 2023-10-09T10:17:09.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929128+0000", 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7369999999999997e-06 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929138+0000", 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9340000000000001e-06 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929143+0000", 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9960000000000001e-06 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.931154+0000", 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020111819999999998 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.931179+0000", 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4958000000000001e-05 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.958180+0000", 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027001421000000001 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.958216+0000", 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5936000000000003e-05 2023-10-09T10:17:09.504 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127798 9.a e355/353 9:5b6bbbbe:::benchmark_data_smithi012_313365_object15323:head v 355'7735, mlcod=0'0)", 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.934647+0000", 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 62.235273163000002, 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023586669000000001, 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934647+0000", 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934647+0000", 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.506 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934646+0000", 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934655+0000", 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0510000000000002e-06 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934668+0000", 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3465000000000001e-05 2023-10-09T10:17:09.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934677+0000", 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1849999999999993e-06 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.940490+0000", 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058127029999999998 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.940519+0000", 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8762999999999999e-05 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.958219+0000", 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017699751999999999 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.958234+0000", 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5293000000000001e-05 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:128048 9.a e358/357 9:5a9a9e1f:::benchmark_data_smithi012_313365_object15573:head v 358'7758, mlcod=0'0)", 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:09.628597+0000", 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 59.541323343000002, 2023-10-09T10:17:09.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023843540999999999, 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628597+0000", 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628597+0000", 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628595+0000", 2023-10-09T10:17:09.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:09.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628606+0000", 2023-10-09T10:17:09.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0376e-05 2023-10-09T10:17:09.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628620+0000", 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3925e-05 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628630+0000", 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0054000000000001e-05 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.634055+0000", 2023-10-09T10:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054254949999999998 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.634092+0000", 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6748999999999997e-05 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.652390+0000", 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018298043 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.652441+0000", 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0386000000000002e-05 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.516 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128066 9.1 9:8603ff6a:::benchmark_data_smithi012_313365_object15591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 59.487762785000001, 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55978837800000003, 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128066 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682155+0000", 2023-10-09T10:17:09.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682168+0000", 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2682e-05 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682184+0000", 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6339000000000001e-05 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682194+0000", 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4420000000000004e-06 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682258+0000", 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4463999999999998e-05 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:09.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682271+0000", 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3166000000000001e-05 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237630+0000", 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55535865299999998 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237768+0000", 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013818000000000001 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237891+0000", 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012287299999999999 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.240596+0000", 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002704559 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241710+0000", 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00111455 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241856+0000", 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014592000000000001 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241905+0000", 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9042000000000002e-05 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241946+0000", 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0655999999999998e-05 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:153125 9.1 9:8513a9b6:::benchmark_data_smithi012_313365_object40650:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:03.723046+0000", 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 5.4468747620000002, 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022738141, 2023-10-09T10:17:09.524 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 153125 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.723046+0000", 2023-10-09T10:17:09.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.723046+0000", 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.723044+0000", 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.723051+0000", 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5980000000000002e-06 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.723065+0000", 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.454e-05 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.723075+0000", 2023-10-09T10:17:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.628e-06 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.723966+0000", 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00089079700000000003 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.724080+0000", 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011479 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.724184+0000", 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103352 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.726460+0000", 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022764809999999999 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.738755+0000", 2023-10-09T10:17:09.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012294920000000001 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.745692+0000", 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0069368219999999996 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.745747+0000", 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5216000000000001e-05 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:03.745784+0000", 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6594999999999998e-05 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156228 9.11 9:8e81769a:::benchmark_data_smithi012_313365_object43753:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:07.974745+0000", 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 1.1951754290000001, 2023-10-09T10:17:09.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023573625000000001, 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156228 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974745+0000", 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974745+0000", 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974744+0000", 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974751+0000", 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3359999999999997e-06 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974767+0000", 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6160000000000001e-05 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974776+0000", 2023-10-09T10:17:09.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8310000000000007e-06 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974836+0000", 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0568e-05 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974950+0000", 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114021 2023-10-09T10:17:09.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.975014+0000", 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3950999999999998e-05 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.976771+0000", 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001756781 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.978790+0000", 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020192069999999999 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.998216+0000", 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019425782999999999 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.998272+0000", 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5448000000000002e-05 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.998319+0000", 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6898999999999999e-05 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156448 9.1 9:835065d8:::benchmark_data_smithi012_313365_object43973:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.228667+0000", 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.94125324099999996, 2023-10-09T10:17:09.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024562891, 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156448 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228667+0000", 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228667+0000", 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228665+0000", 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228678+0000", 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3193999999999999e-05 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228704+0000", 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5791000000000001e-05 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228717+0000", 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3049e-05 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228900+0000", 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000182919 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.229060+0000", 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000160571 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:17:09.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.229211+0000", 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015067499999999999 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.233957+0000", 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0047465529999999997 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.236603+0000", 2023-10-09T10:17:09.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026458010000000001 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253132+0000", 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016529245000000001 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253190+0000", 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7237000000000001e-05 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253230+0000", 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0358000000000002e-05 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156449 9.1 9:87277be4:::benchmark_data_smithi012_313365_object43974:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.228735+0000", 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.94118541, 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024607628999999999, 2023-10-09T10:17:09.545 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156449 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228735+0000", 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228735+0000", 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228734+0000", 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228738+0000", 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2389999999999998e-06 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228744+0000", 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2140000000000001e-06 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228749+0000", 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3299999999999998e-06 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.231538+0000", 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027884709999999998 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.231646+0000", 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108622 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.231745+0000", 2023-10-09T10:17:09.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9099999999999996e-05 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.236559+0000", 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048137290000000001 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.247589+0000", 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011029793 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253267+0000", 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056784360000000002 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253305+0000", 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7728000000000002e-05 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253343+0000", 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7438999999999998e-05 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156450 9.1 9:844cce54:::benchmark_data_smithi012_313365_object43975:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.228754+0000", 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.94116646100000001, 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024882517999999999, 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.552 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156450 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228754+0000", 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.553 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228754+0000", 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228754+0000", 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228756+0000", 2023-10-09T10:17:09.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3420000000000002e-06 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228761+0000", 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1789999999999999e-06 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228765+0000", 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3039999999999998e-06 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.234111+0000", 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053460319999999997 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.234233+0000", 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121143 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.234319+0000", 2023-10-09T10:17:09.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6410999999999996e-05 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.236896+0000", 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002577382 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.247617+0000", 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010720587 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.557 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253572+0000", 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059550430000000001 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253606+0000", 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4199000000000002e-05 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253636+0000", 2023-10-09T10:17:09.558 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0247e-05 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156571 9.11 9:88722a35:::benchmark_data_smithi012_313365_object44096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.76949653699999998, 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.21718912700000001, 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.559 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156571 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.560 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400422+0000", 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400431+0000", 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2720000000000003e-06 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.561 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400450+0000", 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8981e-05 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400462+0000", 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2333e-05 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.562 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.402769+0000", 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023067069999999999 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.402909+0000", 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013978499999999999 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.403031+0000", 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012171300000000001 2023-10-09T10:17:09.563 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.446104+0000", 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.043073587000000003 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575808+0000", 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12970353000000001 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617506+0000", 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041698357999999998 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617561+0000", 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5191000000000001e-05 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617613+0000", 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1649e-05 2023-10-09T10:17:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156578 9.11 9:8e383e1c:::benchmark_data_smithi012_313365_object44103:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.65829861499999998, 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.19283726200000001, 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:09.566 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156578 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:09.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511620+0000", 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511628+0000", 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7139999999999999e-06 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511644+0000", 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6018000000000001e-05 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511654+0000", 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9119999999999993e-06 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:09.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511718+0000", 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3587e-05 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511874+0000", 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000156126 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.512003+0000", 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012904400000000001 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575815+0000", 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063811965999999998 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.656235+0000", 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.080420144999999998 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704358+0000", 2023-10-09T10:17:09.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.048122282000000002 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704415+0000", 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7090999999999999e-05 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704459+0000", 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4347000000000001e-05 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.572 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.573 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.573 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:09.573 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:09.573 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:17:09.574 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:17:09.574 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:17:09.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:09.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:09.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:09.589 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:17:09.611 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:17:09.631 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:17:09.651 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:17:09.672 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:17:09.675 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[147926]: pgmap v1058: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 33276/212157 objects degraded (15.685%); 2606/212157 objects misplaced (1.228%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[147926]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[147926]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[147926]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[147926]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[152263]: pgmap v1058: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 33276/212157 objects degraded (15.685%); 2606/212157 objects misplaced (1.228%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T10:17:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[152263]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T10:17:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[152263]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T10:17:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[152263]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[152263]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:09.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:09 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:09.691 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:17:09.692 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:17:09.693 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:17:09.693 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:17:09.693 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:17:09.693 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:17:09.693 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:17:09.693 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:17:09.693 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:17:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:09 smithi110 ceph-mon[136032]: pgmap v1058: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 17 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 33276/212157 objects degraded (15.685%); 2606/212157 objects misplaced (1.228%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T10:17:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:09 smithi110 ceph-mon[136032]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:09 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T10:17:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:09 smithi110 ceph-mon[136032]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T10:17:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:09 smithi110 ceph-mon[136032]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:09 smithi110 ceph-mon[136032]: failed to encode map e377 with expected crc 2023-10-09T10:17:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:09 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:09 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:09.702 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:09.737 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:09.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:09.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:09.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:09.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:09.904 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:09.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:09.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:10.011 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:10.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:10.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:17:10.072 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:10.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:10.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:10.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:10.204 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:10.239 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:10.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:10.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:10.344 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:10.370 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:10.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T10:17:10.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.403 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:17:10.414 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:10.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:10.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:10.531 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:10.570 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:10.609 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:10.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T10:17:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2926218757' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3314995578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2926218757' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:10.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3314995578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:10.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T10:17:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: failed to encode map e378 with expected crc 2023-10-09T10:17:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2926218757' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:10.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3314995578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:10.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:10.743 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:10.777 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:10.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:10.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:10.882 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:10.908 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:10.946 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:10.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:10.972 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:11.050 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:11.078 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:11.263 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:11.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:11.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:11.366 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:11.392 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:11.432 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:11.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:11.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:11.564 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:11.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: pgmap v1061: 109 pgs: 3 remapped+peering, 3 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23838/194727 objects degraded (12.242%); 2514/194727 objects misplaced (1.291%) 2023-10-09T10:17:11.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:11.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[147926]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: pgmap v1061: 109 pgs: 3 remapped+peering, 3 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23838/194727 objects degraded (12.242%); 2514/194727 objects misplaced (1.291%) 2023-10-09T10:17:11.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:11 smithi012 ceph-mon[152263]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: pgmap v1061: 109 pgs: 3 remapped+peering, 3 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23838/194727 objects degraded (12.242%); 2514/194727 objects misplaced (1.291%) 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: failed to encode map e379 with expected crc 2023-10-09T10:17:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:11 smithi110 ceph-mon[136032]: failed to encode map e379 with expected crc 2023-10-09T10:17:12.016 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:12.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:12.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:12.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:17:12.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:12.162 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:12.300 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:12.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:12.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:12.405 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:12.436 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:12.472 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:12.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:12.576 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:12.604 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:12.645 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:12.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:12 smithi012 ceph-mon[147926]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T10:17:12.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:12 smithi012 ceph-mon[147926]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:12 smithi012 ceph-mon[147926]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:12 smithi012 ceph-mon[147926]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:12 smithi012 ceph-mon[152263]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T10:17:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:12 smithi012 ceph-mon[152263]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:12 smithi012 ceph-mon[152263]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:12 smithi012 ceph-mon[152263]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:12 smithi110 ceph-mon[136032]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T10:17:12.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:12 smithi110 ceph-mon[136032]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:12 smithi110 ceph-mon[136032]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:12 smithi110 ceph-mon[136032]: failed to encode map e380 with expected crc 2023-10-09T10:17:12.749 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:12.775 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:12.810 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:12.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:12.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:12.836 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:17:12.913 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:12.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:12.981 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:12.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:12.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:13.086 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:13.114 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:13.152 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:13.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:13.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:13.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:13.328 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:13.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:13.365 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:13.402 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[152263]: pgmap v1064: 109 pgs: 3 remapped+peering, 4 peering, 8 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22015/193422 objects degraded (11.382%); 2514/193422 objects misplaced (1.300%) 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[152263]: failed to encode map e380 with expected crc 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[152263]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[152263]: osdmap e381: 8 total, 8 up, 8 in 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[152263]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[147926]: pgmap v1064: 109 pgs: 3 remapped+peering, 4 peering, 8 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22015/193422 objects degraded (11.382%); 2514/193422 objects misplaced (1.300%) 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[147926]: failed to encode map e380 with expected crc 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[147926]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[147926]: osdmap e381: 8 total, 8 up, 8 in 2023-10-09T10:17:13.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[147926]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[147926]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[147926]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 22015/193422 objects degraded (11.382%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:13.432 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:13.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:13.490 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:13.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:13.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:13.586 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:13.587 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:13.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:13.624 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:13.664 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:13.667 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:17:13.667 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:17:13.667 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T10:17:13.668 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:17:13.668 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:17:13.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T10:17:13.670 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[152263]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.670 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[152263]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.670 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 22015/193422 objects degraded (11.382%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:13.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:13.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:13 smithi110 ceph-mon[136032]: pgmap v1064: 109 pgs: 3 remapped+peering, 4 peering, 8 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22015/193422 objects degraded (11.382%); 2514/193422 objects misplaced (1.300%) 2023-10-09T10:17:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:13 smithi110 ceph-mon[136032]: failed to encode map e380 with expected crc 2023-10-09T10:17:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:13 smithi110 ceph-mon[136032]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:13 smithi110 ceph-mon[136032]: osdmap e381: 8 total, 8 up, 8 in 2023-10-09T10:17:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:13 smithi110 ceph-mon[136032]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:13 smithi110 ceph-mon[136032]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:13 smithi110 ceph-mon[136032]: failed to encode map e381 with expected crc 2023-10-09T10:17:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 22015/193422 objects degraded (11.382%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:13.772 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:13.798 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:13.831 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:13.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:13.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:13.934 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:13.962 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:14.002 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:14.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:14.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:14.106 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:14.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:17:14.135 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:14.178 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:14.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:14.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:14.283 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:14.311 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:14.406 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:14 smithi012 ceph-mon[147926]: 9.11 scrub starts 2023-10-09T10:17:14.406 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:14 smithi012 ceph-mon[152263]: 9.11 scrub starts 2023-10-09T10:17:14.406 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:14.495 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:14.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:14.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:14.559 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:14.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:14.631 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:14.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:14.681 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:14.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:14.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:14.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:14 smithi110 ceph-mon[136032]: 9.11 scrub starts 2023-10-09T10:17:14.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:14.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:14.824 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:14.862 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:14.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:14.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:14.996 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:15.035 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:15.035 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:15.435 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:15.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:15.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:15.532 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:17:15.544 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:15.553 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:17:15.572 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:17:15.573 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:15.592 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:17:15.611 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:17:15.611 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:15.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:15.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:15.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:17:15.650 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:17:15.668 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:17:15.669 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:17:15.669 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:17:15.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:17:15.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:17:15.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:17:15.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:17:15.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:17:15.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:17:15.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[147926]: pgmap v1066: 109 pgs: 3 remapped+peering, 5 peering, 6 unknown, 4 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 22015/192558 objects degraded (11.433%); 2113/192558 objects misplaced (1.097%); 6.0 MiB/s, 162 objects/s recovering 2023-10-09T10:17:15.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:17:15.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:17:15.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[152263]: pgmap v1066: 109 pgs: 3 remapped+peering, 5 peering, 6 unknown, 4 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 22015/192558 objects degraded (11.433%); 2113/192558 objects misplaced (1.097%); 6.0 MiB/s, 162 objects/s recovering 2023-10-09T10:17:15.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:17:15.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:17:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:15 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:15 smithi110 ceph-mon[136032]: pgmap v1066: 109 pgs: 3 remapped+peering, 5 peering, 6 unknown, 4 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 22015/192558 objects degraded (11.433%); 2113/192558 objects misplaced (1.097%); 6.0 MiB/s, 162 objects/s recovering 2023-10-09T10:17:15.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:15 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:15 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:15 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:17:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:15 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:17:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:15 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:15.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:15.753 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:15.793 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:15.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:15.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:15.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:15.936 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:16.025 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:16.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:16.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:16.162 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:16.198 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:16.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:16.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:16.302 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:16.331 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:16.372 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:16.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:16.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:16.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:16.504 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:16.543 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:16.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:16.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:16.648 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:16.672 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:16 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/6796301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:16.675 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:16.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:16 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/6796301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:16.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:16 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/6796301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:16.711 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:16.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:16.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:16.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:16.844 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:16.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:16.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:16.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:16.984 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:17.010 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:17.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:17.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:17.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:17.148 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:17.180 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:17.217 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:17.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:17.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:17.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:17.352 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:17.424 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:17:17.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:17 smithi012 ceph-mon[152263]: pgmap v1067: 109 pgs: 1 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 1 peering, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.74k op/s; 30990/205122 objects degraded (15.108%); 3654/205122 objects misplaced (1.781%); 19 MiB/s, 356 objects/s recovering 2023-10-09T10:17:17.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:17 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2549202956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:17.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:17 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:17.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:17 smithi012 ceph-mon[147926]: pgmap v1067: 109 pgs: 1 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 1 peering, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.74k op/s; 30990/205122 objects degraded (15.108%); 3654/205122 objects misplaced (1.781%); 19 MiB/s, 356 objects/s recovering 2023-10-09T10:17:17.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:17 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2549202956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:17.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:17 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:17 smithi110 ceph-mon[136032]: pgmap v1067: 109 pgs: 1 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 1 peering, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.74k op/s; 30990/205122 objects degraded (15.108%); 3654/205122 objects misplaced (1.781%); 19 MiB/s, 356 objects/s recovering 2023-10-09T10:17:17.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:17 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2549202956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:17.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:17 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:17:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:17:17] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-10-09T10:17:18.366 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:17:18.366 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:17:18.366 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:17:18.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:17:18.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[152263]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[152263]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:18.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[152263]: osdmap e382: 8 total, 8 up, 8 in 2023-10-09T10:17:18.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[152263]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 30990/205122 objects degraded (15.108%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:18.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:17:18.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[147926]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[147926]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:18.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[147926]: osdmap e382: 8 total, 8 up, 8 in 2023-10-09T10:17:18.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[147926]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 30990/205122 objects degraded (15.108%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:18.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:18 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:17:18.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:18 smithi110 ceph-mon[136032]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:18 smithi110 ceph-mon[136032]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:18 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:18.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:18 smithi110 ceph-mon[136032]: osdmap e382: 8 total, 8 up, 8 in 2023-10-09T10:17:18.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:18 smithi110 ceph-mon[136032]: failed to encode map e382 with expected crc 2023-10-09T10:17:18.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 30990/205122 objects degraded (15.108%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:18.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:17:19.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:19 smithi012 ceph-mon[152263]: pgmap v1069: 109 pgs: 1 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.62k op/s; 30990/205104 objects degraded (15.109%); 3654/205104 objects misplaced (1.782%); 19 MiB/s, 346 objects/s recovering 2023-10-09T10:17:19.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:19 smithi012 ceph-mon[152263]: failed to encode map e382 with expected crc 2023-10-09T10:17:19.534 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:19.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:19.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:19.642 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:19.656 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:17:19.656 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:17:19.656 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:17:19.656 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:17:19.656 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:17:19.675 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:19 smithi012 ceph-mon[147926]: pgmap v1069: 109 pgs: 1 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.62k op/s; 30990/205104 objects degraded (15.109%); 3654/205104 objects misplaced (1.782%); 19 MiB/s, 346 objects/s recovering 2023-10-09T10:17:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:19 smithi012 ceph-mon[147926]: failed to encode map e382 with expected crc 2023-10-09T10:17:19.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:19 smithi110 ceph-mon[136032]: pgmap v1069: 109 pgs: 1 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.62k op/s; 30990/205104 objects degraded (15.109%); 3654/205104 objects misplaced (1.782%); 19 MiB/s, 346 objects/s recovering 2023-10-09T10:17:19.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:19 smithi110 ceph-mon[136032]: failed to encode map e382 with expected crc 2023-10-09T10:17:19.713 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:19.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:19.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:19.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:19.848 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:20.050 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:20.055 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:20.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:20.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:17:20.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:20.199 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:20.249 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:20.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:20.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:20.356 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:20.405 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:20.502 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:20.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:20.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:20.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:20 smithi012 ceph-mon[147926]: 9.1 scrub starts 2023-10-09T10:17:20.581 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:20 smithi012 ceph-mon[152263]: 9.1 scrub starts 2023-10-09T10:17:20.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:20.651 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:20.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:20 smithi110 ceph-mon[136032]: 9.1 scrub starts 2023-10-09T10:17:20.694 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:20.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:20.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:20.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:20.844 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:21.091 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:21.095 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:21.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:21.197 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:21.231 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:21.278 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:21.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:21.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:21.316 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:17:21.348 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:17:21.377 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:17:21.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:21.408 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:17:21.418 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:21.431 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:17:21.454 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:17:21.469 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:21.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:21.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:21.483 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:17:21.510 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:17:21.512 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:17:21.512 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:17:21.512 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:17:21.512 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:17:21.512 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:17:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:17:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:17:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:17:21.575 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:21.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:17:21.602 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:21 smithi110 ceph-mon[136032]: pgmap v1070: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 1.28k op/s; 30920/204444 objects degraded (15.124%); 3642/204444 objects misplaced (1.781%); 15 MiB/s, 274 objects/s recovering 2023-10-09T10:17:21.602 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:21 smithi110 ceph-mon[136032]: 9.21 scrub starts 2023-10-09T10:17:21.602 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1799176004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:21.605 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:21.645 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:21.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:21.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:21 smithi012 ceph-mon[147926]: pgmap v1070: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 1.28k op/s; 30920/204444 objects degraded (15.124%); 3642/204444 objects misplaced (1.781%); 15 MiB/s, 274 objects/s recovering 2023-10-09T10:17:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:21 smithi012 ceph-mon[147926]: 9.21 scrub starts 2023-10-09T10:17:21.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1799176004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:21.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:21 smithi012 ceph-mon[152263]: pgmap v1070: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 1.28k op/s; 30920/204444 objects degraded (15.124%); 3642/204444 objects misplaced (1.781%); 15 MiB/s, 274 objects/s recovering 2023-10-09T10:17:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:21 smithi012 ceph-mon[152263]: 9.21 scrub starts 2023-10-09T10:17:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1799176004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:21.748 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:21.777 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:21.822 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:21.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:21.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:21.976 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:22.020 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:22.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:22.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:22.158 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:22.202 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:22.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:22.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:22.338 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:22.388 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:22.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:22.496 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:22.535 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:22.583 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:22.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:22 smithi012 ceph-mon[147926]: 9.11 deep-scrub starts 2023-10-09T10:17:22.586 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:22 smithi012 ceph-mon[152263]: 9.11 deep-scrub starts 2023-10-09T10:17:22.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:22.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:22.688 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:22.692 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:22.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:22 smithi110 ceph-mon[136032]: 9.11 deep-scrub starts 2023-10-09T10:17:22.720 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:22.738 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:22.738 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:22.758 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:22.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:22.868 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:22.908 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:23.253 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:23.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:23.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:23.358 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:23.400 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:23.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:17:23.502 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:23.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:23.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:23.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:23 smithi012 ceph-mon[147926]: pgmap v1071: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 1.10k op/s; 30890/203997 objects degraded (15.142%); 2654/203997 objects misplaced (1.301%); 17 MiB/s, 276 objects/s recovering 2023-10-09T10:17:23.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 30890/203997 objects degraded (15.142%), 22 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:23.508 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:23 smithi012 ceph-mon[152263]: pgmap v1071: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 1.10k op/s; 30890/203997 objects degraded (15.142%); 2654/203997 objects misplaced (1.301%); 17 MiB/s, 276 objects/s recovering 2023-10-09T10:17:23.509 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 30890/203997 objects degraded (15.142%), 22 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:23.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:23.648 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:23 smithi110 ceph-mon[136032]: pgmap v1071: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 1.10k op/s; 30890/203997 objects degraded (15.142%); 2654/203997 objects misplaced (1.301%); 17 MiB/s, 276 objects/s recovering 2023-10-09T10:17:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 30890/203997 objects degraded (15.142%), 22 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:23.712 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:23.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:23.817 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:23.852 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:23.897 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:23.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:23.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:24.002 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:24.040 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:24.087 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:24.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:24.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:24.199 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:24.226 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:24.270 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:24.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:24.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:24.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:24.394 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:17:24.412 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:24.468 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:17:24.476 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:24.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:24.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:24.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:24.626 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:24.626 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:17:24.626 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:17:24.626 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T10:17:24.626 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:17:24.626 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:17:24.627 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T10:17:24.664 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:24 smithi110 ceph-mon[136032]: 9.1 scrub starts 2023-10-09T10:17:24.664 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:24 smithi110 ceph-mon[136032]: pgmap v1072: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 1.06k op/s; 30890/202461 objects degraded (15.257%); 2654/202461 objects misplaced (1.311%); 16 MiB/s, 253 objects/s recovering 2023-10-09T10:17:24.664 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/923564769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:24.664 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:24 smithi012 ceph-mon[147926]: 9.1 scrub starts 2023-10-09T10:17:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:24 smithi012 ceph-mon[147926]: pgmap v1072: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 1.06k op/s; 30890/202461 objects degraded (15.257%); 2654/202461 objects misplaced (1.311%); 16 MiB/s, 253 objects/s recovering 2023-10-09T10:17:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/923564769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:24.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:24 smithi012 ceph-mon[152263]: 9.1 scrub starts 2023-10-09T10:17:24.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:24 smithi012 ceph-mon[152263]: pgmap v1072: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 1.06k op/s; 30890/202461 objects degraded (15.257%); 2654/202461 objects misplaced (1.311%); 16 MiB/s, 253 objects/s recovering 2023-10-09T10:17:24.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/923564769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:24.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:24.877 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:24.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:24.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:24.983 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:25.016 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:25.059 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:25.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:25.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:25.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:25.197 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:25.326 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:25.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:25.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:25.342 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:25.430 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:25.449 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[147926]: 9.21 scrub starts 2023-10-09T10:17:25.449 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:25.449 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[147926]: failed to encode map e383 with expected crc 2023-10-09T10:17:25.449 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:17:25.449 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[147926]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T10:17:25.449 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[147926]: failed to encode map e383 with expected crc 2023-10-09T10:17:25.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[152263]: 9.21 scrub starts 2023-10-09T10:17:25.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:25.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[152263]: failed to encode map e383 with expected crc 2023-10-09T10:17:25.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:17:25.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[152263]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T10:17:25.450 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:25 smithi012 ceph-mon[152263]: failed to encode map e383 with expected crc 2023-10-09T10:17:25.459 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:25.578 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:25.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:25.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:25.665 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:25.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:25.728 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:25.769 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:25.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:25.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:25.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:25.917 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:25.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:25 smithi110 ceph-mon[136032]: 9.21 scrub starts 2023-10-09T10:17:25.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:25 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:25.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:25 smithi110 ceph-mon[136032]: failed to encode map e383 with expected crc 2023-10-09T10:17:25.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:17:25.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:25 smithi110 ceph-mon[136032]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T10:17:25.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:25 smithi110 ceph-mon[136032]: failed to encode map e383 with expected crc 2023-10-09T10:17:26.010 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:26.011 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:26.027 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:26.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:26.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:26.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:26.164 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:26.483 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:26.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:26.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:26.588 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:26.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:26.658 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:26 smithi012 ceph-mon[147926]: pgmap v1074: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 30473/197886 objects degraded (15.399%); 2029/197886 objects misplaced (1.025%); 15 MiB/s, 248 objects/s recovering 2023-10-09T10:17:26.658 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:26 smithi012 ceph-mon[152263]: pgmap v1074: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 30473/197886 objects degraded (15.399%); 2029/197886 objects misplaced (1.025%); 15 MiB/s, 248 objects/s recovering 2023-10-09T10:17:26.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:17:26.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:26.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:26.770 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:26.799 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:26 smithi110 ceph-mon[136032]: pgmap v1074: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 30473/197886 objects degraded (15.399%); 2029/197886 objects misplaced (1.025%); 15 MiB/s, 248 objects/s recovering 2023-10-09T10:17:26.802 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:27.153 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:17:27.173 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:17:27.200 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:17:27.232 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:17:27.255 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:17:27.274 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:17:27.293 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:17:27.314 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:17:27.315 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:17:27.315 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:17:27.316 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:17:27.316 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:17:27.317 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:17:27.317 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:17:27.317 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:17:27.317 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:17:27.328 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:27.342 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:27.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:27.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:27.494 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:27.537 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:27.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:27.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:27.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2559195946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:27.610 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2559195946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:27.643 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:27.644 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2559195946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:27.677 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:27.702 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:17:27.775 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:27.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:27.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:27.914 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:28.128 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:28.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:28.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:17:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:17:27] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-10-09T10:17:28.234 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:28.273 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:28.337 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:28.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:28.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:28.467 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:28 smithi012 ceph-mon[152263]: pgmap v1075: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 30473/195807 objects degraded (15.563%); 2029/195807 objects misplaced (1.036%); 12 MiB/s, 202 objects/s recovering 2023-10-09T10:17:28.467 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 30473/197886 objects degraded (15.399%), 22 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:28.469 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:28 smithi012 ceph-mon[147926]: pgmap v1075: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 30473/195807 objects degraded (15.563%); 2029/195807 objects misplaced (1.036%); 12 MiB/s, 202 objects/s recovering 2023-10-09T10:17:28.469 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 30473/197886 objects degraded (15.399%), 22 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:28.485 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:28.527 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:28.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:28.556 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:17:28.557 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-10-09T10:17:28.557 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:17:28.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:28.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:28 smithi110 ceph-mon[136032]: pgmap v1075: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 30473/195807 objects degraded (15.563%); 2029/195807 objects misplaced (1.036%); 12 MiB/s, 202 objects/s recovering 2023-10-09T10:17:28.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 30473/197886 objects degraded (15.399%), 22 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:28.670 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:28.686 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:28.701 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:17:28.701 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:17:28.701 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:17:28.701 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:17:28.702 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:17:28.716 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:28.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:28.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:28.825 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:28.859 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:28.899 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:28.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:29.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:29.036 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:29.229 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:29.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:29.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:29.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:17:29.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:29.368 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:29.585 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:29.588 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:29.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:29.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:29.590 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:29.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:29.730 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:29.736 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:29.789 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:29.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:29.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:29.895 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:29.937 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:29.979 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:29.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:30.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:17:30.084 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:30.147 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:30.489 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:30.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:30.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:30.547 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:30.592 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:30.639 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:30.684 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:30.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:30.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:30.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:30.823 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:30 smithi012 ceph-mon[147926]: pgmap v1076: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 30152/192849 objects degraded (15.635%); 1969/192849 objects misplaced (1.021%); 14 MiB/s, 223 objects/s recovering 2023-10-09T10:17:30.824 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:30 smithi012 ceph-mon[152263]: pgmap v1076: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 30152/192849 objects degraded (15.635%); 1969/192849 objects misplaced (1.021%); 14 MiB/s, 223 objects/s recovering 2023-10-09T10:17:30.827 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:30.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:30 smithi110 ceph-mon[136032]: pgmap v1076: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 30152/192849 objects degraded (15.635%); 1969/192849 objects misplaced (1.021%); 14 MiB/s, 223 objects/s recovering 2023-10-09T10:17:31.398 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:31.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:31.488 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:17:31.505 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:31.542 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:31.589 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:31.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:31.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:31 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/227750741' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:31 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:31.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:31 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/227750741' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:31 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:31.696 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:31.731 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:31 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/227750741' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:31.732 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:31 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:31.733 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:31.778 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:31.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:31.884 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:31.923 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:31.970 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:31.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:31.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:32.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:32.115 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:32.156 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:32.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:32.263 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:32.296 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:32.339 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:32.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:32.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:32.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:32.493 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:32.498 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:32.545 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:32.548 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:32 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:32.548 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:32 smithi110 ceph-mon[136032]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.548 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:32 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:32.548 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:32 smithi110 ceph-mon[136032]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.548 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:32 smithi110 ceph-mon[136032]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T10:17:32.548 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:32 smithi110 ceph-mon[136032]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.549 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:32 smithi110 ceph-mon[136032]: pgmap v1078: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 30089/191745 objects degraded (15.692%); 1969/191745 objects misplaced (1.027%); 9.6 MiB/s, 174 objects/s recovering 2023-10-09T10:17:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:32.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:32.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:32.681 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:32.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:32.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[147926]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:32.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[147926]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[147926]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T10:17:32.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[147926]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[147926]: pgmap v1078: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 30089/191745 objects degraded (15.692%); 1969/191745 objects misplaced (1.027%); 9.6 MiB/s, 174 objects/s recovering 2023-10-09T10:17:32.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:32.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[152263]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:32.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[152263]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[152263]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T10:17:32.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[152263]: failed to encode map e384 with expected crc 2023-10-09T10:17:32.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:32 smithi012 ceph-mon[152263]: pgmap v1078: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 30089/191745 objects degraded (15.692%); 1969/191745 objects misplaced (1.027%); 9.6 MiB/s, 174 objects/s recovering 2023-10-09T10:17:33.003 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:33.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:33.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:33.147 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:33.194 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:33.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:33.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:33.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:33.340 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:33.591 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:33.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:33.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:33.698 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:33.739 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:33.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[147926]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T10:17:33.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[147926]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[147926]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[147926]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[147926]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[147926]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[147926]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 30089/191745 objects degraded (15.692%), 22 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[152263]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[152263]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[152263]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[152263]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[152263]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.774 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[152263]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[152263]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.775 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 30089/191745 objects degraded (15.692%), 22 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:33.782 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:33.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:33 smithi110 ceph-mon[136032]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T10:17:33.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:33 smithi110 ceph-mon[136032]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:33 smithi110 ceph-mon[136032]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:33 smithi110 ceph-mon[136032]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:33 smithi110 ceph-mon[136032]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:33 smithi110 ceph-mon[136032]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:33 smithi110 ceph-mon[136032]: failed to encode map e385 with expected crc 2023-10-09T10:17:33.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 30089/191745 objects degraded (15.692%), 22 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:33.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:33.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:33.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:33.920 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:33.929 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:17:33.929 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:17:33.929 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:17:33.929 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.959725 2023-10-09T10:17:33.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0.9597252137014707 2023-10-09T10:17:33.961 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:33.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:33.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:34.026 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:17:34.026 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114935 9.f 9:f4378248:::benchmark_data_smithi012_313365_object2460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 139.85989611799999, 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2871666070000001, 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.027 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114935 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162479+0000", 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162485+0000", 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3759999999999996e-06 2023-10-09T10:17:34.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162497+0000", 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162503+0000", 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9770000000000003e-06 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174344+0000", 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01184062 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174510+0000", 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000166026 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174655+0000", 2023-10-09T10:17:34.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000144563 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.181694+0000", 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007039694 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.214204+0000", 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.03250981 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.222717+0000", 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085130050000000006 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.222730+0000", 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3229e-05 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196525+0000", 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97379490400000002 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196531+0000", 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.879e-06 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.428687+0000", 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.232155509 2023-10-09T10:17:34.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.428882+0000", 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019557999999999999 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.449518+0000", 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020635767999999999 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.449647+0000", 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000129014 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114936 9.a 9:58a6faa8:::benchmark_data_smithi012_313365_object2461:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 139.85986540600001, 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.092944401, 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.036 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114936 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162513+0000", 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1610000000000001e-06 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162518+0000", 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7310000000000001e-06 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162522+0000", 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9920000000000001e-06 2023-10-09T10:17:34.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164493+0000", 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019717620000000002 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164686+0000", 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019211 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:17:34.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164874+0000", 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018879699999999999 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.168656+0000", 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037811670000000002 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.169482+0000", 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082636199999999997 2023-10-09T10:17:34.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.212142+0000", 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042660245999999999 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.212158+0000", 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5977999999999998e-05 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.195199+0000", 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98304075999999996 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.195213+0000", 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4025e-05 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236218+0000", 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0410054230000001 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236372+0000", 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015330899999999999 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255344+0000", 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018972025 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255456+0000", 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111928 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.045 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114940 9.f 9:f2b30a6a:::benchmark_data_smithi012_313365_object2465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 139.848760556, 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2912956169999998, 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114940 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.046 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173614+0000", 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173632+0000", 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8349999999999999e-05 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173651+0000", 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8372e-05 2023-10-09T10:17:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173660+0000", 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0429999999999996e-06 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.177952+0000", 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042923120000000004 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.178136+0000", 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018356399999999999 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.178304+0000", 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016803400000000001 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.181873+0000", 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035693090000000001 2023-10-09T10:17:34.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.214244+0000", 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032370566000000003 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223012+0000", 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087685690000000004 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223019+0000", 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196547+0000", 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97352829299999999 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196551+0000", 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9949999999999999e-06 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.434365+0000", 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2378136799999999 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.434545+0000", 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000180044 2023-10-09T10:17:34.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.464846+0000", 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030301600000000001 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.464912+0000", 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5380000000000001e-05 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.055 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114950 9.f 9:f115a5aa:::benchmark_data_smithi012_313365_object2475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:17:34.055 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:17:34.057 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 139.798749265, 2023-10-09T10:17:34.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2705686389999999, 2023-10-09T10:17:34.057 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.057 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.057 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.057 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114950 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223625+0000", 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223632+0000", 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8639999999999998e-06 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223644+0000", 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1824000000000001e-05 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223651+0000", 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5499999999999997e-06 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223694+0000", 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2765999999999997e-05 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223699+0000", 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2569999999999998e-06 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196570+0000", 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97287036000000005 2023-10-09T10:17:34.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196576+0000", 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5710000000000003e-06 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.439509+0000", 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.242932758 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.440273+0000", 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00076375 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474247+0000", 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033973640999999999 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474363+0000", 2023-10-09T10:17:34.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116765 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474458+0000", 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4829999999999998e-05 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.479012+0000", 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045542890000000004 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.481102+0000", 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020900659999999998 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.489625+0000", 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085221180000000004 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.489680+0000", 2023-10-09T10:17:34.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5405999999999998e-05 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.494196+0000", 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045159930000000003 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115069 9.10 9:0c166afc:::benchmark_data_smithi012_313365_object2594:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:17:34.066 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.47672944199999, 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5496961840000001, 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115069 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545646+0000", 2023-10-09T10:17:34.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545653+0000", 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5399999999999998e-06 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545670+0000", 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7144000000000001e-05 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545679+0000", 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5830000000000007e-06 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545730+0000", 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1313000000000002e-05 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546723+0000", 2023-10-09T10:17:34.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099291799999999997 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079001+0000", 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5322773119999997 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079100+0000", 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9010000000000002e-05 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079372+0000", 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000272782 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.080850+0000", 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014771509999999999 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.082687+0000", 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001837336 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.082730+0000", 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3612999999999997e-05 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093025+0000", 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010295011 2023-10-09T10:17:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093094+0000", 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8725000000000004e-05 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095343+0000", 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022491519999999999 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.074 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115106 9.10 9:0b40069f:::benchmark_data_smithi012_313365_object2631:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 137.33854473700001, 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4133900090000004, 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.075 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115106 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683831+0000", 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683836+0000", 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0440000000000003e-06 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683846+0000", 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9250000000000007e-06 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683854+0000", 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7959999999999996e-06 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683942+0000", 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7602000000000006e-05 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.684891+0000", 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094994899999999995 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089254+0000", 2023-10-09T10:17:34.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4043629649999998 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089396+0000", 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141886 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089758+0000", 2023-10-09T10:17:34.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036181000000000001 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.091841+0000", 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020825930000000002 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095413+0000", 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035718400000000002 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095477+0000", 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4685999999999997e-05 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095503+0000", 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5575000000000002e-05 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095538+0000", 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5116000000000001e-05 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.097222+0000", 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001683941 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122337 9.10 9:0bd6554e:::benchmark_data_smithi012_313365_object9862:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 103.125397302, 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6698403750000002, 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122337 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.084 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896978+0000", 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896984+0000", 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7479999999999999e-06 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896996+0000", 2023-10-09T10:17:34.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.199e-05 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897001+0000", 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8189999999999997e-06 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897056+0000", 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4818999999999998e-05 2023-10-09T10:17:34.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897145+0000", 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8943000000000005e-05 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.517542+0000", 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6203965519999999 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.517672+0000", 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013004499999999999 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.518031+0000", 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035930199999999998 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.536908+0000", 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018877416000000001 2023-10-09T10:17:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.536918+0000", 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9369999999999999e-06 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.537422+0000", 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00050323499999999999 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564538+0000", 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027116735999999999 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564585+0000", 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6592999999999999e-05 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.566820+0000", 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022347140000000001 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122523 9.10 9:0d3d12ba:::benchmark_data_smithi012_313365_object10048:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 102.58004801, 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7874849670000001, 2023-10-09T10:17:34.092 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122523 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:17:34.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442327+0000", 2023-10-09T10:17:34.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442334+0000", 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.221e-06 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442347+0000", 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2809000000000001e-05 2023-10-09T10:17:34.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442355+0000", 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0760000000000003e-06 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442400+0000", 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4570000000000002e-05 2023-10-09T10:17:34.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442467+0000", 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7063999999999994e-05 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194401+0000", 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7519345159999999 2023-10-09T10:17:34.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194518+0000", 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116872 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194929+0000", 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041051599999999998 2023-10-09T10:17:34.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.211495+0000", 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016566498999999998 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.211544+0000", 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9190999999999997e-05 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.214081+0000", 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025366910000000002 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.227445+0000", 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013363808 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.227497+0000", 2023-10-09T10:17:34.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1860000000000002e-05 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.229814+0000", 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023168160000000002 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122561 9.10 9:092dbf94:::benchmark_data_smithi012_313365_object10086:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.101 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 102.480938939, 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1569061710000001, 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122561 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541437+0000", 2023-10-09T10:17:34.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541441+0000", 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1919999999999998e-06 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541449+0000", 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.746e-06 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541454+0000", 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4639999999999997e-06 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541494+0000", 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9505999999999997e-05 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541543+0000", 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9872e-05 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.660487+0000", 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1189437330000001 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.660622+0000", 2023-10-09T10:17:34.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135024 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.661025+0000", 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040268299999999998 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.668459+0000", 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074342699999999998 2023-10-09T10:17:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.668490+0000", 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1279999999999999e-05 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683619+0000", 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015128694999999999 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.696066+0000", 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012446472 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.696109+0000", 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3887999999999997e-05 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698344+0000", 2023-10-09T10:17:34.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022343480000000001 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122689 9.10 9:0a3555ea:::benchmark_data_smithi012_313365_object10214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 102.06752142800001, 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7453082640000002, 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122689 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954853+0000", 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954862+0000", 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2959999999999999e-06 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954874+0000", 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2072000000000001e-05 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954882+0000", 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1030000000000002e-06 2023-10-09T10:17:34.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954938+0000", 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6165999999999997e-05 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.955019+0000", 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1001000000000006e-05 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.666823+0000", 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7118038929999999 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.666922+0000", 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8803999999999998e-05 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.667247+0000", 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032501299999999997 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683704+0000", 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016457423999999998 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683804+0000", 2023-10-09T10:17:34.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100046 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683814+0000", 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0083e-05 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698412+0000", 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014598266 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698458+0000", 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5503999999999997e-05 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.700163+0000", 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001705441 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122759 9.10 9:0b678b9b:::benchmark_data_smithi012_313365_object10284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.89786958800001, 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.605785577, 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122759 2023-10-09T10:17:34.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124506+0000", 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124513+0000", 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124528+0000", 2023-10-09T10:17:34.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4893e-05 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124536+0000", 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1319999999999994e-06 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124601+0000", 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5038999999999999e-05 2023-10-09T10:17:34.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124689+0000", 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7894999999999998e-05 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704065+0000", 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5793759000000001 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704190+0000", 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012584799999999999 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704594+0000", 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040309699999999999 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.727961+0000", 2023-10-09T10:17:34.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023367024 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728012+0000", 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1174000000000002e-05 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728024+0000", 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1966e-05 2023-10-09T10:17:34.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728058+0000", 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3911000000000001e-05 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728098+0000", 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0286e-05 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.730293+0000", 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021946410000000002 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122850 9.10 9:0ca7b949:::benchmark_data_smithi012_313365_object10375:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:52.368637+0000", 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.653739706, 2023-10-09T10:17:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.402450548, 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122850 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368637+0000", 2023-10-09T10:17:34.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368637+0000", 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368636+0000", 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368644+0000", 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0160000000000008e-06 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368661+0000", 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7278999999999998e-05 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368670+0000", 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6900000000000004e-06 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368716+0000", 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5191999999999997e-05 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.368770+0000", 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4789e-05 2023-10-09T10:17:34.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.726225+0000", 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3574544230000001 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.726304+0000", 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9400000000000006e-05 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.726690+0000", 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000386024 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.736145+0000", 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0094551549999999998 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.736206+0000", 2023-10-09T10:17:34.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0946999999999998e-05 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.744450+0000", 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0082431880000000003 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.769110+0000", 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024660049 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.769187+0000", 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7342999999999995e-05 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.771087+0000", 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001900451 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124366 9.10 9:0b423b51:::benchmark_data_smithi012_313365_object11891:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:56.460974+0000", 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 97.561402322999996, 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.401633814, 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124366 2023-10-09T10:17:34.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.460974+0000", 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.460974+0000", 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.460973+0000", 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.460979+0000", 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.37e-06 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.460996+0000", 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6481e-05 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.461004+0000", 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9519999999999994e-06 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.461059+0000", 2023-10-09T10:17:34.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5037999999999997e-05 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.461140+0000", 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1013000000000005e-05 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.737425+0000", 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2762851710000001 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.737568+0000", 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000142931 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.738027+0000", 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000459517 2023-10-09T10:17:34.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.739326+0000", 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012987249999999999 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.741695+0000", 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002368387 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.742766+0000", 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010717910000000001 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.860410+0000", 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11764311299999999 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.860475+0000", 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5828999999999996e-05 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.862608+0000", 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021327619999999999 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.144 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124410 9.10 9:0a0e0dea:::benchmark_data_smithi012_313365_object11935:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 97.319422603999996, 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5937443060000001, 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124410 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.145 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702952+0000", 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702960+0000", 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6920000000000002e-06 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702975+0000", 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4969e-05 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702983+0000", 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7510000000000008e-06 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.707988+0000", 2023-10-09T10:17:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050051440000000004 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.708039+0000", 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0998999999999997e-05 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.196596+0000", 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.488557455 2023-10-09T10:17:34.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.196709+0000", 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000113075 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.197066+0000", 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035631299999999997 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.198754+0000", 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016883169999999999 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.216476+0000", 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017721654 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.216545+0000", 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8888999999999997e-05 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.294372+0000", 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.077827269000000004 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.294431+0000", 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9001e-05 2023-10-09T10:17:34.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.296698+0000", 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00226742 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124495 9.10 9:0df9feaa:::benchmark_data_smithi012_313365_object12020:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 97.019933069000004, 2023-10-09T10:17:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6452444429999999, 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124495 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:17:34.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002442+0000", 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002451+0000", 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6459999999999999e-06 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002465+0000", 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4531e-05 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002478+0000", 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3091999999999999e-05 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002543+0000", 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5003000000000001e-05 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002642+0000", 2023-10-09T10:17:34.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8486000000000005e-05 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604276+0000", 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.601633879 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604435+0000", 2023-10-09T10:17:34.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:34.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000159295 2023-10-09T10:17:34.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604865+0000", 2023-10-09T10:17:34.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043062800000000001 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613177+0000", 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0083113730000000004 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613227+0000", 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9805000000000002e-05 2023-10-09T10:17:34.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613271+0000", 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4691999999999998e-05 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645599+0000", 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032327910000000001 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645659+0000", 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9821000000000002e-05 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647688+0000", 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020288989999999998 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124499 9.10 9:0914df8f:::benchmark_data_smithi012_313365_object12024:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 97.003499286999997, 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6316283970000001, 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124499 2023-10-09T10:17:34.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018876+0000", 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018883+0000", 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2869999999999999e-06 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018893+0000", 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0190999999999999e-05 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018900+0000", 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2099999999999996e-06 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018964+0000", 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.355e-05 2023-10-09T10:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.019054+0000", 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0099999999999995e-05 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611351+0000", 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5922972340000001 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611460+0000", 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108608 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611836+0000", 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037601000000000003 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.621012+0000", 2023-10-09T10:17:34.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009175753 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645527+0000", 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024515678999999999 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645566+0000", 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8294000000000003e-05 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647904+0000", 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023386240000000001 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647967+0000", 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2964000000000002e-05 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.650506+0000", 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025385540000000002 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124527 9.10 9:0cb84cd4:::benchmark_data_smithi012_313365_object12052:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 96.915965744000005, 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5734285209999999, 2023-10-09T10:17:34.172 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124527 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106409+0000", 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106417+0000", 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2060000000000001e-06 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106431+0000", 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4693e-05 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106440+0000", 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4649999999999994e-06 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106537+0000", 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7021000000000005e-05 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106637+0000", 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010021799999999999 2023-10-09T10:17:34.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.617803+0000", 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5111660599999999 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.617924+0000", 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012099600000000001 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.177 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.618302+0000", 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037780300000000001 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643413+0000", 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025111018999999998 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647753+0000", 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043399199999999997 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647820+0000", 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7353000000000004e-05 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.677556+0000", 2023-10-09T10:17:34.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029735927999999998 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.677618+0000", 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2069000000000005e-05 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.679839+0000", 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022211359999999999 2023-10-09T10:17:34.180 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124541 9.10 9:0fc113d5:::benchmark_data_smithi012_313365_object12066:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 96.818003207999993, 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.506778945, 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.181 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124541 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204372+0000", 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204381+0000", 2023-10-09T10:17:34.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9970000000000004e-06 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204402+0000", 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0761999999999999e-05 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204412+0000", 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0368e-05 2023-10-09T10:17:34.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204473+0000", 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1490999999999998e-05 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204558+0000", 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4437999999999994e-05 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643445+0000", 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4388867670000001 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643565+0000", 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120537 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.186 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643947+0000", 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038192800000000002 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647844+0000", 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038965330000000002 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.652834+0000", 2023-10-09T10:17:34.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049904190000000003 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.652889+0000", 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4759000000000002e-05 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.708684+0000", 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.05579493 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.708750+0000", 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6202000000000001e-05 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.711152+0000", 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024023360000000001 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128019 9.a 9:56d43b6a:::benchmark_data_smithi012_313365_object15544:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 86.369087226999994, 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.955188565, 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128019 2023-10-09T10:17:34.190 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653288+0000", 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653296+0000", 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.79e-06 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653308+0000", 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2376e-05 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653323+0000", 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4844000000000001e-05 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653379+0000", 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6054999999999998e-05 2023-10-09T10:17:34.193 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653595+0000", 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021573 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653860+0000", 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026534600000000002 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.696431+0000", 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042570967000000001 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.696443+0000", 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1739000000000001e-05 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.323819+0000", 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.62737651100000003 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.323833+0000", 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3869000000000001e-05 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.494433+0000", 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1706003599999999 2023-10-09T10:17:34.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.494639+0000", 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020562600000000001 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.608361+0000", 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.113721818 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.608478+0000", 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011719500000000001 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128025 9.f 9:f78d4616:::benchmark_data_smithi012_313365_object15550:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 86.206880106, 2023-10-09T10:17:34.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.792284043, 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128025 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:17:34.199 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815495+0000", 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.200 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815501+0000", 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815513+0000", 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1770000000000001e-05 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:17:34.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815520+0000", 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1010000000000003e-06 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815583+0000", 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2779000000000003e-05 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815595+0000", 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2153999999999999e-05 2023-10-09T10:17:34.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.227751+0000", 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.41215553100000002 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.227764+0000", 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3131e-05 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.482847+0000", 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.255083368 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.483568+0000", 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00072077100000000004 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:17:34.204 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.582946+0000", 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.099378280999999999 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.583074+0000", 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012746599999999999 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.583609+0000", 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00053539899999999999 2023-10-09T10:17:34.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.595736+0000", 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012126994 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.599249+0000", 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035130600000000001 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.206 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.602352+0000", 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031027540000000001 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.603699+0000", 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001347284 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.603770+0000", 2023-10-09T10:17:34.207 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1014999999999997e-05 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.607781+0000", 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004010319 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:17:34.208 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:17:34.261 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:34.301 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:17:34.311 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:34.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:34.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:34.327 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:17:34.347 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:17:34.372 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:17:34.398 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:17:34.417 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:34.428 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:17:34.453 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:34.459 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:17:34.484 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:17:34.485 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:17:34.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:17:34.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:17:34.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:17:34.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:17:34.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:17:34.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:17:34.486 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:17:34.507 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:34.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:34.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:34.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[147926]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[147926]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.583 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[147926]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T10:17:34.583 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[147926]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.583 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[147926]: pgmap v1081: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 30089/189597 objects degraded (15.870%); 1969/189597 objects misplaced (1.039%); 3.4 MiB/s, 73 objects/s recovering 2023-10-09T10:17:34.583 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2075796768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:34.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[152263]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[152263]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[152263]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T10:17:34.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[152263]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[152263]: pgmap v1081: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 30089/189597 objects degraded (15.870%); 1969/189597 objects misplaced (1.039%); 3.4 MiB/s, 73 objects/s recovering 2023-10-09T10:17:34.585 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:34 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2075796768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:34.586 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:17:34.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:34.679 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:34.727 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:34.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:34.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:34.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:34.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:34 smithi110 ceph-mon[136032]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.863 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:34 smithi110 ceph-mon[136032]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.864 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:34 smithi110 ceph-mon[136032]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T10:17:34.864 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:34 smithi110 ceph-mon[136032]: failed to encode map e386 with expected crc 2023-10-09T10:17:34.864 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:34 smithi110 ceph-mon[136032]: pgmap v1081: 109 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 30089/189597 objects degraded (15.870%); 1969/189597 objects misplaced (1.039%); 3.4 MiB/s, 73 objects/s recovering 2023-10-09T10:17:34.864 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:34 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2075796768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:34.866 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:34.873 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:34.913 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:34.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:34.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:17:35.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:35.058 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:35.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:35.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:35.198 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:35.236 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:35.308 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:35.325 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:35.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:35.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:35.468 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:35.681 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:35.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:35.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:35.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:35.794 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:35.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:36.079 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:36.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:36.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:36.220 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:36.578 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:36.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:36.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:36.590 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:36.590 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:36.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:36 smithi012 ceph-mon[147926]: pgmap v1082: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 27207/184689 objects degraded (14.731%); 3709/184689 objects misplaced (2.008%); 6.3 MiB/s, 126 objects/s recovering 2023-10-09T10:17:36.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:36 smithi012 ceph-mon[152263]: pgmap v1082: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 27207/184689 objects degraded (14.731%); 3709/184689 objects misplaced (2.008%); 6.3 MiB/s, 126 objects/s recovering 2023-10-09T10:17:36.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/406447081' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]: dispatch 2023-10-09T10:17:36.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]: dispatch 2023-10-09T10:17:36.673 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.4 primary-affinity to 0.959725 (8628962) 2023-10-09T10:17:36.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:36.722 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:36.924 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:36.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:36 smithi110 ceph-mon[136032]: pgmap v1082: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 27207/184689 objects degraded (14.731%); 3709/184689 objects misplaced (2.008%); 6.3 MiB/s, 126 objects/s recovering 2023-10-09T10:17:36.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/406447081' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]: dispatch 2023-10-09T10:17:36.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]: dispatch 2023-10-09T10:17:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/406447081' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]: dispatch 2023-10-09T10:17:36.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]: dispatch 2023-10-09T10:17:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:36.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:36.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:17:36.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:17:37.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:37.066 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:37.223 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:37.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:37.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:37.360 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:37.412 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:37.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:37.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:37.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:37.546 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:37.582 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:37.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:37.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:37.617 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:37.621 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:37.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[147926]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]': finished 2023-10-09T10:17:37.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[147926]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T10:17:37.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[147926]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[147926]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/981928015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:37.660 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:37.674 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:17:37.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:37.734 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:37.915 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:17:37] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-10-09T10:17:37.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[152263]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]': finished 2023-10-09T10:17:37.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[152263]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T10:17:37.916 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[152263]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.916 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[152263]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.916 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/981928015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:37.916 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:37.916 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:37 smithi110 ceph-mon[136032]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.916 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9597252137014707}]': finished 2023-10-09T10:17:37.916 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:37 smithi110 ceph-mon[136032]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T10:17:37.917 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:37 smithi110 ceph-mon[136032]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.917 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:37 smithi110 ceph-mon[136032]: failed to encode map e387 with expected crc 2023-10-09T10:17:37.917 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/981928015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:37.917 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:17:38.093 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:38.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:38.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:38.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:38.245 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:38.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:38.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:38.581 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:38.612 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:38.650 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:38.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:38.744 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: pgmap v1084: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 27207/183123 objects degraded (14.857%); 3709/183123 objects misplaced (2.025%); 5.6 MiB/s, 96 objects/s recovering 2023-10-09T10:17:38.744 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:38.744 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.744 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.744 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.744 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.744 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:17:38.744 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T10:17:38.745 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 27207/183123 objects degraded (14.857%), 22 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:38.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:38.784 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:38.870 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:38.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:38.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: pgmap v1084: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 27207/183123 objects degraded (14.857%); 3709/183123 objects misplaced (2.025%); 5.6 MiB/s, 96 objects/s recovering 2023-10-09T10:17:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:17:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T10:17:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 27207/183123 objects degraded (14.857%), 22 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:38.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: pgmap v1084: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 27207/183123 objects degraded (14.857%); 3709/183123 objects misplaced (2.025%); 5.6 MiB/s, 96 objects/s recovering 2023-10-09T10:17:38.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:38.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: failed to encode map e388 with expected crc 2023-10-09T10:17:38.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:17:38.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T10:17:38.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 27207/183123 objects degraded (14.857%), 22 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:38.966 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:38.966 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:38.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:39.064 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:39.383 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:39.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:39.474 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:17:39.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:39.526 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:39.576 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:17:39.578 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:39.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:39.606 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:17:39.631 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:17:39.655 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:17:39.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:39 smithi110 ceph-mon[136032]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.675 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:39 smithi110 ceph-mon[136032]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T10:17:39.676 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:39 smithi110 ceph-mon[136032]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.676 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:39 smithi110 ceph-mon[136032]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.676 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:39 smithi110 ceph-mon[136032]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.676 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2500331095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:39.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:39.685 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:17:39.713 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:17:39.728 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:39.743 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:17:39.772 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:39.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:39.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:39.780 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:17:39.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:17:39.782 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:17:39.783 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:17:39.783 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:17:39.783 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:17:39.783 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:17:39.783 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:17:39.783 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:17:39.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:39.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[152263]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[152263]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T10:17:39.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[152263]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[152263]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[152263]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.911 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2500331095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:39.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[147926]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[147926]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T10:17:39.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[147926]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[147926]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[147926]: failed to encode map e389 with expected crc 2023-10-09T10:17:39.912 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2500331095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:39.915 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:40.109 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:40.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:40.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:40.208 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:40.214 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:40.251 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:40.350 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:40.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:40.457 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:40.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:40.536 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:40.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:40.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:40.641 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:40.675 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:40.695 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:17:40.729 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:40.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:40.761 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:40 smithi110 ceph-mon[136032]: pgmap v1087: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 26937/180252 objects degraded (14.944%); 3191/180252 objects misplaced (1.770%); 13 MiB/s, 210 objects/s recovering 2023-10-09T10:17:40.761 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:40 smithi110 ceph-mon[136032]: 9.11 scrub starts 2023-10-09T10:17:40.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:40.868 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:40.914 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:40.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:40.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:40 smithi012 ceph-mon[147926]: pgmap v1087: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 26937/180252 objects degraded (14.944%); 3191/180252 objects misplaced (1.770%); 13 MiB/s, 210 objects/s recovering 2023-10-09T10:17:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:40 smithi012 ceph-mon[147926]: 9.11 scrub starts 2023-10-09T10:17:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:40 smithi012 ceph-mon[152263]: pgmap v1087: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 26937/180252 objects degraded (14.944%); 3191/180252 objects misplaced (1.770%); 13 MiB/s, 210 objects/s recovering 2023-10-09T10:17:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:40 smithi012 ceph-mon[152263]: 9.11 scrub starts 2023-10-09T10:17:41.020 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:41.057 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:41.128 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:41.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:41.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:41.280 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:41.326 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:41.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:41.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:41.391 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:41.430 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:41.504 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:41.556 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:41.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:41.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:41.602 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:17:41.602 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:17:41.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:17:41.602 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:17:41.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:85019 8.4 8:26bd7172:::benchmark_data_smithi012_271002_object85018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:46.061719+0000", 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 475.53658938900003, 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.398916023, 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:17:41.603 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 85019 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061719+0000", 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061719+0000", 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061718+0000", 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5325000000000002e-05 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061783+0000", 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7779999999999999e-05 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061801+0000", 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8170000000000001e-06 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061808+0000", 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7277999999999997e-05 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061855+0000", 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.832e-06 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.061862+0000", 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93984965899999995 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001712+0000", 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7869999999999996e-06 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001722+0000", 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4215940549999999 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.423316+0000", 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011345000000000001 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.423429+0000", 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000177066 2023-10-09T10:17:41.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.423606+0000", 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010226247000000001 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433832+0000", 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3725000000000004e-05 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433896+0000", 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022457444 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.456354+0000", 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5872000000000003e-05 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.456409+0000", 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042259159999999997 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.460635+0000", 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.398916023 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.612 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:85022 8.4 8:259e8b82:::benchmark_data_smithi012_271002_object85021:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:46.064695+0000", 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 475.53361391599998, 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.39940138, 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 85022 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.613 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064695+0000", 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064695+0000", 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064693+0000", 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6102999999999999e-05 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064779+0000", 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3779999999999999e-05 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064803+0000", 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.562e-06 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064810+0000", 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4813000000000004e-05 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064875+0000", 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432999999999999e-05 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064887+0000", 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.936856048 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001743+0000", 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8660000000000006e-06 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001751+0000", 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.424300347 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.426051+0000", 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105982 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.426157+0000", 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013454 2023-10-09T10:17:41.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.426292+0000", 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075499429999999999 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433842+0000", 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7359000000000001e-05 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433939+0000", 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026765071000000001 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.460704+0000", 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3587000000000001e-05 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.460748+0000", 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033485020000000002 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.464096+0000", 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.39940138 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.621 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:85023 8.4 8:25543529:::benchmark_data_smithi012_271002_object85022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:46.064828+0000", 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 475.53348094400002, 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.401756979, 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 85023 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064828+0000", 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064828+0000", 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064827+0000", 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7450000000000002e-05 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064865+0000", 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6919999999999997e-06 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064874+0000", 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5569999999999997e-06 2023-10-09T10:17:41.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064879+0000", 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5850999999999999e-05 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064925+0000", 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4500000000000004e-06 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064933+0000", 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93683761200000004 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001771+0000", 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2289999999999997e-06 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001778+0000", 2023-10-09T10:17:41.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.426578484 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.428357+0000", 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132087 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.428489+0000", 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000149829 2023-10-09T10:17:41.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.428638+0000", 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052126619999999999 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433851+0000", 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013186900000000001 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.433983+0000", 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.03017481 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.464158+0000", 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0639000000000002e-05 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.464198+0000", 2023-10-09T10:17:41.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023863640000000002 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.466585+0000", 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.401756979 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.630 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45518.0:85024 8.4 8:2abed64e:::benchmark_data_smithi012_271002_object85023:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:09:46.064887+0000", 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 475.53342198199999, 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4040639640000001, 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45518", 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3315738058", 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 85024 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.631 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064887+0000", 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064887+0000", 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064886+0000", 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4586000000000001e-05 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064911+0000", 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.844e-06 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064918+0000", 2023-10-09T10:17:41.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9160000000000003e-06 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064922+0000", 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9326999999999999e-05 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064951+0000", 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9810000000000003e-06 2023-10-09T10:17:41.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:46.064956+0000", 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93684371 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001800+0000", 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2970000000000004e-06 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:47.001808+0000", 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4321932479999999 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.434001+0000", 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012312900000000001 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.434124+0000", 2023-10-09T10:17:41.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000124043 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.434248+0000", 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070481770000000001 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.441296+0000", 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014025800000000001 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.441437+0000", 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025203030000000001 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.466640+0000", 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8245999999999999e-05 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.466678+0000", 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022727250000000002 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:09:48.468951+0000", 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4040639640000001 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.639 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156576 9.1f 9:fa882c35:::benchmark_data_smithi012_313365_object44101:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.513695+0000", 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 33.084613896, 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8210460340000001, 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156576 2023-10-09T10:17:41.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513695+0000", 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513695+0000", 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513693+0000", 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1532000000000001e-05 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513734+0000", 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9835000000000001e-05 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513774+0000", 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.024e-05 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513784+0000", 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014760687999999999 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.528545+0000", 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1260000000000001e-05 2023-10-09T10:17:41.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.528556+0000", 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0299076679999999 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.558464+0000", 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083750999999999999 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.566839+0000", 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.574253793 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.141093+0000", 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024537180000000001 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.143546+0000", 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056110070000000003 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.149157+0000", 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019102211000000001 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.168260+0000", 2023-10-09T10:17:41.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15842210500000001 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.326682+0000", 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085343699999999997 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.327535+0000", 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001518244 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.329053+0000", 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056875099999999998 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.334741+0000", 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8210460340000001 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156587 9.1f 9:fdc5c06c:::benchmark_data_smithi012_313365_object44112:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.741326+0000", 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.856982592999998, 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6031445180000006, 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156587 2023-10-09T10:17:41.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741326+0000", 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741326+0000", 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741325+0000", 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7740000000000004e-06 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741332+0000", 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6189999999999999e-05 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741349+0000", 2023-10-09T10:17:41.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.879e-06 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741355+0000", 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4529e-05 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741399+0000", 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6150000000000002e-06 2023-10-09T10:17:41.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741406+0000", 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8257677169999997 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.567173+0000", 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073666579999999999 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:41.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.574540+0000", 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.59405215 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.168592+0000", 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029038190000000002 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.171496+0000", 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057499700000000001 2023-10-09T10:17:41.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.177246+0000", 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024581421999999999 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.201827+0000", 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13390748299999999 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.335735+0000", 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00092294899999999995 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.336658+0000", 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001351713 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.338010+0000", 2023-10-09T10:17:41.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064611529999999999 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.344471+0000", 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6031445180000006 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156590 9.f 9:f095e657:::benchmark_data_smithi012_313365_object44115:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e378)", 2023-10-09T10:17:41.657 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.392835+0000", 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.205473572999999, 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9880701189999996, 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156590 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392835+0000", 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392835+0000", 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392835+0000", 2023-10-09T10:17:41.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8289999999999998e-06 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392840+0000", 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0960000000000001e-06 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392843+0000", 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.277e-06 2023-10-09T10:17:41.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392845+0000", 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003385271 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.396230+0000", 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0803e-05 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.661 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.396241+0000", 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2029989060000004 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.599240+0000", 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084562230000000006 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:41.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.607696+0000", 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6506387739999999 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.258335+0000", 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027081309999999999 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.261043+0000", 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055769770000000003 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.266620+0000", 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057250374999999999 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.323871+0000", 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.045280169000000002 2023-10-09T10:17:41.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.369151+0000", 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014548790000000001 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.370606+0000", 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001960318 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.372566+0000", 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083394309999999996 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.380905+0000", 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9880701189999996 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156617 9.1f 9:ff63f14a:::benchmark_data_smithi012_313365_object44142:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.540544+0000", 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.057764476000003, 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8147314209999994, 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156617 2023-10-09T10:17:41.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540544+0000", 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540544+0000", 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:17:41.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540542+0000", 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5820000000000004e-06 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540550+0000", 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4589999999999999e-05 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540565+0000", 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9290000000000003e-06 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540571+0000", 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1217000000000001e-05 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540622+0000", 2023-10-09T10:17:41.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0083e-05 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540632+0000", 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.034325011 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.574957+0000", 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075580600000000001 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.582515+0000", 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6093602599999999 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.191875+0000", 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021833450000000002 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.672 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.194059+0000", 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045656330000000004 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.198624+0000", 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033429538000000002 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.232054+0000", 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.113659181 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.345713+0000", 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010585200000000001 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.346771+0000", 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001606884 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.348378+0000", 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068974500000000003 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.355276+0000", 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8147314209999994 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.675 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156618 9.1f 9:fc73e5f2:::benchmark_data_smithi012_313365_object44143:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.543841+0000", 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.054467346999999, 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8224182819999992, 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156618 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543841+0000", 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543841+0000", 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543840+0000", 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9709999999999998e-06 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543847+0000", 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3609e-05 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543860+0000", 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.032e-06 2023-10-09T10:17:41.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543866+0000", 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4990000000000001e-05 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543911+0000", 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4530000000000002e-06 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543920+0000", 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0390108969999998 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.582931+0000", 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081182119999999997 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:41.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591049+0000", 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6286248130000001 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.219674+0000", 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033205679999999999 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.222994+0000", 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057524799999999999 2023-10-09T10:17:41.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.228747+0000", 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029478516 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.258225+0000", 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098204816 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.682 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.356430+0000", 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099631200000000007 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.357426+0000", 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018869570000000001 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.359313+0000", 2023-10-09T10:17:41.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069463579999999997 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.366260+0000", 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8224182819999992 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.684 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156620 9.1f 9:fa83bd3e:::benchmark_data_smithi012_313365_object44145:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.545391+0000", 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.052917663000002, 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8318410220000008, 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156620 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545391+0000", 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545391+0000", 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545390+0000", 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1890000000000001e-06 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545397+0000", 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4773000000000001e-05 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545412+0000", 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1689999999999997e-06 2023-10-09T10:17:41.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545418+0000", 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0813000000000003e-05 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545469+0000", 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1269999999999992e-06 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545478+0000", 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0459541999999997 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591432+0000", 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080512090000000001 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.599483+0000", 2023-10-09T10:17:41.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6481732660000001 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.247657+0000", 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022613690000000001 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.249918+0000", 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047491729999999998 2023-10-09T10:17:41.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.254667+0000", 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042271461000000003 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.296939+0000", 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.070329569999999994 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.691 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.367268+0000", 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010517650000000001 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.368320+0000", 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001726287 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.370046+0000", 2023-10-09T10:17:41.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071859460000000003 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.377232+0000", 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8318410220000008 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156621 9.f 9:f03746ff:::benchmark_data_smithi012_313365_object44146:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:17:41.693 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.546166+0000", 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.052142920999998, 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8468432579999998, 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156621 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546166+0000", 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546166+0000", 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546165+0000", 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8250000000000001e-06 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546171+0000", 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2918999999999999e-05 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546184+0000", 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0870000000000003e-06 2023-10-09T10:17:41.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546190+0000", 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0841999999999998e-05 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546251+0000", 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4081e-05 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546265+0000", 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.061941139 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.608206+0000", 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090726560000000001 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.617279+0000", 2023-10-09T10:17:41.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6674236599999999 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.284702+0000", 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002600265 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.287302+0000", 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053856440000000002 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.292688+0000", 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031507306999999998 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.324195+0000", 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057879049000000002 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.382074+0000", 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011540610000000001 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.383229+0000", 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001782921 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.385011+0000", 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079976449999999994 2023-10-09T10:17:41.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.393009+0000", 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8468432579999998 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156626 9.f 9:f552e3bf:::benchmark_data_smithi012_313365_object44151:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.557031+0000", 2023-10-09T10:17:41.702 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 32.041277532999999, 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8484499169999999, 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156626 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557031+0000", 2023-10-09T10:17:41.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557031+0000", 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557030+0000", 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4510000000000003e-06 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557037+0000", 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4916000000000001e-05 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557052+0000", 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1659999999999997e-06 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557059+0000", 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4586000000000002e-05 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557114+0000", 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7219999999999994e-06 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557124+0000", 2023-10-09T10:17:41.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0606237529999998 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.617747+0000", 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084044659999999993 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.626152+0000", 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.684881012 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.311033+0000", 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002790294 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.313823+0000", 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058745819999999997 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.319698+0000", 2023-10-09T10:17:41.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052440439999999998 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.324942+0000", 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.069203928999999997 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.394146+0000", 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001441814 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.395587+0000", 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021981000000000001 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.397786+0000", 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076955579999999999 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.405481+0000", 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8484499169999999 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(9.1f 381/377 [PushReplyOp(9:fa882c35:::benchmark_data_smithi012_313365_object44101:head)])", 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:16.568831+0000", 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 25.02947812, 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.571736494, 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.711 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.568831+0000", 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.568831+0000", 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.568829+0000", 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0290000000000005e-06 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.568837+0000", 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5749000000000001e-05 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.713 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.568853+0000", 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5279999999999992e-06 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.568863+0000", 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.558411671 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.127274+0000", 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013292745 2023-10-09T10:17:41.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.140567+0000", 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.571736494 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(9.1f 381/377 [PushReplyOp(9:fdc5c06c:::benchmark_data_smithi012_313365_object44112:head)])", 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:16.575547+0000", 2023-10-09T10:17:41.715 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 25.022761546000002, 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.592715495, 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.575547+0000", 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.575547+0000", 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.575546+0000", 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1929999999999993e-06 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.575555+0000", 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1568e-05 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.575566+0000", 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2889999999999997e-06 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.575571+0000", 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5786533390000002 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.154225+0000", 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014037905999999999 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.168263+0000", 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.592715495 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(9.1f 381/377 [PushReplyOp(9:ff63f14a:::benchmark_data_smithi012_313365_object44142:head)])", 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:16.583509+0000", 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 25.014799836000002, 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.608057845, 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.583509+0000", 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.583509+0000", 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.583508+0000", 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8776000000000001e-05 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.583527+0000", 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3951000000000001e-05 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.583541+0000", 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9630000000000003e-06 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.583546+0000", 2023-10-09T10:17:41.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5969572599999999 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.180504+0000", 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011063030999999999 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.191567+0000", 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.608057845 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.723 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(9.1f 381/377 [PushReplyOp(9:fc73e5f2:::benchmark_data_smithi012_313365_object44143:head)])", 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:16.591944+0000", 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 25.006364423000001, 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6272846969999999, 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591944+0000", 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591944+0000", 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591944+0000", 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6009999999999997e-06 2023-10-09T10:17:41.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591951+0000", 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1596e-05 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591963+0000", 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4019999999999999e-06 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591969+0000", 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6102629400000001 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.202232+0000", 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016996941000000002 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.219229+0000", 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6272846969999999 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(9.1f 381/377 [PushReplyOp(9:fa83bd3e:::benchmark_data_smithi012_313365_object44145:head)])", 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:16.601403+0000", 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 24.996905791, 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.645950638, 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:17:41.728 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.601403+0000", 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.601403+0000", 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.601402+0000", 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8359999999999997e-06 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.601408+0000", 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1711e-05 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.601420+0000", 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4489999999999998e-06 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.601425+0000", 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.631001605 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.232427+0000", 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014926907 2023-10-09T10:17:41.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.247354+0000", 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.645950638 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(9.f 381/378 [PushReplyOp(9:f095e657:::benchmark_data_smithi012_313365_object44115:head)])", 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:16.609259+0000", 2023-10-09T10:17:41.732 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 24.989049636000001, 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6489692680000001, 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.609259+0000", 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.609259+0000", 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.609258+0000", 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.865e-06 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.609263+0000", 2023-10-09T10:17:41.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2962999999999999e-05 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.609276+0000", 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8790000000000002e-06 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.609281+0000", 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6236679839999999 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.232949+0000", 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.025279225999999998 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.258228+0000", 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6489692680000001 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(9.f 381/378 [PushReplyOp(9:f03746ff:::benchmark_data_smithi012_313365_object44146:head)])", 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:16.618696+0000", 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 24.979612675999999, 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.665516929, 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.618696+0000", 2023-10-09T10:17:41.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.618696+0000", 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.618695+0000", 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5409999999999997e-06 2023-10-09T10:17:41.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.618702+0000", 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2918999999999999e-05 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.618715+0000", 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.874e-06 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.618721+0000", 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.651902658 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.270623+0000", 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013589861 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.284213+0000", 2023-10-09T10:17:41.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.665516929 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPushReply(9.f 381/378 [PushReplyOp(9:f552e3bf:::benchmark_data_smithi012_313365_object44151:head)])", 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:16.627497+0000", 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 24.970812243000001, 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6831947519999999, 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:17:41.741 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.627497+0000", 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.627497+0000", 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:17:41.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.627495+0000", 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.379e-06 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.627503+0000", 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6509000000000001e-05 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.627519+0000", 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7880000000000007e-06 2023-10-09T10:17:41.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.627527+0000", 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6692744099999999 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.296801+0000", 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013890026999999999 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:17:41.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:17:41.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.310691+0000", 2023-10-09T10:17:41.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6831947519999999 2023-10-09T10:17:41.745 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.745 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.745 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.745 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:17:41.745 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:17:41.745 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:17:41.745 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:41.783 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:41.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:41.919 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:41 smithi110 ceph-mon[136032]: 9.1c scrub starts 2023-10-09T10:17:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:41.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:41.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:41 smithi012 ceph-mon[147926]: 9.1c scrub starts 2023-10-09T10:17:41.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:41 smithi012 ceph-mon[152263]: 9.1c scrub starts 2023-10-09T10:17:41.985 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:17:42.022 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:42.060 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:42.125 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:42.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:42.137 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:17:42.137 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:17:42.138 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:17:42.138 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.000000 2023-10-09T10:17:42.138 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0 2023-10-09T10:17:42.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:17:42.230 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:42.289 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:42.349 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:42.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:42.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:42.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:42.521 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:42.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:42.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:42.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:42.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:42.742 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:42.802 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:42 smithi012 ceph-mon[147926]: pgmap v1088: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 26874/179415 objects degraded (14.979%); 3191/179415 objects misplaced (1.779%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T10:17:42.803 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:42 smithi012 ceph-mon[152263]: pgmap v1088: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 26874/179415 objects degraded (14.979%); 3191/179415 objects misplaced (1.779%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T10:17:42.813 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:42.871 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:42.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:42.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:42.928 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:42.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:43.027 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:43.073 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:43.074 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:43.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:43.178 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:43.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:42 smithi110 ceph-mon[136032]: pgmap v1088: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 26874/179415 objects degraded (14.979%); 3191/179415 objects misplaced (1.779%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T10:17:43.235 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:43.288 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:43.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:43.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:43.431 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:43.630 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:43.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:43.716 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:17:43.736 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:43.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:43 smithi110 ceph-mon[136032]: 9.1c scrub starts 2023-10-09T10:17:43.786 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2466280946' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:43.787 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 26874/179415 objects degraded (14.979%), 22 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:43.791 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:44.090 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:44.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:44.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:44.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:43 smithi012 ceph-mon[147926]: 9.1c scrub starts 2023-10-09T10:17:44.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2466280946' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:44.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 26874/179415 objects degraded (14.979%), 22 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:44.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:43 smithi012 ceph-mon[152263]: 9.1c scrub starts 2023-10-09T10:17:44.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2466280946' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:17:44.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 26874/179415 objects degraded (14.979%), 22 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:44.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:44.255 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:44.556 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:44.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:44.618 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:17:44.619 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:17:44.619 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:17:44.619 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:17:44.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:44.729 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:44.788 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:44.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:44.896 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: 9.1c scrub starts 2023-10-09T10:17:44.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: pgmap v1089: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 26874/176910 objects degraded (15.191%); 3191/176910 objects misplaced (1.804%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2466280946' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: 9.1 scrub starts 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: 9.1c scrub starts 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: pgmap v1089: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 26874/176910 objects degraded (15.191%); 3191/176910 objects misplaced (1.804%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2466280946' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: 9.1 scrub starts 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:44.937 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: 9.1c scrub starts 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: pgmap v1089: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 26874/176910 objects degraded (15.191%); 3191/176910 objects misplaced (1.804%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2466280946' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: failed to encode map e390 with expected crc 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: 9.1 scrub starts 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:17:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:17:45.027 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:17:45.715 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:45.983 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:45 smithi110 ceph-mon[136032]: failed to encode map e390 with expected crc 2023-10-09T10:17:45.984 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:45 smithi110 ceph-mon[136032]: 9.1c scrub starts 2023-10-09T10:17:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:45 smithi012 ceph-mon[147926]: failed to encode map e390 with expected crc 2023-10-09T10:17:46.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:45 smithi012 ceph-mon[147926]: 9.1c scrub starts 2023-10-09T10:17:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:45 smithi012 ceph-mon[152263]: failed to encode map e390 with expected crc 2023-10-09T10:17:46.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:45 smithi012 ceph-mon[152263]: 9.1c scrub starts 2023-10-09T10:17:46.977 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:46 smithi012 ceph-mon[152263]: pgmap v1091: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 26415/172383 objects degraded (15.323%); 2653/172383 objects misplaced (1.539%); 13 MiB/s, 200 objects/s recovering 2023-10-09T10:17:46.978 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:46 smithi012 ceph-mon[152263]: 9.21 scrub starts 2023-10-09T10:17:46.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:46 smithi012 ceph-mon[147926]: pgmap v1091: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 26415/172383 objects degraded (15.323%); 2653/172383 objects misplaced (1.539%); 13 MiB/s, 200 objects/s recovering 2023-10-09T10:17:46.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:46 smithi012 ceph-mon[147926]: 9.21 scrub starts 2023-10-09T10:17:47.042 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:47.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:47.090 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:17:47.090 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:17:47.090 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:17:47.090 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:17:47.090 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:17:47.114 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:46 smithi110 ceph-mon[136032]: pgmap v1091: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 26415/172383 objects degraded (15.323%); 2653/172383 objects misplaced (1.539%); 13 MiB/s, 200 objects/s recovering 2023-10-09T10:17:47.114 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:46 smithi110 ceph-mon[136032]: 9.21 scrub starts 2023-10-09T10:17:47.147 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:47.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:47.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:47.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:47.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:47.490 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:17:47.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:47.540 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:47.588 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:47.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:47.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:47.734 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:47.808 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:47.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:17:47.814 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:47 smithi012 ceph-mon[152263]: 9.11 scrub starts 2023-10-09T10:17:47.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:47 smithi012 ceph-mon[147926]: 9.11 scrub starts 2023-10-09T10:17:47.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:47.979 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:48.080 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:48.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:48.094 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:17:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:17:47] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-10-09T10:17:48.166 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:17:48.191 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:17:48.195 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:17:48.207 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:47 smithi110 ceph-mon[136032]: 9.11 scrub starts 2023-10-09T10:17:48.220 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:17:48.248 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:48.248 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:17:48.272 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:17:48.292 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:17:48.293 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:48.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:48.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:17:48.302 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:17:48.324 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:17:48.350 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:17:48.353 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:17:48.353 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:17:48.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:17:48.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:17:48.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:17:48.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:17:48.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:17:48.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:17:48.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:48.440 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:48.541 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:48.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:48.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:48.646 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:48.684 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:48.731 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:48.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:48.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:48.754 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.7 primary-affinity to 0 (802) 2023-10-09T10:17:48.836 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:17:48.877 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:48 smithi110 ceph-mon[136032]: pgmap v1092: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 26415/170445 objects degraded (15.498%); 2653/170445 objects misplaced (1.557%); 11 MiB/s, 178 objects/s recovering 2023-10-09T10:17:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:48 smithi110 ceph-mon[136032]: 9.1 deep-scrub starts 2023-10-09T10:17:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1895735979' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T10:17:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:48 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T10:17:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/798631593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 26415/170445 objects degraded (15.498%), 22 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:48.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:17:49.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[147926]: pgmap v1092: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 26415/170445 objects degraded (15.498%); 2653/170445 objects misplaced (1.557%); 11 MiB/s, 178 objects/s recovering 2023-10-09T10:17:49.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[147926]: 9.1 deep-scrub starts 2023-10-09T10:17:49.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1895735979' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T10:17:49.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T10:17:49.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/798631593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:49.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 26415/170445 objects degraded (15.498%), 22 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:49.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[152263]: pgmap v1092: 109 pgs: 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 26415/170445 objects degraded (15.498%); 2653/170445 objects misplaced (1.557%); 11 MiB/s, 178 objects/s recovering 2023-10-09T10:17:49.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[152263]: 9.1 deep-scrub starts 2023-10-09T10:17:49.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1895735979' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T10:17:49.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T10:17:49.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/798631593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:17:49.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 26415/170445 objects degraded (15.498%), 22 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:49.014 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:17:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:49.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:17:49.121 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:49.171 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:49.363 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:49.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:49.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:49.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:49.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:49.582 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:49.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:49.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:49.645 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:49.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:49.717 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:49.767 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:49.770 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:49.770 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:49.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:49.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:49.921 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:49.970 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[147926]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T10:17:49.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:49.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:49.975 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T10:17:49.975 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T10:17:49.975 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.975 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: failed to encode map e391 with expected crc 2023-10-09T10:17:49.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:49 smithi012 ceph-mon[152263]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T10:17:50.076 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:50.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T10:17:50.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T10:17:50.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.099 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.099 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.099 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.099 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.099 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: failed to encode map e391 with expected crc 2023-10-09T10:17:50.099 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:49 smithi110 ceph-mon[136032]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T10:17:50.109 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:50.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:17:50.349 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:50.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:50.453 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:50.483 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:50.790 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:50.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:17:51.037 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: pgmap v1094: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 22908/166017 objects degraded (13.799%); 1991/166017 objects misplaced (1.199%); 14 MiB/s, 223 objects/s recovering 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:50 smithi110 ceph-mon[136032]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T10:17:51.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: pgmap v1094: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 22908/166017 objects degraded (13.799%); 1991/166017 objects misplaced (1.199%); 14 MiB/s, 223 objects/s recovering 2023-10-09T10:17:51.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.110 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[147926]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: pgmap v1094: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 22908/166017 objects degraded (13.799%); 1991/166017 objects misplaced (1.199%); 14 MiB/s, 223 objects/s recovering 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.111 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.112 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.112 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.112 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: failed to encode map e392 with expected crc 2023-10-09T10:17:51.112 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:50 smithi012 ceph-mon[152263]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T10:17:52.026 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:52.026 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:52.054 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:51 smithi110 ceph-mon[136032]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.055 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:51 smithi110 ceph-mon[136032]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.055 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:51 smithi110 ceph-mon[136032]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.055 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:51 smithi110 ceph-mon[136032]: 9.21 scrub starts 2023-10-09T10:17:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:51 smithi012 ceph-mon[147926]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:51 smithi012 ceph-mon[147926]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:51 smithi012 ceph-mon[147926]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:51 smithi012 ceph-mon[147926]: 9.21 scrub starts 2023-10-09T10:17:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:51 smithi012 ceph-mon[152263]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:51 smithi012 ceph-mon[152263]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:51 smithi012 ceph-mon[152263]: failed to encode map e393 with expected crc 2023-10-09T10:17:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:51 smithi012 ceph-mon[152263]: 9.21 scrub starts 2023-10-09T10:17:52.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:17:52.975 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:52.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:52.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:53.079 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:53.121 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:52 smithi110 ceph-mon[136032]: pgmap v1097: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 22908/165429 objects degraded (13.848%); 1991/165429 objects misplaced (1.204%); 9.9 MiB/s, 156 objects/s recovering 2023-10-09T10:17:53.124 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:53.168 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:53.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:53.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:53.175 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:52 smithi012 ceph-mon[147926]: pgmap v1097: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 22908/165429 objects degraded (13.848%); 1991/165429 objects misplaced (1.204%); 9.9 MiB/s, 156 objects/s recovering 2023-10-09T10:17:53.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:52 smithi012 ceph-mon[152263]: pgmap v1097: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 22908/165429 objects degraded (13.848%); 1991/165429 objects misplaced (1.204%); 9.9 MiB/s, 156 objects/s recovering 2023-10-09T10:17:53.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:53.340 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:53.780 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:53 smithi110 ceph-mon[136032]: 9.11 scrub starts 2023-10-09T10:17:53.781 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:53 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 22908/165429 objects degraded (13.848%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:53.909 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:53.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:53.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:53.914 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:54.013 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:17:54.049 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:54.101 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:17:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:54.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:17:54.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:53 smithi012 ceph-mon[147926]: 9.11 scrub starts 2023-10-09T10:17:54.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:53 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 22908/165429 objects degraded (13.848%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:54.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:53 smithi012 ceph-mon[152263]: 9.11 scrub starts 2023-10-09T10:17:54.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:53 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 22908/165429 objects degraded (13.848%), 20 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:54.207 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:17:54.207 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:17:54.207 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T10:17:54.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:17:54.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:17:54.252 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:54.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:54 smithi012 ceph-mon[147926]: 9.1c scrub starts 2023-10-09T10:17:54.913 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:54 smithi012 ceph-mon[147926]: pgmap v1098: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 22908/163401 objects degraded (14.019%); 1991/163401 objects misplaced (1.218%); 9.8 MiB/s, 154 objects/s recovering 2023-10-09T10:17:54.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:54 smithi012 ceph-mon[152263]: 9.1c scrub starts 2023-10-09T10:17:54.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:54 smithi012 ceph-mon[152263]: pgmap v1098: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 22908/163401 objects degraded (14.019%); 1991/163401 objects misplaced (1.218%); 9.8 MiB/s, 154 objects/s recovering 2023-10-09T10:17:55.045 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:17:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:55.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:17:55.098 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:55.098 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:55.129 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:54 smithi110 ceph-mon[136032]: 9.1c scrub starts 2023-10-09T10:17:55.129 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:54 smithi110 ceph-mon[136032]: pgmap v1098: 109 pgs: 2 peering, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 22908/163401 objects degraded (14.019%); 1991/163401 objects misplaced (1.218%); 9.8 MiB/s, 154 objects/s recovering 2023-10-09T10:17:55.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:17:55.198 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:55.256 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:17:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:55.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:17:55.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:17:55.403 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:55.452 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:17:55.458 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:17:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:55.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:17:55.563 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:17:55.608 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:17:55.611 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:55.680 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:17:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:55.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:17:55.785 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:17:55.797 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2631778857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:55.797 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:55 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:55.813 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:17:55.832 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:17:56.166 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:17:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2631778857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:55 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:56.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2631778857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:56.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:55 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:17:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:56 smithi110 ceph-mon[136032]: pgmap v1099: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 25364/159285 objects degraded (15.924%); 1621/159285 objects misplaced (1.018%); 15 MiB/s, 325 objects/s recovering 2023-10-09T10:17:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:56 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:56.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:56 smithi110 ceph-mon[136032]: failed to encode map e394 with expected crc 2023-10-09T10:17:56.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:56 smithi110 ceph-mon[136032]: failed to encode map e394 with expected crc 2023-10-09T10:17:56.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:56 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:17:56.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:56 smithi110 ceph-mon[136032]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T10:17:57.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[147926]: pgmap v1099: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 25364/159285 objects degraded (15.924%); 1621/159285 objects misplaced (1.018%); 15 MiB/s, 325 objects/s recovering 2023-10-09T10:17:57.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:57.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[147926]: failed to encode map e394 with expected crc 2023-10-09T10:17:57.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[147926]: failed to encode map e394 with expected crc 2023-10-09T10:17:57.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:17:57.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[147926]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T10:17:57.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[152263]: pgmap v1099: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 25364/159285 objects degraded (15.924%); 1621/159285 objects misplaced (1.018%); 15 MiB/s, 325 objects/s recovering 2023-10-09T10:17:57.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:17:57.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[152263]: failed to encode map e394 with expected crc 2023-10-09T10:17:57.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[152263]: failed to encode map e394 with expected crc 2023-10-09T10:17:57.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:17:57.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:56 smithi012 ceph-mon[152263]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T10:17:57.415 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:17:57.415 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:17:57.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:17:58.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:17:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:17:57] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-10-09T10:17:58.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:58 smithi012 ceph-mon[147926]: pgmap v1101: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 25364/157188 objects degraded (16.136%); 1621/157188 objects misplaced (1.031%); 5.5 MiB/s, 163 objects/s recovering 2023-10-09T10:17:58.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 25364/157188 objects degraded (16.136%), 22 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:58.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:58 smithi012 ceph-mon[152263]: pgmap v1101: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 25364/157188 objects degraded (16.136%); 1621/157188 objects misplaced (1.031%); 5.5 MiB/s, 163 objects/s recovering 2023-10-09T10:17:58.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 25364/157188 objects degraded (16.136%), 22 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:59.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:58 smithi110 ceph-mon[136032]: pgmap v1101: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 25364/157188 objects degraded (16.136%); 1621/157188 objects misplaced (1.031%); 5.5 MiB/s, 163 objects/s recovering 2023-10-09T10:17:59.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 25364/157188 objects degraded (16.136%), 22 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:17:59.916 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:17:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:17:59.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:00.021 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:00.057 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:00.105 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:00.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:00.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[147926]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T10:18:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[147926]: failed to encode map e395 with expected crc 2023-10-09T10:18:00.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[147926]: failed to encode map e395 with expected crc 2023-10-09T10:18:00.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[152263]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T10:18:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[152263]: failed to encode map e395 with expected crc 2023-10-09T10:18:00.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:17:59 smithi012 ceph-mon[152263]: failed to encode map e395 with expected crc 2023-10-09T10:18:00.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:59 smithi110 ceph-mon[136032]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T10:18:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:59 smithi110 ceph-mon[136032]: failed to encode map e395 with expected crc 2023-10-09T10:18:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:17:59 smithi110 ceph-mon[136032]: failed to encode map e395 with expected crc 2023-10-09T10:18:00.244 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:00.289 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:00.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:00.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:00.441 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:01.323 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:01.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:01.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: failed to encode map e395 with expected crc 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: failed to encode map e395 with expected crc 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: pgmap v1103: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 23864/151785 objects degraded (15.722%); 1527/151785 objects misplaced (1.006%); 9.4 MiB/s, 336 objects/s recovering 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[147926]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: failed to encode map e395 with expected crc 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: failed to encode map e395 with expected crc 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: pgmap v1103: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 23864/151785 objects degraded (15.722%); 1527/151785 objects misplaced (1.006%); 9.4 MiB/s, 336 objects/s recovering 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:00 smithi012 ceph-mon[152263]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.428 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: failed to encode map e395 with expected crc 2023-10-09T10:18:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: failed to encode map e395 with expected crc 2023-10-09T10:18:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: pgmap v1103: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 23864/151785 objects degraded (15.722%); 1527/151785 objects misplaced (1.006%); 9.4 MiB/s, 336 objects/s recovering 2023-10-09T10:18:01.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T10:18:01.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:00 smithi110 ceph-mon[136032]: failed to encode map e396 with expected crc 2023-10-09T10:18:01.470 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:01.516 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:01.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:01.590 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:01.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:01.677 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:01.728 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:01.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:01.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:01.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:01.871 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:01.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:01.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:01.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:02.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:02.057 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:02.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:01 smithi110 ceph-mon[136032]: failed to encode map e396 with expected crc 2023-10-09T10:18:02.388 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:02.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:02.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:02.392 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:01 smithi012 ceph-mon[147926]: failed to encode map e396 with expected crc 2023-10-09T10:18:02.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:01 smithi012 ceph-mon[152263]: failed to encode map e396 with expected crc 2023-10-09T10:18:02.399 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:18:02.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:02.544 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:03.163 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:18:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:02 smithi012 ceph-mon[152263]: pgmap v1105: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 23864/151047 objects degraded (15.799%); 1527/151047 objects misplaced (1.011%); 5.4 MiB/s, 234 objects/s recovering 2023-10-09T10:18:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:02 smithi012 ceph-mon[147926]: pgmap v1105: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 23864/151047 objects degraded (15.799%); 1527/151047 objects misplaced (1.011%); 5.4 MiB/s, 234 objects/s recovering 2023-10-09T10:18:03.189 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:18:03.220 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:18:03.250 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:18:03.261 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:03.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:03.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:03.275 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:18:03.298 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:18:03.327 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:18:03.352 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:18:03.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:18:03.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:18:03.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:18:03.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:18:03.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:18:03.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:18:03.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:18:03.354 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:18:03.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:03.429 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:03.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:02 smithi110 ceph-mon[136032]: pgmap v1105: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 23864/151047 objects degraded (15.799%); 1527/151047 objects misplaced (1.011%); 5.4 MiB/s, 234 objects/s recovering 2023-10-09T10:18:04.050 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:18:04.052 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:18:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1136546639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:04.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 23864/151047 objects degraded (15.799%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:04.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1136546639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:04.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 23864/151047 objects degraded (15.799%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:04.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1136546639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 23864/151047 objects degraded (15.799%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:04.554 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:04.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:04.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:04.664 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:04.702 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:04.753 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:04.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:04.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:04.858 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:04.887 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:04.930 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:04.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:05.035 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:05.065 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:05.109 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:05.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:05.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:05.214 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:05.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:05.298 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:05.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:05.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:05 smithi110 ceph-mon[136032]: pgmap v1106: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 21978/147759 objects degraded (14.874%); 3051/147759 objects misplaced (2.065%); 4.2 MiB/s, 220 objects/s recovering 2023-10-09T10:18:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:05 smithi110 ceph-mon[136032]: from='client.56425 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:18:05.453 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:05.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:05 smithi012 ceph-mon[152263]: pgmap v1106: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 21978/147759 objects degraded (14.874%); 3051/147759 objects misplaced (2.065%); 4.2 MiB/s, 220 objects/s recovering 2023-10-09T10:18:05.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:05 smithi012 ceph-mon[152263]: from='client.56425 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:18:05.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:05 smithi012 ceph-mon[147926]: pgmap v1106: 109 pgs: 1 peering, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 21978/147759 objects degraded (14.874%); 3051/147759 objects misplaced (2.065%); 4.2 MiB/s, 220 objects/s recovering 2023-10-09T10:18:05.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:05 smithi012 ceph-mon[147926]: from='client.56425 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:18:06.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:18:07.569 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:07.573 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:07 smithi012 ceph-mon[147926]: pgmap v1107: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 21384/142788 objects degraded (14.976%); 2964/142788 objects misplaced (2.076%); 8.1 MiB/s, 278 objects/s recovering 2023-10-09T10:18:07.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:07 smithi012 ceph-mon[152263]: pgmap v1107: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 21384/142788 objects degraded (14.976%); 2964/142788 objects misplaced (2.076%); 8.1 MiB/s, 278 objects/s recovering 2023-10-09T10:18:07.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:07.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:07.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:07 smithi110 ceph-mon[136032]: pgmap v1107: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 21384/142788 objects degraded (14.976%); 2964/142788 objects misplaced (2.076%); 8.1 MiB/s, 278 objects/s recovering 2023-10-09T10:18:07.718 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:18:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:18:07] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.33.4" 2023-10-09T10:18:08.331 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:08.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:08.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:08.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:08.471 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:08.512 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:08.515 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 21384/142788 objects degraded (14.976%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:08.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:08.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:08.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:08.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 21384/142788 objects degraded (14.976%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:08.649 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 21384/142788 objects degraded (14.976%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:08.652 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:09.080 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1106,"stamp":"2023-10-09T10:18:03.596813+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3226769336,"num_objects":49253,"num_object_clones":0,"num_object_copies":147759,"num_objects_missing_on_primary":1054,"num_objects_missing":1054,"num_objects_degraded":21978,"num_objects_misplaced":3051,"num_objects_unfound":0,"num_objects_dirty":49253,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":169371,"num_write_kb":6772078,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33002,"num_bytes_recovered":1909795610,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":262684,"ondisk_log_size":262684,"up":327,"acting":318,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":379,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16584212,"kb_used_data":11739924,"kb_used_omap":172,"kb_used_meta":4844051,"kb_avail":733377004,"statfs":{"total":767960285184,"available":750978052096,"internally_reserved":0,"allocated":12021682176,"data_stored":12013055647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":176164,"internal_metadata":4960309212},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-251789312,"num_objects":-3842,"num_object_clones":0,"num_object_copies":-11526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3386,"num_objects_misplaced":1430,"num_objects_unfound":0,"num_objects_dirty":-3842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3842,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1723,"num_bytes_recovered":34603008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.798574"},"pg_stats":[{"pgid":"9.23","version":"394'8420","reported_seq":9116,"reported_epoch":394,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.060079+0000","last_change":"2023-10-09T10:17:49.823791+0000","last_active":"2023-10-09T10:17:59.060079+0000","last_peered":"2023-10-09T10:17:59.060079+0000","last_clean":"2023-10-09T10:15:14.172820+0000","last_became_active":"2023-10-09T10:17:49.774751+0000","last_became_peered":"2023-10-09T10:17:49.774751+0000","last_unstale":"2023-10-09T10:17:59.060079+0000","last_undegraded":"2023-10-09T10:17:49.746768+0000","last_fullsized":"2023-10-09T10:17:59.060079+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":377,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8420,"log_dups_size":0,"ondisk_log_size":8420,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50348032,"num_objects":768,"num_object_clones":0,"num_object_copies":2304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1416,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2716,"num_write_kb":111504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":279,"num_bytes_recovered":18284544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,3,7","objects":739},{"shards":"1,5,7","objects":29}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.22","version":"395'8384","reported_seq":9082,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.272545+0000","last_change":"2023-10-09T10:17:09.513820+0000","last_active":"2023-10-09T10:17:59.272545+0000","last_peered":"2023-10-09T10:17:59.272545+0000","last_clean":"2023-10-09T10:15:14.051568+0000","last_became_active":"2023-10-09T10:17:09.506371+0000","last_became_peered":"2023-10-09T10:17:09.506371+0000","last_unstale":"2023-10-09T10:17:59.272545+0000","last_undegraded":"2023-10-09T10:17:09.450481+0000","last_fullsized":"2023-10-09T10:17:59.272545+0000","mapping_epoch":377,"log_start":"321'588","ondisk_log_start":"321'588","created":377,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7796,"log_dups_size":0,"ondisk_log_size":7796,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50593792,"num_objects":772,"num_object_clones":0,"num_object_copies":2316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1462,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2700,"num_write_kb":111104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":18743296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,3","objects":779},{"shards":"1,2,7","objects":-7}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.21","version":"396'8627","reported_seq":8757,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:00.321183+0000","last_change":"2023-10-09T10:17:50.934467+0000","last_active":"2023-10-09T10:18:00.321183+0000","last_peered":"2023-10-09T10:18:00.321183+0000","last_clean":"2023-10-09T10:18:00.321183+0000","last_became_active":"2023-10-09T10:17:09.424870+0000","last_became_peered":"2023-10-09T10:17:09.424870+0000","last_unstale":"2023-10-09T10:18:00.321183+0000","last_undegraded":"2023-10-09T10:18:00.321183+0000","last_fullsized":"2023-10-09T10:18:00.321183+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":377,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8627,"log_dups_size":0,"ondisk_log_size":8627,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2776,"num_write_kb":114368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.20","version":"394'8401","reported_seq":8587,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.001559+0000","last_change":"2023-10-09T10:17:09.475599+0000","last_active":"2023-10-09T10:17:59.001559+0000","last_peered":"2023-10-09T10:17:59.001559+0000","last_clean":"2023-10-09T10:17:59.001559+0000","last_became_active":"2023-10-09T10:17:09.456790+0000","last_became_peered":"2023-10-09T10:17:09.456790+0000","last_unstale":"2023-10-09T10:17:59.001559+0000","last_undegraded":"2023-10-09T10:17:59.001559+0000","last_fullsized":"2023-10-09T10:17:59.001559+0000","mapping_epoch":377,"log_start":"321'624","ondisk_log_start":"321'624","created":377,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":6,"last_scrub":"339'6671","last_scrub_stamp":"2023-10-09T10:15:03.503975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:15:03.503975+0000","objects_scrubbed":3418,"log_size":7777,"log_dups_size":0,"ondisk_log_size":7777,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.90657236399999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52559872,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1011,"num_write_kb":3328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.10","version":"395'8609","reported_seq":16319,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:17:59.283512+0000","last_change":"2023-10-09T10:17:38.892595+0000","last_active":"2023-10-09T10:17:59.283512+0000","last_peered":"2023-10-09T10:17:59.283512+0000","last_clean":"2023-10-09T10:15:06.274037+0000","last_became_active":"2023-10-09T10:17:38.878568+0000","last_became_peered":"2023-10-09T10:17:38.878568+0000","last_unstale":"2023-10-09T10:17:59.283512+0000","last_undegraded":"2023-10-09T10:17:38.729727+0000","last_fullsized":"2023-10-09T10:17:38.673783+0000","mapping_epoch":388,"log_start":"324'2474","ondisk_log_start":"324'2474","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6135,"log_dups_size":0,"ondisk_log_size":6135,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101548032,"num_objects":1550,"num_object_clones":0,"num_object_copies":4650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3054,"num_objects_misplaced":1527,"num_objects_unfound":0,"num_objects_dirty":1550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5346,"num_write_kb":220640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1899,"num_bytes_recovered":124387328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[1,7],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1527},{"shards":"1,7","objects":23}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":112,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253402+0000","last_change":"2023-10-09T10:17:37.685528+0000","last_active":"2023-10-09T10:17:59.253402+0000","last_peered":"2023-10-09T10:17:59.253402+0000","last_clean":"2023-10-09T10:17:59.253402+0000","last_became_active":"2023-10-09T10:17:37.685092+0000","last_became_peered":"2023-10-09T10:17:37.685092+0000","last_unstale":"2023-10-09T10:17:59.253402+0000","last_undegraded":"2023-10-09T10:17:59.253402+0000","last_fullsized":"2023-10-09T10:17:59.253402+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":320,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.804341+0000","last_change":"2023-10-09T10:16:08.230865+0000","last_active":"2023-10-09T10:17:55.804341+0000","last_peered":"2023-10-09T10:17:55.804341+0000","last_clean":"2023-10-09T10:17:55.804341+0000","last_became_active":"2023-10-09T10:16:08.230559+0000","last_became_peered":"2023-10-09T10:16:08.230559+0000","last_unstale":"2023-10-09T10:17:55.804341+0000","last_undegraded":"2023-10-09T10:17:55.804341+0000","last_fullsized":"2023-10-09T10:17:55.804341+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:02.674870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.13","version":"395'8667","reported_seq":9106,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.303860+0000","last_change":"2023-10-09T10:17:50.801181+0000","last_active":"2023-10-09T10:17:59.303860+0000","last_peered":"2023-10-09T10:17:59.303860+0000","last_clean":"2023-10-09T10:15:14.025151+0000","last_became_active":"2023-10-09T10:17:50.754824+0000","last_became_peered":"2023-10-09T10:17:50.754824+0000","last_unstale":"2023-10-09T10:17:59.303860+0000","last_undegraded":"2023-10-09T10:17:50.750441+0000","last_fullsized":"2023-10-09T10:17:59.303860+0000","mapping_epoch":392,"log_start":"324'2569","ondisk_log_start":"324'2569","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6098,"log_dups_size":0,"ondisk_log_size":6098,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103841792,"num_objects":1585,"num_object_clones":0,"num_object_copies":4755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3306,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5401,"num_write_kb":223520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":154,"num_bytes_recovered":10092544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,3,7","objects":1551},{"shards":"1,5,7","objects":34}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":415,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.804303+0000","last_change":"2023-10-09T10:15:42.314309+0000","last_active":"2023-10-09T10:17:55.804303+0000","last_peered":"2023-10-09T10:17:55.804303+0000","last_clean":"2023-10-09T10:17:55.804303+0000","last_became_active":"2023-10-09T10:15:42.313553+0000","last_became_peered":"2023-10-09T10:15:42.313553+0000","last_unstale":"2023-10-09T10:17:55.804303+0000","last_undegraded":"2023-10-09T10:17:55.804303+0000","last_fullsized":"2023-10-09T10:17:55.804303+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:00.421233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":289,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245449+0000","last_change":"2023-10-09T10:16:08.309799+0000","last_active":"2023-10-09T10:17:59.245449+0000","last_peered":"2023-10-09T10:17:59.245449+0000","last_clean":"2023-10-09T10:17:59.245449+0000","last_became_active":"2023-10-09T10:16:08.309448+0000","last_became_peered":"2023-10-09T10:16:08.309448+0000","last_unstale":"2023-10-09T10:17:59.245449+0000","last_undegraded":"2023-10-09T10:17:59.245449+0000","last_fullsized":"2023-10-09T10:17:59.245449+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:58.976233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"394'8624","reported_seq":15691,"reported_epoch":395,"state":"peering","last_fresh":"2023-10-09T10:17:59.246068+0000","last_change":"2023-10-09T10:17:59.246242+0000","last_active":"2023-10-09T10:17:59.246068+0000","last_peered":"2023-10-09T10:17:59.168867+0000","last_clean":"2023-10-09T10:15:14.060602+0000","last_became_active":"2023-10-09T10:16:08.554288+0000","last_became_peered":"2023-10-09T10:16:08.554288+0000","last_unstale":"2023-10-09T10:17:59.246068+0000","last_undegraded":"2023-10-09T10:17:59.246068+0000","last_fullsized":"2023-10-09T10:17:59.246068+0000","mapping_epoch":395,"log_start":"324'2509","ondisk_log_start":"324'2509","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6115,"log_dups_size":0,"ondisk_log_size":6115,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107151360,"num_objects":1635,"num_object_clones":0,"num_object_copies":4905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5355,"num_write_kb":223680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3475,"num_bytes_recovered":129957888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3,7],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":357,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.247837+0000","last_change":"2023-10-09T10:16:16.789610+0000","last_active":"2023-10-09T10:17:59.247837+0000","last_peered":"2023-10-09T10:17:59.247837+0000","last_clean":"2023-10-09T10:17:59.247837+0000","last_became_active":"2023-10-09T10:16:13.361832+0000","last_became_peered":"2023-10-09T10:16:13.361832+0000","last_unstale":"2023-10-09T10:17:59.247837+0000","last_undegraded":"2023-10-09T10:17:59.247837+0000","last_fullsized":"2023-10-09T10:17:59.247837+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:46.197713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":188,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.806769+0000","last_change":"2023-10-09T10:16:41.473093+0000","last_active":"2023-10-09T10:17:55.806769+0000","last_peered":"2023-10-09T10:17:55.806769+0000","last_clean":"2023-10-09T10:17:55.806769+0000","last_became_active":"2023-10-09T10:16:41.472774+0000","last_became_peered":"2023-10-09T10:16:41.472774+0000","last_unstale":"2023-10-09T10:17:55.806769+0000","last_undegraded":"2023-10-09T10:17:55.806769+0000","last_fullsized":"2023-10-09T10:17:55.806769+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:07.117932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1d","version":"394'10489","reported_seq":10603,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.246772+0000","last_change":"2023-10-09T10:17:09.512906+0000","last_active":"2023-10-09T10:17:59.246772+0000","last_peered":"2023-10-09T10:17:59.246772+0000","last_clean":"2023-10-09T10:17:59.246772+0000","last_became_active":"2023-10-09T10:17:09.498676+0000","last_became_peered":"2023-10-09T10:17:09.498676+0000","last_unstale":"2023-10-09T10:17:59.246772+0000","last_undegraded":"2023-10-09T10:17:59.246772+0000","last_fullsized":"2023-10-09T10:17:59.246772+0000","mapping_epoch":377,"log_start":"334'5326","ondisk_log_start":"334'5326","created":377,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5163,"log_dups_size":0,"ondisk_log_size":5163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101744652,"num_objects":1553,"num_object_clones":0,"num_object_copies":4659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1553,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5543,"num_write_kb":227041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":324,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245883+0000","last_change":"2023-10-09T10:16:18.273288+0000","last_active":"2023-10-09T10:17:59.245883+0000","last_peered":"2023-10-09T10:17:59.245883+0000","last_clean":"2023-10-09T10:17:59.245883+0000","last_became_active":"2023-10-09T10:16:18.272936+0000","last_became_peered":"2023-10-09T10:16:18.272936+0000","last_unstale":"2023-10-09T10:17:59.245883+0000","last_undegraded":"2023-10-09T10:17:59.245883+0000","last_fullsized":"2023-10-09T10:17:59.245883+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:39:34.847781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":341,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.805632+0000","last_change":"2023-10-09T10:16:08.619899+0000","last_active":"2023-10-09T10:17:55.805632+0000","last_peered":"2023-10-09T10:17:55.805632+0000","last_clean":"2023-10-09T10:17:55.805632+0000","last_became_active":"2023-10-09T10:16:08.619549+0000","last_became_peered":"2023-10-09T10:16:08.619549+0000","last_unstale":"2023-10-09T10:17:55.805632+0000","last_undegraded":"2023-10-09T10:17:55.805632+0000","last_fullsized":"2023-10-09T10:17:55.805632+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:07.870610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1c","version":"396'10419","reported_seq":23511,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:03.165468+0000","last_change":"2023-10-09T10:17:53.244929+0000","last_active":"2023-10-09T10:18:03.165468+0000","last_peered":"2023-10-09T10:18:03.165468+0000","last_clean":"2023-10-09T10:18:03.165468+0000","last_became_active":"2023-10-09T10:17:37.672972+0000","last_became_peered":"2023-10-09T10:17:37.672972+0000","last_unstale":"2023-10-09T10:18:03.165468+0000","last_undegraded":"2023-10-09T10:18:03.165468+0000","last_fullsized":"2023-10-09T10:18:03.165468+0000","mapping_epoch":387,"log_start":"324'2492","ondisk_log_start":"324'2492","created":377,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7927,"log_dups_size":0,"ondisk_log_size":7927,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102563840,"num_objects":1565,"num_object_clones":0,"num_object_copies":4695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5496,"num_write_kb":225952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3235,"num_bytes_recovered":196968448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803744+0000","last_change":"2023-10-09T10:16:18.280500+0000","last_active":"2023-10-09T10:17:55.803744+0000","last_peered":"2023-10-09T10:17:55.803744+0000","last_clean":"2023-10-09T10:17:55.803744+0000","last_became_active":"2023-10-09T10:16:18.275348+0000","last_became_peered":"2023-10-09T10:16:18.275348+0000","last_unstale":"2023-10-09T10:17:55.803744+0000","last_undegraded":"2023-10-09T10:17:55.803744+0000","last_fullsized":"2023-10-09T10:17:55.803744+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":305,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:00.251938+0000","last_change":"2023-10-09T10:15:42.396546+0000","last_active":"2023-10-09T10:18:00.251938+0000","last_peered":"2023-10-09T10:18:00.251938+0000","last_clean":"2023-10-09T10:18:00.251938+0000","last_became_active":"2023-10-09T10:15:42.367055+0000","last_became_peered":"2023-10-09T10:15:42.367055+0000","last_unstale":"2023-10-09T10:18:00.251938+0000","last_undegraded":"2023-10-09T10:18:00.251938+0000","last_fullsized":"2023-10-09T10:18:00.251938+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:57.600046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1f","version":"395'10402","reported_seq":16813,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.549097+0000","last_change":"2023-10-09T10:17:18.110271+0000","last_active":"2023-10-09T10:17:59.549097+0000","last_peered":"2023-10-09T10:17:59.549097+0000","last_clean":"2023-10-09T10:15:06.273272+0000","last_became_active":"2023-10-09T10:17:16.203206+0000","last_became_peered":"2023-10-09T10:17:16.203206+0000","last_unstale":"2023-10-09T10:17:59.549097+0000","last_undegraded":"2023-10-09T10:17:14.435030+0000","last_fullsized":"2023-10-09T10:17:59.549097+0000","mapping_epoch":377,"log_start":"334'5299","ondisk_log_start":"334'5299","created":377,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5103,"log_dups_size":0,"ondisk_log_size":5103,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107053056,"num_objects":1634,"num_object_clones":0,"num_object_copies":4902,"num_objects_missing_on_primary":265,"num_objects_missing":265,"num_objects_degraded":1943,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5473,"num_write_kb":227392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1842,"num_bytes_recovered":120619008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,2,5","objects":39},{"shards":"1,3,5","objects":1595}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":27,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.242864+0000","last_change":"2023-10-09T10:17:49.750072+0000","last_active":"2023-10-09T10:17:59.242864+0000","last_peered":"2023-10-09T10:17:59.242864+0000","last_clean":"2023-10-09T10:17:59.242864+0000","last_became_active":"2023-10-09T10:17:49.749683+0000","last_became_peered":"2023-10-09T10:17:49.749683+0000","last_unstale":"2023-10-09T10:17:59.242864+0000","last_undegraded":"2023-10-09T10:17:59.242864+0000","last_fullsized":"2023-10-09T10:17:59.242864+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:24.234208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":393,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.808425+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:17:55.808425+0000","last_peered":"2023-10-09T10:17:55.808425+0000","last_clean":"2023-10-09T10:17:55.808425+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:17:55.808425+0000","last_undegraded":"2023-10-09T10:17:55.808425+0000","last_fullsized":"2023-10-09T10:17:55.808425+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1e","version":"396'10373","reported_seq":10479,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:03.171474+0000","last_change":"2023-10-09T10:17:37.701383+0000","last_active":"2023-10-09T10:18:03.171474+0000","last_peered":"2023-10-09T10:18:03.171474+0000","last_clean":"2023-10-09T10:18:03.171474+0000","last_became_active":"2023-10-09T10:17:37.674107+0000","last_became_peered":"2023-10-09T10:17:37.674107+0000","last_unstale":"2023-10-09T10:18:03.171474+0000","last_undegraded":"2023-10-09T10:18:03.171474+0000","last_fullsized":"2023-10-09T10:18:03.171474+0000","mapping_epoch":387,"log_start":"334'5090","ondisk_log_start":"334'5090","created":377,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5283,"log_dups_size":0,"ondisk_log_size":5283,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96862208,"num_objects":1478,"num_object_clones":0,"num_object_copies":4434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5515,"num_write_kb":223776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":104,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253949+0000","last_change":"2023-10-09T10:17:37.701325+0000","last_active":"2023-10-09T10:17:59.253949+0000","last_peered":"2023-10-09T10:17:59.253949+0000","last_clean":"2023-10-09T10:17:59.253949+0000","last_became_active":"2023-10-09T10:17:37.700655+0000","last_became_peered":"2023-10-09T10:17:37.700655+0000","last_unstale":"2023-10-09T10:17:59.253949+0000","last_undegraded":"2023-10-09T10:17:59.253949+0000","last_fullsized":"2023-10-09T10:17:59.253949+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":144,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253903+0000","last_change":"2023-10-09T10:17:37.691936+0000","last_active":"2023-10-09T10:17:59.253903+0000","last_peered":"2023-10-09T10:17:59.253903+0000","last_clean":"2023-10-09T10:17:59.253903+0000","last_became_active":"2023-10-09T10:17:37.691458+0000","last_became_peered":"2023-10-09T10:17:37.691458+0000","last_unstale":"2023-10-09T10:17:59.253903+0000","last_undegraded":"2023-10-09T10:17:59.253903+0000","last_fullsized":"2023-10-09T10:17:59.253903+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.19","version":"396'9049","reported_seq":9191,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:02.258505+0000","last_change":"2023-10-09T10:16:18.393528+0000","last_active":"2023-10-09T10:18:02.258505+0000","last_peered":"2023-10-09T10:18:02.258505+0000","last_clean":"2023-10-09T10:15:06.355378+0000","last_became_active":"2023-10-09T10:16:18.369005+0000","last_became_peered":"2023-10-09T10:16:18.369005+0000","last_unstale":"2023-10-09T10:18:02.258505+0000","last_undegraded":"2023-10-09T10:16:18.226963+0000","last_fullsized":"2023-10-09T10:16:18.205099+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9049,"log_dups_size":0,"ondisk_log_size":9049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102203392,"num_objects":1560,"num_object_clones":0,"num_object_copies":4680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5666,"num_write_kb":231200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1560}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":187,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253829+0000","last_change":"2023-10-09T10:17:49.756252+0000","last_active":"2023-10-09T10:17:59.253829+0000","last_peered":"2023-10-09T10:17:59.253829+0000","last_clean":"2023-10-09T10:17:59.253829+0000","last_became_active":"2023-10-09T10:17:49.755906+0000","last_became_peered":"2023-10-09T10:17:49.755906+0000","last_unstale":"2023-10-09T10:17:59.253829+0000","last_undegraded":"2023-10-09T10:17:59.253829+0000","last_fullsized":"2023-10-09T10:17:59.253829+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":349,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:00.251399+0000","last_change":"2023-10-09T10:15:42.406082+0000","last_active":"2023-10-09T10:18:00.251399+0000","last_peered":"2023-10-09T10:18:00.251399+0000","last_clean":"2023-10-09T10:18:00.251399+0000","last_became_active":"2023-10-09T10:15:42.405487+0000","last_became_peered":"2023-10-09T10:15:42.405487+0000","last_unstale":"2023-10-09T10:18:00.251399+0000","last_undegraded":"2023-10-09T10:18:00.251399+0000","last_fullsized":"2023-10-09T10:18:00.251399+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:58.713810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.18","version":"396'9101","reported_seq":10845,"reported_epoch":396,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:02.364722+0000","last_change":"2023-10-09T10:17:58.715703+0000","last_active":"2023-10-09T10:18:02.364722+0000","last_peered":"2023-10-09T10:18:02.364722+0000","last_clean":"2023-10-09T10:15:06.273595+0000","last_became_active":"2023-10-09T10:16:27.068683+0000","last_became_peered":"2023-10-09T10:16:27.068683+0000","last_unstale":"2023-10-09T10:18:02.364722+0000","last_undegraded":"2023-10-09T10:16:26.827722+0000","last_fullsized":"2023-10-09T10:16:22.231298+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9101,"log_dups_size":0,"ondisk_log_size":9101,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98369536,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1568,"num_objects_misplaced":1524,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5707,"num_write_kb":230656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":800,"num_bytes_recovered":32571392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1524},{"shards":"0,5","objects":-1591}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":288,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.244878+0000","last_change":"2023-10-09T10:16:08.455134+0000","last_active":"2023-10-09T10:17:59.244878+0000","last_peered":"2023-10-09T10:17:59.244878+0000","last_clean":"2023-10-09T10:17:59.244878+0000","last_became_active":"2023-10-09T10:16:08.454793+0000","last_became_peered":"2023-10-09T10:16:08.454793+0000","last_unstale":"2023-10-09T10:17:59.244878+0000","last_undegraded":"2023-10-09T10:17:59.244878+0000","last_fullsized":"2023-10-09T10:17:59.244878+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:25:33.883382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803763+0000","last_change":"2023-10-09T10:16:18.255609+0000","last_active":"2023-10-09T10:17:55.803763+0000","last_peered":"2023-10-09T10:17:55.803763+0000","last_clean":"2023-10-09T10:17:55.803763+0000","last_became_active":"2023-10-09T10:16:18.244618+0000","last_became_peered":"2023-10-09T10:16:18.244618+0000","last_unstale":"2023-10-09T10:17:55.803763+0000","last_undegraded":"2023-10-09T10:17:55.803763+0000","last_fullsized":"2023-10-09T10:17:55.803763+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1b","version":"396'10350","reported_seq":10510,"reported_epoch":396,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:02.298762+0000","last_change":"2023-10-09T10:17:09.499484+0000","last_active":"2023-10-09T10:18:02.298762+0000","last_peered":"2023-10-09T10:18:02.298762+0000","last_clean":"2023-10-09T10:15:06.321287+0000","last_became_active":"2023-10-09T10:17:09.485087+0000","last_became_peered":"2023-10-09T10:17:09.485087+0000","last_unstale":"2023-10-09T10:18:02.298762+0000","last_undegraded":"2023-10-09T10:17:09.479799+0000","last_fullsized":"2023-10-09T10:18:02.298762+0000","mapping_epoch":377,"log_start":"324'2335","ondisk_log_start":"324'2335","created":377,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8015,"log_dups_size":0,"ondisk_log_size":8015,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99549184,"num_objects":1519,"num_object_clones":0,"num_object_copies":4557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":504,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5495,"num_write_kb":224448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,5,6","objects":1519}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":146,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253069+0000","last_change":"2023-10-09T10:17:37.683049+0000","last_active":"2023-10-09T10:17:59.253069+0000","last_peered":"2023-10-09T10:17:59.253069+0000","last_clean":"2023-10-09T10:17:59.253069+0000","last_became_active":"2023-10-09T10:17:37.681922+0000","last_became_peered":"2023-10-09T10:17:37.681922+0000","last_unstale":"2023-10-09T10:17:59.253069+0000","last_undegraded":"2023-10-09T10:17:59.253069+0000","last_fullsized":"2023-10-09T10:17:59.253069+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":288,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245525+0000","last_change":"2023-10-09T10:16:08.232247+0000","last_active":"2023-10-09T10:17:59.245525+0000","last_peered":"2023-10-09T10:17:59.245525+0000","last_clean":"2023-10-09T10:17:59.245525+0000","last_became_active":"2023-10-09T10:16:08.231927+0000","last_became_peered":"2023-10-09T10:16:08.231927+0000","last_unstale":"2023-10-09T10:17:59.245525+0000","last_undegraded":"2023-10-09T10:17:59.245525+0000","last_fullsized":"2023-10-09T10:17:59.245525+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:20.408880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1a","version":"395'10058","reported_seq":10224,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.485839+0000","last_change":"2023-10-09T10:17:10.501570+0000","last_active":"2023-10-09T10:17:59.485839+0000","last_peered":"2023-10-09T10:17:59.485839+0000","last_clean":"2023-10-09T10:15:06.345974+0000","last_became_active":"2023-10-09T10:17:10.480946+0000","last_became_peered":"2023-10-09T10:17:10.480946+0000","last_unstale":"2023-10-09T10:17:59.485839+0000","last_undegraded":"2023-10-09T10:17:10.418566+0000","last_fullsized":"2023-10-09T10:17:59.485839+0000","mapping_epoch":377,"log_start":"334'4912","ondisk_log_start":"334'4912","created":377,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5146,"log_dups_size":0,"ondisk_log_size":5146,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99123200,"num_objects":1513,"num_object_clones":0,"num_object_copies":4539,"num_objects_missing_on_primary":257,"num_objects_missing":257,"num_objects_degraded":257,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5311,"num_write_kb":218336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":491520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,3,7","objects":1},{"shards":"1,5,7","objects":1512}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":333,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.248035+0000","last_change":"2023-10-09T10:16:26.743265+0000","last_active":"2023-10-09T10:17:59.248035+0000","last_peered":"2023-10-09T10:17:59.248035+0000","last_clean":"2023-10-09T10:16:00.577412+0000","last_became_active":"2023-10-09T10:16:22.206357+0000","last_became_peered":"2023-10-09T10:16:22.206357+0000","last_unstale":"2023-10-09T10:17:59.248035+0000","last_undegraded":"2023-10-09T10:16:18.203819+0000","last_fullsized":"2023-10-09T10:17:59.248035+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:47.851069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":341,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.806719+0000","last_change":"2023-10-09T10:16:08.618851+0000","last_active":"2023-10-09T10:17:55.806719+0000","last_peered":"2023-10-09T10:17:55.806719+0000","last_clean":"2023-10-09T10:17:55.806719+0000","last_became_active":"2023-10-09T10:16:08.618501+0000","last_became_peered":"2023-10-09T10:16:08.618501+0000","last_unstale":"2023-10-09T10:17:55.806719+0000","last_undegraded":"2023-10-09T10:17:55.806719+0000","last_fullsized":"2023-10-09T10:17:55.806719+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:45.133486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"396'8811","reported_seq":10218,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:02.244437+0000","last_change":"2023-10-09T10:16:18.358604+0000","last_active":"2023-10-09T10:18:02.244437+0000","last_peered":"2023-10-09T10:18:02.244437+0000","last_clean":"2023-10-09T10:15:06.284821+0000","last_became_active":"2023-10-09T10:16:18.344039+0000","last_became_peered":"2023-10-09T10:16:18.344039+0000","last_unstale":"2023-10-09T10:18:02.244437+0000","last_undegraded":"2023-10-09T10:16:18.231303+0000","last_fullsized":"2023-10-09T10:16:18.204454+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8811,"log_dups_size":0,"ondisk_log_size":8811,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97452032,"num_objects":1487,"num_object_clones":0,"num_object_copies":4461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5509,"num_write_kb":223872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":644,"num_bytes_recovered":42205184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1487}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":167,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.254271+0000","last_change":"2023-10-09T10:17:49.750648+0000","last_active":"2023-10-09T10:17:59.254271+0000","last_peered":"2023-10-09T10:17:59.254271+0000","last_clean":"2023-10-09T10:17:59.254271+0000","last_became_active":"2023-10-09T10:17:49.750268+0000","last_became_peered":"2023-10-09T10:17:49.750268+0000","last_unstale":"2023-10-09T10:17:59.254271+0000","last_undegraded":"2023-10-09T10:17:59.254271+0000","last_fullsized":"2023-10-09T10:17:59.254271+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.802779+0000","last_change":"2023-10-09T10:16:18.237380+0000","last_active":"2023-10-09T10:17:55.802779+0000","last_peered":"2023-10-09T10:17:55.802779+0000","last_clean":"2023-10-09T10:17:55.802779+0000","last_became_active":"2023-10-09T10:16:18.236585+0000","last_became_peered":"2023-10-09T10:16:18.236585+0000","last_unstale":"2023-10-09T10:17:55.802779+0000","last_undegraded":"2023-10-09T10:17:55.802779+0000","last_fullsized":"2023-10-09T10:17:55.802779+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"395'8718","reported_seq":9274,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.262695+0000","last_change":"2023-10-09T10:16:20.239479+0000","last_active":"2023-10-09T10:17:59.262695+0000","last_peered":"2023-10-09T10:17:59.262695+0000","last_clean":"2023-10-09T10:17:59.262695+0000","last_became_active":"2023-10-09T10:16:20.231381+0000","last_became_peered":"2023-10-09T10:16:20.231381+0000","last_unstale":"2023-10-09T10:17:59.262695+0000","last_undegraded":"2023-10-09T10:17:59.262695+0000","last_fullsized":"2023-10-09T10:17:59.262695+0000","mapping_epoch":363,"log_start":"325'2705","ondisk_log_start":"325'2705","created":317,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6013,"log_dups_size":0,"ondisk_log_size":6013,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5459,"num_write_kb":222656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":215,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":168,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.254318+0000","last_change":"2023-10-09T10:17:49.751128+0000","last_active":"2023-10-09T10:17:59.254318+0000","last_peered":"2023-10-09T10:17:59.254318+0000","last_clean":"2023-10-09T10:17:59.254318+0000","last_became_active":"2023-10-09T10:17:49.750853+0000","last_became_peered":"2023-10-09T10:17:49.750853+0000","last_unstale":"2023-10-09T10:17:59.254318+0000","last_undegraded":"2023-10-09T10:17:59.254318+0000","last_fullsized":"2023-10-09T10:17:59.254318+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":305,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253220+0000","last_change":"2023-10-09T10:17:37.683197+0000","last_active":"2023-10-09T10:17:59.253220+0000","last_peered":"2023-10-09T10:17:59.253220+0000","last_clean":"2023-10-09T10:17:59.253220+0000","last_became_active":"2023-10-09T10:17:37.682297+0000","last_became_peered":"2023-10-09T10:17:37.682297+0000","last_unstale":"2023-10-09T10:17:59.253220+0000","last_undegraded":"2023-10-09T10:17:59.253220+0000","last_fullsized":"2023-10-09T10:17:59.253220+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"395'8703","reported_seq":16407,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.503798+0000","last_change":"2023-10-09T10:17:33.574802+0000","last_active":"2023-10-09T10:17:59.503798+0000","last_peered":"2023-10-09T10:17:59.503798+0000","last_clean":"2023-10-09T10:17:59.503798+0000","last_became_active":"2023-10-09T10:17:33.536935+0000","last_became_peered":"2023-10-09T10:17:33.536935+0000","last_unstale":"2023-10-09T10:17:59.503798+0000","last_undegraded":"2023-10-09T10:17:59.503798+0000","last_fullsized":"2023-10-09T10:17:59.503798+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"339'6595","last_scrub_stamp":"2023-10-09T10:15:01.144444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:15:01.144444+0000","objects_scrubbed":0,"log_size":8703,"log_dups_size":0,"ondisk_log_size":8703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102367232,"num_objects":1562,"num_object_clones":0,"num_object_copies":4686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2136,"num_write_kb":10560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3735,"num_bytes_recovered":226557952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":47,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803652+0000","last_change":"2023-10-09T10:16:18.362151+0000","last_active":"2023-10-09T10:17:55.803652+0000","last_peered":"2023-10-09T10:17:55.803652+0000","last_clean":"2023-10-09T10:17:55.803652+0000","last_became_active":"2023-10-09T10:16:18.323076+0000","last_became_peered":"2023-10-09T10:16:18.323076+0000","last_unstale":"2023-10-09T10:17:55.803652+0000","last_undegraded":"2023-10-09T10:17:55.803652+0000","last_fullsized":"2023-10-09T10:17:55.803652+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":238,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.802309+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:17:55.802309+0000","last_peered":"2023-10-09T10:17:55.802309+0000","last_clean":"2023-10-09T10:17:55.802309+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:17:55.802309+0000","last_undegraded":"2023-10-09T10:17:55.802309+0000","last_fullsized":"2023-10-09T10:17:55.802309+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"395'8803","reported_seq":9993,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:17:59.269056+0000","last_change":"2023-10-09T10:16:18.313517+0000","last_active":"2023-10-09T10:17:59.269056+0000","last_peered":"2023-10-09T10:17:59.269056+0000","last_clean":"2023-10-09T10:15:06.224242+0000","last_became_active":"2023-10-09T10:16:18.305795+0000","last_became_peered":"2023-10-09T10:16:18.305795+0000","last_unstale":"2023-10-09T10:17:59.269056+0000","last_undegraded":"2023-10-09T10:16:18.214537+0000","last_fullsized":"2023-10-09T10:16:18.207078+0000","mapping_epoch":361,"log_start":"324'2713","ondisk_log_start":"324'2713","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6090,"log_dups_size":0,"ondisk_log_size":6090,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100171776,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":500,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5486,"num_write_kb":224480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":536,"num_bytes_recovered":35127296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":1527},{"shards":"1,2,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803366+0000","last_change":"2023-10-09T10:16:18.237242+0000","last_active":"2023-10-09T10:17:55.803366+0000","last_peered":"2023-10-09T10:17:55.803366+0000","last_clean":"2023-10-09T10:17:55.803366+0000","last_became_active":"2023-10-09T10:16:18.236617+0000","last_became_peered":"2023-10-09T10:16:18.236617+0000","last_unstale":"2023-10-09T10:17:55.803366+0000","last_undegraded":"2023-10-09T10:17:55.803366+0000","last_fullsized":"2023-10-09T10:17:55.803366+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":265,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245164+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:17:59.245164+0000","last_peered":"2023-10-09T10:17:59.245164+0000","last_clean":"2023-10-09T10:17:59.245164+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:17:59.245164+0000","last_undegraded":"2023-10-09T10:17:59.245164+0000","last_fullsized":"2023-10-09T10:17:59.245164+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"396'8656","reported_seq":8787,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:00.329393+0000","last_change":"2023-10-09T10:17:47.794470+0000","last_active":"2023-10-09T10:18:00.329393+0000","last_peered":"2023-10-09T10:18:00.329393+0000","last_clean":"2023-10-09T10:18:00.329393+0000","last_became_active":"2023-10-09T10:17:09.403921+0000","last_became_peered":"2023-10-09T10:17:09.403921+0000","last_unstale":"2023-10-09T10:18:00.329393+0000","last_undegraded":"2023-10-09T10:18:00.329393+0000","last_fullsized":"2023-10-09T10:18:00.329393+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8656,"log_dups_size":0,"ondisk_log_size":8656,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50397184,"num_objects":769,"num_object_clones":0,"num_object_copies":2307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2805,"num_write_kb":114368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":268,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245251+0000","last_change":"2023-10-09T10:16:08.363203+0000","last_active":"2023-10-09T10:17:59.245251+0000","last_peered":"2023-10-09T10:17:59.245251+0000","last_clean":"2023-10-09T10:17:59.245251+0000","last_became_active":"2023-10-09T10:16:08.248248+0000","last_became_peered":"2023-10-09T10:16:08.248248+0000","last_unstale":"2023-10-09T10:17:59.245251+0000","last_undegraded":"2023-10-09T10:17:59.245251+0000","last_fullsized":"2023-10-09T10:17:59.245251+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:48.937824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":197,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.254017+0000","last_change":"2023-10-09T10:17:37.701500+0000","last_active":"2023-10-09T10:17:59.254017+0000","last_peered":"2023-10-09T10:17:59.254017+0000","last_clean":"2023-10-09T10:17:59.254017+0000","last_became_active":"2023-10-09T10:17:37.700983+0000","last_became_peered":"2023-10-09T10:17:37.700983+0000","last_unstale":"2023-10-09T10:17:59.254017+0000","last_undegraded":"2023-10-09T10:17:59.254017+0000","last_fullsized":"2023-10-09T10:17:59.254017+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"394'8422","reported_seq":8609,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.237618+0000","last_change":"2023-10-09T10:17:09.428557+0000","last_active":"2023-10-09T10:17:59.237618+0000","last_peered":"2023-10-09T10:17:59.237618+0000","last_clean":"2023-10-09T10:17:59.237618+0000","last_became_active":"2023-10-09T10:17:09.420973+0000","last_became_peered":"2023-10-09T10:17:09.420973+0000","last_unstale":"2023-10-09T10:17:59.237618+0000","last_undegraded":"2023-10-09T10:17:59.237618+0000","last_fullsized":"2023-10-09T10:17:59.237618+0000","mapping_epoch":317,"log_start":"321'624","ondisk_log_start":"321'624","created":317,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"339'6671","last_scrub_stamp":"2023-10-09T10:15:03.503975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:15:03.503975+0000","objects_scrubbed":3418,"log_size":7798,"log_dups_size":0,"ondisk_log_size":7798,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.90657236399999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51183616,"num_objects":781,"num_object_clones":0,"num_object_copies":2343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1032,"num_write_kb":3328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":202,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253477+0000","last_change":"2023-10-09T10:17:37.685357+0000","last_active":"2023-10-09T10:17:59.253477+0000","last_peered":"2023-10-09T10:17:59.253477+0000","last_clean":"2023-10-09T10:17:59.253477+0000","last_became_active":"2023-10-09T10:17:37.684740+0000","last_became_peered":"2023-10-09T10:17:37.684740+0000","last_unstale":"2023-10-09T10:17:59.253477+0000","last_undegraded":"2023-10-09T10:17:59.253477+0000","last_fullsized":"2023-10-09T10:17:59.253477+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":229,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253742+0000","last_change":"2023-10-09T10:17:37.690061+0000","last_active":"2023-10-09T10:17:59.253742+0000","last_peered":"2023-10-09T10:17:59.253742+0000","last_clean":"2023-10-09T10:17:59.253742+0000","last_became_active":"2023-10-09T10:17:37.689771+0000","last_became_peered":"2023-10-09T10:17:37.689771+0000","last_unstale":"2023-10-09T10:17:59.253742+0000","last_undegraded":"2023-10-09T10:17:59.253742+0000","last_fullsized":"2023-10-09T10:17:59.253742+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"394'8460","reported_seq":9238,"reported_epoch":394,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.229504+0000","last_change":"2023-10-09T10:17:49.793224+0000","last_active":"2023-10-09T10:17:59.229504+0000","last_peered":"2023-10-09T10:17:59.229504+0000","last_clean":"2023-10-09T10:15:14.172820+0000","last_became_active":"2023-10-09T10:17:49.775592+0000","last_became_peered":"2023-10-09T10:17:49.775592+0000","last_unstale":"2023-10-09T10:17:59.229504+0000","last_undegraded":"2023-10-09T10:17:49.748297+0000","last_fullsized":"2023-10-09T10:17:59.229504+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8460,"log_dups_size":0,"ondisk_log_size":8460,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47726592,"num_objects":728,"num_object_clones":0,"num_object_copies":2184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1504,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2756,"num_write_kb":111504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":319,"num_bytes_recovered":20905984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,3,7","objects":791},{"shards":"1,5,7","objects":-63}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":172,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253273+0000","last_change":"2023-10-09T10:17:37.683342+0000","last_active":"2023-10-09T10:17:59.253273+0000","last_peered":"2023-10-09T10:17:59.253273+0000","last_clean":"2023-10-09T10:17:59.253273+0000","last_became_active":"2023-10-09T10:17:37.682635+0000","last_became_peered":"2023-10-09T10:17:37.682635+0000","last_unstale":"2023-10-09T10:17:59.253273+0000","last_undegraded":"2023-10-09T10:17:59.253273+0000","last_fullsized":"2023-10-09T10:17:59.253273+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":144,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.254232+0000","last_change":"2023-10-09T10:17:37.700874+0000","last_active":"2023-10-09T10:17:59.254232+0000","last_peered":"2023-10-09T10:17:59.254232+0000","last_clean":"2023-10-09T10:17:59.254232+0000","last_became_active":"2023-10-09T10:17:37.700300+0000","last_became_peered":"2023-10-09T10:17:37.700300+0000","last_unstale":"2023-10-09T10:17:59.254232+0000","last_undegraded":"2023-10-09T10:17:59.254232+0000","last_fullsized":"2023-10-09T10:17:59.254232+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"394'8396","reported_seq":9120,"reported_epoch":394,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.049721+0000","last_change":"2023-10-09T10:17:10.437080+0000","last_active":"2023-10-09T10:17:59.049721+0000","last_peered":"2023-10-09T10:17:59.049721+0000","last_clean":"2023-10-09T10:15:14.051568+0000","last_became_active":"2023-10-09T10:17:10.429080+0000","last_became_peered":"2023-10-09T10:17:10.429080+0000","last_unstale":"2023-10-09T10:17:59.049721+0000","last_undegraded":"2023-10-09T10:17:10.393900+0000","last_fullsized":"2023-10-09T10:17:59.049721+0000","mapping_epoch":378,"log_start":"321'588","ondisk_log_start":"321'588","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7808,"log_dups_size":0,"ondisk_log_size":7808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49807360,"num_objects":760,"num_object_clones":0,"num_object_copies":2280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1446,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2712,"num_write_kb":111104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,3","objects":773},{"shards":"1,2,7","objects":-13}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":343,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.805400+0000","last_change":"2023-10-09T10:16:08.267092+0000","last_active":"2023-10-09T10:17:55.805400+0000","last_peered":"2023-10-09T10:17:55.805400+0000","last_clean":"2023-10-09T10:17:55.805400+0000","last_became_active":"2023-10-09T10:16:08.244668+0000","last_became_peered":"2023-10-09T10:16:08.244668+0000","last_unstale":"2023-10-09T10:17:55.805400+0000","last_undegraded":"2023-10-09T10:17:55.805400+0000","last_fullsized":"2023-10-09T10:17:55.805400+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:35:59.836060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":416,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.806615+0000","last_change":"2023-10-09T10:15:42.317788+0000","last_active":"2023-10-09T10:17:55.806615+0000","last_peered":"2023-10-09T10:17:55.806615+0000","last_clean":"2023-10-09T10:17:55.806615+0000","last_became_active":"2023-10-09T10:15:42.317524+0000","last_became_peered":"2023-10-09T10:15:42.317524+0000","last_unstale":"2023-10-09T10:17:55.806615+0000","last_undegraded":"2023-10-09T10:17:55.806615+0000","last_fullsized":"2023-10-09T10:17:55.806615+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:54.507373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"395'10476","reported_seq":17035,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.523505+0000","last_change":"2023-10-09T10:17:18.218427+0000","last_active":"2023-10-09T10:17:59.523505+0000","last_peered":"2023-10-09T10:17:59.523505+0000","last_clean":"2023-10-09T10:15:06.273272+0000","last_became_active":"2023-10-09T10:17:16.191120+0000","last_became_peered":"2023-10-09T10:17:16.191120+0000","last_unstale":"2023-10-09T10:17:59.523505+0000","last_undegraded":"2023-10-09T10:17:12.581924+0000","last_fullsized":"2023-10-09T10:17:59.523505+0000","mapping_epoch":378,"log_start":"334'5299","ondisk_log_start":"334'5299","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5177,"log_dups_size":0,"ondisk_log_size":5177,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102203392,"num_objects":1559,"num_object_clones":0,"num_object_copies":4677,"num_objects_missing_on_primary":239,"num_objects_missing":239,"num_objects_degraded":1910,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5546,"num_write_kb":227392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125468672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,2,5","objects":-37},{"shards":"1,3,5","objects":1596}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":401,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.804425+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:17:55.804425+0000","last_peered":"2023-10-09T10:17:55.804425+0000","last_clean":"2023-10-09T10:17:55.804425+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:17:55.804425+0000","last_undegraded":"2023-10-09T10:17:55.804425+0000","last_fullsized":"2023-10-09T10:17:55.804425+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803751+0000","last_change":"2023-10-09T10:16:18.323515+0000","last_active":"2023-10-09T10:17:55.803751+0000","last_peered":"2023-10-09T10:17:55.803751+0000","last_clean":"2023-10-09T10:17:55.803751+0000","last_became_active":"2023-10-09T10:16:18.321171+0000","last_became_peered":"2023-10-09T10:16:18.321171+0000","last_unstale":"2023-10-09T10:17:55.803751+0000","last_undegraded":"2023-10-09T10:17:55.803751+0000","last_fullsized":"2023-10-09T10:17:55.803751+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":375,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.809390+0000","last_change":"2023-10-09T10:16:18.261462+0000","last_active":"2023-10-09T10:17:55.809390+0000","last_peered":"2023-10-09T10:17:55.809390+0000","last_clean":"2023-10-09T10:17:55.809390+0000","last_became_active":"2023-10-09T10:16:18.261023+0000","last_became_peered":"2023-10-09T10:16:18.261023+0000","last_unstale":"2023-10-09T10:17:55.809390+0000","last_undegraded":"2023-10-09T10:17:55.809390+0000","last_fullsized":"2023-10-09T10:17:55.809390+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:43.268197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"396'9131","reported_seq":9271,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:02.145494+0000","last_change":"2023-10-09T10:16:18.400385+0000","last_active":"2023-10-09T10:18:02.145494+0000","last_peered":"2023-10-09T10:18:02.145494+0000","last_clean":"2023-10-09T10:15:06.364805+0000","last_became_active":"2023-10-09T10:16:18.392713+0000","last_became_peered":"2023-10-09T10:16:18.392713+0000","last_unstale":"2023-10-09T10:18:02.145494+0000","last_undegraded":"2023-10-09T10:16:18.227890+0000","last_fullsized":"2023-10-09T10:16:18.203511+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9131,"log_dups_size":0,"ondisk_log_size":9131,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95649792,"num_objects":1459,"num_object_clones":0,"num_object_copies":4377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5747,"num_write_kb":230624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1459}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803698+0000","last_change":"2023-10-09T10:16:18.255781+0000","last_active":"2023-10-09T10:17:55.803698+0000","last_peered":"2023-10-09T10:17:55.803698+0000","last_clean":"2023-10-09T10:17:55.803698+0000","last_became_active":"2023-10-09T10:16:18.238531+0000","last_became_peered":"2023-10-09T10:16:18.238531+0000","last_unstale":"2023-10-09T10:17:55.803698+0000","last_undegraded":"2023-10-09T10:17:55.803698+0000","last_fullsized":"2023-10-09T10:17:55.803698+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":103,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253750+0000","last_change":"2023-10-09T10:17:37.691757+0000","last_active":"2023-10-09T10:17:59.253750+0000","last_peered":"2023-10-09T10:17:59.253750+0000","last_clean":"2023-10-09T10:17:59.253750+0000","last_became_active":"2023-10-09T10:17:37.690931+0000","last_became_peered":"2023-10-09T10:17:37.690931+0000","last_unstale":"2023-10-09T10:17:59.253750+0000","last_undegraded":"2023-10-09T10:17:59.253750+0000","last_fullsized":"2023-10-09T10:17:59.253750+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":448,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.804359+0000","last_change":"2023-10-09T10:16:08.230567+0000","last_active":"2023-10-09T10:17:55.804359+0000","last_peered":"2023-10-09T10:17:55.804359+0000","last_clean":"2023-10-09T10:17:55.804359+0000","last_became_active":"2023-10-09T10:16:08.229677+0000","last_became_peered":"2023-10-09T10:16:08.229677+0000","last_unstale":"2023-10-09T10:17:55.804359+0000","last_undegraded":"2023-10-09T10:17:55.804359+0000","last_fullsized":"2023-10-09T10:17:55.804359+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:52.866334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"395'10048","reported_seq":10222,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:17:59.413704+0000","last_change":"2023-10-09T10:17:58.717165+0000","last_active":"2023-10-09T10:17:59.413704+0000","last_peered":"2023-10-09T10:17:59.413704+0000","last_clean":"2023-10-09T10:15:06.345974+0000","last_became_active":"2023-10-09T10:17:10.463863+0000","last_became_peered":"2023-10-09T10:17:10.463863+0000","last_unstale":"2023-10-09T10:17:59.413704+0000","last_undegraded":"2023-10-09T10:17:58.716826+0000","last_fullsized":"2023-10-09T10:17:59.413704+0000","mapping_epoch":378,"log_start":"334'4912","ondisk_log_start":"334'4912","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5136,"log_dups_size":0,"ondisk_log_size":5136,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99778560,"num_objects":1522,"num_object_clones":0,"num_object_copies":4566,"num_objects_missing_on_primary":293,"num_objects_missing":293,"num_objects_degraded":293,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5300,"num_write_kb":218336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":491520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,5,7","objects":1522}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":306,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.248890+0000","last_change":"2023-10-09T10:16:08.269361+0000","last_active":"2023-10-09T10:17:59.248890+0000","last_peered":"2023-10-09T10:17:59.248890+0000","last_clean":"2023-10-09T10:17:59.248890+0000","last_became_active":"2023-10-09T10:16:08.266453+0000","last_became_peered":"2023-10-09T10:16:08.266453+0000","last_unstale":"2023-10-09T10:17:59.248890+0000","last_undegraded":"2023-10-09T10:17:59.248890+0000","last_fullsized":"2023-10-09T10:17:59.248890+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:07.633138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":393,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.805552+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:17:55.805552+0000","last_peered":"2023-10-09T10:17:55.805552+0000","last_clean":"2023-10-09T10:17:55.805552+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:17:55.805552+0000","last_undegraded":"2023-10-09T10:17:55.805552+0000","last_fullsized":"2023-10-09T10:17:55.805552+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":431,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.806563+0000","last_change":"2023-10-09T10:16:18.266234+0000","last_active":"2023-10-09T10:17:55.806563+0000","last_peered":"2023-10-09T10:17:55.806563+0000","last_clean":"2023-10-09T10:17:55.806563+0000","last_became_active":"2023-10-09T10:16:18.265721+0000","last_became_peered":"2023-10-09T10:16:18.265721+0000","last_unstale":"2023-10-09T10:17:55.806563+0000","last_undegraded":"2023-10-09T10:17:55.806563+0000","last_fullsized":"2023-10-09T10:17:55.806563+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:59.391242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"396'9962","reported_seq":17825,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:02.286804+0000","last_change":"2023-10-09T10:17:12.423029+0000","last_active":"2023-10-09T10:18:02.286804+0000","last_peered":"2023-10-09T10:18:02.286804+0000","last_clean":"2023-10-09T10:18:02.286804+0000","last_became_active":"2023-10-09T10:17:12.413033+0000","last_became_peered":"2023-10-09T10:17:12.413033+0000","last_unstale":"2023-10-09T10:18:02.286804+0000","last_undegraded":"2023-10-09T10:18:02.286804+0000","last_fullsized":"2023-10-09T10:18:02.286804+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":1211,"log_size":9962,"log_dups_size":0,"ondisk_log_size":9962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100859904,"num_objects":1539,"num_object_clones":0,"num_object_copies":4617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5647,"num_write_kb":229952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3776,"num_bytes_recovered":228851712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803303+0000","last_change":"2023-10-09T10:16:18.477748+0000","last_active":"2023-10-09T10:17:55.803303+0000","last_peered":"2023-10-09T10:17:55.803303+0000","last_clean":"2023-10-09T10:17:55.803303+0000","last_became_active":"2023-10-09T10:16:18.477398+0000","last_became_peered":"2023-10-09T10:16:18.477398+0000","last_unstale":"2023-10-09T10:17:55.803303+0000","last_undegraded":"2023-10-09T10:17:55.803303+0000","last_fullsized":"2023-10-09T10:17:55.803303+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":405,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.247638+0000","last_change":"2023-10-09T10:17:49.756954+0000","last_active":"2023-10-09T10:17:59.247638+0000","last_peered":"2023-10-09T10:17:59.247638+0000","last_clean":"2023-10-09T10:17:59.247638+0000","last_became_active":"2023-10-09T10:17:49.756562+0000","last_became_peered":"2023-10-09T10:17:49.756562+0000","last_unstale":"2023-10-09T10:17:59.247638+0000","last_undegraded":"2023-10-09T10:17:59.247638+0000","last_fullsized":"2023-10-09T10:17:59.247638+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:37.985381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803779+0000","last_change":"2023-10-09T10:16:18.255461+0000","last_active":"2023-10-09T10:17:55.803779+0000","last_peered":"2023-10-09T10:17:55.803779+0000","last_clean":"2023-10-09T10:17:55.803779+0000","last_became_active":"2023-10-09T10:16:18.247752+0000","last_became_peered":"2023-10-09T10:16:18.247752+0000","last_unstale":"2023-10-09T10:17:55.803779+0000","last_undegraded":"2023-10-09T10:17:55.803779+0000","last_fullsized":"2023-10-09T10:17:55.803779+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":193,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253671+0000","last_change":"2023-10-09T10:17:37.689632+0000","last_active":"2023-10-09T10:17:59.253671+0000","last_peered":"2023-10-09T10:17:59.253671+0000","last_clean":"2023-10-09T10:17:59.253671+0000","last_became_active":"2023-10-09T10:17:37.689105+0000","last_became_peered":"2023-10-09T10:17:37.689105+0000","last_unstale":"2023-10-09T10:17:59.253671+0000","last_undegraded":"2023-10-09T10:17:59.253671+0000","last_fullsized":"2023-10-09T10:17:59.253671+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"396'10327","reported_seq":10489,"reported_epoch":396,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:02.272579+0000","last_change":"2023-10-09T10:17:10.422318+0000","last_active":"2023-10-09T10:18:02.272579+0000","last_peered":"2023-10-09T10:18:02.272579+0000","last_clean":"2023-10-09T10:15:06.321287+0000","last_became_active":"2023-10-09T10:17:10.410872+0000","last_became_peered":"2023-10-09T10:17:10.410872+0000","last_unstale":"2023-10-09T10:18:02.272579+0000","last_undegraded":"2023-10-09T10:17:10.398477+0000","last_fullsized":"2023-10-09T10:18:02.272579+0000","mapping_epoch":378,"log_start":"324'2335","ondisk_log_start":"324'2335","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":7992,"log_dups_size":0,"ondisk_log_size":7992,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101056512,"num_objects":1542,"num_object_clones":0,"num_object_copies":4626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":480,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5472,"num_write_kb":224448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,5,6","objects":1542}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":373,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.807083+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:17:55.807083+0000","last_peered":"2023-10-09T10:17:55.807083+0000","last_clean":"2023-10-09T10:17:55.807083+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:17:55.807083+0000","last_undegraded":"2023-10-09T10:17:55.807083+0000","last_fullsized":"2023-10-09T10:17:55.807083+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":309,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.248967+0000","last_change":"2023-10-09T10:16:18.393880+0000","last_active":"2023-10-09T10:17:59.248967+0000","last_peered":"2023-10-09T10:17:59.248967+0000","last_clean":"2023-10-09T10:17:59.248967+0000","last_became_active":"2023-10-09T10:16:18.375766+0000","last_became_peered":"2023-10-09T10:16:18.375766+0000","last_unstale":"2023-10-09T10:17:59.248967+0000","last_undegraded":"2023-10-09T10:17:59.248967+0000","last_fullsized":"2023-10-09T10:17:59.248967+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:56:50.927077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":313,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245375+0000","last_change":"2023-10-09T10:15:07.378501+0000","last_active":"2023-10-09T10:17:59.245375+0000","last_peered":"2023-10-09T10:17:59.245375+0000","last_clean":"2023-10-09T10:17:59.245375+0000","last_became_active":"2023-10-09T10:15:07.378146+0000","last_became_peered":"2023-10-09T10:15:07.378146+0000","last_unstale":"2023-10-09T10:17:59.245375+0000","last_undegraded":"2023-10-09T10:17:59.245375+0000","last_fullsized":"2023-10-09T10:17:59.245375+0000","mapping_epoch":341,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":342,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:56.827491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"396'10360","reported_seq":10465,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:03.127958+0000","last_change":"2023-10-09T10:17:37.700641+0000","last_active":"2023-10-09T10:18:03.127958+0000","last_peered":"2023-10-09T10:18:03.127958+0000","last_clean":"2023-10-09T10:18:03.127958+0000","last_became_active":"2023-10-09T10:17:37.677480+0000","last_became_peered":"2023-10-09T10:17:37.677480+0000","last_unstale":"2023-10-09T10:18:03.127958+0000","last_undegraded":"2023-10-09T10:18:03.127958+0000","last_fullsized":"2023-10-09T10:18:03.127958+0000","mapping_epoch":387,"log_start":"334'5090","ondisk_log_start":"334'5090","created":317,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5270,"log_dups_size":0,"ondisk_log_size":5270,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97714176,"num_objects":1491,"num_object_clones":0,"num_object_copies":4473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5502,"num_write_kb":223776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":293,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.247102+0000","last_change":"2023-10-09T10:17:49.750539+0000","last_active":"2023-10-09T10:17:59.247102+0000","last_peered":"2023-10-09T10:17:59.247102+0000","last_clean":"2023-10-09T10:17:59.247102+0000","last_became_active":"2023-10-09T10:17:49.750241+0000","last_became_peered":"2023-10-09T10:17:49.750241+0000","last_unstale":"2023-10-09T10:17:59.247102+0000","last_undegraded":"2023-10-09T10:17:59.247102+0000","last_fullsized":"2023-10-09T10:17:59.247102+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:28.658196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":448,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.805717+0000","last_change":"2023-10-09T10:16:18.261069+0000","last_active":"2023-10-09T10:17:55.805717+0000","last_peered":"2023-10-09T10:17:55.805717+0000","last_clean":"2023-10-09T10:17:55.805717+0000","last_became_active":"2023-10-09T10:16:18.260736+0000","last_became_peered":"2023-10-09T10:16:18.260736+0000","last_unstale":"2023-10-09T10:17:55.805717+0000","last_undegraded":"2023-10-09T10:17:55.805717+0000","last_fullsized":"2023-10-09T10:17:55.805717+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:18.217603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":286,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245391+0000","last_change":"2023-10-09T10:16:18.477779+0000","last_active":"2023-10-09T10:17:59.245391+0000","last_peered":"2023-10-09T10:17:59.245391+0000","last_clean":"2023-10-09T10:17:59.245391+0000","last_became_active":"2023-10-09T10:16:18.477390+0000","last_became_peered":"2023-10-09T10:16:18.477390+0000","last_unstale":"2023-10-09T10:17:59.245391+0000","last_undegraded":"2023-10-09T10:17:59.245391+0000","last_fullsized":"2023-10-09T10:17:59.245391+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:06.576807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"394'10479","reported_seq":10598,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245639+0000","last_change":"2023-10-09T10:17:58.710957+0000","last_active":"2023-10-09T10:17:59.245639+0000","last_peered":"2023-10-09T10:17:59.245639+0000","last_clean":"2023-10-09T10:17:59.245639+0000","last_became_active":"2023-10-09T10:17:09.461024+0000","last_became_peered":"2023-10-09T10:17:09.461024+0000","last_unstale":"2023-10-09T10:17:59.245639+0000","last_undegraded":"2023-10-09T10:17:59.245639+0000","last_fullsized":"2023-10-09T10:17:59.245639+0000","mapping_epoch":361,"log_start":"334'5326","ondisk_log_start":"334'5326","created":317,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5153,"log_dups_size":0,"ondisk_log_size":5153,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102400012,"num_objects":1563,"num_object_clones":0,"num_object_copies":4689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5533,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":355,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.243169+0000","last_change":"2023-10-09T10:15:42.391115+0000","last_active":"2023-10-09T10:17:59.243169+0000","last_peered":"2023-10-09T10:17:59.243169+0000","last_clean":"2023-10-09T10:17:59.243169+0000","last_became_active":"2023-10-09T10:15:42.390765+0000","last_became_peered":"2023-10-09T10:15:42.390765+0000","last_unstale":"2023-10-09T10:17:59.243169+0000","last_undegraded":"2023-10-09T10:17:59.243169+0000","last_fullsized":"2023-10-09T10:17:59.243169+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:03.271932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":374,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.806915+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:17:55.806915+0000","last_peered":"2023-10-09T10:17:55.806915+0000","last_clean":"2023-10-09T10:17:55.806915+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:17:55.806915+0000","last_undegraded":"2023-10-09T10:17:55.806915+0000","last_fullsized":"2023-10-09T10:17:55.806915+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":310,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.247732+0000","last_change":"2023-10-09T10:17:49.752128+0000","last_active":"2023-10-09T10:17:59.247732+0000","last_peered":"2023-10-09T10:17:59.247732+0000","last_clean":"2023-10-09T10:17:59.247732+0000","last_became_active":"2023-10-09T10:17:49.751857+0000","last_became_peered":"2023-10-09T10:17:49.751857+0000","last_unstale":"2023-10-09T10:17:59.247732+0000","last_undegraded":"2023-10-09T10:17:59.247732+0000","last_fullsized":"2023-10-09T10:17:59.247732+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:31:59.458570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"396'10515","reported_seq":23597,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:03.138779+0000","last_change":"2023-10-09T10:17:37.701310+0000","last_active":"2023-10-09T10:18:03.138779+0000","last_peered":"2023-10-09T10:18:03.138779+0000","last_clean":"2023-10-09T10:18:03.138779+0000","last_became_active":"2023-10-09T10:17:37.672886+0000","last_became_peered":"2023-10-09T10:17:37.672886+0000","last_unstale":"2023-10-09T10:18:03.138779+0000","last_undegraded":"2023-10-09T10:18:03.138779+0000","last_fullsized":"2023-10-09T10:18:03.138779+0000","mapping_epoch":387,"log_start":"324'2492","ondisk_log_start":"324'2492","created":317,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8023,"log_dups_size":0,"ondisk_log_size":8023,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96272384,"num_objects":1469,"num_object_clones":0,"num_object_copies":4407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5592,"num_write_kb":225952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3234,"num_bytes_recovered":196968448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":303,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803249+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:17:55.803249+0000","last_peered":"2023-10-09T10:17:55.803249+0000","last_clean":"2023-10-09T10:17:55.803249+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:17:55.803249+0000","last_undegraded":"2023-10-09T10:17:55.803249+0000","last_fullsized":"2023-10-09T10:17:55.803249+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":270,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245287+0000","last_change":"2023-10-09T10:08:35.085303+0000","last_active":"2023-10-09T10:17:59.245287+0000","last_peered":"2023-10-09T10:17:59.245287+0000","last_clean":"2023-10-09T10:17:59.245287+0000","last_became_active":"2023-10-09T10:08:35.085095+0000","last_became_peered":"2023-10-09T10:08:35.085095+0000","last_unstale":"2023-10-09T10:17:59.245287+0000","last_undegraded":"2023-10-09T10:17:59.245287+0000","last_fullsized":"2023-10-09T10:17:59.245287+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:55.346665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":131,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.252989+0000","last_change":"2023-10-09T10:17:37.682902+0000","last_active":"2023-10-09T10:17:59.252989+0000","last_peered":"2023-10-09T10:17:59.252989+0000","last_clean":"2023-10-09T10:17:59.252989+0000","last_became_active":"2023-10-09T10:17:37.681476+0000","last_became_peered":"2023-10-09T10:17:37.681476+0000","last_unstale":"2023-10-09T10:17:59.252989+0000","last_undegraded":"2023-10-09T10:17:59.252989+0000","last_fullsized":"2023-10-09T10:17:59.252989+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.11","version":"396'8917","reported_seq":9037,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-10-09T10:18:00.343891+0000","last_change":"2023-10-09T10:17:52.861771+0000","last_active":"2023-10-09T10:18:00.343891+0000","last_peered":"2023-10-09T10:18:00.343891+0000","last_clean":"2023-10-09T10:18:00.343891+0000","last_became_active":"2023-10-09T10:16:22.227875+0000","last_became_peered":"2023-10-09T10:16:22.227875+0000","last_unstale":"2023-10-09T10:18:00.343891+0000","last_undegraded":"2023-10-09T10:18:00.343891+0000","last_fullsized":"2023-10-09T10:18:00.343891+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8917,"log_dups_size":0,"ondisk_log_size":8917,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103415808,"num_objects":1578,"num_object_clones":0,"num_object_copies":4734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5560,"num_write_kb":228416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":233,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":264,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245477+0000","last_change":"2023-10-09T10:08:35.086564+0000","last_active":"2023-10-09T10:17:59.245477+0000","last_peered":"2023-10-09T10:17:59.245477+0000","last_clean":"2023-10-09T10:17:59.245477+0000","last_became_active":"2023-10-09T10:08:35.086213+0000","last_became_peered":"2023-10-09T10:08:35.086213+0000","last_unstale":"2023-10-09T10:17:59.245477+0000","last_undegraded":"2023-10-09T10:17:59.245477+0000","last_fullsized":"2023-10-09T10:17:59.245477+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:37.059045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":359,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.805796+0000","last_change":"2023-10-09T10:16:08.231228+0000","last_active":"2023-10-09T10:17:55.805796+0000","last_peered":"2023-10-09T10:17:55.805796+0000","last_clean":"2023-10-09T10:17:55.805796+0000","last_became_active":"2023-10-09T10:16:08.230940+0000","last_became_peered":"2023-10-09T10:16:08.230940+0000","last_unstale":"2023-10-09T10:17:55.805796+0000","last_undegraded":"2023-10-09T10:17:55.805796+0000","last_fullsized":"2023-10-09T10:17:55.805796+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:52.611954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.16","version":"395'8819","reported_seq":9313,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:17:59.245591+0000","last_change":"2023-10-09T10:16:18.352731+0000","last_active":"2023-10-09T10:17:59.245591+0000","last_peered":"2023-10-09T10:17:59.245591+0000","last_clean":"2023-10-09T10:15:06.314173+0000","last_became_active":"2023-10-09T10:16:18.345487+0000","last_became_peered":"2023-10-09T10:16:18.345487+0000","last_unstale":"2023-10-09T10:17:59.245591+0000","last_undegraded":"2023-10-09T10:16:18.224255+0000","last_fullsized":"2023-10-09T10:16:18.206382+0000","mapping_epoch":361,"log_start":"325'2811","ondisk_log_start":"325'2811","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6008,"log_dups_size":0,"ondisk_log_size":6008,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100433920,"num_objects":1533,"num_object_clones":0,"num_object_copies":4599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":508,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5503,"num_write_kb":225120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":190,"num_bytes_recovered":12451840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":1533}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":47,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803280+0000","last_change":"2023-10-09T10:16:18.299039+0000","last_active":"2023-10-09T10:17:55.803280+0000","last_peered":"2023-10-09T10:17:55.803280+0000","last_clean":"2023-10-09T10:17:55.803280+0000","last_became_active":"2023-10-09T10:16:18.228601+0000","last_became_peered":"2023-10-09T10:16:18.228601+0000","last_unstale":"2023-10-09T10:17:55.803280+0000","last_undegraded":"2023-10-09T10:17:55.803280+0000","last_fullsized":"2023-10-09T10:17:55.803280+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":211,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.253667+0000","last_change":"2023-10-09T10:17:37.676882+0000","last_active":"2023-10-09T10:17:59.253667+0000","last_peered":"2023-10-09T10:17:59.253667+0000","last_clean":"2023-10-09T10:17:59.253667+0000","last_became_active":"2023-10-09T10:17:37.676439+0000","last_became_peered":"2023-10-09T10:17:37.676439+0000","last_unstale":"2023-10-09T10:17:59.253667+0000","last_undegraded":"2023-10-09T10:17:59.253667+0000","last_fullsized":"2023-10-09T10:17:59.253667+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.17","version":"395'8662","reported_seq":16962,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:00.158318+0000","last_change":"2023-10-09T10:16:18.442217+0000","last_active":"2023-10-09T10:18:00.158318+0000","last_peered":"2023-10-09T10:18:00.158318+0000","last_clean":"2023-10-09T10:15:06.303688+0000","last_became_active":"2023-10-09T10:16:18.348754+0000","last_became_peered":"2023-10-09T10:16:18.348754+0000","last_unstale":"2023-10-09T10:18:00.158318+0000","last_undegraded":"2023-10-09T10:16:18.207994+0000","last_fullsized":"2023-10-09T10:16:18.204785+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8662,"log_dups_size":0,"ondisk_log_size":8662,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104333312,"num_objects":1592,"num_object_clones":0,"num_object_copies":4776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":264,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5396,"num_write_kb":223616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3878,"num_bytes_recovered":248971264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,3,7","objects":1},{"shards":"2,7","objects":1591}],"blocked_by":[],"up_primary":5,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":47,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.803657+0000","last_change":"2023-10-09T10:16:18.351145+0000","last_active":"2023-10-09T10:17:55.803657+0000","last_peered":"2023-10-09T10:17:55.803657+0000","last_clean":"2023-10-09T10:17:55.803657+0000","last_became_active":"2023-10-09T10:16:18.321691+0000","last_became_peered":"2023-10-09T10:16:18.321691+0000","last_unstale":"2023-10-09T10:17:55.803657+0000","last_undegraded":"2023-10-09T10:17:55.803657+0000","last_fullsized":"2023-10-09T10:17:55.803657+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":252,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.241964+0000","last_change":"2023-10-09T10:15:42.393203+0000","last_active":"2023-10-09T10:17:59.241964+0000","last_peered":"2023-10-09T10:17:59.241964+0000","last_clean":"2023-10-09T10:17:59.241964+0000","last_became_active":"2023-10-09T10:15:42.381511+0000","last_became_peered":"2023-10-09T10:15:42.381511+0000","last_unstale":"2023-10-09T10:17:59.241964+0000","last_undegraded":"2023-10-09T10:17:59.241964+0000","last_fullsized":"2023-10-09T10:17:59.241964+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:23.196321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"394'8637","reported_seq":9187,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.172283+0000","last_change":"2023-10-09T10:16:10.248177+0000","last_active":"2023-10-09T10:17:59.172283+0000","last_peered":"2023-10-09T10:17:59.172283+0000","last_clean":"2023-10-09T10:17:59.172283+0000","last_became_active":"2023-10-09T10:16:10.240509+0000","last_became_peered":"2023-10-09T10:16:10.240509+0000","last_unstale":"2023-10-09T10:17:59.172283+0000","last_undegraded":"2023-10-09T10:17:59.172283+0000","last_fullsized":"2023-10-09T10:17:59.172283+0000","mapping_epoch":358,"log_start":"324'2503","ondisk_log_start":"324'2503","created":338,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100270080,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5378,"num_write_kb":221056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":49,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-10-09T10:17:55.805973+0000","last_change":"2023-10-09T10:16:08.240555+0000","last_active":"2023-10-09T10:17:55.805973+0000","last_peered":"2023-10-09T10:17:55.805973+0000","last_clean":"2023-10-09T10:17:55.805973+0000","last_became_active":"2023-10-09T10:16:08.239692+0000","last_became_peered":"2023-10-09T10:16:08.239692+0000","last_unstale":"2023-10-09T10:17:55.805973+0000","last_undegraded":"2023-10-09T10:17:55.805973+0000","last_fullsized":"2023-10-09T10:17:55.805973+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:39.501717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":436,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.248816+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:17:59.248816+0000","last_peered":"2023-10-09T10:17:59.248816+0000","last_clean":"2023-10-09T10:17:59.248816+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:17:59.248816+0000","last_undegraded":"2023-10-09T10:17:59.248816+0000","last_fullsized":"2023-10-09T10:17:59.248816+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.15","version":"396'8831","reported_seq":11977,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:02.179175+0000","last_change":"2023-10-09T10:16:18.463184+0000","last_active":"2023-10-09T10:18:02.179175+0000","last_peered":"2023-10-09T10:18:02.179175+0000","last_clean":"2023-10-09T10:15:06.324825+0000","last_became_active":"2023-10-09T10:16:18.336894+0000","last_became_peered":"2023-10-09T10:16:18.336894+0000","last_unstale":"2023-10-09T10:18:02.179175+0000","last_undegraded":"2023-10-09T10:16:18.463001+0000","last_fullsized":"2023-10-09T10:16:18.205452+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8831,"log_dups_size":0,"ondisk_log_size":8831,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":489,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5529,"num_write_kb":225088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1511,"num_bytes_recovered":98041856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1505}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":376,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.245689+0000","last_change":"2023-10-09T10:16:18.354987+0000","last_active":"2023-10-09T10:17:59.245689+0000","last_peered":"2023-10-09T10:17:59.245689+0000","last_clean":"2023-10-09T10:17:59.245689+0000","last_became_active":"2023-10-09T10:16:18.272788+0000","last_became_peered":"2023-10-09T10:16:18.272788+0000","last_unstale":"2023-10-09T10:17:59.245689+0000","last_undegraded":"2023-10-09T10:17:59.245689+0000","last_fullsized":"2023-10-09T10:17:59.245689+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:36.355149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":143,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-10-09T10:17:59.252950+0000","last_change":"2023-10-09T10:17:37.686077+0000","last_active":"2023-10-09T10:17:59.252950+0000","last_peered":"2023-10-09T10:17:59.252950+0000","last_clean":"2023-10-09T10:17:59.252950+0000","last_became_active":"2023-10-09T10:17:37.685754+0000","last_became_peered":"2023-10-09T10:17:37.685754+0000","last_unstale":"2023-10-09T10:17:59.252950+0000","last_undegraded":"2023-10-09T10:17:59.252950+0000","last_fullsized":"2023-10-09T10:17:59.252950+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":10490,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":36,"stat_sum":{"num_bytes":3226173464,"num_objects":49229,"num_object_clones":0,"num_object_copies":147687,"num_objects_missing_on_primary":1054,"num_objects_missing":1054,"num_objects_degraded":21974,"num_objects_misplaced":3051,"num_objects_unfound":0,"num_objects_dirty":49229,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169111,"num_write_kb":6769633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32939,"num_bytes_recovered":1909194752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11934314496,"data_stored":11934302280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":262490,"ondisk_log_size":262490,"up":108,"acting":99,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430580,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2899680,"kb_used_data":1991320,"kb_used_omap":0,"kb_used_meta":908352,"kb_avail":90845472,"statfs":{"total":95995035648,"available":93025763328,"internally_reserved":0,"allocated":2039111680,"data_stored":2038412690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930152448},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:58 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.4410000000000001,"15min":2.0579999999999998},"min":{"1min":0.251,"5min":0.223,"15min":0.217},"max":{"1min":7.7160000000000002,"5min":8.5429999999999993,"15min":50.591999999999999},"last":1.075},{"interface":"front","average":{"1min":1.446,"5min":1.4510000000000001,"15min":2.085},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":10.029,"5min":10.029,"15min":50.747},"last":0.70899999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:17:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.663,"15min":2.0760000000000001},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.22500000000000001},"max":{"1min":7.75,"5min":9.657,"15min":25.338000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.9770000000000001,"5min":2.1360000000000001,"15min":1.964},"min":{"1min":0.32700000000000001,"5min":0.249,"15min":0.249},"max":{"1min":7.6740000000000004,"5min":14.5,"15min":18.27},"last":0.41199999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1230000000000002,"5min":2.0590000000000002,"15min":2.198},"min":{"1min":0.30199999999999999,"5min":0.29499999999999998,"15min":0.246},"max":{"1min":7.8600000000000003,"5min":10.682,"15min":13.775},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.421,"5min":2.1840000000000002,"15min":2.141},"min":{"1min":0.38300000000000001,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":4.3719999999999999,"5min":12.845000000000001,"15min":12.845000000000001},"last":0.60199999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:17:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":1.6240000000000001,"15min":1.5800000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.444,"5min":7.444,"15min":7.444},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.0419999999999998,"5min":1.6930000000000001,"15min":1.635},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.0779999999999994,"5min":8.0779999999999994,"15min":8.0779999999999994},"last":0.47999999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.2889999999999999,"15min":1.163},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":5.4409999999999998,"5min":12.699999999999999,"15min":12.699999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.97599999999999998,"15min":1.008},"min":{"1min":0.313,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.29,"5min":12.003,"15min":12.003},"last":0.496}]},{"osd":5,"last update":"Mon Oct 9 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":1.512,"15min":1.3169999999999999},"min":{"1min":0.36799999999999999,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":7.8300000000000001,"5min":12.048,"15min":25.277999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.397,"15min":1.2190000000000001},"min":{"1min":0.33400000000000002,"5min":0.26800000000000002,"15min":0.22600000000000001},"max":{"1min":4.2519999999999998,"5min":18.547999999999998,"15min":25.754000000000001},"last":0.67500000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.1539999999999999,"15min":1.131},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.26100000000000001},"max":{"1min":6.8390000000000004,"5min":8.8569999999999993,"15min":8.8569999999999993},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.121,"5min":1.3009999999999999,"15min":1.103},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.27200000000000002},"max":{"1min":4.0060000000000002,"5min":10.625,"15min":10.625},"last":0.44900000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626808,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1631184,"kb_used_data":731912,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":92113968,"statfs":{"total":95995035648,"available":94324703232,"internally_reserved":0,"allocated":749477888,"data_stored":748796212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:00 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.4570000000000001,"15min":1.7410000000000001},"min":{"1min":0.24199999999999999,"5min":0.23100000000000001,"15min":0.22600000000000001},"max":{"1min":5.306,"5min":9.6940000000000008,"15min":15.435},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.7509999999999999,"5min":1.774,"15min":1.839},"min":{"1min":0.28199999999999997,"5min":0.22,"15min":0.22},"max":{"1min":8.0999999999999996,"5min":13.385,"15min":15.308999999999999},"last":0.28000000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.7969999999999999,"15min":2.1440000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.23400000000000001},"max":{"1min":7.9800000000000004,"5min":17.925000000000001,"15min":35.609999999999999},"last":0.437},{"interface":"front","average":{"1min":1.992,"5min":1.964,"15min":2.238},"min":{"1min":0.30599999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.3929999999999998,"5min":10.58,"15min":32.396000000000001},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:17:00 2023","interfaces":[{"interface":"back","average":{"1min":3.0840000000000001,"5min":2.0209999999999999,"15min":1.843},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.26200000000000001},"max":{"1min":29.367999999999999,"5min":29.367999999999999,"15min":29.367999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.774,"5min":2.0070000000000001,"15min":2.5449999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.29799999999999999},"max":{"1min":8.4480000000000004,"5min":11.720000000000001,"15min":14.542999999999999},"last":0.55700000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:17:46 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":1.512,"15min":1.6200000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":2.855,"5min":8.0229999999999997,"15min":8.0229999999999997},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.4510000000000001,"5min":1.4870000000000001,"15min":1.4930000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.4189999999999996,"5min":6.4189999999999996,"15min":6.4189999999999996},"last":0.45800000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:17:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.755,"15min":0.71399999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.54,"5min":6.2539999999999996,"15min":10.113},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.76900000000000002,"15min":0.72099999999999997},"min":{"1min":0.37,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.0940000000000003,"5min":9.218,"15min":9.218},"last":0.51700000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:17:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":0.90100000000000002,"15min":0.91800000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.23699999999999999},"max":{"1min":6.6210000000000004,"5min":6.6210000000000004,"15min":7.9260000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.76400000000000001,"15min":0.95399999999999996},"min":{"1min":0.32900000000000001,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":4.3360000000000003,"5min":9.4890000000000008,"15min":14.465999999999999},"last":0.54000000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.111,"15min":0.98799999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.252},"max":{"1min":4.032,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.407,"5min":1.1519999999999999,"15min":1.03},"min":{"1min":0.40000000000000002,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":4.0640000000000001,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":0.58799999999999997}]}]},{"osd":1,"up_from":84,"seq":360777253121,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2617576,"kb_used_data":1941472,"kb_used_omap":9,"kb_used_meta":676086,"kb_avail":91127576,"statfs":{"total":95995035648,"available":93314637824,"internally_reserved":0,"allocated":1988067328,"data_stored":1986644984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":692313065},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.6100000000000001,"15min":2.4670000000000001},"min":{"1min":0.188,"5min":0.188,"15min":0.183},"max":{"1min":8.1370000000000005,"5min":13.303000000000001,"15min":31.007000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.4790000000000001,"5min":1.97,"15min":2.714},"min":{"1min":0.25800000000000001,"5min":0.216,"15min":0.192},"max":{"1min":10.449999999999999,"5min":10.449999999999999,"15min":20.501999999999999},"last":0.34399999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:16:57 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":2.6720000000000002,"15min":3.298},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.217},"max":{"1min":14.223000000000001,"5min":23.105,"15min":23.105},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.3839999999999999,"5min":2.1640000000000001,"15min":3.0219999999999998},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":6.6470000000000002,"5min":28.024999999999999,"15min":28.024999999999999},"last":0.496}]},{"osd":3,"last update":"Mon Oct 9 10:17:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.1619999999999999,"15min":1.1719999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.8419999999999996,"5min":6.673,"15min":6.673},"last":2.6440000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.6080000000000001,"15min":1.6779999999999999},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.4269999999999996,"5min":7.2160000000000002,"15min":7.2160000000000002},"last":0.41999999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":1.228,"15min":1.4279999999999999},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":4.5609999999999999,"5min":7.3789999999999996,"15min":21.620000000000001},"last":2.7050000000000001},{"interface":"front","average":{"1min":1.7669999999999999,"5min":1.6899999999999999,"15min":1.8100000000000001},"min":{"1min":0.38700000000000001,"5min":0.314,"15min":0.308},"max":{"1min":10.497999999999999,"5min":10.497999999999999,"15min":12.904},"last":0.46100000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:17:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":3.1840000000000002,"15min":2.5640000000000001},"min":{"1min":0.372,"5min":0.33300000000000002,"15min":0.29099999999999998},"max":{"1min":5.3470000000000004,"5min":92.177999999999997,"15min":92.177999999999997},"last":0.443},{"interface":"front","average":{"1min":1.804,"5min":3.4249999999999998,"15min":2.8620000000000001},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.29099999999999998},"max":{"1min":5.8760000000000003,"5min":91.869,"15min":91.869},"last":2.4060000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.5069999999999999,"15min":1.948},"min":{"1min":0.42999999999999999,"5min":0.35599999999999998,"15min":0.307},"max":{"1min":6.6210000000000004,"5min":9.0779999999999994,"15min":21.670999999999999},"last":2.5640000000000001},{"interface":"front","average":{"1min":1.9410000000000001,"5min":1.8129999999999999,"15min":2.1099999999999999},"min":{"1min":0.437,"5min":0.39200000000000002,"15min":0.35099999999999998},"max":{"1min":5.3230000000000004,"5min":9.3160000000000007,"15min":14.471},"last":0.57999999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.9299999999999999,"15min":2.3700000000000001},"min":{"1min":0.40600000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5700000000000003,"5min":10.127000000000001,"15min":16.331},"last":2.4729999999999999},{"interface":"front","average":{"1min":1.752,"5min":1.5549999999999999,"15min":2.048},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.371},"max":{"1min":6.5919999999999996,"5min":9.125,"15min":21.593},"last":2.6760000000000002}]}]},{"osd":0,"up_from":79,"seq":339302416646,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494976,"kb_used_data":978104,"kb_used_omap":28,"kb_used_meta":516835,"kb_avail":92250176,"statfs":{"total":95995035648,"available":94464180224,"internally_reserved":0,"allocated":1001578496,"data_stored":1000146341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":529239888},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:17:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.7989999999999999,"15min":2.1379999999999999},"min":{"1min":0.25800000000000001,"5min":0.19800000000000001,"15min":0.16600000000000001},"max":{"1min":5.5090000000000003,"5min":24.399999999999999,"15min":24.399999999999999},"last":1.1419999999999999},{"interface":"front","average":{"1min":1.3,"5min":2.2719999999999998,"15min":3.3370000000000002},"min":{"1min":0.26600000000000001,"5min":0.22800000000000001,"15min":0.21299999999999999},"max":{"1min":6.1340000000000003,"5min":22.734000000000002,"15min":31.925999999999998},"last":0.41099999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":2.1930000000000001,"15min":2.2309999999999999},"min":{"1min":0.29599999999999999,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":4.3799999999999999,"5min":24.43,"15min":24.43},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.0549999999999999,"5min":2.0819999999999999,"15min":2.548},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":4.4299999999999997,"5min":24.459,"15min":24.459},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:17:45 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.133,"15min":1.161},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.6980000000000004,"5min":6.4059999999999997,"15min":6.4059999999999997},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.371,"5min":1.135,"15min":1.0960000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.3929999999999998,"5min":6.3929999999999998,"15min":6.3929999999999998},"last":0.46300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:18:00 2023","interfaces":[{"interface":"back","average":{"1min":4.3579999999999997,"5min":2.117,"15min":2.1739999999999999},"min":{"1min":0.34300000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":57.719000000000001,"5min":57.719000000000001,"15min":57.719000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":4.3780000000000001,"5min":2.0699999999999998,"15min":2.29},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":56.378,"5min":56.378,"15min":56.378},"last":0.75600000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:18:00 2023","interfaces":[{"interface":"back","average":{"1min":4.4059999999999997,"5min":4.6369999999999996,"15min":3.4729999999999999},"min":{"1min":0.34399999999999997,"5min":0.34000000000000002,"15min":0.28000000000000003},"max":{"1min":56.445999999999998,"5min":229.589,"15min":229.589},"last":0.56000000000000005},{"interface":"front","average":{"1min":4.1950000000000003,"5min":4.5359999999999996,"15min":2.6720000000000002},"min":{"1min":0.45100000000000001,"5min":0.30199999999999999,"15min":0.27000000000000002},"max":{"1min":54.411000000000001,"5min":228.88900000000001,"15min":228.88900000000001},"last":2.3660000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:18:00 2023","interfaces":[{"interface":"back","average":{"1min":4.444,"5min":2.5219999999999998,"15min":2.0760000000000001},"min":{"1min":0.42299999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":54.482999999999997,"5min":54.482999999999997,"15min":54.482999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":4.867,"5min":2.3919999999999999,"15min":2.9279999999999999},"min":{"1min":0.45900000000000002,"5min":0.378,"15min":0.35699999999999998},"max":{"1min":57.753,"5min":57.753,"15min":57.753},"last":0.47499999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:18:00 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":2.4079999999999999,"15min":2.839},"min":{"1min":0.46000000000000002,"5min":0.35499999999999998,"15min":0.34300000000000003},"max":{"1min":56.411999999999999,"5min":56.411999999999999,"15min":56.411999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":4.1950000000000003,"5min":2.3439999999999999,"15min":2.1139999999999999},"min":{"1min":0.47399999999999998,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":54.451999999999998,"5min":54.451999999999998,"15min":54.451999999999998},"last":0.65400000000000003}]}]},{"osd":2,"up_from":217,"seq":932007903381,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1364052,"kb_used_data":924108,"kb_used_omap":83,"kb_used_meta":439852,"kb_avail":92381100,"statfs":{"total":95995035648,"available":94598246400,"internally_reserved":0,"allocated":946286592,"data_stored":944943255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":450408943},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:05 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.9890000000000001,"15min":1.9390000000000001},"min":{"1min":0.23899999999999999,"5min":0.185,"15min":0.17899999999999999},"max":{"1min":3.5099999999999998,"5min":17.946000000000002,"15min":19.372},"last":1.488},{"interface":"front","average":{"1min":1.341,"5min":2.391,"15min":2.9489999999999998},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.23000000000000001},"max":{"1min":3.7000000000000002,"5min":20.143000000000001,"15min":20.143000000000001},"last":1.637}]},{"osd":1,"last update":"Mon Oct 9 10:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":2.3519999999999999,"15min":2.5640000000000001},"min":{"1min":0.316,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":3.4830000000000001,"5min":19.283999999999999,"15min":19.283999999999999},"last":1.5569999999999999},{"interface":"front","average":{"1min":1.619,"5min":2.3199999999999998,"15min":2.6970000000000001},"min":{"1min":0.30299999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.0049999999999999,"5min":19.027000000000001,"15min":19.027000000000001},"last":0.629}]},{"osd":3,"last update":"Mon Oct 9 10:17:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1819999999999999,"5min":1.3340000000000001,"15min":1.026},"min":{"1min":0.25700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":35.043999999999997,"5min":35.043999999999997,"15min":35.043999999999997},"last":0.37},{"interface":"front","average":{"1min":4.2889999999999997,"5min":1.6679999999999999,"15min":1.2310000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":50.406999999999996,"5min":50.406999999999996,"15min":50.406999999999996},"last":0.31}]},{"osd":4,"last update":"Mon Oct 9 10:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.724,"15min":1.3500000000000001},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.26600000000000001},"max":{"1min":3.1640000000000001,"5min":18.873999999999999,"15min":18.873999999999999},"last":0.432},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.8640000000000001,"15min":1.4870000000000001},"min":{"1min":0.35799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":24.352,"5min":24.352,"15min":24.352},"last":0.39500000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":2.673,"15min":1.986},"min":{"1min":0.35799999999999998,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":5.6079999999999997,"5min":57.298000000000002,"15min":57.298000000000002},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.83699999999999997,"5min":2.5409999999999999,"15min":1.966},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":2.9609999999999999,"5min":57.828000000000003,"15min":57.828000000000003},"last":1.681}]},{"osd":6,"last update":"Mon Oct 9 10:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":2.331,"15min":1.732},"min":{"1min":0.44700000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.8600000000000003,"5min":18.943999999999999,"15min":18.943999999999999},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.1539999999999999,"5min":2.1549999999999998,"15min":1.9550000000000001},"min":{"1min":0.44400000000000001,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":5.6600000000000001,"5min":15.378,"15min":15.378},"last":0.45200000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":2.5009999999999999,"15min":1.8029999999999999},"min":{"1min":0.41099999999999998,"5min":0.308,"15min":0.308},"max":{"1min":4.9669999999999996,"5min":18.972999999999999,"15min":18.972999999999999},"last":0.496},{"interface":"front","average":{"1min":1.306,"5min":2.677,"15min":2.1299999999999999},"min":{"1min":0.39300000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.6360000000000001,"5min":15.353,"15min":15.353},"last":1.1890000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488221,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2616204,"kb_used_data":2452932,"kb_used_omap":15,"kb_used_meta":163248,"kb_avail":91128948,"statfs":{"total":95995035648,"available":93316042752,"internally_reserved":0,"allocated":2511802368,"data_stored":2510120965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":167166406},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.1080000000000001,"15min":1.119},"min":{"1min":0.223,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.4050000000000002,"5min":7.4050000000000002,"15min":7.4050000000000002},"last":0.24199999999999999},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.1399999999999999,"15min":0.998},"min":{"1min":0.30099999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.4850000000000003,"5min":7.4850000000000003,"15min":7.4850000000000003},"last":0.19900000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":1.2629999999999999,"15min":1.161},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":15.970000000000001,"5min":15.970000000000001,"15min":15.970000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.8300000000000001,"5min":0.91000000000000003,"15min":0.75700000000000001},"min":{"1min":0.32800000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.516,"5min":7.516,"15min":7.516},"last":0.27800000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":2.1349999999999998,"15min":2.1699999999999999},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9779999999999998,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":1.101},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.3700000000000001,"15min":1.27},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":16.018000000000001,"5min":16.018000000000001,"15min":16.018000000000001},"last":0.43099999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":0.82799999999999996,"15min":0.70699999999999996},"min":{"1min":0.36299999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.3630000000000004,"5min":7.3630000000000004,"15min":7.3630000000000004},"last":0.317},{"interface":"front","average":{"1min":2.0339999999999998,"5min":1.429,"15min":1.3280000000000001},"min":{"1min":0.32100000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8689999999999998,"5min":7.8689999999999998,"15min":7.8689999999999998},"last":1.1459999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":0.81999999999999995,"15min":0.73599999999999999},"min":{"1min":0.443,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.3639999999999999,"5min":3.3639999999999999,"15min":3.3639999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.673,"5min":0.89900000000000002,"15min":0.77000000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":7.4539999999999997,"5min":7.4539999999999997,"15min":7.4539999999999997},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":1.5009999999999999,"15min":1.385},"min":{"1min":0.41199999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.9219999999999997,"5min":7.9219999999999997,"15min":7.9219999999999997},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.3380000000000001,"5min":0.98399999999999999,"15min":0.92500000000000004},"min":{"1min":0.48499999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.3929999999999998,"5min":7.3570000000000002,"15min":7.3570000000000002},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.0469999999999999,"15min":0.84899999999999998},"min":{"1min":0.39500000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":10.459,"5min":10.459,"15min":10.459},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.274,"5min":1.532,"15min":1.4079999999999999},"min":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":7.952,"5min":7.952,"15min":7.952},"last":1.1910000000000001}]}]},{"osd":4,"up_from":266,"seq":1142461300851,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905884,"kb_used_data":579028,"kb_used_omap":35,"kb_used_meta":326812,"kb_avail":92839268,"statfs":{"total":95995035648,"available":95067410432,"internally_reserved":0,"allocated":592924672,"data_stored":592237239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":334655726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.95199999999999996,"15min":0.99199999999999999},"min":{"1min":0.248,"5min":0.222,"15min":0.21199999999999999},"max":{"1min":2.6179999999999999,"5min":6.6970000000000001,"15min":10.881},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.3480000000000001,"15min":1.429},"min":{"1min":0.247,"5min":0.247,"15min":0.224},"max":{"1min":2.6480000000000001,"5min":8.298,"15min":12.246},"last":0.31}]},{"osd":1,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.3700000000000001,"15min":1.48},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.25},"max":{"1min":2.7120000000000002,"5min":18.303000000000001,"15min":18.303000000000001},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.3220000000000001,"15min":1.5269999999999999},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":1.774,"5min":18.036000000000001,"15min":18.036000000000001},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.25,"15min":1.3700000000000001},"min":{"1min":0.29399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.78,"5min":6.6689999999999996,"15min":9.2469999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.472,"15min":1.3680000000000001},"min":{"1min":0.314,"5min":0.28100000000000003,"15min":0.27700000000000002},"max":{"1min":2.661,"5min":11.778,"15min":11.778},"last":0.32000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:17:44 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":1.516,"15min":1.6240000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.6819999999999999,"5min":9.6170000000000009,"15min":9.6170000000000009},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.0149999999999999,"15min":1.048},"min":{"1min":0.36799999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.5739999999999998,"5min":3.3119999999999998,"15min":3.3119999999999998},"last":0.57999999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.8280000000000001,"15min":1.2509999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":1.4550000000000001,"5min":99.585999999999999,"15min":99.585999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.57599999999999996,"5min":1.8560000000000001,"15min":1.0569999999999999},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":2.5419999999999998,"5min":99.384,"15min":99.384},"last":0.66900000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.76600000000000001,"15min":0.749},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":2.6930000000000001,"5min":4.6449999999999996,"15min":6.851},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.88900000000000001,"15min":0.85199999999999998},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":2.742,"5min":4.9080000000000004,"15min":12.255000000000001},"last":0.59899999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.871,"15min":0.91600000000000004},"min":{"1min":0.34999999999999998,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":2.5979999999999999,"5min":6.6630000000000003,"15min":12.237},"last":0.622},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.93200000000000005,"15min":0.98899999999999999},"min":{"1min":0.34000000000000002,"5min":0.32100000000000001,"15min":0.30399999999999999},"max":{"1min":1.502,"5min":7.4960000000000004,"15min":14.964},"last":1.121}]}]},{"osd":5,"up_from":36,"seq":154618823035,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3054656,"kb_used_data":2141048,"kb_used_omap":0,"kb_used_meta":913600,"kb_avail":90690496,"statfs":{"total":95995035648,"available":92867067904,"internally_reserved":0,"allocated":2192433152,"data_stored":2191753961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":935526400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:46 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.486,"15min":1.669},"min":{"1min":0.23100000000000001,"5min":0.219,"15min":0.21099999999999999},"max":{"1min":5.1050000000000004,"5min":16.079000000000001,"15min":16.079000000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.111,"5min":1.379,"15min":1.6499999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":7.0499999999999998,"5min":15.907,"15min":15.907},"last":0.27200000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:17:25 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.8,"15min":1.75},"min":{"1min":0.26200000000000001,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":8.0890000000000004,"5min":16.056999999999999,"15min":16.056999999999999},"last":1.778},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.7330000000000001,"15min":1.8680000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.1280000000000001,"5min":23.099,"15min":23.099},"last":0.38600000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:17:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1150000000000002,"5min":1.976,"15min":2.2040000000000002},"min":{"1min":0.35099999999999998,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":12.553000000000001,"5min":15.973000000000001,"15min":15.973000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":1.4990000000000001,"5min":1.498,"15min":1.829},"min":{"1min":0.317,"5min":0.317,"15min":0.313},"max":{"1min":10.007999999999999,"5min":15.929,"15min":15.929},"last":0.55900000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:17:44 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":1.52,"15min":1.49},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":8.8689999999999998,"5min":8.8689999999999998,"15min":8.8689999999999998},"last":1.8859999999999999},{"interface":"front","average":{"1min":1.742,"5min":1.524,"15min":1.488},"min":{"1min":0.442,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":7.0960000000000001,"5min":7.0960000000000001,"15min":7.0960000000000001},"last":0.434}]},{"osd":4,"last update":"Mon Oct 9 10:17:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97299999999999998,"15min":0.84799999999999998},"min":{"1min":0.26400000000000001,"5min":0.25800000000000001,"15min":0.24299999999999999},"max":{"1min":8.468,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":1.855},{"interface":"front","average":{"1min":1.0549999999999999,"5min":0.73699999999999999,"15min":0.745},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":4.6029999999999998,"5min":4.7110000000000003,"15min":8.5559999999999992},"last":0.41299999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:17:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.0640000000000001,"15min":1.0620000000000001},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.25},"max":{"1min":2.0640000000000001,"5min":10.222,"15min":13.483000000000001},"last":2.0219999999999998},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.23,"15min":1.1379999999999999},"min":{"1min":0.28399999999999997,"5min":0.26700000000000002,"15min":0.22600000000000001},"max":{"1min":5.04,"5min":15.997,"15min":18.600999999999999},"last":0.88700000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:17:13 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":1.24,"15min":1.129},"min":{"1min":0.38400000000000001,"5min":0.311,"15min":0.25600000000000001},"max":{"1min":5.5019999999999998,"5min":16.030999999999999,"15min":16.030999999999999},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.127,"5min":0.67900000000000005,"15min":0.82299999999999995},"min":{"1min":0.32800000000000001,"5min":0.27500000000000002,"15min":0.26800000000000002},"max":{"1min":5.3239999999999998,"5min":5.3239999999999998,"15min":8.5220000000000002},"last":1.52}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":994181120,"data_stored":994181120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1976832000,"data_stored":1976827928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":936378368,"data_stored":936378368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2503409664,"data_stored":2503409664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":584318976,"data_stored":584318976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2167803904,"data_stored":2167799832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":742129664,"data_stored":742129664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2029260800,"data_stored":2029256728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:18:09.089 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-09T10:18:09.288 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:09.294 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:09.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:09.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:09.396 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:09.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:09.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:09 smithi012 ceph-mon[147926]: pgmap v1108: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 21384/140676 objects degraded (15.201%); 2528/140676 objects misplaced (1.797%); 11 MiB/s, 318 objects/s recovering 2023-10-09T10:18:09.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:09 smithi012 ceph-mon[152263]: pgmap v1108: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 21384/140676 objects degraded (15.201%); 2528/140676 objects misplaced (1.797%); 11 MiB/s, 318 objects/s recovering 2023-10-09T10:18:09.452 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:09 smithi110 ceph-mon[136032]: pgmap v1108: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 21384/140676 objects degraded (15.201%); 2528/140676 objects misplaced (1.797%); 11 MiB/s, 318 objects/s recovering 2023-10-09T10:18:09.925 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:09.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:09.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:10.033 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:10.073 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:10.127 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:10.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:10.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:10.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:10.281 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:10.332 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:10.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:10.437 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:10.468 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:10.508 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:10.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:10.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:10.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:10.634 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:10.665 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:10.980 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.1f 396/377 [PushOp(9:fcb5697b:::benchmark_data_smithi012_313365_object69008:head, version: 330'4321, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:fcb5697b:::benchmark_data_smithi012_313365_object69008:head@330'4321, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:10.976958+0000", 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.001234813, 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012880420000000001, 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:10.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:10.976958+0000", 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:10.976958+0000", 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:10.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:10.976957+0000", 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:10.977024+0000", 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7226999999999999e-05 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:10.977049+0000", 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5528e-05 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:10.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:10.977066+0000", 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6646e-05 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:10.977154+0000", 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8123999999999999e-05 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:10.977170+0000", 2023-10-09T10:18:10.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5934e-05 2023-10-09T10:18:10.985 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:10.985 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:10.985 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:10.985 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:10.985 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:18:10.986 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 1 2023-10-09T10:18:10.986 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:18:11.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:11 smithi012 ceph-mon[147926]: pgmap v1109: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 20934/136329 objects degraded (15.356%); 2435/136329 objects misplaced (1.786%); 8.2 MiB/s, 161 objects/s recovering 2023-10-09T10:18:11.260 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:11 smithi012 ceph-mon[152263]: pgmap v1109: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 20934/136329 objects degraded (15.356%); 2435/136329 objects misplaced (1.786%); 8.2 MiB/s, 161 objects/s recovering 2023-10-09T10:18:11.301 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:11.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:11.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:11.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:11.481 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:11.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:11 smithi110 ceph-mon[136032]: pgmap v1109: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 20934/136329 objects degraded (15.356%); 2435/136329 objects misplaced (1.786%); 8.2 MiB/s, 161 objects/s recovering 2023-10-09T10:18:11.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:18:12.300 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:12.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:12.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:12.406 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:12.454 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:12.509 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:12.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:12.621 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:12.662 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:12.704 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:12.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:12.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:12.810 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:12.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:12.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:12.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:13.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:13.048 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:13.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:13 smithi012 ceph-mon[152263]: pgmap v1110: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 20754/134130 objects degraded (15.473%); 2435/134130 objects misplaced (1.815%); 8.2 MiB/s, 158 objects/s recovering 2023-10-09T10:18:13.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 20754/134130 objects degraded (15.473%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:13.605 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:13 smithi012 ceph-mon[147926]: pgmap v1110: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 20754/134130 objects degraded (15.473%); 2435/134130 objects misplaced (1.815%); 8.2 MiB/s, 158 objects/s recovering 2023-10-09T10:18:13.606 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 20754/134130 objects degraded (15.473%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:13.624 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:13.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:13.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:13.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:13 smithi110 ceph-mon[136032]: pgmap v1110: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 20754/134130 objects degraded (15.473%); 2435/134130 objects misplaced (1.815%); 8.2 MiB/s, 158 objects/s recovering 2023-10-09T10:18:13.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 20754/134130 objects degraded (15.473%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:13.731 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:13.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:13.831 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:13.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:13.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:13.936 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:13.984 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:14.026 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:14.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:14.130 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:14.144 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:18:14.184 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:18:14.185 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:14.188 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:14.224 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:18:14.263 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:18:14.294 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:18:14.295 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:18:14.323 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:18:14.351 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:18:14.362 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:14.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:14.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:14.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:14.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2209241514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:14.371 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3046346873' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:14.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:14.372 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:14.372 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2209241514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:14.372 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3046346873' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:14.372 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:14.391 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:18:14.392 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:18:14.392 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:18:14.392 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:18:14.392 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:18:14.392 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:18:14.392 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:18:14.392 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:18:14.392 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:18:14.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2209241514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3046346873' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:14.472 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:14.530 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:14.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:14.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:14.681 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T10:18:14.684 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:14.724 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:14.776 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:14.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:14.884 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:14.914 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:14.964 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:14.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:14.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:15.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:15.102 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: pgmap v1111: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 20754/130869 objects degraded (15.859%); 1967/130869 objects misplaced (1.503%); 10 MiB/s, 188 objects/s recovering 2023-10-09T10:18:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T10:18:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/9554003' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: pgmap v1111: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 20754/130869 objects degraded (15.859%); 1967/130869 objects misplaced (1.503%); 10 MiB/s, 188 objects/s recovering 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/9554003' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:18:15.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:18:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: pgmap v1111: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 20754/130869 objects degraded (15.859%); 1967/130869 objects misplaced (1.503%); 10 MiB/s, 188 objects/s recovering 2023-10-09T10:18:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T10:18:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: failed to encode map e397 with expected crc 2023-10-09T10:18:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/9554003' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:18:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T10:18:16.510 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:16.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:16.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:16.617 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:16.653 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:16 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:18:16.658 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:16.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:16 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:18:16.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:16 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:18:16.715 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:16.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:16.823 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:16.892 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:17.000 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:17.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:17.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:17.143 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:17.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:17 smithi012 ceph-mon[147926]: pgmap v1113: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 20124/125664 objects degraded (16.014%); 1874/125664 objects misplaced (1.491%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:18:17.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:17 smithi012 ceph-mon[152263]: pgmap v1113: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 20124/125664 objects degraded (16.014%); 1874/125664 objects misplaced (1.491%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:18:17.447 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T10:18:17.524 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:17.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:17.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:17.663 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:17 smithi110 ceph-mon[136032]: pgmap v1113: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 20124/125664 objects degraded (16.014%); 1874/125664 objects misplaced (1.491%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:18:17.667 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:17.732 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:17.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:17.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:17.838 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:17.883 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:17.990 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:17.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:17.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:18.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:18.149 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:18:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:18:17] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.33.4" 2023-10-09T10:18:18.152 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:18.206 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:18.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:18.276 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:18:18.276 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:18:18.304 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 20124/125664 objects degraded (16.014%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:18.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:18.366 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:18.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 20124/125664 objects degraded (16.014%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:18.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 20124/125664 objects degraded (16.014%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:18.429 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:18.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:18.539 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:18.576 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:18.627 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:18.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:18.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:18:18.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:18.808 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:18.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:18.866 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:18.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:18.966 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:19.003 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:19.057 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:19.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:19.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:19.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:18:19.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:19.233 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:19.294 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:19.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:19.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:19.306 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:19 smithi110 ceph-mon[136032]: pgmap v1114: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 20124/123471 objects degraded (16.299%); 1368/123471 objects misplaced (1.108%); 13 MiB/s, 204 objects/s recovering 2023-10-09T10:18:19.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:19.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:19 smithi012 ceph-mon[147926]: pgmap v1114: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 20124/123471 objects degraded (16.299%); 1368/123471 objects misplaced (1.108%); 13 MiB/s, 204 objects/s recovering 2023-10-09T10:18:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:19 smithi012 ceph-mon[152263]: pgmap v1114: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 20124/123471 objects degraded (16.299%); 1368/123471 objects misplaced (1.108%); 13 MiB/s, 204 objects/s recovering 2023-10-09T10:18:19.452 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:19.489 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:19.517 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:19.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:19.625 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:19.685 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:19.761 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:19.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:19.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:19.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:18:19.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:19.958 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:20.018 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:20.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:20.027 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:20.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:20.129 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:20.186 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:21 smithi012 ceph-mon[152263]: pgmap v1115: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 19698/119151 objects degraded (16.532%); 1283/119151 objects misplaced (1.077%); 13 MiB/s, 203 objects/s recovering 2023-10-09T10:18:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:21 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:21.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:21 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:21.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:21 smithi012 ceph-mon[147926]: pgmap v1115: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 19698/119151 objects degraded (16.532%); 1283/119151 objects misplaced (1.077%); 13 MiB/s, 203 objects/s recovering 2023-10-09T10:18:21.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:21 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:21.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:21 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:21.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:21 smithi110 ceph-mon[136032]: pgmap v1115: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 19698/119151 objects degraded (16.532%); 1283/119151 objects misplaced (1.077%); 13 MiB/s, 203 objects/s recovering 2023-10-09T10:18:21.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:21 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:21.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:21 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:21.769 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:21.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:21.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:21.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:21.922 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:22.425 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:22.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:22.448 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:18:22.448 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:18:22.448 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.6220185358336316 2023-10-09T10:18:22.448 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 0.6220185358336316 2023-10-09T10:18:22.533 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:22.580 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:22.674 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:22.969 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:22.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:23.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:23.121 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:23.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:23 smithi012 ceph-mon[147926]: pgmap v1116: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 19512/117042 objects degraded (16.671%); 1283/117042 objects misplaced (1.096%); 13 MiB/s, 203 objects/s recovering 2023-10-09T10:18:23.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 19512/117042 objects degraded (16.671%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:23.599 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:23 smithi012 ceph-mon[152263]: pgmap v1116: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 19512/117042 objects degraded (16.671%); 1283/117042 objects misplaced (1.096%); 13 MiB/s, 203 objects/s recovering 2023-10-09T10:18:23.599 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 19512/117042 objects degraded (16.671%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:23.617 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:23.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:23.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:23 smithi110 ceph-mon[136032]: pgmap v1116: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 19512/117042 objects degraded (16.671%); 1283/117042 objects misplaced (1.096%); 13 MiB/s, 203 objects/s recovering 2023-10-09T10:18:23.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 19512/117042 objects degraded (16.671%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:23.723 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:23.787 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:23.861 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:23.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:23.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:24.004 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:24.160 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:24.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:24.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:24.294 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:24.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:24.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:24.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:24.498 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:25.156 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:25.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:25.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:25.260 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:25.297 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:25.342 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:25.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:25.448 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:25.482 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:25.545 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:25.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:25.556 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:25 smithi110 ceph-mon[136032]: pgmap v1117: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 19133/113529 objects degraded (16.853%); 1169/113529 objects misplaced (1.030%); 11 MiB/s, 210 objects/s recovering 2023-10-09T10:18:25.651 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:25 smithi012 ceph-mon[147926]: pgmap v1117: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 19133/113529 objects degraded (16.853%); 1169/113529 objects misplaced (1.030%); 11 MiB/s, 210 objects/s recovering 2023-10-09T10:18:25.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:25 smithi012 ceph-mon[152263]: pgmap v1117: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 19133/113529 objects degraded (16.853%); 1169/113529 objects misplaced (1.030%); 11 MiB/s, 210 objects/s recovering 2023-10-09T10:18:25.703 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:25.771 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:25.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:25.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:25.877 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:25.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:25.975 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:25.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:26.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:26.119 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:26.202 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:18:26.203 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:18:26.987 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:26.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:26.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:27.042 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:18:27.052 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:27.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:27.151 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:27.215 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:27.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:27.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:27.398 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:27.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:27 smithi110 ceph-mon[136032]: pgmap v1118: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 18398/107334 objects degraded (17.141%); 1062/107334 objects misplaced (0.989%); 13 MiB/s, 241 objects/s recovering 2023-10-09T10:18:27.458 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:27.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:27.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:27.570 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:27.619 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:27 smithi012 ceph-mon[147926]: pgmap v1118: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 18398/107334 objects degraded (17.141%); 1062/107334 objects misplaced (0.989%); 13 MiB/s, 241 objects/s recovering 2023-10-09T10:18:27.619 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:27 smithi012 ceph-mon[152263]: pgmap v1118: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 18398/107334 objects degraded (17.141%); 1062/107334 objects misplaced (0.989%); 13 MiB/s, 241 objects/s recovering 2023-10-09T10:18:27.622 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:27.836 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:18:27.885 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:27.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:27.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:28.003 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:28.045 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:28.110 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:28.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:28.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:28.148 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:18:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:18:27] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.33.4" 2023-10-09T10:18:28.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:28.292 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:28.385 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:18:28.416 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4046663327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:18:28.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:28 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:18:28.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 18398/107334 objects degraded (17.141%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:28.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4046663327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:18:28.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:28 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:18:28.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 18398/107334 objects degraded (17.141%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:28.434 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:18:28.455 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:18:28.489 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:18:28.528 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:18:28.568 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:18:28.600 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:18:28.632 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:18:28.668 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:18:28.669 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:18:28.669 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:18:28.669 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:18:28.669 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:18:28.669 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:18:28.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:18:28.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:18:28.670 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:18:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4046663327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:18:28.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:28 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:18:28.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 18398/107334 objects degraded (17.141%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:29.281 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:18:29.281 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:18:29.281 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:18:29.282 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:18:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: pgmap v1119: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 17749/105015 objects degraded (16.901%); 1062/105015 objects misplaced (1.011%); 9.2 MiB/s, 237 objects/s recovering 2023-10-09T10:18:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:18:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T10:18:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/718527858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: pgmap v1119: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 17749/105015 objects degraded (16.901%); 1062/105015 objects misplaced (1.011%); 9.2 MiB/s, 237 objects/s recovering 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:18:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T10:18:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/718527858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: pgmap v1119: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 17749/105015 objects degraded (16.901%); 1062/105015 objects misplaced (1.011%); 9.2 MiB/s, 237 objects/s recovering 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: failed to encode map e398 with expected crc 2023-10-09T10:18:29.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/718527858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:29.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:29.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:18:30.547 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:18:30.770 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:30.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:30.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:30.875 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:30.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:31.009 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:31.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:31.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:31.176 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:31.323 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:31.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:31.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:31.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:31 smithi110 ceph-mon[136032]: pgmap v1121: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 17262/100182 objects degraded (17.231%); 949/100182 objects misplaced (0.947%); 8.1 MiB/s, 237 objects/s recovering 2023-10-09T10:18:31.355 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:31 smithi110 ceph-mon[136032]: 9.1c scrub starts 2023-10-09T10:18:31.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:31 smithi012 ceph-mon[147926]: pgmap v1121: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 17262/100182 objects degraded (17.231%); 949/100182 objects misplaced (0.947%); 8.1 MiB/s, 237 objects/s recovering 2023-10-09T10:18:31.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:31 smithi012 ceph-mon[147926]: 9.1c scrub starts 2023-10-09T10:18:31.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:31 smithi012 ceph-mon[152263]: pgmap v1121: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 17262/100182 objects degraded (17.231%); 949/100182 objects misplaced (0.947%); 8.1 MiB/s, 237 objects/s recovering 2023-10-09T10:18:31.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:31 smithi012 ceph-mon[152263]: 9.1c scrub starts 2023-10-09T10:18:31.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:31.474 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:31.629 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:31.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:31.734 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:31.789 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:31.854 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:31.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:31.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:31.960 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:32.011 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:32.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:32.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:32.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:32.168 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:18:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:18:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:18:32.168 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:18:32.168 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:18:32.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:32.250 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:32.309 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:32.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:32.415 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:32.464 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:32.580 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:32.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:32.675 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:18:32.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:32.725 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:32.768 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:32.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:32.873 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:32.923 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:32.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:18:32.982 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:32.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:32.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:33.087 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:33.145 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:33.207 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:33.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:33.269 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.0 to 0.622019 (9f3c) 2023-10-09T10:18:33.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:33.371 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:33.434 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:18:33.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:33 smithi110 ceph-mon[136032]: pgmap v1122: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 17043/97614 objects degraded (17.460%); 949/97614 objects misplaced (0.972%); 8.3 MiB/s, 240 objects/s recovering 2023-10-09T10:18:33.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:33 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:33 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1710775992' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.6220185358336316}]: dispatch 2023-10-09T10:18:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:33 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:18:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:33 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:18:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:33 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.667 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:33.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[147926]: pgmap v1122: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 17043/97614 objects degraded (17.460%); 949/97614 objects misplaced (0.972%); 8.3 MiB/s, 240 objects/s recovering 2023-10-09T10:18:33.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1710775992' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.6220185358336316}]: dispatch 2023-10-09T10:18:33.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:18:33.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:18:33.670 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.671 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[152263]: pgmap v1122: 109 pgs: 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 17043/97614 objects degraded (17.460%); 949/97614 objects misplaced (0.972%); 8.3 MiB/s, 240 objects/s recovering 2023-10-09T10:18:33.671 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.671 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.671 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1710775992' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.6220185358336316}]: dispatch 2023-10-09T10:18:33.671 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:18:33.671 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:18:33.671 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:33 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:33.672 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:33.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:33.774 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:33.802 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:33.838 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:18:33.886 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:33.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:33.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:33.992 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:34.022 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:34.113 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:34.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:34.216 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:34.248 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1710775992' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.6220185358336316}]': finished 2023-10-09T10:18:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[147926]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T10:18:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[147926]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[147926]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[147926]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17043/97614 objects degraded (17.460%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:34.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[147926]: 9.e scrub starts 2023-10-09T10:18:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1710775992' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.6220185358336316}]': finished 2023-10-09T10:18:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[152263]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T10:18:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[152263]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[152263]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[152263]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17043/97614 objects degraded (17.460%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:34.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:34 smithi012 ceph-mon[152263]: 9.e scrub starts 2023-10-09T10:18:34.486 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:34.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:34.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:34 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1710775992' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.6220185358336316}]': finished 2023-10-09T10:18:34.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:34 smithi110 ceph-mon[136032]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T10:18:34.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:34 smithi110 ceph-mon[136032]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:34 smithi110 ceph-mon[136032]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:34 smithi110 ceph-mon[136032]: failed to encode map e399 with expected crc 2023-10-09T10:18:34.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:34 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17043/97614 objects degraded (17.460%), 21 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:34.529 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:34 smithi110 ceph-mon[136032]: 9.e scrub starts 2023-10-09T10:18:34.591 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:34.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:34.715 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:35.126 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:18:35.127 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:18:35.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:18:35.127 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:18:35.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.127 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114937 9.c 9:3de29534:::benchmark_data_smithi012_313365_object2462:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:18:35.127 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:18:35.127 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 200.84172928699999, 2023-10-09T10:18:35.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0079670510000001, 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114937 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215214+0000", 2023-10-09T10:18:35.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9450000000000005e-06 2023-10-09T10:18:35.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215224+0000", 2023-10-09T10:18:35.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4849999999999992e-06 2023-10-09T10:18:35.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215232+0000", 2023-10-09T10:18:35.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3200000000000002e-06 2023-10-09T10:18:35.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215239+0000", 2023-10-09T10:18:35.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8306000000000002e-05 2023-10-09T10:18:35.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215278+0000", 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3389999999999998e-06 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215284+0000", 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98804788700000001 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.203332+0000", 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00200256 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.205335+0000", 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015448999999999999 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.205489+0000", 2023-10-09T10:18:35.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013383071999999999 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218872+0000", 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041514890000000004 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223024+0000", 2023-10-09T10:18:35.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7812999999999997e-05 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223081+0000", 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100445 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223182+0000", 2023-10-09T10:18:35.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0079670510000001 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114938 9.c 9:317dc30b:::benchmark_data_smithi012_313365_object2463:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 200.84169597100001, 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0080620979999999, 2023-10-09T10:18:35.138 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114938 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:18:35.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:18:35.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.979e-06 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215250+0000", 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5199999999999997e-06 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215258+0000", 2023-10-09T10:18:35.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1250000000000001e-06 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215261+0000", 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8232000000000001e-05 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215320+0000", 2023-10-09T10:18:35.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1710000000000002e-06 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215328+0000", 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99764452400000003 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.212972+0000", 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010863 2023-10-09T10:18:35.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213081+0000", 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012440199999999999 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213205+0000", 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056800169999999999 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218885+0000", 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043305829999999998 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223216+0000", 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1077000000000001e-05 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223247+0000", 2023-10-09T10:18:35.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3231000000000005e-05 2023-10-09T10:18:35.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223310+0000", 2023-10-09T10:18:35.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0080620979999999 2023-10-09T10:18:35.147 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.147 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.147 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.147 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114946 9.c 9:3c5b83c8:::benchmark_data_smithi012_313365_object2471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:18:35.148 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:18:35.148 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 200.84166277099999, 2023-10-09T10:18:35.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0081399559999999, 2023-10-09T10:18:35.148 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.148 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.148 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.148 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.149 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.149 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114946 2023-10-09T10:18:35.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.149 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:18:35.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:18:35.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:35.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:18:35.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6449999999999999e-06 2023-10-09T10:18:35.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215283+0000", 2023-10-09T10:18:35.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7349999999999998e-06 2023-10-09T10:18:35.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215286+0000", 2023-10-09T10:18:35.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7470000000000001e-06 2023-10-09T10:18:35.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215289+0000", 2023-10-09T10:18:35.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2694000000000001e-05 2023-10-09T10:18:35.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215342+0000", 2023-10-09T10:18:35.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2049999999999998e-06 2023-10-09T10:18:35.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:18:35.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215345+0000", 2023-10-09T10:18:35.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.000627545 2023-10-09T10:18:35.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215973+0000", 2023-10-09T10:18:35.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011626899999999999 2023-10-09T10:18:35.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216089+0000", 2023-10-09T10:18:35.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132232 2023-10-09T10:18:35.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:18:35.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216221+0000", 2023-10-09T10:18:35.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027664209999999998 2023-10-09T10:18:35.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218988+0000", 2023-10-09T10:18:35.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043527119999999999 2023-10-09T10:18:35.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223340+0000", 2023-10-09T10:18:35.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9572000000000001e-05 2023-10-09T10:18:35.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223370+0000", 2023-10-09T10:18:35.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1471000000000002e-05 2023-10-09T10:18:35.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223421+0000", 2023-10-09T10:18:35.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0081399559999999 2023-10-09T10:18:35.160 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.160 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.161 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.161 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128021 9.c 9:3ed78aba:::benchmark_data_smithi012_313365_object15546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:18:35.161 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:18:35.161 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 147.35372102100001, 2023-10-09T10:18:35.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999, 2023-10-09T10:18:35.162 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.162 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.162 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.162 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.162 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.162 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128021 2023-10-09T10:18:35.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.163 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:18:35.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:18:35.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:35.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703222+0000", 2023-10-09T10:18:35.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7700000000000004e-06 2023-10-09T10:18:35.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703229+0000", 2023-10-09T10:18:35.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1178e-05 2023-10-09T10:18:35.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703240+0000", 2023-10-09T10:18:35.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0509999999999996e-06 2023-10-09T10:18:35.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703245+0000", 2023-10-09T10:18:35.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9240000000000002e-05 2023-10-09T10:18:35.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703304+0000", 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.429e-05 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703318+0000", 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.498177753 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201496+0000", 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2643e-05 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201519+0000", 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1325606320000001 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334079+0000", 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016077600000000001 2023-10-09T10:18:35.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334240+0000", 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016287699999999999 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334403+0000", 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050087500000000002 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339412+0000", 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1050000000000007e-05 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339473+0000", 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013735 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339610+0000", 2023-10-09T10:18:35.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041248470000000001 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343735+0000", 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9188999999999999e-05 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343784+0000", 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111032 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343895+0000", 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128030 9.c 9:39ef457f:::benchmark_data_smithi012_313365_object15555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 147.21370994500001, 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999, 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.173 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128030 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843233+0000", 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843241+0000", 2023-10-09T10:18:35.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3162e-05 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843254+0000", 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5490000000000001e-06 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843261+0000", 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2747999999999997e-05 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843314+0000", 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7970000000000006e-06 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843323+0000", 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35848033200000001 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201803+0000", 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6119999999999999e-06 2023-10-09T10:18:35.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201812+0000", 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1353121740000001 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337124+0000", 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8503000000000003e-05 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337212+0000", 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0490999999999998e-05 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337303+0000", 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002260742 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339564+0000", 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1705999999999994e-05 2023-10-09T10:18:35.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339635+0000", 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002741145 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.342376+0000", 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001560173 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343937+0000", 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6062000000000001e-05 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343973+0000", 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4776000000000002e-05 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.344068+0000", 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.181 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128782 9.c 9:3e7b84df:::benchmark_data_smithi012_313365_object16307:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:28.004865+0000", 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 127.05207869199999, 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.071993426999999999, 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.182 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128782 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004865+0000", 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004865+0000", 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:35.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004864+0000", 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0590000000000003e-06 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004872+0000", 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5293000000000001e-05 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004887+0000", 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7459999999999997e-06 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004893+0000", 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8399000000000006e-05 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.004971+0000", 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109824 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.005081+0000", 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000124888 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.005206+0000", 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025613400000000001 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.007767+0000", 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3368000000000005e-05 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.007851+0000", 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00040009399999999998 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.008251+0000", 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.068476722000000004 2023-10-09T10:18:35.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.076728+0000", 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9867999999999998e-05 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.076778+0000", 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1296000000000002e-05 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:28.076859+0000", 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.071993426999999999 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:129554 9.c 9:3ee6beeb:::benchmark_data_smithi012_313365_object17079:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:30.306417+0000", 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 124.750527191, 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038617190000000003, 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.189 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 129554 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306417+0000", 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306417+0000", 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306415+0000", 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7190000000000007e-06 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306424+0000", 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8678e-05 2023-10-09T10:18:35.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306442+0000", 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1040000000000001e-06 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306449+0000", 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4561000000000007e-05 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306524+0000", 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000189667 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306714+0000", 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017995500000000001 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.306894+0000", 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036508880000000001 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.310544+0000", 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002046572 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.312591+0000", 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024919819999999998 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.315083+0000", 2023-10-09T10:18:35.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029807935000000001 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.344891+0000", 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4691999999999997e-05 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.344946+0000", 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8405000000000005e-05 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.345034+0000", 2023-10-09T10:18:35.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038617190000000003 2023-10-09T10:18:35.197 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.197 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.197 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:129562 9.c 9:39b51c53:::benchmark_data_smithi012_313365_object17087:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:30.333070+0000", 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 124.72387412800001, 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031552197999999997, 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.198 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 129562 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333070+0000", 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333070+0000", 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333068+0000", 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1009000000000001e-05 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333079+0000", 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0988999999999998e-05 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333100+0000", 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9189999999999999e-06 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333108+0000", 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0591000000000002e-05 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333178+0000", 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0890000000000006e-06 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-10-09T10:18:35.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333187+0000", 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011867011 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.345054+0000", 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103452 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.345158+0000", 2023-10-09T10:18:35.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103909 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.345262+0000", 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086899980000000009 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.353952+0000", 2023-10-09T10:18:35.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1491e-05 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.353993+0000", 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029029170000000001 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.356896+0000", 2023-10-09T10:18:35.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075623560000000001 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364459+0000", 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3745999999999996e-05 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364522+0000", 2023-10-09T10:18:35.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9827999999999994e-05 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364622+0000", 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031552197999999997 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.207 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:129563 9.c 9:3efd6b8d:::benchmark_data_smithi012_313365_object17088:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:30.333122+0000", 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 124.723821833, 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031645813000000002, 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 129563 2023-10-09T10:18:35.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333122+0000", 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333122+0000", 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:35.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333122+0000", 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0860000000000002e-06 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333125+0000", 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7100000000000004e-06 2023-10-09T10:18:35.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333131+0000", 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.805e-06 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333134+0000", 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102803 2023-10-09T10:18:35.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333237+0000", 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7409999999999992e-06 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.333245+0000", 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014888682 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.348134+0000", 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013137199999999999 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.348265+0000", 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100295 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:18:35.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.348365+0000", 2023-10-09T10:18:35.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056663199999999999 2023-10-09T10:18:35.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.354032+0000", 2023-10-09T10:18:35.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1625999999999998e-05 2023-10-09T10:18:35.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.354073+0000", 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003101683 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.357175+0000", 2023-10-09T10:18:35.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0074756780000000004 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364651+0000", 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3782000000000003e-05 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364684+0000", 2023-10-09T10:18:35.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3739999999999994e-05 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:30.364768+0000", 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031645813000000002 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:167531 9.1c 9:3c67e90c:::benchmark_data_smithi012_313365_object55056:head [delete] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:43.461584+0000", 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 51.595360274000001, 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054644947999999999, 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 167531 2023-10-09T10:18:35.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.461584+0000", 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.461584+0000", 2023-10-09T10:18:35.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.461581+0000", 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2233e-05 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.461594+0000", 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4108e-05 2023-10-09T10:18:35.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.461618+0000", 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.414e-06 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.461627+0000", 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6498e-05 2023-10-09T10:18:35.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.461684+0000", 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015814400000000001 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.461842+0000", 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000409505 2023-10-09T10:18:35.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.462251+0000", 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001757155 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464008+0000", 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000214806 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464223+0000", 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051875174000000003 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.516098+0000", 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4790999999999995e-05 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.516183+0000", 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5504999999999999e-05 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.516229+0000", 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054644947999999999 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.226 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:167534 9.c 9:363c0663:::benchmark_data_smithi012_313365_object55059:head [delete] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:43.464415+0000", 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 51.592529067999997, 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054338614, 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.227 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.228 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.228 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.228 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 167534 2023-10-09T10:18:35.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.228 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464415+0000", 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464415+0000", 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464414+0000", 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.412e-06 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464419+0000", 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1175999999999999e-05 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464431+0000", 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5279999999999997e-06 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464435+0000", 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1390000000000003e-05 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464466+0000", 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4332000000000004e-05 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464561+0000", 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031403600000000001 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-10-09T10:18:35.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.464875+0000", 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001866471 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.466741+0000", 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045477400000000002 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.467196+0000", 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051465626 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.518662+0000", 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0111999999999997e-05 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.518712+0000", 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1786000000000003e-05 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.518754+0000", 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.054338614 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:167538 9.1c 9:3ba1d3ca:::benchmark_data_smithi012_313365_object55063:head [delete] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:43.474400+0000", 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 51.582544005999999, 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044330205999999997, 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.236 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 167538 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474400+0000", 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474400+0000", 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474398+0000", 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1419999999999993e-06 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474407+0000", 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6277e-05 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474423+0000", 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.81e-06 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474430+0000", 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9314000000000003e-05 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474479+0000", 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013349600000000001 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474613+0000", 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000381486 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.474994+0000", 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023773969999999998 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.477371+0000", 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011105329999999999 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.478482+0000", 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040165643000000001 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.518648+0000", 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7472999999999998e-05 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.518695+0000", 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5243999999999997e-05 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:43.518730+0000", 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044330205999999997 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:177684 9.1e 9:7a4d27dd:::benchmark_data_smithi012_313365_object65209:head [delete] snapc 0=[] ondisk+write+known_if_redirected e396)", 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:04.126703+0000", 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.930241212999999, 2023-10-09T10:18:35.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13911110600000001, 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 177684 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.126703+0000", 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.126703+0000", 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.126701+0000", 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4520000000000003e-06 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.126710+0000", 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7907999999999999e-05 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.126728+0000", 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.374e-06 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.126736+0000", 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2726999999999998e-05 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.126788+0000", 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013269599999999999 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.126921+0000", 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015092400000000001 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:18:35.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.127072+0000", 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001248896 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.128321+0000", 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058757700000000004 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.128909+0000", 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.136818419 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.265727+0000", 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3300999999999998e-05 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.265770+0000", 2023-10-09T10:18:35.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3736999999999998e-05 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.265814+0000", 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13911110600000001 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:177688 9.1c 9:3cd7cafb:::benchmark_data_smithi012_313365_object65213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e396)", 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:04.183639+0000", 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 30.873305069000001, 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082278039999999997, 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 177688 2023-10-09T10:18:35.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.183639+0000", 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.183639+0000", 2023-10-09T10:18:35.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.183637+0000", 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1989999999999995e-06 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.183646+0000", 2023-10-09T10:18:35.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9190000000000001e-05 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.183666+0000", 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4700000000000002e-06 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.183674+0000", 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4249000000000002e-05 2023-10-09T10:18:35.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.183728+0000", 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012404699999999999 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.183852+0000", 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035548699999999998 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.184208+0000", 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016878279999999999 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.185896+0000", 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00069899899999999995 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.186595+0000", 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079253967999999994 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.265849+0000", 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4387000000000001e-05 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.265883+0000", 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3935e-05 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:04.265917+0000", 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082278039999999997 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.260 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:193763 9.e 9:764fbf12:::benchmark_data_smithi012_313365_object81288:head [delete] snapc 0=[] ondisk+write+known_if_redirected e398)", 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:30.541546+0000", 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.5153985780000001, 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.075581388999999999, 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.261 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 193763 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541546+0000", 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541546+0000", 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541544+0000", 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5499999999999997e-06 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541551+0000", 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5231e-05 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541566+0000", 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3060000000000003e-06 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541573+0000", 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3418999999999998e-05 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541626+0000", 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146427 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541773+0000", 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000136076 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.541909+0000", 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001983402 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.543892+0000", 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00051736999999999996 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.544409+0000", 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.072622501000000006 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.617032+0000", 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4395999999999999e-05 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.617086+0000", 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0546000000000001e-05 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.617127+0000", 2023-10-09T10:18:35.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.075581388999999999 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:193764 9.1e 9:78570eeb:::benchmark_data_smithi012_313365_object81289:head [delete] snapc 0=[] ondisk+write+known_if_redirected e398)", 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:30.542123+0000", 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.51482063, 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076135829000000002, 2023-10-09T10:18:35.269 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 193764 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542123+0000", 2023-10-09T10:18:35.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542123+0000", 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542123+0000", 2023-10-09T10:18:35.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4290000000000003e-06 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542127+0000", 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.293e-05 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542140+0000", 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3289999999999999e-06 2023-10-09T10:18:35.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542146+0000", 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0448999999999999e-05 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542177+0000", 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7632000000000005e-05 2023-10-09T10:18:35.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542255+0000", 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3277999999999999e-05 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.542328+0000", 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014482970000000001 2023-10-09T10:18:35.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.543776+0000", 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00074505000000000003 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.544521+0000", 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.073646428999999999 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.618168+0000", 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4184999999999997e-05 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.618222+0000", 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7518999999999998e-05 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.618259+0000", 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076135829000000002 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:193770 9.1c 9:390462ca:::benchmark_data_smithi012_313365_object81295:head [delete] snapc 0=[] ondisk+write+known_if_redirected e398)", 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:30.573074+0000", 2023-10-09T10:18:35.277 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.4838698900000002, 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046248953000000002, 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 193770 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573074+0000", 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573074+0000", 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999967 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573071+0000", 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3592e-05 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573084+0000", 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2504999999999999e-05 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573107+0000", 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0276e-05 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573117+0000", 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9914999999999999e-05 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573167+0000", 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8392000000000003e-05 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573265+0000", 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042724100000000002 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.573693+0000", 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022295800000000001 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.575922+0000", 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00080939800000000002 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.576732+0000", 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042512965999999999 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.619245+0000", 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4112e-05 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.619289+0000", 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4455999999999997e-05 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.619323+0000", 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.046248953000000002 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:194021 9.10 e398/388 9:0e235a69:::benchmark_data_smithi012_313365_object81546:head v 398'9227, mlcod=334'5051)", 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:30.934253+0000", 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.1226911429999999, 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030420847000000001, 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.934253+0000", 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.934253+0000", 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.934253+0000", 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7849999999999992e-06 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.934260+0000", 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8039999999999998e-06 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.934266+0000", 2023-10-09T10:18:35.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7950000000000001e-06 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.934270+0000", 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016537919000000002 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.950808+0000", 2023-10-09T10:18:35.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2638999999999997e-05 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.950841+0000", 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013787491000000001 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.964628+0000", 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5697e-05 2023-10-09T10:18:35.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:30.964674+0000", 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030420847000000001 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.291 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:194306 9.1c 9:3c41c917:::benchmark_data_smithi012_313365_object81831:head [delete] snapc 0=[] ondisk+write+known_if_redirected e398)", 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:31.248335+0000", 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 3.8086093249999999, 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056020388999999997, 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 194306 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.292 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248335+0000", 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248335+0000", 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:18:35.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248334+0000", 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5960000000000003e-06 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248339+0000", 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1299e-05 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248351+0000", 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0549999999999999e-06 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248356+0000", 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7663000000000002e-05 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248393+0000", 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5550000000000005e-05 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248489+0000", 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035778399999999999 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.248847+0000", 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015164359999999999 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.250363+0000", 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070229999999999999 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.251065+0000", 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0531931 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.304258+0000", 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3974000000000002e-05 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.304312+0000", 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2749e-05 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.304355+0000", 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.056020388999999997 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:194308 9.1c 9:3cb73bee:::benchmark_data_smithi012_313365_object81833:head [delete] snapc 0=[] ondisk+write+known_if_redirected e398)", 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:31.257856+0000", 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 3.7990884710000001, 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.050303050000000002, 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:18:35.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 194308 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.257856+0000", 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:18:35.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.257856+0000", 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.257854+0000", 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5069999999999997e-06 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:18:35.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.257861+0000", 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2697000000000001e-05 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.257874+0000", 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5520000000000004e-06 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:18:35.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.257879+0000", 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7997999999999998e-05 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.257917+0000", 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9037999999999997e-05 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.258006+0000", 2023-10-09T10:18:35.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028506899999999999 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,7", 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.258291+0000", 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016215540000000001 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.259913+0000", 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001054199 2023-10-09T10:18:35.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.260967+0000", 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047102324000000001 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.308069+0000", 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8727999999999998e-05 2023-10-09T10:18:35.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.308118+0000", 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0519000000000003e-05 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:31.308159+0000", 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.050303050000000002 2023-10-09T10:18:35.307 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.308 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.308 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.308 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:18:35.308 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:18:35.308 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:18:35.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:35.309 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:35.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:35.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:35.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:35 smithi012 ceph-mon[147926]: pgmap v1124: 109 pgs: 11 active+recovery_wait+degraded, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 16499/93426 objects degraded (17.660%); 949/93426 objects misplaced (1.016%); 4.6 MiB/s, 223 objects/s recovering 2023-10-09T10:18:35.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:35 smithi012 ceph-mon[147926]: failed to encode map e400 with expected crc 2023-10-09T10:18:35.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:35 smithi012 ceph-mon[147926]: failed to encode map e400 with expected crc 2023-10-09T10:18:35.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:35 smithi012 ceph-mon[147926]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T10:18:35.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:35 smithi012 ceph-mon[152263]: pgmap v1124: 109 pgs: 11 active+recovery_wait+degraded, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 16499/93426 objects degraded (17.660%); 949/93426 objects misplaced (1.016%); 4.6 MiB/s, 223 objects/s recovering 2023-10-09T10:18:35.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:35 smithi012 ceph-mon[152263]: failed to encode map e400 with expected crc 2023-10-09T10:18:35.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:35 smithi012 ceph-mon[152263]: failed to encode map e400 with expected crc 2023-10-09T10:18:35.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:35 smithi012 ceph-mon[152263]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T10:18:35.444 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:35.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:35 smithi110 ceph-mon[136032]: pgmap v1124: 109 pgs: 11 active+recovery_wait+degraded, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 16499/93426 objects degraded (17.660%); 949/93426 objects misplaced (1.016%); 4.6 MiB/s, 223 objects/s recovering 2023-10-09T10:18:35.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:35 smithi110 ceph-mon[136032]: failed to encode map e400 with expected crc 2023-10-09T10:18:35.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:35 smithi110 ceph-mon[136032]: failed to encode map e400 with expected crc 2023-10-09T10:18:35.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:35 smithi110 ceph-mon[136032]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T10:18:36.274 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:18:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T10:18:36.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: 9.c starting backfill to osd.6 from (0'0,0'0] MAX to 398'11029 2023-10-09T10:18:36.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/315355037' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:36.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:36.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:36 smithi110 ceph-mon[136032]: 9.1c starting backfill to osd.6 from (0'0,0'0] MAX to 398'11052 2023-10-09T10:18:36.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T10:18:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: 9.c starting backfill to osd.6 from (0'0,0'0] MAX to 398'11029 2023-10-09T10:18:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/315355037' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[152263]: 9.1c starting backfill to osd.6 from (0'0,0'0] MAX to 398'11052 2023-10-09T10:18:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T10:18:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: failed to encode map e401 with expected crc 2023-10-09T10:18:36.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: 9.c starting backfill to osd.6 from (0'0,0'0] MAX to 398'11029 2023-10-09T10:18:36.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/315355037' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:36.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:36.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:36 smithi012 ceph-mon[147926]: 9.1c starting backfill to osd.6 from (0'0,0'0] MAX to 398'11052 2023-10-09T10:18:37.244 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:37.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:37.349 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:37.377 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:37.676 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: pgmap v1127: 109 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 14094/88815 objects degraded (15.869%); 949/88815 objects misplaced (1.069%); 6.5 MiB/s, 242 objects/s recovering 2023-10-09T10:18:37.676 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:18:37.677 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.677 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.677 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:37.677 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T10:18:37.677 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.677 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.677 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:37 smithi110 ceph-mon[136032]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: pgmap v1127: 109 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 14094/88815 objects degraded (15.869%); 949/88815 objects misplaced (1.069%); 6.5 MiB/s, 242 objects/s recovering 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[147926]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: pgmap v1127: 109 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 14094/88815 objects degraded (15.869%); 949/88815 objects misplaced (1.069%); 6.5 MiB/s, 242 objects/s recovering 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-mon[152263]: failed to encode map e402 with expected crc 2023-10-09T10:18:37.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:18:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:18:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:18:37] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.33.4" 2023-10-09T10:18:38.674 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:18:38.674 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:18:38.675 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:18:38.675 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2023-10-09T10:18:38.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0 2023-10-09T10:18:38.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[147926]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[147926]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[147926]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T10:18:38.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[147926]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 14094/88815 objects degraded (15.869%), 14 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:38.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[147926]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T10:18:38.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[152263]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[152263]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[152263]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T10:18:38.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[152263]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 14094/88815 objects degraded (15.869%), 14 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:38.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:38 smithi012 ceph-mon[152263]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T10:18:38.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:38 smithi110 ceph-mon[136032]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:38 smithi110 ceph-mon[136032]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:38 smithi110 ceph-mon[136032]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T10:18:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:38 smithi110 ceph-mon[136032]: failed to encode map e403 with expected crc 2023-10-09T10:18:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 14094/88815 objects degraded (15.869%), 14 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:38 smithi110 ceph-mon[136032]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T10:18:39.103 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:39.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:39.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:39.237 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:39.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[147926]: pgmap v1130: 109 pgs: 2 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 8 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 13672/88464 objects degraded (15.455%); 951/88464 objects misplaced (1.075%); 6.3 MiB/s, 278 objects/s recovering 2023-10-09T10:18:39.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[147926]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[147926]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[147926]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[147926]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[152263]: pgmap v1130: 109 pgs: 2 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 8 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 13672/88464 objects degraded (15.455%); 951/88464 objects misplaced (1.075%); 6.3 MiB/s, 278 objects/s recovering 2023-10-09T10:18:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[152263]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[152263]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[152263]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:39 smithi012 ceph-mon[152263]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:39 smithi110 ceph-mon[136032]: pgmap v1130: 109 pgs: 2 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 8 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 13672/88464 objects degraded (15.455%); 951/88464 objects misplaced (1.075%); 6.3 MiB/s, 278 objects/s recovering 2023-10-09T10:18:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:39 smithi110 ceph-mon[136032]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:39 smithi110 ceph-mon[136032]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:39 smithi110 ceph-mon[136032]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:39 smithi110 ceph-mon[136032]: failed to encode map e404 with expected crc 2023-10-09T10:18:39.930 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:39.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:39.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:40.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:40.074 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:40.164 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:40.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:40.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:40.199 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:40.270 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:40.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[147926]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[147926]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T10:18:40.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[147926]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[147926]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[147926]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[147926]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[147926]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.295 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[147926]: 9.11 scrub starts 2023-10-09T10:18:40.312 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:40.376 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:40.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:40.487 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:40.573 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:40 smithi110 ceph-mon[136032]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.573 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:40 smithi110 ceph-mon[136032]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T10:18:40.573 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:40 smithi110 ceph-mon[136032]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.573 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:40 smithi110 ceph-mon[136032]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.574 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:40 smithi110 ceph-mon[136032]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.574 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:40 smithi110 ceph-mon[136032]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.574 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:40 smithi110 ceph-mon[136032]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.574 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:40 smithi110 ceph-mon[136032]: 9.11 scrub starts 2023-10-09T10:18:40.576 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:40.641 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:40.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[152263]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[152263]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T10:18:40.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[152263]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[152263]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[152263]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[152263]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[152263]: failed to encode map e405 with expected crc 2023-10-09T10:18:40.651 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:40 smithi012 ceph-mon[152263]: 9.11 scrub starts 2023-10-09T10:18:40.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:40.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:40.753 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:40.796 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:40.947 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:41.099 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:41.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:41.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:41.254 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:41.277 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:18:41.295 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: pgmap v1133: 109 pgs: 2 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 8 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 31 op/s; 13671/88434 objects degraded (15.459%); 951/88434 objects misplaced (1.075%); 16 KiB/s, 105 objects/s recovering 2023-10-09T10:18:41.295 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T10:18:41.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: 9.1 scrub starts 2023-10-09T10:18:41.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1902855627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:41.303 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:18:41.331 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:18:41.362 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:18:41.390 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:18:41.421 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:18:41.450 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:18:41.482 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:18:41.483 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:18:41.483 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:18:41.483 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:18:41.483 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:18:41.483 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:18:41.483 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:18:41.484 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:18:41.484 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:18:41.583 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: pgmap v1133: 109 pgs: 2 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 8 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 31 op/s; 13671/88434 objects degraded (15.459%); 951/88434 objects misplaced (1.075%); 16 KiB/s, 105 objects/s recovering 2023-10-09T10:18:41.583 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T10:18:41.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: 9.1 scrub starts 2023-10-09T10:18:41.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1902855627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:41.657 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:41.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:41.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:41.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: pgmap v1133: 109 pgs: 2 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 8 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 31 op/s; 13671/88434 objects degraded (15.459%); 951/88434 objects misplaced (1.075%); 16 KiB/s, 105 objects/s recovering 2023-10-09T10:18:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T10:18:41.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: failed to encode map e406 with expected crc 2023-10-09T10:18:41.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: 9.1 scrub starts 2023-10-09T10:18:41.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1902855627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:41.762 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:41.830 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:41.934 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:41.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:41.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:41.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:18:42.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:42.129 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:42.387 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:42 smithi012 ceph-mon[147926]: 9.21 scrub starts 2023-10-09T10:18:42.393 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:42 smithi012 ceph-mon[152263]: 9.21 scrub starts 2023-10-09T10:18:42.401 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:42.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:42.507 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:42.557 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:42 smithi110 ceph-mon[136032]: 9.21 scrub starts 2023-10-09T10:18:42.566 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:42.636 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:42.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:42.744 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:42.787 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:42.986 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:42.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:43.071 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:18:43.072 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:18:43.092 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:43.130 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:43.171 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:43.176 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:43.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:43.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:43.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: pgmap v1135: 109 pgs: 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 22265/87822 objects degraded (25.352%); 931/87822 objects misplaced (1.060%); 29 MiB/s, 790 objects/s recovering 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: 9.11 scrub starts 2023-10-09T10:18:43.311 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 22265/87822 objects degraded (25.352%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:43.322 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:43.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: pgmap v1135: 109 pgs: 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 22265/87822 objects degraded (25.352%); 931/87822 objects misplaced (1.060%); 29 MiB/s, 790 objects/s recovering 2023-10-09T10:18:43.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T10:18:43.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: 9.11 scrub starts 2023-10-09T10:18:43.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 22265/87822 objects degraded (25.352%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:43.466 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:43.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:43.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:18:43.571 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:43.572 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: pgmap v1135: 109 pgs: 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 22265/87822 objects degraded (25.352%); 931/87822 objects misplaced (1.060%); 29 MiB/s, 790 objects/s recovering 2023-10-09T10:18:43.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T10:18:43.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: failed to encode map e407 with expected crc 2023-10-09T10:18:43.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: 9.11 scrub starts 2023-10-09T10:18:43.574 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 22265/87822 objects degraded (25.352%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:18:43.609 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:43.714 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:43.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:43.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:43.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:43.900 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:43.983 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:43.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:44.092 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:44.165 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:44.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:44.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:44.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:44.328 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.3 primary-affinity to 0 (802) 2023-10-09T10:18:44.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:44.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:44 smithi110 ceph-mon[136032]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:44 smithi110 ceph-mon[136032]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:44 smithi110 ceph-mon[136032]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T10:18:44.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:44 smithi110 ceph-mon[136032]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1252489692' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T10:18:44.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:44.437 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:44.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:44.510 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:44.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:18:44.535 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:44.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:44.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:44.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[147926]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[147926]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.556 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[147926]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T10:18:44.556 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[147926]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.556 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1252489692' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T10:18:44.556 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:44.556 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:44.557 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[152263]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.557 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[152263]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.557 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[152263]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T10:18:44.558 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[152263]: failed to encode map e408 with expected crc 2023-10-09T10:18:44.558 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1252489692' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T10:18:44.558 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:44.558 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:44.644 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:44.711 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:44.785 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:44.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:44.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:44.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:44.952 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:45.257 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:45.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:45.345 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:18:45.362 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:45.415 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: pgmap v1138: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 21128/87069 objects degraded (24.266%); 931/87069 objects misplaced (1.069%); 29 MiB/s, 890 objects/s recovering 2023-10-09T10:18:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1252489692' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T10:18:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T10:18:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: 9.1 scrub starts 2023-10-09T10:18:45.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1922002502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: pgmap v1138: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 21128/87069 objects degraded (24.266%); 931/87069 objects misplaced (1.069%); 29 MiB/s, 890 objects/s recovering 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1252489692' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: 9.1 scrub starts 2023-10-09T10:18:45.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1922002502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:18:45.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:45.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:45.588 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:45.633 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: pgmap v1138: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 21128/87069 objects degraded (24.266%); 931/87069 objects misplaced (1.069%); 29 MiB/s, 890 objects/s recovering 2023-10-09T10:18:45.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1252489692' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T10:18:45.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T10:18:45.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: failed to encode map e409 with expected crc 2023-10-09T10:18:45.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: 9.1 scrub starts 2023-10-09T10:18:45.634 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1922002502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:18:45.641 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:45.700 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:45.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:45.806 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:45.862 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:45.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:45.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:45.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:45.927 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:46.022 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:46.055 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:46.119 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:46.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:46.223 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:46.261 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:46.318 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:46.319 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[147926]: failed to encode map e410 with expected crc 2023-10-09T10:18:46.319 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1922002502' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:18:46.319 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[147926]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T10:18:46.319 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[147926]: failed to encode map e410 with expected crc 2023-10-09T10:18:46.355 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:46.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:46.441 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:18:46.441 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:18:46.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:46.504 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:46.562 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:46.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:46.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[152263]: failed to encode map e410 with expected crc 2023-10-09T10:18:46.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1922002502' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:18:46.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[152263]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T10:18:46.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:46 smithi012 ceph-mon[152263]: failed to encode map e410 with expected crc 2023-10-09T10:18:46.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:46.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:46.670 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:46 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:46 smithi110 ceph-mon[136032]: failed to encode map e410 with expected crc 2023-10-09T10:18:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:46 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1922002502' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:18:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:46 smithi110 ceph-mon[136032]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T10:18:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:46 smithi110 ceph-mon[136032]: failed to encode map e410 with expected crc 2023-10-09T10:18:46.742 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:46.806 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:46.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:46.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:46.868 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:18:46.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:46.958 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:47.014 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:47.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:47.119 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:47.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:47.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:47.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:47.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:47.372 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:47.412 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:47.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:47 smithi110 ceph-mon[136032]: pgmap v1141: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 20644/80892 objects degraded (25.520%); 857/80892 objects misplaced (1.059%); 7.6 MiB/s, 348 objects/s recovering 2023-10-09T10:18:47.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:47 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:18:47.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:47 smithi110 ceph-mon[136032]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:47 smithi110 ceph-mon[136032]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T10:18:47.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:47 smithi110 ceph-mon[136032]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:47 smithi110 ceph-mon[136032]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:47 smithi110 ceph-mon[136032]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.473 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:47.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:47.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:47.582 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:47.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[147926]: pgmap v1141: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 20644/80892 objects degraded (25.520%); 857/80892 objects misplaced (1.059%); 7.6 MiB/s, 348 objects/s recovering 2023-10-09T10:18:47.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:18:47.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[147926]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[147926]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T10:18:47.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[147926]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[147926]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[147926]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.662 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[152263]: pgmap v1141: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 20644/80892 objects degraded (25.520%); 857/80892 objects misplaced (1.059%); 7.6 MiB/s, 348 objects/s recovering 2023-10-09T10:18:47.662 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:18:47.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[152263]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[152263]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T10:18:47.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[152263]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[152263]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-mon[152263]: failed to encode map e411 with expected crc 2023-10-09T10:18:47.668 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:47.738 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:47.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:47.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:47.831 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:18:47.851 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:47.860 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:47.913 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:47.924 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:18:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:18:47] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.33.4" 2023-10-09T10:18:47.930 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:18:47.959 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:18:47.971 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:47.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:47.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:47.987 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:18:48.012 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:18:48.047 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:18:48.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:48.079 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:18:48.121 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:18:48.127 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:48.158 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:18:48.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:18:48.185 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:18:48.185 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:18:48.185 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:18:48.185 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:18:48.185 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:18:48.185 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:18:48.185 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:18:48.190 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:48.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:48.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:48.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:48.338 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:48.388 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:48.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:48.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:48.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2569583679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:48.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 20644/80892 objects degraded (25.520%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:18:48.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2569583679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:48.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 20644/80892 objects degraded (25.520%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:18:48.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:48.581 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2569583679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:48.581 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 20644/80892 objects degraded (25.520%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:18:48.584 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:48.600 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:48.675 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:48.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:48.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:48.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:48.817 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:49.016 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:49.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:49.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:49.161 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:49.264 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:49.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:49.373 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:49.448 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:49.522 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:49.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:49.555 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:49 smithi012 ceph-mon[147926]: pgmap v1143: 109 pgs: 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 20644/79419 objects degraded (25.994%); 857/79419 objects misplaced (1.079%); 7.1 MiB/s, 194 objects/s recovering 2023-10-09T10:18:49.555 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:49 smithi012 ceph-mon[152263]: pgmap v1143: 109 pgs: 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 20644/79419 objects degraded (25.994%); 857/79419 objects misplaced (1.079%); 7.1 MiB/s, 194 objects/s recovering 2023-10-09T10:18:49.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:49.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:49 smithi110 ceph-mon[136032]: pgmap v1143: 109 pgs: 2 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 20644/79419 objects degraded (25.994%); 857/79419 objects misplaced (1.079%); 7.1 MiB/s, 194 objects/s recovering 2023-10-09T10:18:49.711 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:49.762 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:49.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:49.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:49.782 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:18:49.782 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:18:49.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:18:49.782 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:18:49.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.782 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121890 9.1 9.4309a8a1 (undecoded) ondisk+retry+write+known_if_redirected e352)", 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 188.47931927600001, 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99073293299999998, 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121890 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300149+0000", 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300155+0000", 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2280000000000003e-06 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300163+0000", 2023-10-09T10:18:49.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8429999999999993e-06 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300170+0000", 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5989999999999998e-06 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300211+0000", 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1511000000000003e-05 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300219+0000", 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6190000000000002e-06 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.290871+0000", 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99065166100000002 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.290884+0000", 2023-10-09T10:18:49.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2998e-05 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121893 9.11 9.c051ab51 (undecoded) ondisk+write+known_if_redirected e352)", 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 188.47341063100001, 2023-10-09T10:18:49.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98772148100000001, 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121893 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306058+0000", 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306064+0000", 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1700000000000002e-06 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306074+0000", 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0094e-05 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306084+0000", 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6579999999999997e-06 2023-10-09T10:18:49.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306174+0000", 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0655000000000005e-05 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306187+0000", 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2376e-05 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.293771+0000", 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98758401100000004 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.293781+0000", 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0017000000000001e-05 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121898 9.c 9.187a12cc (undecoded) ondisk+write+known_if_redirected e352)", 2023-10-09T10:18:49.793 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 188.44617243799999, 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94351869099999996, 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121898 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.794 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333295+0000", 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333306+0000", 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0361e-05 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333346+0000", 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0145999999999998e-05 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333357+0000", 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1377e-05 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333428+0000", 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0835999999999997e-05 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333441+0000", 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2449000000000001e-05 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.276804+0000", 2023-10-09T10:18:49.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94336297999999996 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.276816+0000", 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2670999999999999e-05 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:123521 9.2 e354/353 9:46c69193:::benchmark_data_smithi012_313365_object11046:head v 354'7084, mlcod=344'6827)", 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:54.203163+0000", 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 175.57630706200001, 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023058531, 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203163+0000", 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203163+0000", 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203162+0000", 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203167+0000", 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7600000000000002e-06 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203174+0000", 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1180000000000003e-06 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.203180+0000", 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5310000000000001e-06 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.206390+0000", 2023-10-09T10:18:49.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032101149999999999 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.206421+0000", 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1452999999999999e-05 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.226183+0000", 2023-10-09T10:18:49.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019761772 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.226221+0000", 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8231e-05 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:123709 9.a e354/353 9:5c5183a2:::benchmark_data_smithi012_313365_object11234:head v 354'7475, mlcod=0'0)", 2023-10-09T10:18:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:54.758440+0000", 2023-10-09T10:18:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 175.02102984699999, 2023-10-09T10:18:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025809789999999999, 2023-10-09T10:18:49.809 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758440+0000", 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758440+0000", 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758440+0000", 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758444+0000", 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5940000000000004e-06 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758451+0000", 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7009999999999999e-06 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.758456+0000", 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9150000000000004e-06 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.766498+0000", 2023-10-09T10:18:49.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0080422059999999997 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.766526+0000", 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8189000000000001e-05 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.784204+0000", 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017677531999999999 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:54.784250+0000", 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5983999999999999e-05 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:125277 9.a e354/353 9:5960c75e:::benchmark_data_smithi012_313365_object12802:head v 354'7584, mlcod=0'0)", 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:59.479531+0000", 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 170.29993886099999, 2023-10-09T10:18:49.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023230032000000001, 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479531+0000", 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479531+0000", 2023-10-09T10:18:49.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479530+0000", 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479538+0000", 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1099999999999997e-06 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479548+0000", 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0301e-05 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.479555+0000", 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2389999999999996e-06 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.483886+0000", 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043309020000000002 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.483915+0000", 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8530999999999998e-05 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.502627+0000", 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018712876 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.502761+0000", 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000133719 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:125279 9.2 e354/353 9:40d3f81e:::benchmark_data_smithi012_313365_object12804:head v 354'7141, mlcod=344'6827)", 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:59.482769+0000", 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 170.29670104100001, 2023-10-09T10:18:49.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029954650999999999, 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482769+0000", 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482769+0000", 2023-10-09T10:18:49.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482768+0000", 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482777+0000", 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5490000000000006e-06 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482790+0000", 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2818e-05 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482798+0000", 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3820000000000004e-06 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.491998+0000", 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0092004579999999999 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.492024+0000", 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5789e-05 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512700+0000", 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020676132999999999 2023-10-09T10:18:49.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512724+0000", 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3291000000000001e-05 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:125278 9.a e354/353 9:59d42cbe:::benchmark_data_smithi012_313365_object12803:head v 354'7585, mlcod=0'0)", 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:59.482806+0000", 2023-10-09T10:18:49.824 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 170.29666385900001, 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.02989023, 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482806+0000", 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482806+0000", 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482806+0000", 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482810+0000", 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0949999999999998e-06 2023-10-09T10:18:49.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482815+0000", 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4489999999999998e-06 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.482819+0000", 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0759999999999996e-06 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.494191+0000", 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011372044 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.494219+0000", 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7787e-05 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512667+0000", 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018447519999999999 2023-10-09T10:18:49.828 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512696+0000", 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9553e-05 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:125284 9.2 e354/353 9:476a4c30:::benchmark_data_smithi012_313365_object12809:head v 354'7142, mlcod=344'6827)", 2023-10-09T10:18:49.829 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:59.488043+0000", 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 170.291427414, 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024718561999999999, 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488043+0000", 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488043+0000", 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488042+0000", 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488051+0000", 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7359999999999996e-06 2023-10-09T10:18:49.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488066+0000", 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4744e-05 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.488075+0000", 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.234e-06 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.832 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.504341+0000", 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016265601000000001 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.504364+0000", 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3397999999999999e-05 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512726+0000", 2023-10-09T10:18:49.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0083614750000000002 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:59.512761+0000", 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5423999999999998e-05 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127796 9.a e355/353 9:5b549510:::benchmark_data_smithi012_313365_object15321:head v 355'7734, mlcod=0'0)", 2023-10-09T10:18:49.834 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.929123+0000", 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 162.85034677600001, 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029092829000000001, 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929123+0000", 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929123+0000", 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929123+0000", 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929128+0000", 2023-10-09T10:18:49.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7369999999999997e-06 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929138+0000", 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9340000000000001e-06 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.929143+0000", 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9960000000000001e-06 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.931154+0000", 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020111819999999998 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.931179+0000", 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4958000000000001e-05 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.958180+0000", 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027001421000000001 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.958216+0000", 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5936000000000003e-05 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.839 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127798 9.a e355/353 9:5b6bbbbe:::benchmark_data_smithi012_313365_object15323:head v 355'7735, mlcod=0'0)", 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.934647+0000", 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 162.84482281699999, 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023586669000000001, 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934647+0000", 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.840 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934647+0000", 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934646+0000", 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934655+0000", 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0510000000000002e-06 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934668+0000", 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3465000000000001e-05 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.934677+0000", 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1849999999999993e-06 2023-10-09T10:18:49.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.940490+0000", 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058127029999999998 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.940519+0000", 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8762999999999999e-05 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.958219+0000", 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017699751999999999 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.958234+0000", 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5293000000000001e-05 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.844 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:128048 9.a e358/357 9:5a9a9e1f:::benchmark_data_smithi012_313365_object15573:head v 358'7758, mlcod=0'0)", 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:09.628597+0000", 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 160.15087299699999, 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023843540999999999, 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628597+0000", 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628597+0000", 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628595+0000", 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628606+0000", 2023-10-09T10:18:49.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0376e-05 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628620+0000", 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3925e-05 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.628630+0000", 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0054000000000001e-05 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.634055+0000", 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054254949999999998 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.634092+0000", 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6748999999999997e-05 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.848 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.652390+0000", 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018298043 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.652441+0000", 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0386000000000002e-05 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.849 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128066 9.1 9:8603ff6a:::benchmark_data_smithi012_313365_object15591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 160.09731243900001, 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55978837800000003, 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128066 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.850 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.851 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682155+0000", 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682168+0000", 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2682e-05 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682184+0000", 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6339000000000001e-05 2023-10-09T10:18:49.852 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682194+0000", 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4420000000000004e-06 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682258+0000", 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4463999999999998e-05 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.853 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682271+0000", 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3166000000000001e-05 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237630+0000", 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55535865299999998 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237768+0000", 2023-10-09T10:18:49.854 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013818000000000001 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237891+0000", 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012287299999999999 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.240596+0000", 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002704559 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.855 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241710+0000", 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00111455 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241856+0000", 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014592000000000001 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241905+0000", 2023-10-09T10:18:49.856 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9042000000000002e-05 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241946+0000", 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0655999999999998e-05 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156228 9.11 9:8e81769a:::benchmark_data_smithi012_313365_object43753:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:18:49.857 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:07.974745+0000", 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.80472508299999, 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023573625000000001, 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156228 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.858 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974745+0000", 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974745+0000", 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974744+0000", 2023-10-09T10:18:49.859 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974751+0000", 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3359999999999997e-06 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974767+0000", 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6160000000000001e-05 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.860 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974776+0000", 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8310000000000007e-06 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974836+0000", 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0568e-05 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.974950+0000", 2023-10-09T10:18:49.861 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114021 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.975014+0000", 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3950999999999998e-05 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.976771+0000", 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001756781 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.862 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.978790+0000", 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020192069999999999 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.998216+0000", 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.019425782999999999 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.998272+0000", 2023-10-09T10:18:49.863 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5448000000000002e-05 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:07.998319+0000", 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6898999999999999e-05 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156448 9.1 9:835065d8:::benchmark_data_smithi012_313365_object43973:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:18:49.864 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.228667+0000", 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.550802895, 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024562891, 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156448 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.865 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228667+0000", 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228667+0000", 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228665+0000", 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:18:49.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228678+0000", 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3193999999999999e-05 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228704+0000", 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5791000000000001e-05 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.867 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228717+0000", 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3049e-05 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228900+0000", 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000182919 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.868 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.229060+0000", 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000160571 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.229211+0000", 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015067499999999999 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.233957+0000", 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0047465529999999997 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.869 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.236603+0000", 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026458010000000001 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253132+0000", 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016529245000000001 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.870 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253190+0000", 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7237000000000001e-05 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253230+0000", 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0358000000000002e-05 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.871 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156449 9.1 9:87277be4:::benchmark_data_smithi012_313365_object43974:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.228735+0000", 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.55073506399999, 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024607628999999999, 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156449 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.872 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228735+0000", 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228735+0000", 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.873 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228734+0000", 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228738+0000", 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2389999999999998e-06 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228744+0000", 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2140000000000001e-06 2023-10-09T10:18:49.874 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228749+0000", 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3299999999999998e-06 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.231538+0000", 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027884709999999998 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.875 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.231646+0000", 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108622 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.231745+0000", 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9099999999999996e-05 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.236559+0000", 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048137290000000001 2023-10-09T10:18:49.876 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.247589+0000", 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011029793 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253267+0000", 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056784360000000002 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.877 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253305+0000", 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7728000000000002e-05 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253343+0000", 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7438999999999998e-05 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.878 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156450 9.1 9:844cce54:::benchmark_data_smithi012_313365_object43975:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.228754+0000", 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.550716115, 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024882517999999999, 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156450 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.879 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228754+0000", 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228754+0000", 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.880 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228754+0000", 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228756+0000", 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3420000000000002e-06 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228761+0000", 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1789999999999999e-06 2023-10-09T10:18:49.881 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.228765+0000", 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3039999999999998e-06 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.234111+0000", 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053460319999999997 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.882 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.234233+0000", 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121143 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.234319+0000", 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6410999999999996e-05 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.236896+0000", 2023-10-09T10:18:49.883 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002577382 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.247617+0000", 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010720587 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253572+0000", 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0059550430000000001 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.884 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253606+0000", 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4199000000000002e-05 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.253636+0000", 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0247e-05 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.885 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156571 9.11 9:88722a35:::benchmark_data_smithi012_313365_object44096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.379046191, 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.21718912700000001, 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156571 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.886 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.887 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400422+0000", 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400431+0000", 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2720000000000003e-06 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400450+0000", 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8981e-05 2023-10-09T10:18:49.888 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400462+0000", 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2333e-05 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.402769+0000", 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023067069999999999 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.889 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.402909+0000", 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013978499999999999 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.403031+0000", 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012171300000000001 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.446104+0000", 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.043073587000000003 2023-10-09T10:18:49.890 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575808+0000", 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12970353000000001 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617506+0000", 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041698357999999998 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.891 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617561+0000", 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5191000000000001e-05 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617613+0000", 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1649e-05 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.892 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156574 9.21 9:87a74460:::benchmark_data_smithi012_313365_object44099:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.374033539, 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0291358530000001, 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156574 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.893 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.894 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405435+0000", 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405441+0000", 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7290000000000003e-06 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405454+0000", 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2811e-05 2023-10-09T10:18:49.895 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405465+0000", 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0448000000000001e-05 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575963+0000", 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17049810900000001 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.896 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575975+0000", 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.221e-05 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.430808+0000", 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85483296499999994 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.430955+0000", 2023-10-09T10:18:49.897 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014732499999999999 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.431079+0000", 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012408499999999999 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.432894+0000", 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001814593 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434249+0000", 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001355537 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434463+0000", 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000213165 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434525+0000", 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2205000000000005e-05 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434572+0000", 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7500000000000003e-05 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156578 9.11 9:8e383e1c:::benchmark_data_smithi012_313365_object44103:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:18:49.900 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 101.267848269, 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.19283726200000001, 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156578 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.901 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511620+0000", 2023-10-09T10:18:49.902 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511628+0000", 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7139999999999999e-06 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511644+0000", 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6018000000000001e-05 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.903 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511654+0000", 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9119999999999993e-06 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511718+0000", 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3587e-05 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511874+0000", 2023-10-09T10:18:49.904 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000156126 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.512003+0000", 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012904400000000001 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575815+0000", 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063811965999999998 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.905 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.656235+0000", 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.080420144999999998 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704358+0000", 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.048122282000000002 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704415+0000", 2023-10-09T10:18:49.906 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7090999999999999e-05 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704459+0000", 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4347000000000001e-05 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:49.907 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:18:49.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:49.976 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:50.035 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:50.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:50.143 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:50.234 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:50.297 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:18:50.298 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:18:50.298 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T10:18:50.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:18:50.303 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:50.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:50.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:18:50.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:50.459 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:50.495 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:50.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:50.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:50.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:50.637 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:50.688 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:50.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:50.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:50.793 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:50.829 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:50.875 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:50.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:50.982 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:51.004 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:51.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:18:51.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:51.100 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:51.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:51.146 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:51.208 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:51.279 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:51.335 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:51.340 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:51 smithi012 ceph-mon[147926]: pgmap v1144: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 20369/77004 objects degraded (26.452%); 857/77004 objects misplaced (1.113%); 7.9 MiB/s, 185 objects/s recovering 2023-10-09T10:18:51.340 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2741367045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:51.340 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:51.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:51.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:51.358 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:18:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:51 smithi110 ceph-mon[136032]: pgmap v1144: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 20369/77004 objects degraded (26.452%); 857/77004 objects misplaced (1.113%); 7.9 MiB/s, 185 objects/s recovering 2023-10-09T10:18:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2741367045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:51.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:18:51.487 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:51.554 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:18:51.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:51.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:18:51.661 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:51.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:51 smithi012 ceph-mon[152263]: pgmap v1144: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 20369/77004 objects degraded (26.452%); 857/77004 objects misplaced (1.113%); 7.9 MiB/s, 185 objects/s recovering 2023-10-09T10:18:51.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2741367045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:51.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:18:51.721 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:51.784 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:51.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:51.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:51.891 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:51.927 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:52.276 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:52.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:52.382 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:52.432 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:52.492 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:52.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:52.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:52.600 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:52.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:52.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:52.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[147926]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T10:18:52.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[147926]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[147926]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[147926]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[147926]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[147926]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:52.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:52.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[152263]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T10:18:52.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[152263]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[152263]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[152263]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[152263]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:52 smithi012 ceph-mon[152263]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.632 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:52 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:18:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:52 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:18:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:52 smithi110 ceph-mon[136032]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T10:18:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:52 smithi110 ceph-mon[136032]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:52 smithi110 ceph-mon[136032]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:52 smithi110 ceph-mon[136032]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:52 smithi110 ceph-mon[136032]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:52 smithi110 ceph-mon[136032]: failed to encode map e412 with expected crc 2023-10-09T10:18:52.720 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:52.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:52.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:52.727 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:52.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:52.867 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:52.957 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:52.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:52.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:53.065 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:53.122 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:53.201 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:53.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:53.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:53.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:18:53.385 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:53.385 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:53 smithi110 ceph-mon[136032]: pgmap v1146: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 20098/72471 objects degraded (27.732%); 750/72471 objects misplaced (1.035%); 5.6 MiB/s, 145 objects/s recovering 2023-10-09T10:18:53.385 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:53 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:18:53.385 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:53 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 20098/72471 objects degraded (27.732%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:18:53.436 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:18:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:53.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:18:53.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:53.579 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:53 smithi012 ceph-mon[147926]: pgmap v1146: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 20098/72471 objects degraded (27.732%); 750/72471 objects misplaced (1.035%); 5.6 MiB/s, 145 objects/s recovering 2023-10-09T10:18:53.579 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:53 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:18:53.580 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:53 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 20098/72471 objects degraded (27.732%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:18:53.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:53 smithi012 ceph-mon[152263]: pgmap v1146: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 20098/72471 objects degraded (27.732%); 750/72471 objects misplaced (1.035%); 5.6 MiB/s, 145 objects/s recovering 2023-10-09T10:18:53.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:53 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:18:53.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:53 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 20098/72471 objects degraded (27.732%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:18:53.583 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:53.632 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:53.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:53.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:53.738 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:18:53.754 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:18:53.766 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:18:53.812 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:53.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:18:53.872 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:18:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:53.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:18:53.982 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:54.030 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:54.136 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:54.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:54.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:54.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:18:54.282 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:54.599 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:204013 9.9 9:91a9cab4:::benchmark_data_smithi012_313365_object91538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e412)", 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:54.598763+0000", 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00022641499999999999, 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00024864400000000001, 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:18:54.600 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 204013 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:54.598763+0000", 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:18:54.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:54.598763+0000", 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:54.598761+0000", 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:54.598770+0000", 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9709999999999993e-06 2023-10-09T10:18:54.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:54.598786+0000", 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6382999999999999e-05 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:54.598797+0000", 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0355e-05 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:18:54.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:54.598862+0000", 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5505000000000004e-05 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:54.598956+0000", 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4308999999999994e-05 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:18:54.604 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 1 2023-10-09T10:18:54.605 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:18:55.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:55 smithi012 ceph-mon[152263]: pgmap v1147: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 20098/70899 objects degraded (28.347%); 750/70899 objects misplaced (1.058%); 4.4 MiB/s, 114 objects/s recovering 2023-10-09T10:18:55.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:55 smithi012 ceph-mon[152263]: from='client.36354 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:18:55.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:55 smithi012 ceph-mon[147926]: pgmap v1147: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 20098/70899 objects degraded (28.347%); 750/70899 objects misplaced (1.058%); 4.4 MiB/s, 114 objects/s recovering 2023-10-09T10:18:55.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:55 smithi012 ceph-mon[147926]: from='client.36354 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:18:55.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:55 smithi110 ceph-mon[136032]: pgmap v1147: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 20098/70899 objects degraded (28.347%); 750/70899 objects misplaced (1.058%); 4.4 MiB/s, 114 objects/s recovering 2023-10-09T10:18:55.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:55 smithi110 ceph-mon[136032]: from='client.36354 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:18:57.212 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1147,"stamp":"2023-10-09T10:18:53.614383+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1547737016,"num_objects":23633,"num_object_clones":0,"num_object_copies":70899,"num_objects_missing_on_primary":2312,"num_objects_missing":2312,"num_objects_degraded":20098,"num_objects_misplaced":750,"num_objects_unfound":0,"num_objects_dirty":23633,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":194991,"num_write_kb":6772078,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45907,"num_bytes_recovered":2452630500,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":273364,"ondisk_log_size":273364,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":384,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11638040,"kb_used_data":6729560,"kb_used_omap":172,"kb_used_meta":4908051,"kb_avail":738323176,"statfs":{"total":767960285184,"available":756042932224,"internally_reserved":0,"allocated":6891069440,"data_stored":6882191534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":176164,"internal_metadata":5025845212},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-218300416,"num_objects":-3331,"num_object_clones":0,"num_object_copies":-9993,"num_objects_missing_on_primary":-161,"num_objects_missing":-161,"num_objects_degraded":-546,"num_objects_misplaced":-107,"num_objects_unfound":0,"num_objects_dirty":-3331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3331,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":917,"num_bytes_recovered":36634624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002219"},"pg_stats":[{"pgid":"9.23","version":"411'8808","reported_seq":10296,"reported_epoch":411,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:49.360158+0000","last_change":"2023-10-09T10:17:49.823791+0000","last_active":"2023-10-09T10:18:49.360158+0000","last_peered":"2023-10-09T10:18:49.360158+0000","last_clean":"2023-10-09T10:15:14.172820+0000","last_became_active":"2023-10-09T10:17:49.774751+0000","last_became_peered":"2023-10-09T10:17:49.774751+0000","last_unstale":"2023-10-09T10:18:49.360158+0000","last_undegraded":"2023-10-09T10:17:49.746768+0000","last_fullsized":"2023-10-09T10:18:49.360158+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":377,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8808,"log_dups_size":0,"ondisk_log_size":8808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24920064,"num_objects":380,"num_object_clones":0,"num_object_copies":1140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1027,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3104,"num_write_kb":111504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":668,"num_bytes_recovered":43778048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,3,7","objects":350},{"shards":"1,5,7","objects":30}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.22","version":"411'8775","reported_seq":10271,"reported_epoch":411,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:49.350001+0000","last_change":"2023-10-09T10:17:09.513820+0000","last_active":"2023-10-09T10:18:49.350001+0000","last_peered":"2023-10-09T10:18:49.350001+0000","last_clean":"2023-10-09T10:15:14.051568+0000","last_became_active":"2023-10-09T10:17:09.506371+0000","last_became_peered":"2023-10-09T10:17:09.506371+0000","last_unstale":"2023-10-09T10:18:49.350001+0000","last_undegraded":"2023-10-09T10:17:09.450481+0000","last_fullsized":"2023-10-09T10:18:49.350001+0000","mapping_epoch":377,"log_start":"321'588","ondisk_log_start":"321'588","created":377,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1071,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3091,"num_write_kb":111104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":677,"num_bytes_recovered":44367872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,3","objects":388},{"shards":"1,2,7","objects":-7}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.21","version":"411'9046","reported_seq":9192,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:50.266610+0000","last_change":"2023-10-09T10:18:41.591287+0000","last_active":"2023-10-09T10:18:50.266610+0000","last_peered":"2023-10-09T10:18:50.266610+0000","last_clean":"2023-10-09T10:18:50.266610+0000","last_became_active":"2023-10-09T10:17:09.424870+0000","last_became_peered":"2023-10-09T10:17:09.424870+0000","last_unstale":"2023-10-09T10:18:50.266610+0000","last_undegraded":"2023-10-09T10:18:50.266610+0000","last_fullsized":"2023-10-09T10:18:50.266610+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":377,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9046,"log_dups_size":0,"ondisk_log_size":9046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24838144,"num_objects":379,"num_object_clones":0,"num_object_copies":1137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3195,"num_write_kb":114368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.20","version":"411'8805","reported_seq":9007,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.030628+0000","last_change":"2023-10-09T10:17:09.475599+0000","last_active":"2023-10-09T10:18:49.030628+0000","last_peered":"2023-10-09T10:18:49.030628+0000","last_clean":"2023-10-09T10:18:49.030628+0000","last_became_active":"2023-10-09T10:17:09.456790+0000","last_became_peered":"2023-10-09T10:17:09.456790+0000","last_unstale":"2023-10-09T10:18:49.030628+0000","last_undegraded":"2023-10-09T10:18:49.030628+0000","last_fullsized":"2023-10-09T10:18:49.030628+0000","mapping_epoch":377,"log_start":"321'624","ondisk_log_start":"321'624","created":377,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":6,"last_scrub":"339'6671","last_scrub_stamp":"2023-10-09T10:15:03.503975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:15:03.503975+0000","objects_scrubbed":3418,"log_size":8181,"log_dups_size":0,"ondisk_log_size":8181,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.90657236399999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26083328,"num_objects":398,"num_object_clones":0,"num_object_copies":1194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1415,"num_write_kb":3328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.10","version":"411'9440","reported_seq":20343,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:49.752581+0000","last_change":"2023-10-09T10:18:39.755302+0000","last_active":"2023-10-09T10:18:49.752581+0000","last_peered":"2023-10-09T10:18:49.752581+0000","last_clean":"2023-10-09T10:15:06.274037+0000","last_became_active":"2023-10-09T10:18:37.564918+0000","last_became_peered":"2023-10-09T10:18:37.564918+0000","last_unstale":"2023-10-09T10:18:49.752581+0000","last_undegraded":"2023-10-09T10:18:37.532863+0000","last_fullsized":"2023-10-09T10:18:35.261566+0000","mapping_epoch":400,"log_start":"328'3808","ondisk_log_start":"328'3808","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5632,"log_dups_size":0,"ondisk_log_size":5632,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47087616,"num_objects":719,"num_object_clones":0,"num_object_copies":2157,"num_objects_missing_on_primary":691,"num_objects_missing":691,"num_objects_degraded":3955,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6177,"num_write_kb":220640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2731,"num_bytes_recovered":178913280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,2,4,6","objects":4},{"shards":"1,2,6","objects":691},{"shards":"4,7","objects":24}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":127,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333807+0000","last_change":"2023-10-09T10:17:37.685528+0000","last_active":"2023-10-09T10:18:45.333807+0000","last_peered":"2023-10-09T10:18:45.333807+0000","last_clean":"2023-10-09T10:18:45.333807+0000","last_became_active":"2023-10-09T10:17:37.685092+0000","last_became_peered":"2023-10-09T10:17:37.685092+0000","last_unstale":"2023-10-09T10:18:45.333807+0000","last_undegraded":"2023-10-09T10:18:45.333807+0000","last_fullsized":"2023-10-09T10:18:45.333807+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":336,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338807+0000","last_change":"2023-10-09T10:16:08.230865+0000","last_active":"2023-10-09T10:18:45.338807+0000","last_peered":"2023-10-09T10:18:45.338807+0000","last_clean":"2023-10-09T10:18:45.338807+0000","last_became_active":"2023-10-09T10:16:08.230559+0000","last_became_peered":"2023-10-09T10:16:08.230559+0000","last_unstale":"2023-10-09T10:18:45.338807+0000","last_undegraded":"2023-10-09T10:18:45.338807+0000","last_fullsized":"2023-10-09T10:18:45.338807+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:02.674870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.13","version":"411'9447","reported_seq":11464,"reported_epoch":411,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:49.341872+0000","last_change":"2023-10-09T10:17:50.801181+0000","last_active":"2023-10-09T10:18:49.341872+0000","last_peered":"2023-10-09T10:18:49.341872+0000","last_clean":"2023-10-09T10:15:14.025151+0000","last_became_active":"2023-10-09T10:17:50.754824+0000","last_became_peered":"2023-10-09T10:17:50.754824+0000","last_unstale":"2023-10-09T10:18:49.341872+0000","last_undegraded":"2023-10-09T10:17:50.750441+0000","last_fullsized":"2023-10-09T10:18:49.341872+0000","mapping_epoch":392,"log_start":"331'4580","ondisk_log_start":"331'4580","created":338,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":4867,"log_dups_size":0,"ondisk_log_size":4867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52723712,"num_objects":805,"num_object_clones":0,"num_object_copies":2415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2525,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6181,"num_write_kb":223520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":935,"num_bytes_recovered":61276160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,3,7","objects":770},{"shards":"1,5,7","objects":35}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":432,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.337483+0000","last_change":"2023-10-09T10:15:42.314309+0000","last_active":"2023-10-09T10:18:45.337483+0000","last_peered":"2023-10-09T10:18:45.337483+0000","last_clean":"2023-10-09T10:18:45.337483+0000","last_became_active":"2023-10-09T10:15:42.313553+0000","last_became_peered":"2023-10-09T10:15:42.313553+0000","last_unstale":"2023-10-09T10:18:45.337483+0000","last_undegraded":"2023-10-09T10:18:45.337483+0000","last_fullsized":"2023-10-09T10:18:45.337483+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:00.421233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":306,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:46.339941+0000","last_change":"2023-10-09T10:16:08.309799+0000","last_active":"2023-10-09T10:18:46.339941+0000","last_peered":"2023-10-09T10:18:46.339941+0000","last_clean":"2023-10-09T10:18:46.339941+0000","last_became_active":"2023-10-09T10:16:08.309448+0000","last_became_peered":"2023-10-09T10:16:08.309448+0000","last_unstale":"2023-10-09T10:18:46.339941+0000","last_undegraded":"2023-10-09T10:18:46.339941+0000","last_fullsized":"2023-10-09T10:18:46.339941+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:58.976233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"411'9478","reported_seq":16569,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.307918+0000","last_change":"2023-10-09T10:18:00.278902+0000","last_active":"2023-10-09T10:18:49.307918+0000","last_peered":"2023-10-09T10:18:49.307918+0000","last_clean":"2023-10-09T10:18:49.307918+0000","last_became_active":"2023-10-09T10:18:00.265061+0000","last_became_peered":"2023-10-09T10:18:00.265061+0000","last_unstale":"2023-10-09T10:18:49.307918+0000","last_undegraded":"2023-10-09T10:18:49.307918+0000","last_fullsized":"2023-10-09T10:18:49.307918+0000","mapping_epoch":395,"log_start":"332'4630","ondisk_log_start":"332'4630","created":338,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51183616,"num_objects":781,"num_object_clones":0,"num_object_copies":2343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6209,"num_write_kb":223680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3475,"num_bytes_recovered":129957888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":335,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.337807+0000","last_change":"2023-10-09T10:18:34.268776+0000","last_active":"2023-10-09T10:18:45.337807+0000","last_peered":"2023-10-09T10:18:45.337807+0000","last_clean":"2023-10-09T10:18:45.337807+0000","last_became_active":"2023-10-09T10:18:34.268454+0000","last_became_peered":"2023-10-09T10:18:34.268454+0000","last_unstale":"2023-10-09T10:18:45.337807+0000","last_undegraded":"2023-10-09T10:18:45.337807+0000","last_fullsized":"2023-10-09T10:18:45.337807+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:40.225209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339062+0000","last_change":"2023-10-09T10:16:41.473093+0000","last_active":"2023-10-09T10:18:45.339062+0000","last_peered":"2023-10-09T10:18:45.339062+0000","last_clean":"2023-10-09T10:18:45.339062+0000","last_became_active":"2023-10-09T10:16:41.472774+0000","last_became_peered":"2023-10-09T10:16:41.472774+0000","last_unstale":"2023-10-09T10:18:45.339062+0000","last_undegraded":"2023-10-09T10:18:45.339062+0000","last_fullsized":"2023-10-09T10:18:45.339062+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:07.117932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1d","version":"411'11290","reported_seq":11420,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.241843+0000","last_change":"2023-10-09T10:17:09.512906+0000","last_active":"2023-10-09T10:18:49.241843+0000","last_peered":"2023-10-09T10:18:49.241843+0000","last_clean":"2023-10-09T10:18:49.241843+0000","last_became_active":"2023-10-09T10:17:09.498676+0000","last_became_peered":"2023-10-09T10:17:09.498676+0000","last_unstale":"2023-10-09T10:18:49.241843+0000","last_undegraded":"2023-10-09T10:18:49.241843+0000","last_fullsized":"2023-10-09T10:18:49.241843+0000","mapping_epoch":377,"log_start":"334'5326","ondisk_log_start":"334'5326","created":377,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49250316,"num_objects":752,"num_object_clones":0,"num_object_copies":2256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":752,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6344,"num_write_kb":227041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":340,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:46.340119+0000","last_change":"2023-10-09T10:16:18.273288+0000","last_active":"2023-10-09T10:18:46.340119+0000","last_peered":"2023-10-09T10:18:46.340119+0000","last_clean":"2023-10-09T10:18:46.340119+0000","last_became_active":"2023-10-09T10:16:18.272936+0000","last_became_peered":"2023-10-09T10:16:18.272936+0000","last_unstale":"2023-10-09T10:18:46.340119+0000","last_undegraded":"2023-10-09T10:18:46.340119+0000","last_fullsized":"2023-10-09T10:18:46.340119+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:39:34.847781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":357,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.337933+0000","last_change":"2023-10-09T10:16:08.619899+0000","last_active":"2023-10-09T10:18:45.337933+0000","last_peered":"2023-10-09T10:18:45.337933+0000","last_clean":"2023-10-09T10:18:45.337933+0000","last_became_active":"2023-10-09T10:16:08.619549+0000","last_became_peered":"2023-10-09T10:16:08.619549+0000","last_unstale":"2023-10-09T10:18:45.337933+0000","last_undegraded":"2023-10-09T10:18:45.337933+0000","last_fullsized":"2023-10-09T10:18:45.337933+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:07.870610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1c","version":"411'11226","reported_seq":26238,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.614657+0000","last_change":"2023-10-09T10:18:40.332324+0000","last_active":"2023-10-09T10:18:49.614657+0000","last_peered":"2023-10-09T10:18:49.614657+0000","last_clean":"2023-10-09T10:18:49.614657+0000","last_became_active":"2023-10-09T10:18:40.309978+0000","last_became_peered":"2023-10-09T10:18:40.309978+0000","last_unstale":"2023-10-09T10:18:49.614657+0000","last_undegraded":"2023-10-09T10:18:49.614657+0000","last_fullsized":"2023-10-09T10:18:49.614657+0000","mapping_epoch":405,"log_start":"324'2492","ondisk_log_start":"324'2492","created":377,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8734,"log_dups_size":0,"ondisk_log_size":8734,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49676288,"num_objects":758,"num_object_clones":0,"num_object_copies":2274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6303,"num_write_kb":225952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4175,"num_bytes_recovered":258572288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338933+0000","last_change":"2023-10-09T10:16:18.280500+0000","last_active":"2023-10-09T10:18:45.338933+0000","last_peered":"2023-10-09T10:18:45.338933+0000","last_clean":"2023-10-09T10:18:45.338933+0000","last_became_active":"2023-10-09T10:16:18.275348+0000","last_became_peered":"2023-10-09T10:16:18.275348+0000","last_unstale":"2023-10-09T10:18:45.338933+0000","last_undegraded":"2023-10-09T10:18:45.338933+0000","last_fullsized":"2023-10-09T10:18:45.338933+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":320,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.337616+0000","last_change":"2023-10-09T10:15:42.396546+0000","last_active":"2023-10-09T10:18:45.337616+0000","last_peered":"2023-10-09T10:18:45.337616+0000","last_clean":"2023-10-09T10:18:45.337616+0000","last_became_active":"2023-10-09T10:15:42.367055+0000","last_became_peered":"2023-10-09T10:15:42.367055+0000","last_unstale":"2023-10-09T10:18:45.337616+0000","last_undegraded":"2023-10-09T10:18:45.337616+0000","last_fullsized":"2023-10-09T10:18:45.337616+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:57.600046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1f","version":"411'11234","reported_seq":19326,"reported_epoch":411,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:49.665864+0000","last_change":"2023-10-09T10:18:34.295112+0000","last_active":"2023-10-09T10:18:49.665864+0000","last_peered":"2023-10-09T10:18:49.665864+0000","last_clean":"2023-10-09T10:15:06.273272+0000","last_became_active":"2023-10-09T10:17:16.203206+0000","last_became_peered":"2023-10-09T10:17:16.203206+0000","last_unstale":"2023-10-09T10:18:49.665864+0000","last_undegraded":"2023-10-09T10:18:34.292223+0000","last_fullsized":"2023-10-09T10:18:49.665864+0000","mapping_epoch":377,"log_start":"334'5299","ondisk_log_start":"334'5299","created":377,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5935,"log_dups_size":0,"ondisk_log_size":5935,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52527104,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":265,"num_objects_missing":265,"num_objects_degraded":1112,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6305,"num_write_kb":227392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2673,"num_bytes_recovered":175079424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,2,5","objects":38},{"shards":"1,3,5","objects":764}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":170,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.341490+0000","last_change":"2023-10-09T10:18:45.341154+0000","last_active":"2023-10-09T10:18:45.341490+0000","last_peered":"2023-10-09T10:18:45.341490+0000","last_clean":"2023-10-09T10:18:45.341490+0000","last_became_active":"2023-10-09T10:18:45.340778+0000","last_became_peered":"2023-10-09T10:18:45.340778+0000","last_unstale":"2023-10-09T10:18:45.341490+0000","last_undegraded":"2023-10-09T10:18:45.341490+0000","last_fullsized":"2023-10-09T10:18:45.341490+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":409,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339121+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:18:45.339121+0000","last_peered":"2023-10-09T10:18:45.339121+0000","last_clean":"2023-10-09T10:18:45.339121+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:18:45.339121+0000","last_undegraded":"2023-10-09T10:18:45.339121+0000","last_fullsized":"2023-10-09T10:18:45.339121+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1e","version":"411'11091","reported_seq":11212,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.720076+0000","last_change":"2023-10-09T10:17:37.701383+0000","last_active":"2023-10-09T10:18:49.720076+0000","last_peered":"2023-10-09T10:18:49.720076+0000","last_clean":"2023-10-09T10:18:49.720076+0000","last_became_active":"2023-10-09T10:17:37.674107+0000","last_became_peered":"2023-10-09T10:17:37.674107+0000","last_unstale":"2023-10-09T10:18:49.720076+0000","last_undegraded":"2023-10-09T10:18:49.720076+0000","last_fullsized":"2023-10-09T10:18:49.720076+0000","mapping_epoch":387,"log_start":"334'5090","ondisk_log_start":"334'5090","created":377,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49807360,"num_objects":760,"num_object_clones":0,"num_object_copies":2280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6233,"num_write_kb":223776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":119,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.334877+0000","last_change":"2023-10-09T10:17:37.701325+0000","last_active":"2023-10-09T10:18:45.334877+0000","last_peered":"2023-10-09T10:18:45.334877+0000","last_clean":"2023-10-09T10:18:45.334877+0000","last_became_active":"2023-10-09T10:17:37.700655+0000","last_became_peered":"2023-10-09T10:17:37.700655+0000","last_unstale":"2023-10-09T10:18:45.334877+0000","last_undegraded":"2023-10-09T10:18:45.334877+0000","last_fullsized":"2023-10-09T10:18:45.334877+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":159,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333570+0000","last_change":"2023-10-09T10:17:37.691936+0000","last_active":"2023-10-09T10:18:45.333570+0000","last_peered":"2023-10-09T10:18:45.333570+0000","last_clean":"2023-10-09T10:18:45.333570+0000","last_became_active":"2023-10-09T10:17:37.691458+0000","last_became_peered":"2023-10-09T10:17:37.691458+0000","last_unstale":"2023-10-09T10:18:45.333570+0000","last_undegraded":"2023-10-09T10:18:45.333570+0000","last_fullsized":"2023-10-09T10:18:45.333570+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.19","version":"412'9880","reported_seq":10601,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-10-09T10:18:52.326428+0000","last_change":"2023-10-09T10:18:38.311220+0000","last_active":"2023-10-09T10:18:52.326428+0000","last_peered":"2023-10-09T10:18:52.326428+0000","last_clean":"2023-10-09T10:18:52.326428+0000","last_became_active":"2023-10-09T10:18:38.293238+0000","last_became_peered":"2023-10-09T10:18:38.293238+0000","last_unstale":"2023-10-09T10:18:52.326428+0000","last_undegraded":"2023-10-09T10:18:52.326428+0000","last_fullsized":"2023-10-09T10:18:52.326428+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9880,"log_dups_size":0,"ondisk_log_size":9880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47742976,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6497,"num_write_kb":231200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":277,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":202,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333544+0000","last_change":"2023-10-09T10:17:49.756252+0000","last_active":"2023-10-09T10:18:45.333544+0000","last_peered":"2023-10-09T10:18:45.333544+0000","last_clean":"2023-10-09T10:18:45.333544+0000","last_became_active":"2023-10-09T10:17:49.755906+0000","last_became_peered":"2023-10-09T10:17:49.755906+0000","last_unstale":"2023-10-09T10:18:45.333544+0000","last_undegraded":"2023-10-09T10:18:45.333544+0000","last_fullsized":"2023-10-09T10:18:45.333544+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":309,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.342967+0000","last_change":"2023-10-09T10:18:45.342547+0000","last_active":"2023-10-09T10:18:45.342967+0000","last_peered":"2023-10-09T10:18:45.342967+0000","last_clean":"2023-10-09T10:18:45.342967+0000","last_became_active":"2023-10-09T10:18:45.342206+0000","last_became_peered":"2023-10-09T10:18:45.342206+0000","last_unstale":"2023-10-09T10:18:45.342967+0000","last_undegraded":"2023-10-09T10:18:45.342967+0000","last_fullsized":"2023-10-09T10:18:45.342967+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:32.932519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.18","version":"412'9911","reported_seq":17955,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-10-09T10:18:52.305846+0000","last_change":"2023-10-09T10:18:34.301620+0000","last_active":"2023-10-09T10:18:52.305846+0000","last_peered":"2023-10-09T10:18:52.305846+0000","last_clean":"2023-10-09T10:18:52.305846+0000","last_became_active":"2023-10-09T10:18:34.274390+0000","last_became_peered":"2023-10-09T10:18:34.274390+0000","last_unstale":"2023-10-09T10:18:52.305846+0000","last_undegraded":"2023-10-09T10:18:52.305846+0000","last_fullsized":"2023-10-09T10:18:52.305846+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9911,"log_dups_size":0,"ondisk_log_size":9911,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45285376,"num_objects":691,"num_object_clones":0,"num_object_copies":2073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6517,"num_write_kb":230656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3937,"num_bytes_recovered":132448256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":305,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-10-09T10:18:51.372413+0000","last_change":"2023-10-09T10:16:08.455134+0000","last_active":"2023-10-09T10:18:51.372413+0000","last_peered":"2023-10-09T10:18:51.372413+0000","last_clean":"2023-10-09T10:18:51.372413+0000","last_became_active":"2023-10-09T10:16:08.454793+0000","last_became_peered":"2023-10-09T10:16:08.454793+0000","last_unstale":"2023-10-09T10:18:51.372413+0000","last_undegraded":"2023-10-09T10:18:51.372413+0000","last_fullsized":"2023-10-09T10:18:51.372413+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:25:33.883382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.343185+0000","last_change":"2023-10-09T10:16:18.255609+0000","last_active":"2023-10-09T10:18:45.343185+0000","last_peered":"2023-10-09T10:18:45.343185+0000","last_clean":"2023-10-09T10:18:45.343185+0000","last_became_active":"2023-10-09T10:16:18.244618+0000","last_became_peered":"2023-10-09T10:16:18.244618+0000","last_unstale":"2023-10-09T10:18:45.343185+0000","last_undegraded":"2023-10-09T10:18:45.343185+0000","last_fullsized":"2023-10-09T10:18:45.343185+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1b","version":"411'11143","reported_seq":11328,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:49.565674+0000","last_change":"2023-10-09T10:18:39.337585+0000","last_active":"2023-10-09T10:18:49.565674+0000","last_peered":"2023-10-09T10:18:49.565674+0000","last_clean":"2023-10-09T10:15:06.321287+0000","last_became_active":"2023-10-09T10:18:38.775539+0000","last_became_peered":"2023-10-09T10:18:38.775539+0000","last_unstale":"2023-10-09T10:18:49.565674+0000","last_undegraded":"2023-10-09T10:18:38.739805+0000","last_fullsized":"2023-10-09T10:18:35.259214+0000","mapping_epoch":400,"log_start":"324'2335","ondisk_log_start":"324'2335","created":377,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8808,"log_dups_size":0,"ondisk_log_size":8808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47579136,"num_objects":726,"num_object_clones":0,"num_object_copies":2178,"num_objects_missing_on_primary":504,"num_objects_missing":504,"num_objects_degraded":4052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6288,"num_write_kb":224448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,5,6","objects":754},{"shards":"5,6","objects":-201}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":161,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333161+0000","last_change":"2023-10-09T10:17:37.683049+0000","last_active":"2023-10-09T10:18:45.333161+0000","last_peered":"2023-10-09T10:18:45.333161+0000","last_clean":"2023-10-09T10:18:45.333161+0000","last_became_active":"2023-10-09T10:17:37.681922+0000","last_became_peered":"2023-10-09T10:17:37.681922+0000","last_unstale":"2023-10-09T10:18:45.333161+0000","last_undegraded":"2023-10-09T10:18:45.333161+0000","last_fullsized":"2023-10-09T10:18:45.333161+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":262,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.344414+0000","last_change":"2023-10-09T10:18:34.264136+0000","last_active":"2023-10-09T10:18:45.344414+0000","last_peered":"2023-10-09T10:18:45.344414+0000","last_clean":"2023-10-09T10:18:45.344414+0000","last_became_active":"2023-10-09T10:18:34.263812+0000","last_became_peered":"2023-10-09T10:18:34.263812+0000","last_unstale":"2023-10-09T10:18:45.344414+0000","last_undegraded":"2023-10-09T10:18:45.344414+0000","last_fullsized":"2023-10-09T10:18:45.344414+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.1a","version":"411'10820","reported_seq":11520,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.678570+0000","last_change":"2023-10-09T10:18:32.940758+0000","last_active":"2023-10-09T10:18:49.678570+0000","last_peered":"2023-10-09T10:18:49.678570+0000","last_clean":"2023-10-09T10:18:49.678570+0000","last_became_active":"2023-10-09T10:17:10.480946+0000","last_became_peered":"2023-10-09T10:17:10.480946+0000","last_unstale":"2023-10-09T10:18:49.678570+0000","last_undegraded":"2023-10-09T10:18:49.678570+0000","last_fullsized":"2023-10-09T10:18:49.678570+0000","mapping_epoch":377,"log_start":"334'4912","ondisk_log_start":"334'4912","created":377,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49184768,"num_objects":751,"num_object_clones":0,"num_object_copies":2253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6073,"num_write_kb":218336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":266,"num_bytes_recovered":557056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":326,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338112+0000","last_change":"2023-10-09T10:18:34.286718+0000","last_active":"2023-10-09T10:18:45.338112+0000","last_peered":"2023-10-09T10:18:45.338112+0000","last_clean":"2023-10-09T10:18:45.338112+0000","last_became_active":"2023-10-09T10:18:34.271799+0000","last_became_peered":"2023-10-09T10:18:34.271799+0000","last_unstale":"2023-10-09T10:18:45.338112+0000","last_undegraded":"2023-10-09T10:18:45.338112+0000","last_fullsized":"2023-10-09T10:18:45.338112+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":357,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338986+0000","last_change":"2023-10-09T10:16:08.618851+0000","last_active":"2023-10-09T10:18:45.338986+0000","last_peered":"2023-10-09T10:18:45.338986+0000","last_clean":"2023-10-09T10:18:45.338986+0000","last_became_active":"2023-10-09T10:16:08.618501+0000","last_became_peered":"2023-10-09T10:16:08.618501+0000","last_unstale":"2023-10-09T10:18:45.338986+0000","last_undegraded":"2023-10-09T10:18:45.338986+0000","last_fullsized":"2023-10-09T10:18:45.338986+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:45.133486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"412'9643","reported_seq":12092,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-10-09T10:18:52.177130+0000","last_change":"2023-10-09T10:18:46.377662+0000","last_active":"2023-10-09T10:18:52.177130+0000","last_peered":"2023-10-09T10:18:52.177130+0000","last_clean":"2023-10-09T10:18:52.177130+0000","last_became_active":"2023-10-09T10:18:46.353730+0000","last_became_peered":"2023-10-09T10:18:46.353730+0000","last_unstale":"2023-10-09T10:18:52.177130+0000","last_undegraded":"2023-10-09T10:18:52.177130+0000","last_fullsized":"2023-10-09T10:18:52.177130+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9643,"log_dups_size":0,"ondisk_log_size":9643,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42926080,"num_objects":655,"num_object_clones":0,"num_object_copies":1965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6341,"num_write_kb":223872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1150,"num_bytes_recovered":42205184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":195,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.334383+0000","last_change":"2023-10-09T10:18:34.321877+0000","last_active":"2023-10-09T10:18:45.334383+0000","last_peered":"2023-10-09T10:18:45.334383+0000","last_clean":"2023-10-09T10:18:45.334383+0000","last_became_active":"2023-10-09T10:18:34.266260+0000","last_became_peered":"2023-10-09T10:18:34.266260+0000","last_unstale":"2023-10-09T10:18:45.334383+0000","last_undegraded":"2023-10-09T10:18:45.334383+0000","last_fullsized":"2023-10-09T10:18:45.334383+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":59,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338135+0000","last_change":"2023-10-09T10:16:18.237380+0000","last_active":"2023-10-09T10:18:45.338135+0000","last_peered":"2023-10-09T10:18:45.338135+0000","last_clean":"2023-10-09T10:18:45.338135+0000","last_became_active":"2023-10-09T10:16:18.236585+0000","last_became_peered":"2023-10-09T10:16:18.236585+0000","last_unstale":"2023-10-09T10:18:45.338135+0000","last_undegraded":"2023-10-09T10:18:45.338135+0000","last_fullsized":"2023-10-09T10:18:45.338135+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"411'9497","reported_seq":10069,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.311536+0000","last_change":"2023-10-09T10:16:20.239479+0000","last_active":"2023-10-09T10:18:49.311536+0000","last_peered":"2023-10-09T10:18:49.311536+0000","last_clean":"2023-10-09T10:18:49.311536+0000","last_became_active":"2023-10-09T10:16:20.231381+0000","last_became_peered":"2023-10-09T10:16:20.231381+0000","last_unstale":"2023-10-09T10:18:49.311536+0000","last_undegraded":"2023-10-09T10:18:49.311536+0000","last_fullsized":"2023-10-09T10:18:49.311536+0000","mapping_epoch":363,"log_start":"332'4717","ondisk_log_start":"332'4717","created":317,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":4780,"log_dups_size":0,"ondisk_log_size":4780,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47185920,"num_objects":720,"num_object_clones":0,"num_object_copies":2160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6238,"num_write_kb":222656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":215,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":183,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333735+0000","last_change":"2023-10-09T10:17:49.751128+0000","last_active":"2023-10-09T10:18:45.333735+0000","last_peered":"2023-10-09T10:18:45.333735+0000","last_clean":"2023-10-09T10:18:45.333735+0000","last_became_active":"2023-10-09T10:17:49.750853+0000","last_became_peered":"2023-10-09T10:17:49.750853+0000","last_unstale":"2023-10-09T10:18:45.333735+0000","last_undegraded":"2023-10-09T10:18:45.333735+0000","last_fullsized":"2023-10-09T10:18:45.333735+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":320,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333256+0000","last_change":"2023-10-09T10:17:37.683197+0000","last_active":"2023-10-09T10:18:45.333256+0000","last_peered":"2023-10-09T10:18:45.333256+0000","last_clean":"2023-10-09T10:18:45.333256+0000","last_became_active":"2023-10-09T10:17:37.682297+0000","last_became_peered":"2023-10-09T10:17:37.682297+0000","last_unstale":"2023-10-09T10:18:45.333256+0000","last_undegraded":"2023-10-09T10:18:45.333256+0000","last_fullsized":"2023-10-09T10:18:45.333256+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"411'9542","reported_seq":17262,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.685791+0000","last_change":"2023-10-09T10:17:33.574802+0000","last_active":"2023-10-09T10:18:49.685791+0000","last_peered":"2023-10-09T10:18:49.685791+0000","last_clean":"2023-10-09T10:18:49.685791+0000","last_became_active":"2023-10-09T10:17:33.536935+0000","last_became_peered":"2023-10-09T10:17:33.536935+0000","last_unstale":"2023-10-09T10:18:49.685791+0000","last_undegraded":"2023-10-09T10:18:49.685791+0000","last_fullsized":"2023-10-09T10:18:49.685791+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"339'6595","last_scrub_stamp":"2023-10-09T10:15:01.144444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:15:01.144444+0000","objects_scrubbed":0,"log_size":9542,"log_dups_size":0,"ondisk_log_size":9542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47382528,"num_objects":723,"num_object_clones":0,"num_object_copies":2169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2975,"num_write_kb":10560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3735,"num_bytes_recovered":226557952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":63,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339923+0000","last_change":"2023-10-09T10:16:18.362151+0000","last_active":"2023-10-09T10:18:45.339923+0000","last_peered":"2023-10-09T10:18:45.339923+0000","last_clean":"2023-10-09T10:18:45.339923+0000","last_became_active":"2023-10-09T10:16:18.323076+0000","last_became_peered":"2023-10-09T10:16:18.323076+0000","last_unstale":"2023-10-09T10:18:45.339923+0000","last_undegraded":"2023-10-09T10:18:45.339923+0000","last_fullsized":"2023-10-09T10:18:45.339923+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":254,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.343364+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:18:45.343364+0000","last_peered":"2023-10-09T10:18:45.343364+0000","last_clean":"2023-10-09T10:18:45.343364+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:18:45.343364+0000","last_undegraded":"2023-10-09T10:18:45.343364+0000","last_fullsized":"2023-10-09T10:18:45.343364+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"411'9595","reported_seq":11813,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.711300+0000","last_change":"2023-10-09T10:18:37.318875+0000","last_active":"2023-10-09T10:18:49.711300+0000","last_peered":"2023-10-09T10:18:49.711300+0000","last_clean":"2023-10-09T10:18:49.711300+0000","last_became_active":"2023-10-09T10:18:37.297054+0000","last_became_peered":"2023-10-09T10:18:37.297054+0000","last_unstale":"2023-10-09T10:18:49.711300+0000","last_undegraded":"2023-10-09T10:18:49.711300+0000","last_fullsized":"2023-10-09T10:18:49.711300+0000","mapping_epoch":402,"log_start":"330'4427","ondisk_log_start":"330'4427","created":317,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5168,"log_dups_size":0,"ondisk_log_size":5168,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48267264,"num_objects":736,"num_object_clones":0,"num_object_copies":2208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6278,"num_write_kb":224480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1036,"num_bytes_recovered":35192832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338798+0000","last_change":"2023-10-09T10:16:18.237242+0000","last_active":"2023-10-09T10:18:45.338798+0000","last_peered":"2023-10-09T10:18:45.338798+0000","last_clean":"2023-10-09T10:18:45.338798+0000","last_became_active":"2023-10-09T10:16:18.236617+0000","last_became_peered":"2023-10-09T10:16:18.236617+0000","last_unstale":"2023-10-09T10:18:45.338798+0000","last_undegraded":"2023-10-09T10:18:45.338798+0000","last_fullsized":"2023-10-09T10:18:45.338798+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":282,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:46.339343+0000","last_change":"2023-10-09T10:08:35.096998+0000","last_active":"2023-10-09T10:18:46.339343+0000","last_peered":"2023-10-09T10:18:46.339343+0000","last_clean":"2023-10-09T10:18:46.339343+0000","last_became_active":"2023-10-09T10:08:35.096692+0000","last_became_peered":"2023-10-09T10:08:35.096692+0000","last_unstale":"2023-10-09T10:18:46.339343+0000","last_undegraded":"2023-10-09T10:18:46.339343+0000","last_fullsized":"2023-10-09T10:18:46.339343+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:53.661052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"411'9055","reported_seq":9203,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.836461+0000","last_change":"2023-10-09T10:18:44.442753+0000","last_active":"2023-10-09T10:18:49.836461+0000","last_peered":"2023-10-09T10:18:49.836461+0000","last_clean":"2023-10-09T10:18:49.836461+0000","last_became_active":"2023-10-09T10:17:09.403921+0000","last_became_peered":"2023-10-09T10:17:09.403921+0000","last_unstale":"2023-10-09T10:18:49.836461+0000","last_undegraded":"2023-10-09T10:18:49.836461+0000","last_fullsized":"2023-10-09T10:18:49.836461+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9055,"log_dups_size":0,"ondisk_log_size":9055,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24248320,"num_objects":370,"num_object_clones":0,"num_object_copies":1110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3204,"num_write_kb":114368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":179,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333651+0000","last_change":"2023-10-09T10:18:34.328014+0000","last_active":"2023-10-09T10:18:45.333651+0000","last_peered":"2023-10-09T10:18:45.333651+0000","last_clean":"2023-10-09T10:18:45.333651+0000","last_became_active":"2023-10-09T10:18:34.266363+0000","last_became_peered":"2023-10-09T10:18:34.266363+0000","last_unstale":"2023-10-09T10:18:45.333651+0000","last_undegraded":"2023-10-09T10:18:45.333651+0000","last_fullsized":"2023-10-09T10:18:45.333651+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":212,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.334952+0000","last_change":"2023-10-09T10:17:37.701500+0000","last_active":"2023-10-09T10:18:45.334952+0000","last_peered":"2023-10-09T10:18:45.334952+0000","last_clean":"2023-10-09T10:18:45.334952+0000","last_became_active":"2023-10-09T10:17:37.700983+0000","last_became_peered":"2023-10-09T10:17:37.700983+0000","last_unstale":"2023-10-09T10:18:45.334952+0000","last_undegraded":"2023-10-09T10:18:45.334952+0000","last_fullsized":"2023-10-09T10:18:45.334952+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"411'8820","reported_seq":9023,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.193468+0000","last_change":"2023-10-09T10:17:09.428557+0000","last_active":"2023-10-09T10:18:49.193468+0000","last_peered":"2023-10-09T10:18:49.193468+0000","last_clean":"2023-10-09T10:18:49.193468+0000","last_became_active":"2023-10-09T10:17:09.420973+0000","last_became_peered":"2023-10-09T10:17:09.420973+0000","last_unstale":"2023-10-09T10:18:49.193468+0000","last_undegraded":"2023-10-09T10:18:49.193468+0000","last_fullsized":"2023-10-09T10:18:49.193468+0000","mapping_epoch":317,"log_start":"321'624","ondisk_log_start":"321'624","created":317,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"339'6671","last_scrub_stamp":"2023-10-09T10:15:03.503975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:15:03.503975+0000","objects_scrubbed":3418,"log_size":8196,"log_dups_size":0,"ondisk_log_size":8196,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.90657236399999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25100288,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1430,"num_write_kb":3328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":217,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333488+0000","last_change":"2023-10-09T10:17:37.685357+0000","last_active":"2023-10-09T10:18:45.333488+0000","last_peered":"2023-10-09T10:18:45.333488+0000","last_clean":"2023-10-09T10:18:45.333488+0000","last_became_active":"2023-10-09T10:17:37.684740+0000","last_became_peered":"2023-10-09T10:17:37.684740+0000","last_unstale":"2023-10-09T10:18:45.333488+0000","last_undegraded":"2023-10-09T10:18:45.333488+0000","last_fullsized":"2023-10-09T10:18:45.333488+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":244,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333692+0000","last_change":"2023-10-09T10:17:37.690061+0000","last_active":"2023-10-09T10:18:45.333692+0000","last_peered":"2023-10-09T10:18:45.333692+0000","last_clean":"2023-10-09T10:18:45.333692+0000","last_became_active":"2023-10-09T10:17:37.689771+0000","last_became_peered":"2023-10-09T10:17:37.689771+0000","last_unstale":"2023-10-09T10:18:45.333692+0000","last_undegraded":"2023-10-09T10:18:45.333692+0000","last_fullsized":"2023-10-09T10:18:45.333692+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"411'8863","reported_seq":10463,"reported_epoch":411,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:49.039733+0000","last_change":"2023-10-09T10:17:49.793224+0000","last_active":"2023-10-09T10:18:49.039733+0000","last_peered":"2023-10-09T10:18:49.039733+0000","last_clean":"2023-10-09T10:15:14.172820+0000","last_became_active":"2023-10-09T10:17:49.775592+0000","last_became_peered":"2023-10-09T10:17:49.775592+0000","last_unstale":"2023-10-09T10:18:49.039733+0000","last_undegraded":"2023-10-09T10:17:49.748297+0000","last_fullsized":"2023-10-09T10:18:49.039733+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8863,"log_dups_size":0,"ondisk_log_size":8863,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21315584,"num_objects":325,"num_object_clones":0,"num_object_copies":975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1101,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3159,"num_write_kb":111504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":722,"num_bytes_recovered":47316992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,3,7","objects":388},{"shards":"1,5,7","objects":-63}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":187,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333293+0000","last_change":"2023-10-09T10:17:37.683342+0000","last_active":"2023-10-09T10:18:45.333293+0000","last_peered":"2023-10-09T10:18:45.333293+0000","last_clean":"2023-10-09T10:18:45.333293+0000","last_became_active":"2023-10-09T10:17:37.682635+0000","last_became_peered":"2023-10-09T10:17:37.682635+0000","last_unstale":"2023-10-09T10:18:45.333293+0000","last_undegraded":"2023-10-09T10:18:45.333293+0000","last_fullsized":"2023-10-09T10:18:45.333293+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333663+0000","last_change":"2023-10-09T10:17:37.700874+0000","last_active":"2023-10-09T10:18:45.333663+0000","last_peered":"2023-10-09T10:18:45.333663+0000","last_clean":"2023-10-09T10:18:45.333663+0000","last_became_active":"2023-10-09T10:17:37.700300+0000","last_became_peered":"2023-10-09T10:17:37.700300+0000","last_unstale":"2023-10-09T10:18:45.333663+0000","last_undegraded":"2023-10-09T10:18:45.333663+0000","last_fullsized":"2023-10-09T10:18:45.333663+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"411'8801","reported_seq":10351,"reported_epoch":411,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:49.248069+0000","last_change":"2023-10-09T10:17:10.437080+0000","last_active":"2023-10-09T10:18:49.248069+0000","last_peered":"2023-10-09T10:18:49.248069+0000","last_clean":"2023-10-09T10:15:14.051568+0000","last_became_active":"2023-10-09T10:17:10.429080+0000","last_became_peered":"2023-10-09T10:17:10.429080+0000","last_unstale":"2023-10-09T10:18:49.248069+0000","last_undegraded":"2023-10-09T10:17:10.393900+0000","last_fullsized":"2023-10-09T10:18:49.248069+0000","mapping_epoch":378,"log_start":"321'588","ondisk_log_start":"321'588","created":317,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8213,"log_dups_size":0,"ondisk_log_size":8213,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23265280,"num_objects":355,"num_object_clones":0,"num_object_copies":1065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1041,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3117,"num_write_kb":111104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":703,"num_bytes_recovered":46071808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,3","objects":368},{"shards":"1,2,7","objects":-13}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":359,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.337748+0000","last_change":"2023-10-09T10:16:08.267092+0000","last_active":"2023-10-09T10:18:45.337748+0000","last_peered":"2023-10-09T10:18:45.337748+0000","last_clean":"2023-10-09T10:18:45.337748+0000","last_became_active":"2023-10-09T10:16:08.244668+0000","last_became_peered":"2023-10-09T10:16:08.244668+0000","last_unstale":"2023-10-09T10:18:45.337748+0000","last_undegraded":"2023-10-09T10:18:45.337748+0000","last_fullsized":"2023-10-09T10:18:45.337748+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:35:59.836060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":432,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338905+0000","last_change":"2023-10-09T10:15:42.317788+0000","last_active":"2023-10-09T10:18:45.338905+0000","last_peered":"2023-10-09T10:18:45.338905+0000","last_clean":"2023-10-09T10:18:45.338905+0000","last_became_active":"2023-10-09T10:15:42.317524+0000","last_became_peered":"2023-10-09T10:15:42.317524+0000","last_unstale":"2023-10-09T10:18:45.338905+0000","last_undegraded":"2023-10-09T10:18:45.338905+0000","last_fullsized":"2023-10-09T10:18:45.338905+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:54.507373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"411'11324","reported_seq":19990,"reported_epoch":411,"state":"active+recovering","last_fresh":"2023-10-09T10:18:49.728937+0000","last_change":"2023-10-09T10:18:38.025063+0000","last_active":"2023-10-09T10:18:49.728937+0000","last_peered":"2023-10-09T10:18:49.728937+0000","last_clean":"2023-10-09T10:15:06.273272+0000","last_became_active":"2023-10-09T10:17:16.191120+0000","last_became_peered":"2023-10-09T10:17:16.191120+0000","last_unstale":"2023-10-09T10:18:49.728937+0000","last_undegraded":"2023-10-09T10:18:49.728937+0000","last_fullsized":"2023-10-09T10:18:49.728937+0000","mapping_epoch":378,"log_start":"334'5299","ondisk_log_start":"334'5299","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6025,"log_dups_size":0,"ondisk_log_size":6025,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46628864,"num_objects":711,"num_object_clones":0,"num_object_copies":2133,"num_objects_missing_on_primary":79,"num_objects_missing":79,"num_objects_degraded":0,"num_objects_misplaced":748,"num_objects_unfound":0,"num_objects_dirty":711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6394,"num_write_kb":227392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2923,"num_bytes_recovered":181043200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,2,5","objects":-37},{"shards":"1,3,5","objects":748}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":417,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339488+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:18:45.339488+0000","last_peered":"2023-10-09T10:18:45.339488+0000","last_clean":"2023-10-09T10:18:45.339488+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:18:45.339488+0000","last_undegraded":"2023-10-09T10:18:45.339488+0000","last_fullsized":"2023-10-09T10:18:45.339488+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":59,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.340080+0000","last_change":"2023-10-09T10:16:18.323515+0000","last_active":"2023-10-09T10:18:45.340080+0000","last_peered":"2023-10-09T10:18:45.340080+0000","last_clean":"2023-10-09T10:18:45.340080+0000","last_became_active":"2023-10-09T10:16:18.321171+0000","last_became_peered":"2023-10-09T10:16:18.321171+0000","last_unstale":"2023-10-09T10:18:45.340080+0000","last_undegraded":"2023-10-09T10:18:45.340080+0000","last_fullsized":"2023-10-09T10:18:45.340080+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":391,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339390+0000","last_change":"2023-10-09T10:16:18.261462+0000","last_active":"2023-10-09T10:18:45.339390+0000","last_peered":"2023-10-09T10:18:45.339390+0000","last_clean":"2023-10-09T10:18:45.339390+0000","last_became_active":"2023-10-09T10:16:18.261023+0000","last_became_peered":"2023-10-09T10:16:18.261023+0000","last_unstale":"2023-10-09T10:18:45.339390+0000","last_undegraded":"2023-10-09T10:18:45.339390+0000","last_fullsized":"2023-10-09T10:18:45.339390+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:43.268197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"412'9934","reported_seq":10677,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-10-09T10:18:52.352863+0000","last_change":"2023-10-09T10:18:40.326021+0000","last_active":"2023-10-09T10:18:52.352863+0000","last_peered":"2023-10-09T10:18:52.352863+0000","last_clean":"2023-10-09T10:18:52.352863+0000","last_became_active":"2023-10-09T10:18:40.306343+0000","last_became_peered":"2023-10-09T10:18:40.306343+0000","last_unstale":"2023-10-09T10:18:52.352863+0000","last_undegraded":"2023-10-09T10:18:52.352863+0000","last_fullsized":"2023-10-09T10:18:52.352863+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9934,"log_dups_size":0,"ondisk_log_size":9934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43024384,"num_objects":656,"num_object_clones":0,"num_object_copies":1968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6550,"num_write_kb":230624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":288,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":59,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338550+0000","last_change":"2023-10-09T10:16:18.255781+0000","last_active":"2023-10-09T10:18:45.338550+0000","last_peered":"2023-10-09T10:18:45.338550+0000","last_clean":"2023-10-09T10:18:45.338550+0000","last_became_active":"2023-10-09T10:16:18.238531+0000","last_became_peered":"2023-10-09T10:16:18.238531+0000","last_unstale":"2023-10-09T10:18:45.338550+0000","last_undegraded":"2023-10-09T10:18:45.338550+0000","last_fullsized":"2023-10-09T10:18:45.338550+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":118,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333474+0000","last_change":"2023-10-09T10:17:37.691757+0000","last_active":"2023-10-09T10:18:45.333474+0000","last_peered":"2023-10-09T10:18:45.333474+0000","last_clean":"2023-10-09T10:18:45.333474+0000","last_became_active":"2023-10-09T10:17:37.690931+0000","last_became_peered":"2023-10-09T10:17:37.690931+0000","last_unstale":"2023-10-09T10:18:45.333474+0000","last_undegraded":"2023-10-09T10:18:45.333474+0000","last_fullsized":"2023-10-09T10:18:45.333474+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":464,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339415+0000","last_change":"2023-10-09T10:16:08.230567+0000","last_active":"2023-10-09T10:18:45.339415+0000","last_peered":"2023-10-09T10:18:45.339415+0000","last_clean":"2023-10-09T10:18:45.339415+0000","last_became_active":"2023-10-09T10:16:08.229677+0000","last_became_peered":"2023-10-09T10:16:08.229677+0000","last_unstale":"2023-10-09T10:18:45.339415+0000","last_undegraded":"2023-10-09T10:18:45.339415+0000","last_fullsized":"2023-10-09T10:18:45.339415+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:52.866334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"411'10841","reported_seq":11031,"reported_epoch":411,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:18:49.705132+0000","last_change":"2023-10-09T10:17:58.717165+0000","last_active":"2023-10-09T10:18:49.705132+0000","last_peered":"2023-10-09T10:18:49.705132+0000","last_clean":"2023-10-09T10:15:06.345974+0000","last_became_active":"2023-10-09T10:17:10.463863+0000","last_became_peered":"2023-10-09T10:17:10.463863+0000","last_unstale":"2023-10-09T10:18:49.705132+0000","last_undegraded":"2023-10-09T10:17:58.716826+0000","last_fullsized":"2023-10-09T10:18:49.705132+0000","mapping_epoch":378,"log_start":"334'4912","ondisk_log_start":"334'4912","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5929,"log_dups_size":0,"ondisk_log_size":5929,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47808512,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":293,"num_objects_missing":293,"num_objects_degraded":293,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6093,"num_write_kb":218336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":491520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,5,7","objects":729}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":322,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:46.339675+0000","last_change":"2023-10-09T10:16:08.269361+0000","last_active":"2023-10-09T10:18:46.339675+0000","last_peered":"2023-10-09T10:18:46.339675+0000","last_clean":"2023-10-09T10:18:46.339675+0000","last_became_active":"2023-10-09T10:16:08.266453+0000","last_became_peered":"2023-10-09T10:16:08.266453+0000","last_unstale":"2023-10-09T10:18:46.339675+0000","last_undegraded":"2023-10-09T10:18:46.339675+0000","last_fullsized":"2023-10-09T10:18:46.339675+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:07.633138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":410,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:46.351756+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:18:46.351756+0000","last_peered":"2023-10-09T10:18:46.351756+0000","last_clean":"2023-10-09T10:18:46.351756+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:18:46.351756+0000","last_undegraded":"2023-10-09T10:18:46.351756+0000","last_fullsized":"2023-10-09T10:18:46.351756+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":457,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338839+0000","last_change":"2023-10-09T10:18:34.270806+0000","last_active":"2023-10-09T10:18:45.338839+0000","last_peered":"2023-10-09T10:18:45.338839+0000","last_clean":"2023-10-09T10:18:45.338839+0000","last_became_active":"2023-10-09T10:18:34.270502+0000","last_became_peered":"2023-10-09T10:18:34.270502+0000","last_unstale":"2023-10-09T10:18:45.338839+0000","last_undegraded":"2023-10-09T10:18:45.338839+0000","last_fullsized":"2023-10-09T10:18:45.338839+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:44.839937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"412'10785","reported_seq":18663,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-10-09T10:18:52.352860+0000","last_change":"2023-10-09T10:17:12.423029+0000","last_active":"2023-10-09T10:18:52.352860+0000","last_peered":"2023-10-09T10:18:52.352860+0000","last_clean":"2023-10-09T10:18:52.352860+0000","last_became_active":"2023-10-09T10:17:12.413033+0000","last_became_peered":"2023-10-09T10:17:12.413033+0000","last_unstale":"2023-10-09T10:18:52.352860+0000","last_undegraded":"2023-10-09T10:18:52.352860+0000","last_fullsized":"2023-10-09T10:18:52.352860+0000","mapping_epoch":380,"log_start":"321'784","ondisk_log_start":"321'784","created":317,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":1211,"log_size":10001,"log_dups_size":0,"ondisk_log_size":10001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46923776,"num_objects":716,"num_object_clones":0,"num_object_copies":2148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6470,"num_write_kb":229952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3776,"num_bytes_recovered":228851712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":56,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.346911+0000","last_change":"2023-10-09T10:16:18.477748+0000","last_active":"2023-10-09T10:18:45.346911+0000","last_peered":"2023-10-09T10:18:45.346911+0000","last_clean":"2023-10-09T10:18:45.346911+0000","last_became_active":"2023-10-09T10:16:18.477398+0000","last_became_peered":"2023-10-09T10:16:18.477398+0000","last_unstale":"2023-10-09T10:18:45.346911+0000","last_undegraded":"2023-10-09T10:18:45.346911+0000","last_fullsized":"2023-10-09T10:18:45.346911+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":427,"reported_epoch":411,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:18:46.339023+0000","last_change":"2023-10-09T10:18:35.268317+0000","last_active":"2023-10-09T10:18:46.339023+0000","last_peered":"2023-10-09T10:18:46.339023+0000","last_clean":"2023-10-09T10:18:28.366996+0000","last_became_active":"2023-10-09T10:18:35.267979+0000","last_became_peered":"2023-10-09T10:18:35.267979+0000","last_unstale":"2023-10-09T10:18:46.339023+0000","last_undegraded":"2023-10-09T10:18:46.339023+0000","last_fullsized":"2023-10-09T10:18:46.339023+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:30.730819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[0,7,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":59,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338659+0000","last_change":"2023-10-09T10:16:18.255461+0000","last_active":"2023-10-09T10:18:45.338659+0000","last_peered":"2023-10-09T10:18:45.338659+0000","last_clean":"2023-10-09T10:18:45.338659+0000","last_became_active":"2023-10-09T10:16:18.247752+0000","last_became_peered":"2023-10-09T10:16:18.247752+0000","last_unstale":"2023-10-09T10:18:45.338659+0000","last_undegraded":"2023-10-09T10:18:45.338659+0000","last_fullsized":"2023-10-09T10:18:45.338659+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":208,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333603+0000","last_change":"2023-10-09T10:17:37.689632+0000","last_active":"2023-10-09T10:18:45.333603+0000","last_peered":"2023-10-09T10:18:45.333603+0000","last_clean":"2023-10-09T10:18:45.333603+0000","last_became_active":"2023-10-09T10:17:37.689105+0000","last_became_peered":"2023-10-09T10:17:37.689105+0000","last_unstale":"2023-10-09T10:18:45.333603+0000","last_undegraded":"2023-10-09T10:18:45.333603+0000","last_fullsized":"2023-10-09T10:18:45.333603+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"411'11127","reported_seq":11313,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:18:49.695872+0000","last_change":"2023-10-09T10:18:39.332330+0000","last_active":"2023-10-09T10:18:49.695872+0000","last_peered":"2023-10-09T10:18:49.695872+0000","last_clean":"2023-10-09T10:15:06.321287+0000","last_became_active":"2023-10-09T10:18:38.773956+0000","last_became_peered":"2023-10-09T10:18:38.773956+0000","last_unstale":"2023-10-09T10:18:49.695872+0000","last_undegraded":"2023-10-09T10:18:38.398027+0000","last_fullsized":"2023-10-09T10:18:35.261258+0000","mapping_epoch":400,"log_start":"324'2335","ondisk_log_start":"324'2335","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8792,"log_dups_size":0,"ondisk_log_size":8792,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48627712,"num_objects":742,"num_object_clones":0,"num_object_copies":2226,"num_objects_missing_on_primary":480,"num_objects_missing":480,"num_objects_degraded":3921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6272,"num_write_kb":224448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,5,6","objects":714},{"shards":"5,6","objects":-169}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":390,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338010+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:18:45.338010+0000","last_peered":"2023-10-09T10:18:45.338010+0000","last_clean":"2023-10-09T10:18:45.338010+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:18:45.338010+0000","last_undegraded":"2023-10-09T10:18:45.338010+0000","last_fullsized":"2023-10-09T10:18:45.338010+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":325,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:46.339760+0000","last_change":"2023-10-09T10:16:18.393880+0000","last_active":"2023-10-09T10:18:46.339760+0000","last_peered":"2023-10-09T10:18:46.339760+0000","last_clean":"2023-10-09T10:18:46.339760+0000","last_became_active":"2023-10-09T10:16:18.375766+0000","last_became_peered":"2023-10-09T10:16:18.375766+0000","last_unstale":"2023-10-09T10:18:46.339760+0000","last_undegraded":"2023-10-09T10:18:46.339760+0000","last_fullsized":"2023-10-09T10:18:46.339760+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:56:50.927077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":271,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.340004+0000","last_change":"2023-10-09T10:18:34.268478+0000","last_active":"2023-10-09T10:18:45.340004+0000","last_peered":"2023-10-09T10:18:45.340004+0000","last_clean":"2023-10-09T10:18:45.340004+0000","last_became_active":"2023-10-09T10:18:34.268098+0000","last_became_peered":"2023-10-09T10:18:34.268098+0000","last_unstale":"2023-10-09T10:18:45.340004+0000","last_undegraded":"2023-10-09T10:18:45.340004+0000","last_fullsized":"2023-10-09T10:18:45.340004+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.625264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"411'11121","reported_seq":11243,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.714707+0000","last_change":"2023-10-09T10:18:33.275167+0000","last_active":"2023-10-09T10:18:49.714707+0000","last_peered":"2023-10-09T10:18:49.714707+0000","last_clean":"2023-10-09T10:18:49.714707+0000","last_became_active":"2023-10-09T10:17:37.677480+0000","last_became_peered":"2023-10-09T10:17:37.677480+0000","last_unstale":"2023-10-09T10:18:49.714707+0000","last_undegraded":"2023-10-09T10:18:49.714707+0000","last_fullsized":"2023-10-09T10:18:49.714707+0000","mapping_epoch":387,"log_start":"334'5090","ondisk_log_start":"334'5090","created":317,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":6031,"log_dups_size":0,"ondisk_log_size":6031,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47841280,"num_objects":730,"num_object_clones":0,"num_object_copies":2190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6263,"num_write_kb":223776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":302,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333907+0000","last_change":"2023-10-09T10:18:34.266592+0000","last_active":"2023-10-09T10:18:45.333907+0000","last_peered":"2023-10-09T10:18:45.333907+0000","last_clean":"2023-10-09T10:18:45.333907+0000","last_became_active":"2023-10-09T10:18:34.266239+0000","last_became_peered":"2023-10-09T10:18:34.266239+0000","last_unstale":"2023-10-09T10:18:45.333907+0000","last_undegraded":"2023-10-09T10:18:45.333907+0000","last_fullsized":"2023-10-09T10:18:45.333907+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":464,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339571+0000","last_change":"2023-10-09T10:16:18.261069+0000","last_active":"2023-10-09T10:18:45.339571+0000","last_peered":"2023-10-09T10:18:45.339571+0000","last_clean":"2023-10-09T10:18:45.339571+0000","last_became_active":"2023-10-09T10:16:18.260736+0000","last_became_peered":"2023-10-09T10:16:18.260736+0000","last_unstale":"2023-10-09T10:18:45.339571+0000","last_undegraded":"2023-10-09T10:18:45.339571+0000","last_fullsized":"2023-10-09T10:18:45.339571+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:18.217603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":302,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:46.339471+0000","last_change":"2023-10-09T10:16:18.477779+0000","last_active":"2023-10-09T10:18:46.339471+0000","last_peered":"2023-10-09T10:18:46.339471+0000","last_clean":"2023-10-09T10:18:46.339471+0000","last_became_active":"2023-10-09T10:16:18.477390+0000","last_became_peered":"2023-10-09T10:16:18.477390+0000","last_unstale":"2023-10-09T10:18:46.339471+0000","last_undegraded":"2023-10-09T10:18:46.339471+0000","last_fullsized":"2023-10-09T10:18:46.339471+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:06.576807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"411'11313","reported_seq":11449,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.229099+0000","last_change":"2023-10-09T10:17:58.710957+0000","last_active":"2023-10-09T10:18:49.229099+0000","last_peered":"2023-10-09T10:18:49.229099+0000","last_clean":"2023-10-09T10:18:49.229099+0000","last_became_active":"2023-10-09T10:17:09.461024+0000","last_became_peered":"2023-10-09T10:17:09.461024+0000","last_unstale":"2023-10-09T10:18:49.229099+0000","last_undegraded":"2023-10-09T10:18:49.229099+0000","last_fullsized":"2023-10-09T10:18:49.229099+0000","mapping_epoch":361,"log_start":"334'5326","ondisk_log_start":"334'5326","created":317,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5987,"log_dups_size":0,"ondisk_log_size":5987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47742988,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6367,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":196,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:47.388700+0000","last_change":"2023-10-09T10:18:45.347229+0000","last_active":"2023-10-09T10:18:47.388700+0000","last_peered":"2023-10-09T10:18:47.388700+0000","last_clean":"2023-10-09T10:18:47.388700+0000","last_became_active":"2023-10-09T10:18:45.346901+0000","last_became_peered":"2023-10-09T10:18:45.346901+0000","last_unstale":"2023-10-09T10:18:47.388700+0000","last_undegraded":"2023-10-09T10:18:47.388700+0000","last_fullsized":"2023-10-09T10:18:47.388700+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:53.628052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":391,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.340260+0000","last_change":"2023-10-09T10:08:35.093527+0000","last_active":"2023-10-09T10:18:45.340260+0000","last_peered":"2023-10-09T10:18:45.340260+0000","last_clean":"2023-10-09T10:18:45.340260+0000","last_became_active":"2023-10-09T10:08:35.092434+0000","last_became_peered":"2023-10-09T10:08:35.092434+0000","last_unstale":"2023-10-09T10:18:45.340260+0000","last_undegraded":"2023-10-09T10:18:45.340260+0000","last_fullsized":"2023-10-09T10:18:45.340260+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:56.340661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":319,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.337446+0000","last_change":"2023-10-09T10:18:34.274789+0000","last_active":"2023-10-09T10:18:45.337446+0000","last_peered":"2023-10-09T10:18:45.337446+0000","last_clean":"2023-10-09T10:18:45.337446+0000","last_became_active":"2023-10-09T10:18:34.274396+0000","last_became_peered":"2023-10-09T10:18:34.274396+0000","last_unstale":"2023-10-09T10:18:45.337446+0000","last_undegraded":"2023-10-09T10:18:45.337446+0000","last_fullsized":"2023-10-09T10:18:45.337446+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:37.788599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.c","version":"411'11205","reported_seq":26197,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.634973+0000","last_change":"2023-10-09T10:18:38.303665+0000","last_active":"2023-10-09T10:18:49.634973+0000","last_peered":"2023-10-09T10:18:49.634973+0000","last_clean":"2023-10-09T10:18:49.634973+0000","last_became_active":"2023-10-09T10:18:38.282335+0000","last_became_peered":"2023-10-09T10:18:38.282335+0000","last_unstale":"2023-10-09T10:18:49.634973+0000","last_undegraded":"2023-10-09T10:18:49.634973+0000","last_fullsized":"2023-10-09T10:18:49.634973+0000","mapping_epoch":403,"log_start":"324'2492","ondisk_log_start":"324'2492","created":317,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8713,"log_dups_size":0,"ondisk_log_size":8713,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51052544,"num_objects":779,"num_object_clones":0,"num_object_copies":2337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6282,"num_write_kb":225952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4170,"num_bytes_recovered":258310144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":319,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.342896+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:18:45.342896+0000","last_peered":"2023-10-09T10:18:45.342896+0000","last_clean":"2023-10-09T10:18:45.342896+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:18:45.342896+0000","last_undegraded":"2023-10-09T10:18:45.342896+0000","last_fullsized":"2023-10-09T10:18:45.342896+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":81,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.340389+0000","last_change":"2023-10-09T10:18:34.277306+0000","last_active":"2023-10-09T10:18:45.340389+0000","last_peered":"2023-10-09T10:18:45.340389+0000","last_clean":"2023-10-09T10:18:45.340389+0000","last_became_active":"2023-10-09T10:18:34.268115+0000","last_became_peered":"2023-10-09T10:18:34.268115+0000","last_unstale":"2023-10-09T10:18:45.340389+0000","last_undegraded":"2023-10-09T10:18:45.340389+0000","last_fullsized":"2023-10-09T10:18:45.340389+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:50.982778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":146,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333082+0000","last_change":"2023-10-09T10:17:37.682902+0000","last_active":"2023-10-09T10:18:45.333082+0000","last_peered":"2023-10-09T10:18:45.333082+0000","last_clean":"2023-10-09T10:18:45.333082+0000","last_became_active":"2023-10-09T10:17:37.681476+0000","last_became_peered":"2023-10-09T10:17:37.681476+0000","last_unstale":"2023-10-09T10:18:45.333082+0000","last_undegraded":"2023-10-09T10:18:45.333082+0000","last_fullsized":"2023-10-09T10:18:45.333082+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.11","version":"411'9752","reported_seq":9889,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:50.266466+0000","last_change":"2023-10-09T10:18:42.400780+0000","last_active":"2023-10-09T10:18:50.266466+0000","last_peered":"2023-10-09T10:18:50.266466+0000","last_clean":"2023-10-09T10:18:50.266466+0000","last_became_active":"2023-10-09T10:16:22.227875+0000","last_became_peered":"2023-10-09T10:16:22.227875+0000","last_unstale":"2023-10-09T10:18:50.266466+0000","last_undegraded":"2023-10-09T10:18:50.266466+0000","last_fullsized":"2023-10-09T10:18:50.266466+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9752,"log_dups_size":0,"ondisk_log_size":9752,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48693248,"num_objects":743,"num_object_clones":0,"num_object_copies":2229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6395,"num_write_kb":228416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":233,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":34,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333373+0000","last_change":"2023-10-09T10:18:34.265332+0000","last_active":"2023-10-09T10:18:45.333373+0000","last_peered":"2023-10-09T10:18:45.333373+0000","last_clean":"2023-10-09T10:18:45.333373+0000","last_became_active":"2023-10-09T10:18:34.264895+0000","last_became_peered":"2023-10-09T10:18:34.264895+0000","last_unstale":"2023-10-09T10:18:45.333373+0000","last_undegraded":"2023-10-09T10:18:45.333373+0000","last_fullsized":"2023-10-09T10:18:45.333373+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":385,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339655+0000","last_change":"2023-10-09T10:18:34.267952+0000","last_active":"2023-10-09T10:18:45.339655+0000","last_peered":"2023-10-09T10:18:45.339655+0000","last_clean":"2023-10-09T10:18:45.339655+0000","last_became_active":"2023-10-09T10:18:34.266288+0000","last_became_peered":"2023-10-09T10:18:34.266288+0000","last_unstale":"2023-10-09T10:18:45.339655+0000","last_undegraded":"2023-10-09T10:18:45.339655+0000","last_fullsized":"2023-10-09T10:18:45.339655+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:49.494760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.16","version":"411'9656","reported_seq":10174,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.702174+0000","last_change":"2023-10-09T10:18:39.314032+0000","last_active":"2023-10-09T10:18:49.702174+0000","last_peered":"2023-10-09T10:18:49.702174+0000","last_clean":"2023-10-09T10:18:49.702174+0000","last_became_active":"2023-10-09T10:18:39.298363+0000","last_became_peered":"2023-10-09T10:18:39.298363+0000","last_unstale":"2023-10-09T10:18:49.702174+0000","last_undegraded":"2023-10-09T10:18:49.702174+0000","last_fullsized":"2023-10-09T10:18:49.702174+0000","mapping_epoch":404,"log_start":"330'4533","ondisk_log_start":"330'4533","created":338,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":5123,"log_dups_size":0,"ondisk_log_size":5123,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45580288,"num_objects":696,"num_object_clones":0,"num_object_copies":2088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6340,"num_write_kb":225120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":698,"num_bytes_recovered":12451840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.338714+0000","last_change":"2023-10-09T10:16:18.299039+0000","last_active":"2023-10-09T10:18:45.338714+0000","last_peered":"2023-10-09T10:18:45.338714+0000","last_clean":"2023-10-09T10:18:45.338714+0000","last_became_active":"2023-10-09T10:16:18.228601+0000","last_became_peered":"2023-10-09T10:16:18.228601+0000","last_unstale":"2023-10-09T10:18:45.338714+0000","last_undegraded":"2023-10-09T10:18:45.338714+0000","last_fullsized":"2023-10-09T10:18:45.338714+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":226,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.333378+0000","last_change":"2023-10-09T10:17:37.676882+0000","last_active":"2023-10-09T10:18:45.333378+0000","last_peered":"2023-10-09T10:18:45.333378+0000","last_clean":"2023-10-09T10:18:45.333378+0000","last_became_active":"2023-10-09T10:17:37.676439+0000","last_became_peered":"2023-10-09T10:17:37.676439+0000","last_unstale":"2023-10-09T10:18:45.333378+0000","last_undegraded":"2023-10-09T10:18:45.333378+0000","last_fullsized":"2023-10-09T10:18:45.333378+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.17","version":"411'9441","reported_seq":17800,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.669015+0000","last_change":"2023-10-09T10:18:35.300602+0000","last_active":"2023-10-09T10:18:49.669015+0000","last_peered":"2023-10-09T10:18:49.669015+0000","last_clean":"2023-10-09T10:18:49.669015+0000","last_became_active":"2023-10-09T10:18:35.276589+0000","last_became_peered":"2023-10-09T10:18:35.276589+0000","last_unstale":"2023-10-09T10:18:49.669015+0000","last_undegraded":"2023-10-09T10:18:49.669015+0000","last_fullsized":"2023-10-09T10:18:49.669015+0000","mapping_epoch":400,"log_start":"322'1237","ondisk_log_start":"322'1237","created":338,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":8204,"log_dups_size":0,"ondisk_log_size":8204,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53280768,"num_objects":813,"num_object_clones":0,"num_object_copies":2439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6175,"num_write_kb":223616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3895,"num_bytes_recovered":249036800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":63,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.340004+0000","last_change":"2023-10-09T10:16:18.351145+0000","last_active":"2023-10-09T10:18:45.340004+0000","last_peered":"2023-10-09T10:18:45.340004+0000","last_clean":"2023-10-09T10:18:45.340004+0000","last_became_active":"2023-10-09T10:16:18.321691+0000","last_became_peered":"2023-10-09T10:16:18.321691+0000","last_unstale":"2023-10-09T10:18:45.340004+0000","last_undegraded":"2023-10-09T10:18:45.340004+0000","last_fullsized":"2023-10-09T10:18:45.340004+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":214,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:47.388605+0000","last_change":"2023-10-09T10:18:45.347651+0000","last_active":"2023-10-09T10:18:47.388605+0000","last_peered":"2023-10-09T10:18:47.388605+0000","last_clean":"2023-10-09T10:18:47.388605+0000","last_became_active":"2023-10-09T10:18:45.347306+0000","last_became_peered":"2023-10-09T10:18:45.347306+0000","last_unstale":"2023-10-09T10:18:47.388605+0000","last_undegraded":"2023-10-09T10:18:47.388605+0000","last_fullsized":"2023-10-09T10:18:47.388605+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:28.046520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.14","version":"411'9421","reported_seq":9987,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:49.236074+0000","last_change":"2023-10-09T10:16:10.248177+0000","last_active":"2023-10-09T10:18:49.236074+0000","last_peered":"2023-10-09T10:18:49.236074+0000","last_clean":"2023-10-09T10:18:49.236074+0000","last_became_active":"2023-10-09T10:16:10.240509+0000","last_became_peered":"2023-10-09T10:16:10.240509+0000","last_unstale":"2023-10-09T10:18:49.236074+0000","last_undegraded":"2023-10-09T10:18:49.236074+0000","last_fullsized":"2023-10-09T10:18:49.236074+0000","mapping_epoch":358,"log_start":"331'4508","ondisk_log_start":"331'4508","created":338,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":4913,"log_dups_size":0,"ondisk_log_size":4913,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48889856,"num_objects":746,"num_object_clones":0,"num_object_copies":2238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6162,"num_write_kb":221056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":66,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.339938+0000","last_change":"2023-10-09T10:16:08.240555+0000","last_active":"2023-10-09T10:18:45.339938+0000","last_peered":"2023-10-09T10:18:45.339938+0000","last_clean":"2023-10-09T10:18:45.339938+0000","last_became_active":"2023-10-09T10:16:08.239692+0000","last_became_peered":"2023-10-09T10:16:08.239692+0000","last_unstale":"2023-10-09T10:18:45.339938+0000","last_undegraded":"2023-10-09T10:18:45.339938+0000","last_fullsized":"2023-10-09T10:18:45.339938+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:39.501717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":452,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-10-09T10:18:46.339598+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:18:46.339598+0000","last_peered":"2023-10-09T10:18:46.339598+0000","last_clean":"2023-10-09T10:18:46.339598+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:18:46.339598+0000","last_undegraded":"2023-10-09T10:18:46.339598+0000","last_fullsized":"2023-10-09T10:18:46.339598+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.15","version":"412'9596","reported_seq":13746,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-10-09T10:18:52.364970+0000","last_change":"2023-10-09T10:18:43.338964+0000","last_active":"2023-10-09T10:18:52.364970+0000","last_peered":"2023-10-09T10:18:52.364970+0000","last_clean":"2023-10-09T10:18:52.364970+0000","last_became_active":"2023-10-09T10:18:43.318456+0000","last_became_peered":"2023-10-09T10:18:43.318456+0000","last_unstale":"2023-10-09T10:18:52.364970+0000","last_undegraded":"2023-10-09T10:18:52.364970+0000","last_fullsized":"2023-10-09T10:18:52.364970+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":338,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:13:31.547361+0000","last_clean_scrub_stamp":"2023-10-09T10:13:31.547361+0000","objects_scrubbed":0,"log_size":9596,"log_dups_size":0,"ondisk_log_size":9596,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48496640,"num_objects":740,"num_object_clones":0,"num_object_copies":2220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6294,"num_write_kb":225088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2000,"num_bytes_recovered":98041856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":355,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.362782+0000","last_change":"2023-10-09T10:18:34.433731+0000","last_active":"2023-10-09T10:18:45.362782+0000","last_peered":"2023-10-09T10:18:45.362782+0000","last_clean":"2023-10-09T10:18:45.362782+0000","last_became_active":"2023-10-09T10:18:34.278023+0000","last_became_peered":"2023-10-09T10:18:34.278023+0000","last_unstale":"2023-10-09T10:18:45.362782+0000","last_undegraded":"2023-10-09T10:18:45.362782+0000","last_fullsized":"2023-10-09T10:18:45.362782+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":155,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-10-09T10:18:45.337888+0000","last_change":"2023-10-09T10:17:37.686077+0000","last_active":"2023-10-09T10:18:45.337888+0000","last_peered":"2023-10-09T10:18:45.337888+0000","last_clean":"2023-10-09T10:18:45.337888+0000","last_became_active":"2023-10-09T10:17:37.685754+0000","last_became_peered":"2023-10-09T10:17:37.685754+0000","last_unstale":"2023-10-09T10:18:45.337888+0000","last_undegraded":"2023-10-09T10:18:45.337888+0000","last_fullsized":"2023-10-09T10:18:45.337888+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":10692,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":36,"stat_sum":{"num_bytes":1547141144,"num_objects":23609,"num_object_clones":0,"num_object_copies":70827,"num_objects_missing_on_primary":2312,"num_objects_missing":2312,"num_objects_degraded":20098,"num_objects_misplaced":748,"num_objects_unfound":0,"num_objects_dirty":23609,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":194731,"num_write_kb":6769633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45836,"num_bytes_recovered":2452029440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6775713792,"data_stored":6775701576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":273170,"ondisk_log_size":273170,"up":108,"acting":105,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430590,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1918824,"kb_used_data":1002272,"kb_used_omap":0,"kb_used_meta":916544,"kb_avail":91826328,"statfs":{"total":95995035648,"available":94030159872,"internally_reserved":0,"allocated":1026326528,"data_stored":1025617300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":938541056},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:58 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.4410000000000001,"15min":2.0579999999999998},"min":{"1min":0.251,"5min":0.223,"15min":0.217},"max":{"1min":7.7160000000000002,"5min":8.5429999999999993,"15min":50.591999999999999},"last":1.921},{"interface":"front","average":{"1min":1.446,"5min":1.4510000000000001,"15min":2.085},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":10.029,"5min":10.029,"15min":50.747},"last":0.66600000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:18:24 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.325,"15min":1.9490000000000001},"min":{"1min":0.26900000000000002,"5min":0.23799999999999999,"15min":0.22500000000000001},"max":{"1min":3.9940000000000002,"5min":9.657,"15min":16.352},"last":3.6150000000000002},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.464,"15min":1.982},"min":{"1min":0.27000000000000002,"5min":0.249,"15min":0.249},"max":{"1min":4.258,"5min":9.1110000000000007,"15min":18.27},"last":1.425}]},{"osd":2,"last update":"Mon Oct 9 10:18:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.4990000000000001,"15min":2.0960000000000001},"min":{"1min":0.315,"5min":0.30199999999999999,"15min":0.246},"max":{"1min":4.0419999999999998,"5min":7.8600000000000003,"15min":13.775},"last":3.4089999999999998},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.5960000000000001,"15min":2.044},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28699999999999998},"max":{"1min":2.9369999999999998,"5min":12.845000000000001,"15min":12.845000000000001},"last":1.6639999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.492,"15min":1.536},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.7069999999999999,"5min":7.444,"15min":7.444},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.6639999999999999,"15min":1.625},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.7720000000000002,"5min":8.0779999999999994,"15min":8.0779999999999994},"last":1.887}]},{"osd":4,"last update":"Mon Oct 9 10:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.2889999999999999,"15min":1.163},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":5.4409999999999998,"5min":12.699999999999999,"15min":12.699999999999999},"last":1.966},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.97599999999999998,"15min":1.008},"min":{"1min":0.313,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.29,"5min":12.003,"15min":12.003},"last":0.69499999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:18:14 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.2689999999999999,"15min":1.3080000000000001},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":4.9539999999999997,"5min":12.048,"15min":25.277999999999999},"last":1.944},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.0089999999999999,"15min":1.214},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.22600000000000001},"max":{"1min":3.194,"5min":13.083,"15min":25.754000000000001},"last":8.157}]},{"osd":6,"last update":"Mon Oct 9 10:18:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.91100000000000003,"15min":1.107},"min":{"1min":0.35999999999999999,"5min":0.28399999999999997,"15min":0.26100000000000001},"max":{"1min":4.7869999999999999,"5min":6.8390000000000004,"15min":8.8569999999999993},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.123,"5min":1.0900000000000001,"15min":1.107},"min":{"1min":0.372,"5min":0.307,"15min":0.27200000000000002},"max":{"1min":5.806,"5min":10.625,"15min":10.625},"last":1.8320000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626818,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1466012,"kb_used_data":566676,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":92279140,"statfs":{"total":95995035648,"available":94493839360,"internally_reserved":0,"allocated":580276224,"data_stored":579580768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.0149999999999999,"15min":1.671},"min":{"1min":0.246,"5min":0.23100000000000001,"15min":0.22600000000000001},"max":{"1min":3.5190000000000001,"5min":8.8079999999999998,"15min":15.435},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.454,"15min":1.788},"min":{"1min":0.28000000000000003,"5min":0.251,"15min":0.22},"max":{"1min":6.351,"5min":10.539999999999999,"15min":15.308999999999999},"last":5.1379999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:18:19 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.4370000000000001,"15min":2.0600000000000001},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.23400000000000001},"max":{"1min":1.9450000000000001,"5min":17.925000000000001,"15min":35.609999999999999},"last":0.371},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.605,"15min":2.1459999999999999},"min":{"1min":0.29299999999999998,"5min":0.27100000000000002,"15min":0.24199999999999999},"max":{"1min":2.8919999999999999,"5min":10.58,"15min":32.396000000000001},"last":6.8929999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.8560000000000001,"15min":1.7969999999999999},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.26200000000000001},"max":{"1min":4.5810000000000004,"5min":29.367999999999999,"15min":29.367999999999999},"last":1.6850000000000001},{"interface":"front","average":{"1min":0.623,"5min":1.4399999999999999,"15min":2.3540000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4359999999999999,"5min":8.7669999999999995,"15min":14.542999999999999},"last":0.34200000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.3089999999999999,"15min":1.552},"min":{"1min":0.40400000000000003,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.9379999999999999,"5min":8.0229999999999997,"15min":8.0229999999999997},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.3700000000000001,"15min":1.454},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.5089999999999999,"5min":6.4189999999999996,"15min":6.4189999999999996},"last":5.218}]},{"osd":4,"last update":"Mon Oct 9 10:17:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.755,"15min":0.71399999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.54,"5min":6.2539999999999996,"15min":10.113},"last":1.651},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.76900000000000002,"15min":0.72099999999999997},"min":{"1min":0.37,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.0940000000000003,"5min":9.218,"15min":9.218},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.96099999999999997,"15min":0.94699999999999995},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.23699999999999999},"max":{"1min":3.5859999999999999,"5min":6.6210000000000004,"15min":7.9260000000000002},"last":1.617},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.65000000000000002,"15min":0.96199999999999997},"min":{"1min":0.47199999999999998,"5min":0.28199999999999997,"15min":0.25800000000000001},"max":{"1min":1.9059999999999999,"5min":4.3360000000000003,"15min":14.465999999999999},"last":1.4350000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:18:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.0940000000000001,"15min":0.98199999999999998},"min":{"1min":0.29899999999999999,"5min":0.29199999999999998,"15min":0.252},"max":{"1min":6.0279999999999996,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.1379999999999999,"15min":1.024},"min":{"1min":0.35699999999999998,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":6.0629999999999997,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":0.72399999999999998}]}]},{"osd":1,"up_from":84,"seq":360777253131,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1719588,"kb_used_data":1021212,"kb_used_omap":9,"kb_used_meta":698230,"kb_avail":92025564,"statfs":{"total":95995035648,"available":94234177536,"internally_reserved":0,"allocated":1045721088,"data_stored":1044246964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":714988521},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:18:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.403,"15min":2.3959999999999999},"min":{"1min":0.20499999999999999,"5min":0.188,"15min":0.183},"max":{"1min":15.913,"5min":15.913,"15min":31.007000000000001},"last":0.216},{"interface":"front","average":{"1min":1.5680000000000001,"5min":1.6799999999999999,"15min":2.681},"min":{"1min":0.27000000000000002,"5min":0.216,"15min":0.216},"max":{"1min":16.015000000000001,"5min":16.015000000000001,"15min":20.501999999999999},"last":0.23300000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":2.2629999999999999,"15min":3.1000000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":4.5069999999999997,"5min":23.105,"15min":23.105},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.78400000000000003,"5min":1.48,"15min":2.7839999999999998},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":2.9569999999999999,"5min":28.024999999999999,"15min":28.024999999999999},"last":0.442}]},{"osd":3,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.0469999999999999,"15min":1.133},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.6440000000000001,"5min":6.673,"15min":6.673},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.4379999999999999,"15min":1.6220000000000001},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.464,"5min":7.2160000000000002,"15min":7.2160000000000002},"last":0.39300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":1.228,"15min":1.4279999999999999},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":4.5609999999999999,"5min":7.3789999999999996,"15min":21.620000000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":1.7669999999999999,"5min":1.6899999999999999,"15min":1.8100000000000001},"min":{"1min":0.38700000000000001,"5min":0.314,"15min":0.308},"max":{"1min":10.497999999999999,"5min":10.497999999999999,"15min":12.904},"last":0.498}]},{"osd":5,"last update":"Mon Oct 9 10:18:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":2.7810000000000001,"15min":2.4769999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":2.4830000000000001,"5min":92.177999999999997,"15min":92.177999999999997},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.304,"5min":2.9660000000000002,"15min":2.7400000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29099999999999998},"max":{"1min":3.6520000000000001,"5min":91.869,"15min":91.869},"last":1.228}]},{"osd":6,"last update":"Mon Oct 9 10:18:36 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.337,"15min":1.897},"min":{"1min":0.38700000000000001,"5min":0.35599999999999998,"15min":0.34699999999999998},"max":{"1min":6.4409999999999998,"5min":7.4409999999999998,"15min":21.670999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.667,"5min":1.5529999999999999,"15min":2.089},"min":{"1min":0.45400000000000001,"5min":0.39200000000000002,"15min":0.35099999999999998},"max":{"1min":9.1660000000000004,"5min":9.1660000000000004,"15min":14.471},"last":0.53500000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:18:36 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.6739999999999999,"15min":2.3180000000000001},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8200000000000003,"5min":10.127000000000001,"15min":16.331},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.3320000000000001,"15min":1.9630000000000001},"min":{"1min":0.40899999999999997,"5min":0.371,"15min":0.371},"max":{"1min":3.298,"5min":7.4119999999999999,"15min":21.593},"last":0.41699999999999998}]}]},{"osd":0,"up_from":79,"seq":339302416656,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1046344,"kb_used_data":511424,"kb_used_omap":28,"kb_used_meta":534883,"kb_avail":92698808,"statfs":{"total":95995035648,"available":94923579392,"internally_reserved":0,"allocated":523698176,"data_stored":522195480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":547721040},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:18:31 2023","interfaces":[{"interface":"back","average":{"1min":3.4470000000000001,"5min":1.907,"15min":2.2320000000000002},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.16600000000000001},"max":{"1min":54.518000000000001,"5min":54.518000000000001,"15min":54.518000000000001},"last":0.98399999999999999},{"interface":"front","average":{"1min":3.7000000000000002,"5min":1.9650000000000001,"15min":3.383},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":56.950000000000003,"5min":56.950000000000003,"15min":56.950000000000003},"last":0.71199999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:18:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1509999999999998,"5min":2,"15min":2.2869999999999999},"min":{"1min":0.26800000000000002,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":53.988999999999997,"5min":53.988999999999997,"15min":53.988999999999997},"last":0.28299999999999997},{"interface":"front","average":{"1min":3.2570000000000001,"5min":1.877,"15min":2.5329999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.23300000000000001},"max":{"1min":54.119,"5min":54.119,"15min":54.119},"last":0.51700000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.663,"15min":1.3380000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":56.984999999999999,"5min":56.984999999999999,"15min":56.984999999999999},"last":2.004},{"interface":"front","average":{"1min":3.5,"5min":1.6200000000000001,"15min":1.258},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":56.475999999999999,"5min":56.475999999999999,"15min":56.475999999999999},"last":0.33100000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:18:00 2023","interfaces":[{"interface":"back","average":{"1min":4.3579999999999997,"5min":2.117,"15min":2.1739999999999999},"min":{"1min":0.34300000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":57.719000000000001,"5min":57.719000000000001,"15min":57.719000000000001},"last":0.432},{"interface":"front","average":{"1min":4.3780000000000001,"5min":2.0699999999999998,"15min":2.29},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":56.378,"5min":56.378,"15min":56.378},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:18:00 2023","interfaces":[{"interface":"back","average":{"1min":4.4059999999999997,"5min":4.6369999999999996,"15min":3.4729999999999999},"min":{"1min":0.34399999999999997,"5min":0.34000000000000002,"15min":0.28000000000000003},"max":{"1min":56.445999999999998,"5min":229.589,"15min":229.589},"last":0.376},{"interface":"front","average":{"1min":4.1950000000000003,"5min":4.5359999999999996,"15min":2.6720000000000002},"min":{"1min":0.45100000000000001,"5min":0.30199999999999999,"15min":0.27000000000000002},"max":{"1min":54.411000000000001,"5min":228.88900000000001,"15min":228.88900000000001},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:18:00 2023","interfaces":[{"interface":"back","average":{"1min":4.444,"5min":2.5219999999999998,"15min":2.0760000000000001},"min":{"1min":0.42299999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":54.482999999999997,"5min":54.482999999999997,"15min":54.482999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":4.867,"5min":2.3919999999999999,"15min":2.9279999999999999},"min":{"1min":0.45900000000000002,"5min":0.378,"15min":0.35699999999999998},"max":{"1min":57.753,"5min":57.753,"15min":57.753},"last":0.48099999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:18:00 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":2.4079999999999999,"15min":2.839},"min":{"1min":0.46000000000000002,"5min":0.35499999999999998,"15min":0.34300000000000003},"max":{"1min":56.411999999999999,"5min":56.411999999999999,"15min":56.411999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":4.1950000000000003,"5min":2.3439999999999999,"15min":2.1139999999999999},"min":{"1min":0.47399999999999998,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":54.451999999999998,"5min":54.451999999999998,"15min":54.451999999999998},"last":0.66400000000000003}]}]},{"osd":2,"up_from":217,"seq":932007903391,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":997232,"kb_used_data":551528,"kb_used_omap":83,"kb_used_meta":445612,"kb_avail":92747920,"statfs":{"total":95995035648,"available":94973870080,"internally_reserved":0,"allocated":564764672,"data_stored":563399572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":456307183},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:18:07 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":1.2050000000000001,"15min":1.8480000000000001},"min":{"1min":0.23799999999999999,"5min":0.185,"15min":0.17899999999999999},"max":{"1min":1.488,"5min":11.327,"15min":19.372},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.6240000000000001,"15min":2.8140000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":4.5039999999999996,"5min":10.929,"15min":20.143000000000001},"last":0.38300000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:18:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.569,"15min":2.54},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":34.893999999999998,"5min":34.893999999999998,"15min":34.893999999999998},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.3220000000000001,"5min":2.073,"15min":2.6720000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.254},"max":{"1min":34.962000000000003,"5min":34.962000000000003,"15min":34.962000000000003},"last":0.41699999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:18:43 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.26,"15min":1.0009999999999999},"min":{"1min":0.25700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.393,"5min":35.043999999999997,"15min":35.043999999999997},"last":0.316},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.677,"15min":1.234},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.4039999999999999,"5min":50.406999999999996,"15min":50.406999999999996},"last":0.47399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:17:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.724,"15min":1.3500000000000001},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.26600000000000001},"max":{"1min":3.1640000000000001,"5min":18.873999999999999,"15min":18.873999999999999},"last":1.004},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.8640000000000001,"15min":1.4870000000000001},"min":{"1min":0.35799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":24.352,"5min":24.352,"15min":24.352},"last":0.443}]},{"osd":5,"last update":"Mon Oct 9 10:18:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":2.23,"15min":2.0190000000000001},"min":{"1min":0.35299999999999998,"5min":0.30199999999999999,"15min":0.27700000000000002},"max":{"1min":24.23,"5min":57.298000000000002,"15min":57.298000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.76200000000000001,"5min":1.8029999999999999,"15min":1.8999999999999999},"min":{"1min":0.33700000000000002,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":3.5609999999999999,"5min":57.828000000000003,"15min":57.828000000000003},"last":1.1619999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:18:07 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":1.7250000000000001,"15min":1.72},"min":{"1min":0.36599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.125,"5min":12.257,"15min":18.943999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.109,"5min":1.655,"15min":1.9790000000000001},"min":{"1min":0.45100000000000001,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":24.379000000000001,"5min":24.379000000000001,"15min":24.379000000000001},"last":0.5}]},{"osd":7,"last update":"Mon Oct 9 10:18:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.1080000000000001,"15min":1.9039999999999999},"min":{"1min":0.47899999999999998,"5min":0.308,"15min":0.308},"max":{"1min":35.008000000000003,"5min":35.008000000000003,"15min":35.008000000000003},"last":4.9420000000000002},{"interface":"front","average":{"1min":2.383,"5min":2.028,"15min":2.1850000000000001},"min":{"1min":0.40999999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":24.323,"5min":24.323,"15min":24.323},"last":3.1360000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488231,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1974796,"kb_used_data":1797572,"kb_used_omap":15,"kb_used_meta":177200,"kb_avail":91770356,"statfs":{"total":95995035648,"available":93972844544,"internally_reserved":0,"allocated":1840713728,"data_stored":1838969526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":181453254},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.1080000000000001,"15min":1.119},"min":{"1min":0.223,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.4050000000000002,"5min":7.4050000000000002,"15min":7.4050000000000002},"last":3.2490000000000001},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.1399999999999999,"15min":0.998},"min":{"1min":0.30099999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.4850000000000003,"5min":7.4850000000000003,"15min":7.4850000000000003},"last":0.314}]},{"osd":1,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":1.2629999999999999,"15min":1.161},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":15.970000000000001,"5min":15.970000000000001,"15min":15.970000000000001},"last":3.3420000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":0.91000000000000003,"15min":0.75700000000000001},"min":{"1min":0.32800000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.516,"5min":7.516,"15min":7.516},"last":0.27100000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":2.1349999999999998,"15min":2.1699999999999999},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9779999999999998,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.3700000000000001,"15min":1.27},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":16.018000000000001,"5min":16.018000000000001,"15min":16.018000000000001},"last":3.3690000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":0.82799999999999996,"15min":0.70699999999999996},"min":{"1min":0.36299999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.3630000000000004,"5min":7.3630000000000004,"15min":7.3630000000000004},"last":2.3799999999999999},{"interface":"front","average":{"1min":2.0339999999999998,"5min":1.429,"15min":1.3280000000000001},"min":{"1min":0.32100000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8689999999999998,"5min":7.8689999999999998,"15min":7.8689999999999998},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":0.81999999999999995,"15min":0.73599999999999999},"min":{"1min":0.443,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.3639999999999999,"5min":3.3639999999999999,"15min":3.3639999999999999},"last":3.4649999999999999},{"interface":"front","average":{"1min":1.673,"5min":0.89900000000000002,"15min":0.77000000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":7.4539999999999997,"5min":7.4539999999999997,"15min":7.4539999999999997},"last":0.624}]},{"osd":6,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":1.5009999999999999,"15min":1.385},"min":{"1min":0.41199999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.9219999999999997,"5min":7.9219999999999997,"15min":7.9219999999999997},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.3380000000000001,"5min":0.98399999999999999,"15min":0.92500000000000004},"min":{"1min":0.48499999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.3929999999999998,"5min":7.3570000000000002,"15min":7.3570000000000002},"last":3.5049999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:17:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.0469999999999999,"15min":0.84899999999999998},"min":{"1min":0.39500000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":10.459,"5min":10.459,"15min":10.459},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.274,"5min":1.532,"15min":1.4079999999999999},"min":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":7.952,"5min":7.952,"15min":7.952},"last":1.2609999999999999}]}]},{"osd":4,"up_from":266,"seq":1142461300860,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":628636,"kb_used_data":297684,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":93116516,"statfs":{"total":95995035648,"available":95351312384,"internally_reserved":0,"allocated":304828416,"data_stored":304130838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.95199999999999996,"15min":0.99199999999999999},"min":{"1min":0.248,"5min":0.222,"15min":0.21199999999999999},"max":{"1min":2.6179999999999999,"5min":6.6970000000000001,"15min":10.881},"last":2.4870000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.3480000000000001,"15min":1.429},"min":{"1min":0.247,"5min":0.247,"15min":0.224},"max":{"1min":2.6480000000000001,"5min":8.298,"15min":12.246},"last":2.6070000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.3700000000000001,"15min":1.48},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.25},"max":{"1min":2.7120000000000002,"5min":18.303000000000001,"15min":18.303000000000001},"last":2.6230000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.3220000000000001,"15min":1.5269999999999999},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":1.774,"5min":18.036000000000001,"15min":18.036000000000001},"last":0.5}]},{"osd":2,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.25,"15min":1.3700000000000001},"min":{"1min":0.29399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.78,"5min":6.6689999999999996,"15min":9.2469999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.472,"15min":1.3680000000000001},"min":{"1min":0.314,"5min":0.28100000000000003,"15min":0.27700000000000002},"max":{"1min":2.661,"5min":11.778,"15min":11.778},"last":2.5569999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:18:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.4470000000000001,"15min":1.601},"min":{"1min":0.34300000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.899,"5min":9.6170000000000009,"15min":9.6170000000000009},"last":2.6509999999999998},{"interface":"front","average":{"1min":1.171,"5min":1.0369999999999999,"15min":1.0549999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.5830000000000002,"5min":3.5830000000000002,"15min":3.5830000000000002},"last":0.84699999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.8280000000000001,"15min":1.2509999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":1.4550000000000001,"5min":99.585999999999999,"15min":99.585999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.57599999999999996,"5min":1.8560000000000001,"15min":1.0569999999999999},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":2.5419999999999998,"5min":99.384,"15min":99.384},"last":5.5830000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.76600000000000001,"15min":0.749},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":2.6930000000000001,"5min":4.6449999999999996,"15min":6.851},"last":2.5150000000000001},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.88900000000000001,"15min":0.85199999999999998},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":2.742,"5min":4.9080000000000004,"15min":12.255000000000001},"last":2.637}]},{"osd":7,"last update":"Mon Oct 9 10:17:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.871,"15min":0.91600000000000004},"min":{"1min":0.34999999999999998,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":2.5979999999999999,"5min":6.6630000000000003,"15min":12.237},"last":3.613},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.93200000000000005,"15min":0.98899999999999999},"min":{"1min":0.34000000000000002,"5min":0.32100000000000001,"15min":0.30399999999999999},"max":{"1min":1.502,"5min":7.4960000000000004,"15min":14.964},"last":0.70599999999999996}]}]},{"osd":5,"up_from":36,"seq":154618823045,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1886608,"kb_used_data":981192,"kb_used_omap":0,"kb_used_meta":905408,"kb_avail":91858544,"statfs":{"total":95995035648,"available":94063149056,"internally_reserved":0,"allocated":1004740608,"data_stored":1004051086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927137792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:18:46 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.4099999999999999,"15min":1.661},"min":{"1min":0.251,"5min":0.219,"15min":0.21099999999999999},"max":{"1min":8.0180000000000007,"5min":16.079000000000001,"15min":16.079000000000001},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.355,"15min":1.569},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":7.0800000000000001,"5min":15.907,"15min":15.907},"last":7.0800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:18:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.4510000000000001,"15min":1.7310000000000001},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.23699999999999999},"max":{"1min":8.0530000000000008,"5min":16.056999999999999,"15min":16.056999999999999},"last":3.6339999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.5069999999999999,"15min":1.728},"min":{"1min":0.26200000000000001,"5min":0.248,"15min":0.248},"max":{"1min":3.927,"5min":23.099,"15min":23.099},"last":7.1280000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:18:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":1.675,"15min":2.1080000000000001},"min":{"1min":0.32200000000000001,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":8.0760000000000005,"5min":15.973000000000001,"15min":15.973000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.198,"5min":1.385,"15min":1.74},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.313},"max":{"1min":6.2990000000000004,"5min":15.929,"15min":15.929},"last":7.1449999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:18:46 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.389,"15min":1.446},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.851,"5min":8.8689999999999998,"15min":8.8689999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.546,"15min":1.4950000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":9.4480000000000004,"5min":9.4480000000000004,"15min":9.4480000000000004},"last":9.4480000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:17:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97299999999999998,"15min":0.84799999999999998},"min":{"1min":0.26400000000000001,"5min":0.25800000000000001,"15min":0.24299999999999999},"max":{"1min":8.468,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.0549999999999999,"5min":0.73699999999999999,"15min":0.745},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":4.6029999999999998,"5min":4.7110000000000003,"15min":8.5559999999999992},"last":7.1100000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:18:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.077,"15min":1.0880000000000001},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.25},"max":{"1min":2.8820000000000001,"5min":10.222,"15min":13.483000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.425,"15min":1.202},"min":{"1min":0.33000000000000002,"5min":0.26700000000000002,"15min":0.22600000000000001},"max":{"1min":7.9539999999999997,"5min":15.997,"15min":18.600999999999999},"last":4.7229999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:18:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":1.2809999999999999,"15min":1.1679999999999999},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.25600000000000001},"max":{"1min":8.093,"5min":16.030999999999999,"15min":16.030999999999999},"last":0.499},{"interface":"front","average":{"1min":1.3080000000000001,"5min":0.81999999999999995,"15min":0.82899999999999996},"min":{"1min":0.40200000000000002,"5min":0.27500000000000002,"15min":0.26800000000000002},"max":{"1min":7.4249999999999998,"5min":7.4249999999999998,"15min":8.5220000000000002},"last":7.1600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":517734400,"data_stored":517734400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1038225408,"data_stored":1038221336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":550764544,"data_stored":550764544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1819738112,"data_stored":1819738112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":293601280,"data_stored":293601280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":976097280,"data_stored":976093208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":568983552,"data_stored":568983552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1010569216,"data_stored":1010565144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:18:57.220 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:18:57.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-10-09T10:18:57.311 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:18:57.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:57.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:18:57.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:18:57.416 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:57.537 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:57.573 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:18:57.611 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:18:57.626 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:57.637 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:57.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:57.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:57 smithi012 ceph-mon[147926]: pgmap v1148: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 19540/63849 objects degraded (30.603%); 655/63849 objects misplaced (1.026%); 7.4 MiB/s, 164 objects/s recovering 2023-10-09T10:18:57.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:57 smithi012 ceph-mon[152263]: pgmap v1148: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 19540/63849 objects degraded (30.603%); 655/63849 objects misplaced (1.026%); 7.4 MiB/s, 164 objects/s recovering 2023-10-09T10:18:57.649 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:18:57.688 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:18:57.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:57 smithi110 ceph-mon[136032]: pgmap v1148: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 19540/63849 objects degraded (30.603%); 655/63849 objects misplaced (1.026%); 7.4 MiB/s, 164 objects/s recovering 2023-10-09T10:18:57.721 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:18:57.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:57.757 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:18:57.787 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:18:57.811 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:57.819 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:18:57.826 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:18:57.826 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:18:57.826 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:18:57.826 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:18:57.826 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:18:57.826 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:18:57.826 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:18:57.826 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:18:57.887 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:57.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:57.918 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:18:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:18:57] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.33.4" 2023-10-09T10:18:57.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:58.031 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:58.098 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:18:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:58.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:18:58.209 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:18:58.267 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:58.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/355346748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:58.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 19540/63849 objects degraded (30.603%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:18:58.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/355346748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:58.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 19540/63849 objects degraded (30.603%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:18:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/355346748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:18:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 19540/63849 objects degraded (30.603%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:18:58.725 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:18:58.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:58.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:18:58.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:58.858 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:58.885 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:58.967 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:58.973 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:18:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:58.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:59.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:18:59.130 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:59.330 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:18:59.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:59.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:18:59.440 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:18:59.509 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:18:59.571 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:18:59.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:18:59.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:18:59.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:59 smithi012 ceph-mon[147926]: pgmap v1149: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 19540/61926 objects degraded (31.554%); 655/61926 objects misplaced (1.058%); 6.9 MiB/s, 123 objects/s recovering 2023-10-09T10:18:59.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:59.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:18:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:59.593 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:59 smithi012 ceph-mon[152263]: pgmap v1149: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 19540/61926 objects degraded (31.554%); 655/61926 objects misplaced (1.058%); 6.9 MiB/s, 123 objects/s recovering 2023-10-09T10:18:59.593 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:59.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:18:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:59.692 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:18:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:59 smithi110 ceph-mon[136032]: pgmap v1149: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 19540/61926 objects degraded (31.554%); 655/61926 objects misplaced (1.058%); 6.9 MiB/s, 123 objects/s recovering 2023-10-09T10:18:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:18:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:18:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:18:59.741 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:00.067 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:00.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:00.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:00.214 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:00.347 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:00.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:00.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:00.408 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:19:00.452 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:00.503 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:00.584 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:00.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:00.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:00.596 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3714356034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:00.596 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:00 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:00.596 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3714356034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:00.597 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:00 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:00.693 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3714356034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:00 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:00.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:19:00.746 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:01.102 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T10:19:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: pgmap v1150: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 19198/58971 objects degraded (32.555%); 655/58971 objects misplaced (1.111%); 7.3 MiB/s, 130 objects/s recovering 2023-10-09T10:19:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:19:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T10:19:01.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: 9.11 scrub starts 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: 9.11 scrub ok 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/245236879' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: pgmap v1150: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 19198/58971 objects degraded (32.555%); 655/58971 objects misplaced (1.111%); 7.3 MiB/s, 130 objects/s recovering 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T10:19:01.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: 9.11 scrub starts 2023-10-09T10:19:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: 9.11 scrub ok 2023-10-09T10:19:01.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/245236879' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:19:01.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: pgmap v1150: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 19198/58971 objects degraded (32.555%); 655/58971 objects misplaced (1.111%); 7.3 MiB/s, 130 objects/s recovering 2023-10-09T10:19:01.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:01.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:19:01.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T10:19:01.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: failed to encode map e413 with expected crc 2023-10-09T10:19:01.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: 9.11 scrub starts 2023-10-09T10:19:01.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: 9.11 scrub ok 2023-10-09T10:19:01.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/245236879' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:19:01.917 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:01.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:02.020 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T10:19:02.078 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:02.141 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:02.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:02.248 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:02.299 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:02.321 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:19:02.321 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:19:02.366 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:02.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:02.386 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:02.475 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:02.535 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:02.615 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:02.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:02.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:02.767 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:02.962 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:02.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:03.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:19:03.067 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:03.108 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:03.179 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:03.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:03.291 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:03.335 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:03.388 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:03.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:03.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:03.395 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:03 smithi110 ceph-mon[136032]: pgmap v1152: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 18911/53133 objects degraded (35.592%); 554/53133 objects misplaced (1.043%); 7.5 MiB/s, 127 objects/s recovering 2023-10-09T10:19:03.395 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:03 smithi110 ceph-mon[136032]: 9.1 scrub starts 2023-10-09T10:19:03.395 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:03 smithi110 ceph-mon[136032]: 9.1 scrub ok 2023-10-09T10:19:03.395 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 18911/53133 objects degraded (35.592%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:03.495 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:03.559 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:03.632 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:03.633 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:03 smithi012 ceph-mon[152263]: pgmap v1152: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 18911/53133 objects degraded (35.592%); 554/53133 objects misplaced (1.043%); 7.5 MiB/s, 127 objects/s recovering 2023-10-09T10:19:03.633 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:03 smithi012 ceph-mon[152263]: 9.1 scrub starts 2023-10-09T10:19:03.633 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:03 smithi012 ceph-mon[152263]: 9.1 scrub ok 2023-10-09T10:19:03.633 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 18911/53133 objects degraded (35.592%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:03.633 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:03 smithi012 ceph-mon[147926]: pgmap v1152: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 18911/53133 objects degraded (35.592%); 554/53133 objects misplaced (1.043%); 7.5 MiB/s, 127 objects/s recovering 2023-10-09T10:19:03.633 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:03 smithi012 ceph-mon[147926]: 9.1 scrub starts 2023-10-09T10:19:03.633 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:03 smithi012 ceph-mon[147926]: 9.1 scrub ok 2023-10-09T10:19:03.633 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 18911/53133 objects degraded (35.592%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:03.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:03.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:03.743 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:03.833 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:03.906 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:03.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:03.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:03.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:19:03.948 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:19:03.975 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:19:03.998 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:19:04.011 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:04.015 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:04.022 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:19:04.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:19:04.056 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:19:04.059 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:04.088 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:19:04.112 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:04.114 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:19:04.119 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:19:04.119 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:19:04.119 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:19:04.119 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:19:04.119 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:19:04.119 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:19:04.119 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:19:04.119 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:19:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:04.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:04.224 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:04.297 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:04.350 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:04.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:04.412 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3723592069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:04.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:04.516 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:04.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3723592069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:04.640 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3723592069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:04.966 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:04.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:04.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:05.022 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:05.073 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:05.157 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:05.227 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:05.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:05.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:05.408 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:05.484 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:05.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:05.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:05.575 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:05 smithi012 ceph-mon[147926]: pgmap v1153: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 18911/51183 objects degraded (36.948%); 554/51183 objects misplaced (1.082%); 7.5 MiB/s, 127 objects/s recovering 2023-10-09T10:19:05.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:05 smithi012 ceph-mon[152263]: pgmap v1153: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 18911/51183 objects degraded (36.948%); 554/51183 objects misplaced (1.082%); 7.5 MiB/s, 127 objects/s recovering 2023-10-09T10:19:05.598 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:05.633 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:05.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:05 smithi110 ceph-mon[136032]: pgmap v1153: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 18911/51183 objects degraded (36.948%); 554/51183 objects misplaced (1.082%); 7.5 MiB/s, 127 objects/s recovering 2023-10-09T10:19:05.914 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:05.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:06.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:06.053 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:19:06.053 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:19:06.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:19:06.053 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:19:06.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.053 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114948 9.18 9:1ed3790b:::benchmark_data_smithi012_313365_object2473:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 231.82625081200001, 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9028626509999995, 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114948 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223501+0000", 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223504+0000", 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.932e-06 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223510+0000", 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6890000000000001e-06 2023-10-09T10:19:06.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223515+0000", 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7820000000000001e-06 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223563+0000", 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8705000000000002e-05 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:06.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223573+0000", 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3149999999999998e-06 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.185664+0000", 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96209155300000004 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.185676+0000", 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2165e-05 2023-10-09T10:19:06.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.092908+0000", 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9072312829999998 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093028+0000", 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120679 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093121+0000", 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2170999999999996e-05 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.105712+0000", 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012591150000000001 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124350+0000", 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018638268 2023-10-09T10:19:06.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124362+0000", 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2362e-05 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124426+0000", 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3637000000000001e-05 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.126364+0000", 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019383549999999999 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114976 9.8 9:131c68e0:::benchmark_data_smithi012_313365_object2501:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 230.74387165499999, 2023-10-09T10:19:06.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.83748516, 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114976 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:19:06.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305879+0000", 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305887+0000", 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7619999999999995e-06 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305900+0000", 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3369e-05 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305912+0000", 2023-10-09T10:19:06.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2313000000000001e-05 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.309645+0000", 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003732815 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.309659+0000", 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3890000000000001e-05 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.063120+0000", 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7534609549999995 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.073887+0000", 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010766901000000001 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.106337+0000", 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032450537000000002 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.110335+0000", 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003997212 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.118546+0000", 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0082116299999999993 2023-10-09T10:19:06.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.127135+0000", 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085889409999999992 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.128467+0000", 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001332176 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.130828+0000", 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00236046 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.133144+0000", 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023158419999999998 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.143366+0000", 2023-10-09T10:19:06.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010222337 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114990 9.8 9:10d5a285:::benchmark_data_smithi012_313365_object2515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.78727275400001, 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0309713049999996, 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114990 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262478+0000", 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262492+0000", 2023-10-09T10:19:06.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4115e-05 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.267636+0000", 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051444480000000002 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.267653+0000", 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6095000000000001e-05 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.144210+0000", 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8765578109999996 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155494+0000", 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011283985999999999 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.201082+0000", 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.045587640999999998 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.205016+0000", 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039338280000000003 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.214090+0000", 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090739989999999993 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.243654+0000", 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029564373000000001 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.265457+0000", 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021802591 2023-10-09T10:19:06.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.267777+0000", 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023206949999999998 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.271653+0000", 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038760219999999998 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.293451+0000", 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021797512000000002 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115028 9.18 9:1ec11435:::benchmark_data_smithi012_313365_object2553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.64547043100001, 2023-10-09T10:19:06.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7778839790000003, 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115028 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404280+0000", 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404289+0000", 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5809999999999997e-06 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404316+0000", 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6720000000000002e-05 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404326+0000", 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0453e-05 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155549+0000", 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7512224199999995 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155657+0000", 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108597 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155763+0000", 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106253 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.167754+0000", 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011990404 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179961+0000", 2023-10-09T10:19:06.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012206995999999999 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179968+0000", 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9029999999999998e-06 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.180006+0000", 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8136000000000002e-05 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.182166+0000", 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021601250000000002 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115046 9.8 9:10ac3fd1:::benchmark_data_smithi012_313365_object2571:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.59122166200001, 2023-10-09T10:19:06.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8549557910000001, 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115046 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:19:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458530+0000", 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458533+0000", 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.79e-06 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458539+0000", 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4199999999999998e-06 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458543+0000", 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8060000000000002e-06 2023-10-09T10:19:06.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.168371+0000", 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.709827089 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179894+0000", 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011523197000000001 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.244426+0000", 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064532795000000004 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.249276+0000", 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004849969 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.257311+0000", 2023-10-09T10:19:06.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0080343040000000008 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.295944+0000", 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.038633611999999998 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.297677+0000", 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001732803 2023-10-09T10:19:06.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.298328+0000", 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00065045599999999999 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.301789+0000", 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034609559999999998 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313487+0000", 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011697982000000001 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115063 9.8 9:12e13310:::benchmark_data_smithi012_313365_object2588:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.528178623, 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9951984300000003, 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115063 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521572+0000", 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521582+0000", 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3870000000000001e-06 2023-10-09T10:19:06.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521600+0000", 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8029e-05 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521609+0000", 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7890000000000006e-06 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.314437+0000", 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7928272840000004 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330107+0000", 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015670738 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.392854+0000", 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062746280000000001 2023-10-09T10:19:06.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.396715+0000", 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038610379999999998 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.405209+0000", 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084946369999999993 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.468006+0000", 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062796789000000006 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.492245+0000", 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024238563000000001 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.495737+0000", 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034923879999999999 2023-10-09T10:19:06.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.499136+0000", 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033991400000000001 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.516772+0000", 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017636013999999998 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.101 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115064 9.18 9:1d8cd058:::benchmark_data_smithi012_313365_object2589:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.52438152799999, 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8195911320000002, 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.102 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115064 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525369+0000", 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525377+0000", 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4749999999999993e-06 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525393+0000", 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6319000000000001e-05 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525405+0000", 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1484000000000001e-05 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313537+0000", 2023-10-09T10:19:06.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7881318369999999 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313674+0000", 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137074 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313794+0000", 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120281 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330174+0000", 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016380212000000002 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330198+0000", 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3269e-05 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342558+0000", 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012360884000000001 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342624+0000", 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5277000000000005e-05 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.344962+0000", 2023-10-09T10:19:06.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023383079999999999 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 347/342 [PushOp(9:0c166afc:::benchmark_data_smithi012_313365_object2594:head, version: 320'166, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0c166afc:::benchmark_data_smithi012_313365_object2594:head@320'166, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.502918744, 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4620482260000003, 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:19:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546828+0000", 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999948 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546890+0000", 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1514999999999996e-05 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546920+0000", 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0239e-05 2023-10-09T10:19:06.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546947+0000", 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7192e-05 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.007036+0000", 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4600890619999998 2023-10-09T10:19:06.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.007054+0000", 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.785e-05 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.008882+0000", 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001827841 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115082 9.18 9:1d0726ed:::benchmark_data_smithi012_313365_object2607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.42018677900001, 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8115987789999997, 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.114 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115082 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629565+0000", 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629569+0000", 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.377e-06 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629577+0000", 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0830000000000004e-06 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629584+0000", 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315693+0000", 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6861087850000001 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315831+0000", 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013798599999999999 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315971+0000", 2023-10-09T10:19:06.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000139887 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330217+0000", 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014246325000000001 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330237+0000", 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9962e-05 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.438747+0000", 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10850990200000001 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.438804+0000", 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6765000000000003e-05 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.441164+0000", 2023-10-09T10:19:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023603669999999999 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115093 9.8 9:161687e0:::benchmark_data_smithi012_313365_object2618:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.40198922900001, 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8841582529999998, 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.121 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115093 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647762+0000", 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647769+0000", 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2479999999999999e-06 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647781+0000", 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2565000000000001e-05 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647791+0000", 2023-10-09T10:19:06.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3360000000000001e-06 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330722+0000", 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6829310240000002 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342528+0000", 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011806542999999999 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.439337+0000", 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.096808401000000002 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.445587+0000", 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062502110000000003 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.458972+0000", 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013385052 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.493763+0000", 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.034790897000000001 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.517445+0000", 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023681722999999998 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.519054+0000", 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016091409999999999 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.521336+0000", 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022823409999999998 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.531921+0000", 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010585331 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115107 9.8 9:160e4e76:::benchmark_data_smithi012_313365_object2632:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.374223796, 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9200908160000001, 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.129 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115107 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675527+0000", 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675535+0000", 2023-10-09T10:19:06.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.695e-06 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675546+0000", 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0796e-05 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675553+0000", 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0149999999999998e-06 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.343412+0000", 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6678597110000002 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.360622+0000", 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017209067000000002 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532751+0000", 2023-10-09T10:19:06.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.172129479 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.539246+0000", 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0064948999999999996 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.549540+0000", 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010293656999999999 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.134 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.577693+0000", 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028153504999999999 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578635+0000", 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094198399999999999 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.581004+0000", 2023-10-09T10:19:06.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023692980000000002 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.584057+0000", 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003052768 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595619+0000", 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011562329999999999 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.136 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 347/342 [PushOp(9:0b40069f:::benchmark_data_smithi012_313365_object2631:head, version: 320'167, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0b40069f:::benchmark_data_smithi012_313365_object2631:head@320'167, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.36428191600001, 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3934248150000004, 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685469+0000", 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:06.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685554+0000", 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5025999999999995e-05 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685577+0000", 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.319e-05 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685596+0000", 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8238000000000001e-05 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.077623+0000", 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3920269019999996 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.077631+0000", 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0590000000000003e-06 2023-10-09T10:19:06.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.078895+0000", 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001264668 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115110 9.18 9:189e6899:::benchmark_data_smithi012_313365_object2635:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:19:06.141 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.361532694, 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.870415833, 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115110 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688219+0000", 2023-10-09T10:19:06.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688222+0000", 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2199999999999999e-06 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688226+0000", 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9859999999999996e-06 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688229+0000", 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2629999999999999e-06 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.531968+0000", 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8437393259999997 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532080+0000", 2023-10-09T10:19:06.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111876 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532180+0000", 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9580999999999994e-05 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.536128+0000", 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039479809999999997 2023-10-09T10:19:06.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.536166+0000", 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8197e-05 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556671+0000", 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020505332000000001 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556708+0000", 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6332999999999999e-05 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.558636+0000", 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019280180000000001 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115117 9.18 9:1bb15d0c:::benchmark_data_smithi012_313365_object2642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.34495266900001, 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8692873749999999, 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115117 2023-10-09T10:19:06.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:19:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704799+0000", 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704805+0000", 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0100000000000001e-06 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704817+0000", 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2054e-05 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704823+0000", 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9900000000000002e-06 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.533734+0000", 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.828911078 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.533880+0000", 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014652400000000001 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.534039+0000", 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015923399999999999 2023-10-09T10:19:06.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556665+0000", 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022625698 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.558646+0000", 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019810460000000002 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571804+0000", 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013158174 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571859+0000", 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5074000000000002e-05 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.574087+0000", 2023-10-09T10:19:06.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002227672 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115136 9.8 9:102eeaef:::benchmark_data_smithi012_313365_object2661:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.268707549, 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9345335690000001, 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.156 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115136 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781043+0000", 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781051+0000", 2023-10-09T10:19:06.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2010000000000002e-06 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781062+0000", 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1688000000000001e-05 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781071+0000", 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3089999999999996e-06 2023-10-09T10:19:06.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.557513+0000", 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7764423310000002 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571744+0000", 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014230893 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.636487+0000", 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064743060000000005 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.642690+0000", 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062035450000000004 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.660819+0000", 2023-10-09T10:19:06.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018128339 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.700703+0000", 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.039884233999999998 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.702344+0000", 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016405899999999999 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.702817+0000", 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000473578 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.705696+0000", 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028788009999999998 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.715578+0000", 2023-10-09T10:19:06.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0098824919999999997 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115142 9.18 9:1de397b1:::benchmark_data_smithi012_313365_object2667:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 229.239208502, 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7874058789999996, 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:06.164 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115142 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810542+0000", 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810549+0000", 2023-10-09T10:19:06.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7070000000000003e-06 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810562+0000", 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2543e-05 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810570+0000", 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6279999999999994e-06 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578683+0000", 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7681127830000003 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578869+0000", 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018629700000000001 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.579038+0000", 2023-10-09T10:19:06.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000168355 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.582715+0000", 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0036773040000000002 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.582723+0000", 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0139999999999998e-06 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595701+0000", 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012977700999999999 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595774+0000", 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2996000000000004e-05 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.597950+0000", 2023-10-09T10:19:06.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021762019999999999 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPushReply(9.8 349/346 [PushReplyOp(9:12cf827d:::benchmark_data_smithi012_313365_object76143:head)])", 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:33.511271+0000", 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 212.53848189600001, 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.487463483, 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:19:06.171 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511271+0000", 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511271+0000", 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511269+0000", 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511278+0000", 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3359999999999995e-06 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511298+0000", 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0188999999999999e-05 2023-10-09T10:19:06.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511320+0000", 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2050999999999999e-05 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:35.976548+0000", 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4652284409999998 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:35.998734+0000", 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022185547 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 354/342 [PushOp(9:0d3d12ba:::benchmark_data_smithi012_313365_object10048:head, version: 321'604, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0d3d12ba:::benchmark_data_smithi012_313365_object10048:head@321'604, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.142237+0000", 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 194.90751590900001, 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0200850130000001, 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.175 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142237+0000", 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142237+0000", 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142236+0000", 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142266+0000", 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0227e-05 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.177 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142281+0000", 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4569999999999999e-05 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142298+0000", 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7076999999999999e-05 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.160817+0000", 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0185196150000002 2023-10-09T10:19:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.160830+0000", 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2294e-05 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.162322+0000", 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014918100000000001 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 354/342 [PushOp(9:092dbf94:::benchmark_data_smithi012_313365_object10086:head, version: 321'606, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:092dbf94:::benchmark_data_smithi012_313365_object10086:head@321'606, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.525640+0000", 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 194.52411220499999, 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1090250359999998, 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525640+0000", 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525640+0000", 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525639+0000", 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525682+0000", 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2657999999999997e-05 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525701+0000", 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8756000000000001e-05 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525717+0000", 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6877000000000001e-05 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.633112+0000", 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1073941170000001 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.633126+0000", 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4323e-05 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.634665+0000", 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001539428 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 354/342 [PushOp(9:0a3555ea:::benchmark_data_smithi012_313365_object10214:head, version: 321'613, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0a3555ea:::benchmark_data_smithi012_313365_object10214:head@321'613, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.639337+0000", 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 194.41041589899999, 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9968960120000001, 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639337+0000", 2023-10-09T10:19:06.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639337+0000", 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639336+0000", 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.186 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639443+0000", 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107241 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639486+0000", 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3460000000000001e-05 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639510+0000", 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4189000000000002e-05 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.634718+0000", 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9952079889999998 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.634731+0000", 2023-10-09T10:19:06.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2605e-05 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.636233+0000", 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0015015009999999999 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:06.189 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:19:06.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:06.191 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:06.191 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:06.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:06.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:06.331 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:06.481 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:06.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:06.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:06.591 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:06.682 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:06.755 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:06.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:06.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:19:06.814 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:19:06.860 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:06.937 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:06.998 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:07.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:07.043 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 4, 0, 2, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:19:07.044 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:19:07.044 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [1, 3, 6, 7, 4, 0, 2, 5] 2023-10-09T10:19:07.044 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T10:19:07.106 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:07.184 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:07.245 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:07.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:07.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:07.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:07.410 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:07.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:07 smithi012 ceph-mon[147926]: pgmap v1154: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 904 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 18326/43431 objects degraded (42.196%); 455/43431 objects misplaced (1.048%); 7.7 MiB/s, 89 objects/s recovering 2023-10-09T10:19:07.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/727666573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:07.426 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:07.461 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:19:07.478 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:07.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:07.519 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:07.570 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:07.589 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:07.646 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:07.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:07 smithi012 ceph-mon[152263]: pgmap v1154: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 904 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 18326/43431 objects degraded (42.196%); 455/43431 objects misplaced (1.048%); 7.7 MiB/s, 89 objects/s recovering 2023-10-09T10:19:07.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/727666573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:07.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:07.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:07 smithi110 ceph-mon[136032]: pgmap v1154: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 904 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 18326/43431 objects degraded (42.196%); 455/43431 objects misplaced (1.048%); 7.7 MiB/s, 89 objects/s recovering 2023-10-09T10:19:07.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/727666573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:07.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:07.812 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:07.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:07.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:07.918 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:07.992 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:08.082 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:08.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:19:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:19:07] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.33.4" 2023-10-09T10:19:08.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:09.277 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:19:09.277 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.278 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:09.278 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: osdmap e414: 8 total, 8 up, 8 in 2023-10-09T10:19:09.278 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.278 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.278 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.278 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.278 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 18326/43431 objects degraded (42.196%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:09.279 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:09.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:19:09.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:09.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: osdmap e414: 8 total, 8 up, 8 in 2023-10-09T10:19:09.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.279 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 18326/43431 objects degraded (42.196%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: osdmap e414: 8 total, 8 up, 8 in 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: failed to encode map e414 with expected crc 2023-10-09T10:19:09.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 18326/43431 objects degraded (42.196%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:09.338 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:09.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:09 smithi012 ceph-mon[147926]: pgmap v1156: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 864 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 18326/41529 objects degraded (44.128%); 455/41529 objects misplaced (1.096%); 6.9 MiB/s, 68 objects/s recovering 2023-10-09T10:19:09.737 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:09 smithi012 ceph-mon[152263]: pgmap v1156: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 864 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 18326/41529 objects degraded (44.128%); 455/41529 objects misplaced (1.096%); 6.9 MiB/s, 68 objects/s recovering 2023-10-09T10:19:09.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:19:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:09 smithi110 ceph-mon[136032]: pgmap v1156: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 864 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 18326/41529 objects degraded (44.128%); 455/41529 objects misplaced (1.096%); 6.9 MiB/s, 68 objects/s recovering 2023-10-09T10:19:10.088 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:10.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:10.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:10.276 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:10.328 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:19:10.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:19:10.379 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:10.385 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:19:10.385 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:19:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:10.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:10.388 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:19:10.421 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:19:10.449 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:19:10.487 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:10.490 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:19:10.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:10 smithi012 ceph-mon[147926]: pgmap v1157: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 803 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 17994/38595 objects degraded (46.623%); 455/38595 objects misplaced (1.179%); 6.2 MiB/s, 61 objects/s recovering 2023-10-09T10:19:10.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3031349308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:10.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:10 smithi012 ceph-mon[152263]: pgmap v1157: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 803 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 17994/38595 objects degraded (46.623%); 455/38595 objects misplaced (1.179%); 6.2 MiB/s, 61 objects/s recovering 2023-10-09T10:19:10.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3031349308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:10.531 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:10.539 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:19:10.572 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:19:10.575 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:19:10.575 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:19:10.575 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:19:10.575 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:19:10.575 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:19:10.576 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:19:10.576 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:19:10.576 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:19:10.602 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:10.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:10.711 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:10.772 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:10 smithi110 ceph-mon[136032]: pgmap v1157: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 803 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 17994/38595 objects degraded (46.623%); 455/38595 objects misplaced (1.179%); 6.2 MiB/s, 61 objects/s recovering 2023-10-09T10:19:10.772 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3031349308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:10.777 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:10.835 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:10.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:10.943 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:11.002 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:11.084 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:11.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:11.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:11.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:11.225 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:19:11.087+0000 7f85a99d3700 -1 mon.a@0(leader).osd e414 definitely_dead 0 2023-10-09T10:19:11.230 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:11.383 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:11.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:11.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:19:11.422 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:11.493 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:11.503 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-mon[147926]: 9.21 scrub starts 2023-10-09T10:19:11.503 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-mon[147926]: 9.21 scrub ok 2023-10-09T10:19:11.503 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3000306122' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:19:11.503 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-mon[147926]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T10:19:11.514 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.4. 2023-10-09T10:19:11.566 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:11.609 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:11.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:11.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:11.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:11.750 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:11.755 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-mon[152263]: 9.21 scrub starts 2023-10-09T10:19:11.755 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-mon[152263]: 9.21 scrub ok 2023-10-09T10:19:11.755 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3000306122' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:19:11.756 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:11 smithi012 ceph-mon[152263]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T10:19:11.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:11 smithi110 ceph-mon[136032]: 9.21 scrub starts 2023-10-09T10:19:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:11 smithi110 ceph-mon[136032]: 9.21 scrub ok 2023-10-09T10:19:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3000306122' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T10:19:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:11 smithi110 ceph-mon[136032]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T10:19:12.343 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:12.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3000306122' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T10:19:12.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T10:19:12.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.830 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[152263]: pgmap v1159: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 688 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 17679/33063 objects degraded (53.471%); 346/33063 objects misplaced (1.046%); 9.7 MiB/s, 111 objects/s recovering 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3000306122' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.832 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.832 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:12 smithi012 ceph-mon[147926]: pgmap v1159: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 688 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 17679/33063 objects degraded (53.471%); 346/33063 objects misplaced (1.046%); 9.7 MiB/s, 111 objects/s recovering 2023-10-09T10:19:12.875 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:12.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3000306122' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T10:19:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T10:19:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: failed to encode map e415 with expected crc 2023-10-09T10:19:12.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:12 smithi110 ceph-mon[136032]: pgmap v1159: 109 pgs: 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 688 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 17679/33063 objects degraded (53.471%); 346/33063 objects misplaced (1.046%); 9.7 MiB/s, 111 objects/s recovering 2023-10-09T10:19:12.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:13.012 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:13.159 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:13.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:13.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:13.263 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:13.296 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:13.339 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:13.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:13.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:13.393 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:19:13.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:13.511 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:13.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:13.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T10:19:13.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: 9.10 starting backfill to osd.5 from (0'0,0'0] MAX to 414'9862 2023-10-09T10:19:13.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17679/33063 objects degraded (53.471%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:13.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:13 smithi110 ceph-mon[136032]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T10:19:13.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:13.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:13.713 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.713 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.713 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T10:19:13.713 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.713 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: 9.10 starting backfill to osd.5 from (0'0,0'0] MAX to 414'9862 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17679/33063 objects degraded (53.471%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[147926]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T10:19:13.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: failed to encode map e416 with expected crc 2023-10-09T10:19:13.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: 9.10 starting backfill to osd.5 from (0'0,0'0] MAX to 414'9862 2023-10-09T10:19:13.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17679/33063 objects degraded (53.471%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:13.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:13 smithi012 ceph-mon[152263]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T10:19:13.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:13.784 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:14.124 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:14.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:14.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:14.197 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:19:14.197 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:19:14.228 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:14.267 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:14.285 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:19:14.293 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:14.309 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:14.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:14.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:14.417 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:14.443 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:14.484 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:14.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:14.588 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:14.612 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:14.645 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:14.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 414'9943 2023-10-09T10:19:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: 9.1e starting backfill to osd.0 from (0'0,0'0] MAX to 414'11552 2023-10-09T10:19:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: 9.14 starting backfill to osd.0 from (0'0,0'0] MAX to 414'9889 2023-10-09T10:19:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 414'11581 2023-10-09T10:19:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: 9.c starting backfill to osd.5 from (0'0,0'0] MAX to 414'11648 2023-10-09T10:19:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: 9.1c starting backfill to osd.5 from (0'0,0'0] MAX to 414'11697 2023-10-09T10:19:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: pgmap v1162: 109 pgs: 7 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 652 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 17679/31350 objects degraded (56.392%); 346/31350 objects misplaced (1.104%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T10:19:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3269505237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:19:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:14 smithi110 ceph-mon[136032]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T10:19:14.749 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:14.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:14.829 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:14.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:14.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 414'9943 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: 9.1e starting backfill to osd.0 from (0'0,0'0] MAX to 414'11552 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: 9.14 starting backfill to osd.0 from (0'0,0'0] MAX to 414'9889 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 414'11581 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: 9.c starting backfill to osd.5 from (0'0,0'0] MAX to 414'11648 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: 9.1c starting backfill to osd.5 from (0'0,0'0] MAX to 414'11697 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: pgmap v1162: 109 pgs: 7 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 652 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 17679/31350 objects degraded (56.392%); 346/31350 objects misplaced (1.104%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3269505237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[152263]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 414'9943 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: 9.1e starting backfill to osd.0 from (0'0,0'0] MAX to 414'11552 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: 9.14 starting backfill to osd.0 from (0'0,0'0] MAX to 414'9889 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: failed to encode map e417 with expected crc 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 414'11581 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: 9.c starting backfill to osd.5 from (0'0,0'0] MAX to 414'11648 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: 9.1c starting backfill to osd.5 from (0'0,0'0] MAX to 414'11697 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: pgmap v1162: 109 pgs: 7 peering, 1 active+recovery_wait+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 652 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 17679/31350 objects degraded (56.392%); 346/31350 objects misplaced (1.104%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3269505237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:14.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:14.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:19:14.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:14 smithi012 ceph-mon[147926]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T10:19:14.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:14.983 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:15.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[152263]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[152263]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[152263]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[152263]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:19:15.611 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[147926]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[147926]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[147926]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[147926]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:15 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:19:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:15 smithi110 ceph-mon[136032]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:15 smithi110 ceph-mon[136032]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:15 smithi110 ceph-mon[136032]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:15 smithi110 ceph-mon[136032]: failed to encode map e418 with expected crc 2023-10-09T10:19:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:15 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:19:16.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:16 smithi012 ceph-mon[147926]: pgmap v1164: 109 pgs: 7 peering, 6 active+recovery_wait+degraded, 2 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 578 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 13941/27801 objects degraded (50.146%); 4309/27801 objects misplaced (15.499%); 6.8 MiB/s, 9 keys/s, 230 objects/s recovering 2023-10-09T10:19:16.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:16 smithi012 ceph-mon[152263]: pgmap v1164: 109 pgs: 7 peering, 6 active+recovery_wait+degraded, 2 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 578 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 13941/27801 objects degraded (50.146%); 4309/27801 objects misplaced (15.499%); 6.8 MiB/s, 9 keys/s, 230 objects/s recovering 2023-10-09T10:19:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:16 smithi110 ceph-mon[136032]: pgmap v1164: 109 pgs: 7 peering, 6 active+recovery_wait+degraded, 2 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 578 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 13941/27801 objects degraded (50.146%); 4309/27801 objects misplaced (15.499%); 6.8 MiB/s, 9 keys/s, 230 objects/s recovering 2023-10-09T10:19:17.138 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5] out_osds: [4] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:19:17.139 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:19:17.139 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.7460059837482835 2023-10-09T10:19:17.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 0.7460059837482835 2023-10-09T10:19:17.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:17 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:19:17.994 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:19:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:19:17] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-10-09T10:19:17.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:17 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:19:18.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:17 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:19:18.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:19:18.539 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:18.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:18.645 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:18.709 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:18.777 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:18.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:18.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:18.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:18 smithi012 ceph-mon[147926]: pgmap v1165: 109 pgs: 6 active+recovery_wait+degraded, 2 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 558 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 13941/26853 objects degraded (51.916%); 4309/26853 objects misplaced (16.047%); 4.7 MiB/s, 6 keys/s, 158 objects/s recovering 2023-10-09T10:19:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 13941/26853 objects degraded (51.916%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:18.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:18 smithi012 ceph-mon[152263]: pgmap v1165: 109 pgs: 6 active+recovery_wait+degraded, 2 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 558 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 13941/26853 objects degraded (51.916%); 4309/26853 objects misplaced (16.047%); 4.7 MiB/s, 6 keys/s, 158 objects/s recovering 2023-10-09T10:19:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 13941/26853 objects degraded (51.916%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:18.936 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:18.950 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:19.011 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:19.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:19.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:19.124 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:19.183 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:18 smithi110 ceph-mon[136032]: pgmap v1165: 109 pgs: 6 active+recovery_wait+degraded, 2 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 558 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 13941/26853 objects degraded (51.916%); 4309/26853 objects misplaced (16.047%); 4.7 MiB/s, 6 keys/s, 158 objects/s recovering 2023-10-09T10:19:19.184 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 13941/26853 objects degraded (51.916%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:19.186 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:19.260 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:19.264 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:19:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:19.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:19.294 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:19:19.325 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:19:19.363 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:19:19.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:19.389 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:19.397 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:19:19.437 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:19:19.439 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:19.471 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:19:19.498 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:19:19.500 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:19:19.500 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:19:19.500 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:19:19.500 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:19:19.500 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:19:19.501 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:19:19.501 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:19:19.501 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:19:19.508 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:19.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:19.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:19.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:19.675 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:19.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1535275193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:19.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1535275193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:20.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1535275193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:20.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:19:20.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:20 smithi012 ceph-mon[147926]: pgmap v1166: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 489 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 13682/23541 objects degraded (58.120%); 3931/23541 objects misplaced (16.699%); 6.2 MiB/s, 5 keys/s, 181 objects/s recovering 2023-10-09T10:19:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:20 smithi012 ceph-mon[152263]: pgmap v1166: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 489 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 13682/23541 objects degraded (58.120%); 3931/23541 objects misplaced (16.699%); 6.2 MiB/s, 5 keys/s, 181 objects/s recovering 2023-10-09T10:19:21.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:20 smithi110 ceph-mon[136032]: pgmap v1166: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 489 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 13682/23541 objects degraded (58.120%); 3931/23541 objects misplaced (16.699%); 6.2 MiB/s, 5 keys/s, 181 objects/s recovering 2023-10-09T10:19:21.714 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:21.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:21.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:21.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:21.866 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:21.922 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:21.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:21.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:22.035 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:22.104 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:22.183 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:22.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:22.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:22.295 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:22.364 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:22.419 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:22.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:22.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:22.629 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:22.690 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:22.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:22.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:22.794 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:22.864 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:22.867 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:22 smithi110 ceph-mon[136032]: pgmap v1167: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 387 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 13272/18603 objects degraded (71.343%); 3680/18603 objects misplaced (19.782%); 7.7 MiB/s, 4 keys/s, 201 objects/s recovering 2023-10-09T10:19:22.944 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:22.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:22.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:23.051 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:23.110 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:23.164 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:23.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:23.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:23.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:22 smithi012 ceph-mon[152263]: pgmap v1167: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 387 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 13272/18603 objects degraded (71.343%); 3680/18603 objects misplaced (19.782%); 7.7 MiB/s, 4 keys/s, 201 objects/s recovering 2023-10-09T10:19:23.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:22 smithi012 ceph-mon[147926]: pgmap v1167: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 387 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 13272/18603 objects degraded (71.343%); 3680/18603 objects misplaced (19.782%); 7.7 MiB/s, 4 keys/s, 201 objects/s recovering 2023-10-09T10:19:23.275 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:23.345 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:23.410 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:23.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:23.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:23.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:23.576 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:23.642 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:23.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:23.748 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:23.814 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:23.887 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:23.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:23.893 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 13272/18603 objects degraded (71.343%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:23.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:23.995 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:24.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:19:24.048 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:24.071 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 13272/18603 objects degraded (71.343%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:24.071 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 13272/18603 objects degraded (71.343%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:24.115 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:24.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:24.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:24.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:24.279 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:24.969 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:24.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:25.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:25.112 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:25.276 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:25 smithi012 ceph-mon[147926]: pgmap v1168: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 347 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 13272/16725 objects degraded (79.354%); 3680/16725 objects misplaced (22.003%); 6.4 MiB/s, 3 keys/s, 168 objects/s recovering 2023-10-09T10:19:25.296 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:25.304 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:25.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:25.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:25.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:25.446 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:25.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:25 smithi012 ceph-mon[152263]: pgmap v1168: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 347 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 13272/16725 objects degraded (79.354%); 3680/16725 objects misplaced (22.003%); 6.4 MiB/s, 3 keys/s, 168 objects/s recovering 2023-10-09T10:19:25.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:25 smithi110 ceph-mon[136032]: pgmap v1168: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 347 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 13272/16725 objects degraded (79.354%); 3680/16725 objects misplaced (22.003%); 6.4 MiB/s, 3 keys/s, 168 objects/s recovering 2023-10-09T10:19:26.290 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:26.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:26.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:26.401 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:26.473 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:26.522 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:26.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:26.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:26.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:26.728 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:26.789 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:26.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:26.899 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:26.968 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:26.986 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:19:26.986 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:19:26.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:19:26.986 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:19:26.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.986 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114935 9.f 9:f4378248:::benchmark_data_smithi012_313365_object2460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:19:26.987 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:19:26.987 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 252.81917051299999, 2023-10-09T10:19:26.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2871666070000001, 2023-10-09T10:19:26.987 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:26.987 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:19:26.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:26.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:26.987 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:26.988 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114935 2023-10-09T10:19:26.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.988 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:26.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162479+0000", 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162485+0000", 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3759999999999996e-06 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162497+0000", 2023-10-09T10:19:26.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162503+0000", 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9770000000000003e-06 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174344+0000", 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01184062 2023-10-09T10:19:26.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174510+0000", 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000166026 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174655+0000", 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000144563 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.181694+0000", 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007039694 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.214204+0000", 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.03250981 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:26.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.222717+0000", 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085130050000000006 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.222730+0000", 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3229e-05 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196525+0000", 2023-10-09T10:19:26.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97379490400000002 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196531+0000", 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.879e-06 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.428687+0000", 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.232155509 2023-10-09T10:19:26.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.428882+0000", 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019557999999999999 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.449518+0000", 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020635767999999999 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:26.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.449647+0000", 2023-10-09T10:19:26.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000129014 2023-10-09T10:19:26.997 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:26.997 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:26.997 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:26.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114936 9.a 9:58a6faa8:::benchmark_data_smithi012_313365_object2461:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 252.81913980100001, 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.092944401, 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:26.998 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:26.999 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114936 2023-10-09T10:19:26.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:26.999 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:26.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:26.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:26.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:19:26.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:26.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:19:27.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:27.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162513+0000", 2023-10-09T10:19:27.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1610000000000001e-06 2023-10-09T10:19:27.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162518+0000", 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7310000000000001e-06 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162522+0000", 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9920000000000001e-06 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164493+0000", 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019717620000000002 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164686+0000", 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019211 2023-10-09T10:19:27.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164874+0000", 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018879699999999999 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.168656+0000", 2023-10-09T10:19:27.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037811670000000002 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.169482+0000", 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082636199999999997 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.212142+0000", 2023-10-09T10:19:27.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042660245999999999 2023-10-09T10:19:27.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.212158+0000", 2023-10-09T10:19:27.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5977999999999998e-05 2023-10-09T10:19:27.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.195199+0000", 2023-10-09T10:19:27.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98304075999999996 2023-10-09T10:19:27.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.195213+0000", 2023-10-09T10:19:27.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4025e-05 2023-10-09T10:19:27.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236218+0000", 2023-10-09T10:19:27.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0410054230000001 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236372+0000", 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015330899999999999 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255344+0000", 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018972025 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255456+0000", 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111928 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114940 9.f 9:f2b30a6a:::benchmark_data_smithi012_313365_object2465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 252.808034951, 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2912956169999998, 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114940 2023-10-09T10:19:27.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173614+0000", 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173632+0000", 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8349999999999999e-05 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173651+0000", 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8372e-05 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173660+0000", 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0429999999999996e-06 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.177952+0000", 2023-10-09T10:19:27.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042923120000000004 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.178136+0000", 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018356399999999999 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.178304+0000", 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016803400000000001 2023-10-09T10:19:27.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.181873+0000", 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035693090000000001 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.214244+0000", 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032370566000000003 2023-10-09T10:19:27.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223012+0000", 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087685690000000004 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223019+0000", 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196547+0000", 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97352829299999999 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196551+0000", 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9949999999999999e-06 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.434365+0000", 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2378136799999999 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.434545+0000", 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000180044 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.464846+0000", 2023-10-09T10:19:27.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030301600000000001 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.464912+0000", 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5380000000000001e-05 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114950 9.f 9:f115a5aa:::benchmark_data_smithi012_313365_object2475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:19:27.020 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 252.75802365999999, 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2705686389999999, 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114950 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223625+0000", 2023-10-09T10:19:27.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223632+0000", 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8639999999999998e-06 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223644+0000", 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1824000000000001e-05 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223651+0000", 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5499999999999997e-06 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223694+0000", 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2765999999999997e-05 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223699+0000", 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2569999999999998e-06 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196570+0000", 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97287036000000005 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196576+0000", 2023-10-09T10:19:27.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5710000000000003e-06 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.439509+0000", 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.242932758 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.440273+0000", 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00076375 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474247+0000", 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033973640999999999 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474363+0000", 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116765 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:19:27.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474458+0000", 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4829999999999998e-05 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.479012+0000", 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045542890000000004 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.481102+0000", 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020900659999999998 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.489625+0000", 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085221180000000004 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.489680+0000", 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5405999999999998e-05 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.494196+0000", 2023-10-09T10:19:27.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045159930000000003 2023-10-09T10:19:27.030 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.030 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.030 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.030 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115069 9.10 9:0c166afc:::benchmark_data_smithi012_313365_object2594:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 250.43600383699999, 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5496961840000001, 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115069 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.031 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545646+0000", 2023-10-09T10:19:27.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545653+0000", 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5399999999999998e-06 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545670+0000", 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7144000000000001e-05 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545679+0000", 2023-10-09T10:19:27.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5830000000000007e-06 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545730+0000", 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1313000000000002e-05 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546723+0000", 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099291799999999997 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079001+0000", 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5322773119999997 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079100+0000", 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9010000000000002e-05 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079372+0000", 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000272782 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.080850+0000", 2023-10-09T10:19:27.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014771509999999999 2023-10-09T10:19:27.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.082687+0000", 2023-10-09T10:19:27.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001837336 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.082730+0000", 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3612999999999997e-05 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093025+0000", 2023-10-09T10:19:27.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010295011 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093094+0000", 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8725000000000004e-05 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095343+0000", 2023-10-09T10:19:27.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022491519999999999 2023-10-09T10:19:27.041 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.041 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.041 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.041 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115106 9.10 9:0b40069f:::benchmark_data_smithi012_313365_object2631:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:19:27.041 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:19:27.042 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 250.297819132, 2023-10-09T10:19:27.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4133900090000004, 2023-10-09T10:19:27.042 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.042 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.042 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115106 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683831+0000", 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683836+0000", 2023-10-09T10:19:27.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0440000000000003e-06 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683846+0000", 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9250000000000007e-06 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683854+0000", 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7959999999999996e-06 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683942+0000", 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7602000000000006e-05 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.684891+0000", 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094994899999999995 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089254+0000", 2023-10-09T10:19:27.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4043629649999998 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089396+0000", 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141886 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089758+0000", 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036181000000000001 2023-10-09T10:19:27.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.091841+0000", 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020825930000000002 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095413+0000", 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035718400000000002 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095477+0000", 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4685999999999997e-05 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095503+0000", 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5575000000000002e-05 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095538+0000", 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5116000000000001e-05 2023-10-09T10:19:27.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.097222+0000", 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001683941 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122337 9.10 9:0bd6554e:::benchmark_data_smithi012_313365_object9862:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:19:27.050 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 216.084671697, 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6698403750000002, 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122337 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896978+0000", 2023-10-09T10:19:27.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896984+0000", 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7479999999999999e-06 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896996+0000", 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.199e-05 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897001+0000", 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8189999999999997e-06 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897056+0000", 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4818999999999998e-05 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897145+0000", 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8943000000000005e-05 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.517542+0000", 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6203965519999999 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.517672+0000", 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013004499999999999 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.518031+0000", 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035930199999999998 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.536908+0000", 2023-10-09T10:19:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018877416000000001 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.536918+0000", 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9369999999999999e-06 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.537422+0000", 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00050323499999999999 2023-10-09T10:19:27.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564538+0000", 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027116735999999999 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564585+0000", 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6592999999999999e-05 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.566820+0000", 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022347140000000001 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122523 9.10 9:0d3d12ba:::benchmark_data_smithi012_313365_object10048:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 215.53932240500001, 2023-10-09T10:19:27.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7874849670000001, 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122523 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:19:27.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442327+0000", 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442334+0000", 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.221e-06 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442347+0000", 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2809000000000001e-05 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442355+0000", 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0760000000000003e-06 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442400+0000", 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4570000000000002e-05 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442467+0000", 2023-10-09T10:19:27.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7063999999999994e-05 2023-10-09T10:19:27.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194401+0000", 2023-10-09T10:19:27.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7519345159999999 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194518+0000", 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116872 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194929+0000", 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041051599999999998 2023-10-09T10:19:27.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.211495+0000", 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016566498999999998 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.211544+0000", 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9190999999999997e-05 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.214081+0000", 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025366910000000002 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.227445+0000", 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013363808 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.227497+0000", 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1860000000000002e-05 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.229814+0000", 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023168160000000002 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122561 9.10 9:092dbf94:::benchmark_data_smithi012_313365_object10086:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 215.44021333399999, 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1569061710000001, 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.069 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.070 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122561 2023-10-09T10:19:27.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.070 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:19:27.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541437+0000", 2023-10-09T10:19:27.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541441+0000", 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1919999999999998e-06 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541449+0000", 2023-10-09T10:19:27.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.746e-06 2023-10-09T10:19:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541454+0000", 2023-10-09T10:19:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4639999999999997e-06 2023-10-09T10:19:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541494+0000", 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9505999999999997e-05 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541543+0000", 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9872e-05 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.660487+0000", 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1189437330000001 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.660622+0000", 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135024 2023-10-09T10:19:27.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.661025+0000", 2023-10-09T10:19:27.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040268299999999998 2023-10-09T10:19:27.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.668459+0000", 2023-10-09T10:19:27.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074342699999999998 2023-10-09T10:19:27.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.668490+0000", 2023-10-09T10:19:27.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1279999999999999e-05 2023-10-09T10:19:27.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683619+0000", 2023-10-09T10:19:27.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015128694999999999 2023-10-09T10:19:27.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.696066+0000", 2023-10-09T10:19:27.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012446472 2023-10-09T10:19:27.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.696109+0000", 2023-10-09T10:19:27.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3887999999999997e-05 2023-10-09T10:19:27.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698344+0000", 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022343480000000001 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122689 9.10 9:0a3555ea:::benchmark_data_smithi012_313365_object10214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:19:27.080 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 215.02679582299999, 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7453082640000002, 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122689 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954853+0000", 2023-10-09T10:19:27.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954862+0000", 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2959999999999999e-06 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954874+0000", 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2072000000000001e-05 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954882+0000", 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1030000000000002e-06 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954938+0000", 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6165999999999997e-05 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.084 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.955019+0000", 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1001000000000006e-05 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.666823+0000", 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7118038929999999 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.666922+0000", 2023-10-09T10:19:27.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8803999999999998e-05 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.667247+0000", 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032501299999999997 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683704+0000", 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016457423999999998 2023-10-09T10:19:27.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683804+0000", 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100046 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683814+0000", 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0083e-05 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698412+0000", 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014598266 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698458+0000", 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5503999999999997e-05 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.700163+0000", 2023-10-09T10:19:27.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001705441 2023-10-09T10:19:27.090 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.090 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.090 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122759 9.10 9:0b678b9b:::benchmark_data_smithi012_313365_object10284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 214.85714398299999, 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.605785577, 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.091 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122759 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:19:27.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124506+0000", 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124513+0000", 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124528+0000", 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4893e-05 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124536+0000", 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1319999999999994e-06 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124601+0000", 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5038999999999999e-05 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124689+0000", 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7894999999999998e-05 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704065+0000", 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5793759000000001 2023-10-09T10:19:27.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704190+0000", 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012584799999999999 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704594+0000", 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040309699999999999 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.727961+0000", 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023367024 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728012+0000", 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1174000000000002e-05 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728024+0000", 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1966e-05 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728058+0000", 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3911000000000001e-05 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728098+0000", 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0286e-05 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.730293+0000", 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021946410000000002 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.099 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124410 9.10 9:0a0e0dea:::benchmark_data_smithi012_313365_object11935:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 210.278696999, 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5937443060000001, 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124410 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702952+0000", 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702960+0000", 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6920000000000002e-06 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702975+0000", 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4969e-05 2023-10-09T10:19:27.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702983+0000", 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7510000000000008e-06 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.707988+0000", 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050051440000000004 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.708039+0000", 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0998999999999997e-05 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.196596+0000", 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.488557455 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.196709+0000", 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000113075 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.197066+0000", 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035631299999999997 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.198754+0000", 2023-10-09T10:19:27.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016883169999999999 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.216476+0000", 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017721654 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.216545+0000", 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8888999999999997e-05 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.294372+0000", 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.077827269000000004 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.294431+0000", 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9001e-05 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.296698+0000", 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00226742 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124495 9.10 9:0df9feaa:::benchmark_data_smithi012_313365_object12020:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 209.97920746400001, 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6452444429999999, 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.108 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124495 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002442+0000", 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002451+0000", 2023-10-09T10:19:27.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6459999999999999e-06 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002465+0000", 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4531e-05 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002478+0000", 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3091999999999999e-05 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002543+0000", 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5003000000000001e-05 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002642+0000", 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8486000000000005e-05 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604276+0000", 2023-10-09T10:19:27.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.601633879 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604435+0000", 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000159295 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604865+0000", 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043062800000000001 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613177+0000", 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0083113730000000004 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613227+0000", 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9805000000000002e-05 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613271+0000", 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4691999999999998e-05 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645599+0000", 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032327910000000001 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645659+0000", 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9821000000000002e-05 2023-10-09T10:19:27.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647688+0000", 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020288989999999998 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124499 9.10 9:0914df8f:::benchmark_data_smithi012_313365_object12024:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.116 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 209.96277368200001, 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6316283970000001, 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124499 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018876+0000", 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018883+0000", 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2869999999999999e-06 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018893+0000", 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0190999999999999e-05 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018900+0000", 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2099999999999996e-06 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018964+0000", 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.355e-05 2023-10-09T10:19:27.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.019054+0000", 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0099999999999995e-05 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611351+0000", 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5922972340000001 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611460+0000", 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108608 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611836+0000", 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037601000000000003 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.621012+0000", 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009175753 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645527+0000", 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024515678999999999 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645566+0000", 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8294000000000003e-05 2023-10-09T10:19:27.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647904+0000", 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023386240000000001 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647967+0000", 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2964000000000002e-05 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.650506+0000", 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025385540000000002 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124527 9.10 9:0cb84cd4:::benchmark_data_smithi012_313365_object12052:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 209.875240139, 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5734285209999999, 2023-10-09T10:19:27.125 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124527 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106409+0000", 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106417+0000", 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2060000000000001e-06 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106431+0000", 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4693e-05 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106440+0000", 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4649999999999994e-06 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106537+0000", 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7021000000000005e-05 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106637+0000", 2023-10-09T10:19:27.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010021799999999999 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.617803+0000", 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5111660599999999 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.617924+0000", 2023-10-09T10:19:27.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012099600000000001 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.618302+0000", 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037780300000000001 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643413+0000", 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025111018999999998 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647753+0000", 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043399199999999997 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647820+0000", 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7353000000000004e-05 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.677556+0000", 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029735927999999998 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.677618+0000", 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2069000000000005e-05 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.679839+0000", 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022211359999999999 2023-10-09T10:19:27.133 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124541 9.10 9:0fc113d5:::benchmark_data_smithi012_313365_object12066:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 209.77727760299999, 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.506778945, 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124541 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204372+0000", 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204381+0000", 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9970000000000004e-06 2023-10-09T10:19:27.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204402+0000", 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0761999999999999e-05 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204412+0000", 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0368e-05 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204473+0000", 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1490999999999998e-05 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204558+0000", 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4437999999999994e-05 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643445+0000", 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4388867670000001 2023-10-09T10:19:27.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643565+0000", 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120537 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643947+0000", 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038192800000000002 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647844+0000", 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038965330000000002 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.652834+0000", 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049904190000000003 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.652889+0000", 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4759000000000002e-05 2023-10-09T10:19:27.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.708684+0000", 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.05579493 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.708750+0000", 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6202000000000001e-05 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.711152+0000", 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024023360000000001 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128019 9.a 9:56d43b6a:::benchmark_data_smithi012_313365_object15544:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 199.32836162199999, 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.955188565, 2023-10-09T10:19:27.142 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128019 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653288+0000", 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653296+0000", 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.79e-06 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653308+0000", 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2376e-05 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653323+0000", 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4844000000000001e-05 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653379+0000", 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6054999999999998e-05 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653595+0000", 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021573 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:19:27.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653860+0000", 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026534600000000002 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.696431+0000", 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042570967000000001 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.696443+0000", 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1739000000000001e-05 2023-10-09T10:19:27.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.323819+0000", 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.62737651100000003 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.323833+0000", 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3869000000000001e-05 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.494433+0000", 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1706003599999999 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.494639+0000", 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020562600000000001 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.608361+0000", 2023-10-09T10:19:27.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.113721818 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.608478+0000", 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011719500000000001 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.150 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128025 9.f 9:f78d4616:::benchmark_data_smithi012_313365_object15550:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 199.16615450099999, 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.792284043, 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128025 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815495+0000", 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815501+0000", 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815513+0000", 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1770000000000001e-05 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815520+0000", 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1010000000000003e-06 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815583+0000", 2023-10-09T10:19:27.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2779000000000003e-05 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815595+0000", 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2153999999999999e-05 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.227751+0000", 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.41215553100000002 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.227764+0000", 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3131e-05 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.482847+0000", 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.255083368 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.483568+0000", 2023-10-09T10:19:27.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00072077100000000004 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.582946+0000", 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.099378280999999999 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.583074+0000", 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012746599999999999 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.583609+0000", 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00053539899999999999 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.595736+0000", 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012126994 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.599249+0000", 2023-10-09T10:19:27.158 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035130600000000001 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.602352+0000", 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031027540000000001 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.603699+0000", 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001347284 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.603770+0000", 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1014999999999997e-05 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.607781+0000", 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004010319 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:195679 9.3 9:c12d9ed3:::benchmark_data_smithi012_313365_object83204:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 51.545432624999997, 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9465798859999999, 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.161 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195679 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:19:27.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436217+0000", 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436223+0000", 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9850000000000003e-06 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436235+0000", 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1229e-05 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436243+0000", 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9789999999999993e-06 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.164 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436300+0000", 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7265999999999997e-05 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436951+0000", 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00065154099999999995 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345069+0000", 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.90811741 2023-10-09T10:19:27.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345197+0000", 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128496 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345622+0000", 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042431400000000002 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.166 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.347037+0000", 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014153449999999999 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.348160+0000", 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001123097 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382157+0000", 2023-10-09T10:19:27.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033996849000000003 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382214+0000", 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7012000000000002e-05 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382798+0000", 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058430099999999996 2023-10-09T10:19:27.168 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:195683 9.13 9:c8f1253c:::benchmark_data_smithi012_313365_object83208:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 51.540210584, 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9973604489999999, 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:27.169 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195683 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441439+0000", 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:19:27.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441447+0000", 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4549999999999995e-06 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441463+0000", 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5675000000000002e-05 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441471+0000", 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3100000000000001e-06 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441521+0000", 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9709999999999997e-05 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.442142+0000", 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062057699999999998 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:19:27.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383458+0000", 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.941316144 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383530+0000", 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2574e-05 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383875+0000", 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00034479399999999999 2023-10-09T10:19:27.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.386609+0000", 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027342930000000001 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.386879+0000", 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026999800000000003 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.437748+0000", 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.050869079999999997 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.437820+0000", 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1427999999999994e-05 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.438801+0000", 2023-10-09T10:19:27.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00098100300000000004 2023-10-09T10:19:27.177 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.177 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.177 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.177 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:19:27.177 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:19:27.177 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:19:27.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:27.178 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:27.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:27.207 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:27.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:27.330 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:27.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:27 smithi012 ceph-mon[147926]: pgmap v1169: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 173 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 12550/8370 objects degraded (149.940%); 3021/8370 objects misplaced (36.093%); 9.1 MiB/s, 3 keys/s, 209 objects/s recovering 2023-10-09T10:19:27.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:27 smithi012 ceph-mon[152263]: pgmap v1169: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 173 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 12550/8370 objects degraded (149.940%); 3021/8370 objects misplaced (36.093%); 9.1 MiB/s, 3 keys/s, 209 objects/s recovering 2023-10-09T10:19:27.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:27 smithi110 ceph-mon[136032]: pgmap v1169: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 173 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 12550/8370 objects degraded (149.940%); 3021/8370 objects misplaced (36.093%); 9.1 MiB/s, 3 keys/s, 209 objects/s recovering 2023-10-09T10:19:27.627 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:27.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:27.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:27.734 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:27.804 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:27.835 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:27.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:27.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:27.939 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:27.962 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Removed 112472 objects 2023-10-09T10:19:27.963 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Clean up completed and total clean up time :258.638 2023-10-09T10:19:27.966 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:28.006 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:28.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:28.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:28.111 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:28.135 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:28.168 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:28.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:28.176 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:19:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:19:27] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-10-09T10:19:28.271 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:28.307 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.2 to 0.746006 (befa) 2023-10-09T10:19:28.310 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:28.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3570920012' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7460059837482835}]: dispatch 2023-10-09T10:19:28.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 12550/8370 objects degraded (149.940%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3570920012' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7460059837482835}]: dispatch 2023-10-09T10:19:28.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 12550/8370 objects degraded (149.940%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:28.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3570920012' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7460059837482835}]: dispatch 2023-10-09T10:19:28.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 12550/8370 objects degraded (149.940%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[147926]: pgmap v1170: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 135 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 12550/6555 objects degraded (191.457%); 3021/6555 objects misplaced (46.087%); 6.3 MiB/s, 119 objects/s recovering 2023-10-09T10:19:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[147926]: failed to encode map e419 with expected crc 2023-10-09T10:19:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[147926]: failed to encode map e419 with expected crc 2023-10-09T10:19:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3570920012' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7460059837482835}]': finished 2023-10-09T10:19:29.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[147926]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[152263]: pgmap v1170: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 135 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 12550/6555 objects degraded (191.457%); 3021/6555 objects misplaced (46.087%); 6.3 MiB/s, 119 objects/s recovering 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[152263]: failed to encode map e419 with expected crc 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[152263]: failed to encode map e419 with expected crc 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3570920012' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7460059837482835}]': finished 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[152263]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:29.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:29 smithi110 ceph-mon[136032]: pgmap v1170: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 135 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 12550/6555 objects degraded (191.457%); 3021/6555 objects misplaced (46.087%); 6.3 MiB/s, 119 objects/s recovering 2023-10-09T10:19:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:29 smithi110 ceph-mon[136032]: failed to encode map e419 with expected crc 2023-10-09T10:19:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:29 smithi110 ceph-mon[136032]: failed to encode map e419 with expected crc 2023-10-09T10:19:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3570920012' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7460059837482835}]': finished 2023-10-09T10:19:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:29 smithi110 ceph-mon[136032]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T10:19:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:30.050 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:19:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[152263]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[152263]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[152263]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T10:19:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[152263]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[152263]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.316 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[152263]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:30 smithi110 ceph-mon[136032]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:30 smithi110 ceph-mon[136032]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:30 smithi110 ceph-mon[136032]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T10:19:30.456 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:30 smithi110 ceph-mon[136032]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:30 smithi110 ceph-mon[136032]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.457 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:30 smithi110 ceph-mon[136032]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[147926]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[147926]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[147926]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T10:19:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[147926]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[147926]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:30 smithi012 ceph-mon[147926]: failed to encode map e420 with expected crc 2023-10-09T10:19:30.758 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:31.431 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:31.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:31.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:31.467 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T10:19:31.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T10:19:31.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:31.566 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:31.603 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:31.605 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: pgmap v1173: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 88 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 12357/4254 objects degraded (290.480%); 2754/4254 objects misplaced (64.739%); 6.4 MiB/s, 111 objects/s recovering 2023-10-09T10:19:31.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T10:19:31.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 418'10318 2023-10-09T10:19:31.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: 9.b starting backfill to osd.1 from (0'0,0'0] MAX to 418'11841 2023-10-09T10:19:31.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: 9.1b starting backfill to osd.1 from (0'0,0'0] MAX to 418'11897 2023-10-09T10:19:31.606 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:31 smithi110 ceph-mon[136032]: 9.16 starting backfill to osd.6 from (0'0,0'0] MAX to 418'10365 2023-10-09T10:19:31.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:31.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: pgmap v1173: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 88 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 12357/4254 objects degraded (290.480%); 2754/4254 objects misplaced (64.739%); 6.4 MiB/s, 111 objects/s recovering 2023-10-09T10:19:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T10:19:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 418'10318 2023-10-09T10:19:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: 9.b starting backfill to osd.1 from (0'0,0'0] MAX to 418'11841 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: 9.1b starting backfill to osd.1 from (0'0,0'0] MAX to 418'11897 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[147926]: 9.16 starting backfill to osd.6 from (0'0,0'0] MAX to 418'10365 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: pgmap v1173: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 6 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 88 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 12357/4254 objects degraded (290.480%); 2754/4254 objects misplaced (64.739%); 6.4 MiB/s, 111 objects/s recovering 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: failed to encode map e421 with expected crc 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 418'10318 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: 9.b starting backfill to osd.1 from (0'0,0'0] MAX to 418'11841 2023-10-09T10:19:31.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: 9.1b starting backfill to osd.1 from (0'0,0'0] MAX to 418'11897 2023-10-09T10:19:31.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:31 smithi012 ceph-mon[152263]: 9.16 starting backfill to osd.6 from (0'0,0'0] MAX to 418'10365 2023-10-09T10:19:31.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:31.736 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:32.072 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:19:32.072 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:19:32.072 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:19:32.072 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:19:32.253 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:32.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:32.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:32.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:32.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:32.470 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:32.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:32.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:19:32.576 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:32.602 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:32.638 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:32.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:32.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:32.743 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:32.769 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:32.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:32.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:32.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:32.906 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:32.931 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:32.968 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:32.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:32.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:33.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:33.105 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:33.106 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:33.140 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:33.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:33.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:33.189 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:33.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:33.277 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:33.417 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:33.417 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:33 smithi012 ceph-mon[147926]: pgmap v1175: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait, 6 active+remapped+backfill_wait, 88 active+clean; 29 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 4150/1449 objects degraded (286.404%); 38/1449 objects misplaced (2.622%); 4.1 MiB/s, 79 objects/s recovering 2023-10-09T10:19:33.417 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:33 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:19:33.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4150/1449 objects degraded (286.404%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:33.418 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:33 smithi012 ceph-mon[152263]: pgmap v1175: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait, 6 active+remapped+backfill_wait, 88 active+clean; 29 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 4150/1449 objects degraded (286.404%); 38/1449 objects misplaced (2.622%); 4.1 MiB/s, 79 objects/s recovering 2023-10-09T10:19:33.418 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:33 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:19:33.418 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4150/1449 objects degraded (286.404%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:33.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:33.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:33.551 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:33.577 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:33 smithi110 ceph-mon[136032]: pgmap v1175: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait, 6 active+remapped+backfill_wait, 88 active+clean; 29 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 4150/1449 objects degraded (286.404%); 38/1449 objects misplaced (2.622%); 4.1 MiB/s, 79 objects/s recovering 2023-10-09T10:19:33.577 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:33 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:19:33.577 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4150/1449 objects degraded (286.404%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:33.684 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:33.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:33.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:33.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:33.819 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:33.853 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:33.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:33.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:33.985 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:34.218 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:34.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:34.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:34.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:34.335 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:19:34.361 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:34.421 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:19:34.445 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:19:34.462 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:19:34.483 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:19:34.497 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:34 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2351309728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:34.497 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:34 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:34.501 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:19:34.519 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:19:34.538 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:19:34.556 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:19:34.557 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:19:34.557 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:19:34.557 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:19:34.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:19:34.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:19:34.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:19:34.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:19:34.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:19:34.564 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:34.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:34.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:34.584 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:34 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2351309728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:34.584 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:34 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:34.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:34 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2351309728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:34.584 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:34 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:34.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:34.702 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:34.739 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:19:34.740 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:19:34.989 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:34.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:34.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:35.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:35.122 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:35.241 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:19:35.262 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:35.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:35.366 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:35.394 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:35.436 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:35.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:35.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:35.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[152263]: pgmap v1176: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait, 6 active+remapped+backfill_wait, 88 active+clean; 27 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 328 op/s; 4150/1344 objects degraded (308.780%); 38/1344 objects misplaced (2.827%); 4.1 MiB/s, 79 objects/s recovering 2023-10-09T10:19:35.566 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:35.566 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[152263]: failed to encode map e422 with expected crc 2023-10-09T10:19:35.566 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[152263]: failed to encode map e422 with expected crc 2023-10-09T10:19:35.566 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:35.566 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[152263]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T10:19:35.566 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4029813214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:35.566 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[147926]: pgmap v1176: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait, 6 active+remapped+backfill_wait, 88 active+clean; 27 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 328 op/s; 4150/1344 objects degraded (308.780%); 38/1344 objects misplaced (2.827%); 4.1 MiB/s, 79 objects/s recovering 2023-10-09T10:19:35.566 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:35.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[147926]: failed to encode map e422 with expected crc 2023-10-09T10:19:35.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[147926]: failed to encode map e422 with expected crc 2023-10-09T10:19:35.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:35.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[147926]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T10:19:35.567 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4029813214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:35.568 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:35.609 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:35.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:35.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:35.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:35 smithi110 ceph-mon[136032]: pgmap v1176: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 7 active+recovery_wait+degraded, 1 active+recovery_wait, 6 active+remapped+backfill_wait, 88 active+clean; 27 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 328 op/s; 4150/1344 objects degraded (308.780%); 38/1344 objects misplaced (2.827%); 4.1 MiB/s, 79 objects/s recovering 2023-10-09T10:19:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:35 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:35 smithi110 ceph-mon[136032]: failed to encode map e422 with expected crc 2023-10-09T10:19:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:35 smithi110 ceph-mon[136032]: failed to encode map e422 with expected crc 2023-10-09T10:19:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:35 smithi110 ceph-mon[136032]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T10:19:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4029813214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:35.713 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:35.739 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:35.773 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:35.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:35.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:35.877 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:35.907 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:36.049 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:36.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:36.098 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:36.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:36.181 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:36.218 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:36.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:36.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:36.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:36.323 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:36 smithi110 ceph-mon[136032]: failed to encode map e422 with expected crc 2023-10-09T10:19:36.323 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:36 smithi110 ceph-mon[136032]: failed to encode map e422 with expected crc 2023-10-09T10:19:36.323 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4214855432' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:19:36.323 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:19:36.337 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_3' removed 2023-10-09T10:19:36.373 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:36.491 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:36 smithi012 ceph-mon[147926]: failed to encode map e422 with expected crc 2023-10-09T10:19:36.492 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:36 smithi012 ceph-mon[147926]: failed to encode map e422 with expected crc 2023-10-09T10:19:36.492 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4214855432' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:19:36.492 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:19:36.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:36 smithi012 ceph-mon[152263]: failed to encode map e422 with expected crc 2023-10-09T10:19:36.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:36 smithi012 ceph-mon[152263]: failed to encode map e422 with expected crc 2023-10-09T10:19:36.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4214855432' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:19:36.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:19:36.702 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:19:36.702 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:19:36.702 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T10:19:36.703 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T10:19:36.729 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:36.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:36.832 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:36.863 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:36.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:19:37.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:19:37.303 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5] out_osds: [4] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:19:37.304 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:19:37.304 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T10:19:37.304 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T10:19:37.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[152263]: pgmap v1178: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 89 active+clean; 27 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 12035/1344 objects degraded (895.461%); 51/1344 objects misplaced (3.795%); 3.9 MiB/s, 93 objects/s recovering 2023-10-09T10:19:37.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[152263]: failed to encode map e423 with expected crc 2023-10-09T10:19:37.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:19:37.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[152263]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T10:19:37.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[152263]: failed to encode map e423 with expected crc 2023-10-09T10:19:37.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[147926]: pgmap v1178: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 89 active+clean; 27 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 12035/1344 objects degraded (895.461%); 51/1344 objects misplaced (3.795%); 3.9 MiB/s, 93 objects/s recovering 2023-10-09T10:19:37.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[147926]: failed to encode map e423 with expected crc 2023-10-09T10:19:37.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:19:37.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[147926]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T10:19:37.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-mon[147926]: failed to encode map e423 with expected crc 2023-10-09T10:19:37.674 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:37 smithi110 ceph-mon[136032]: pgmap v1178: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 89 active+clean; 27 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 12035/1344 objects degraded (895.461%); 51/1344 objects misplaced (3.795%); 3.9 MiB/s, 93 objects/s recovering 2023-10-09T10:19:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:37 smithi110 ceph-mon[136032]: failed to encode map e423 with expected crc 2023-10-09T10:19:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:19:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:37 smithi110 ceph-mon[136032]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T10:19:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:37 smithi110 ceph-mon[136032]: failed to encode map e423 with expected crc 2023-10-09T10:19:37.758 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:37.859 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:38.171 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:19:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:19:37] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.33.4" 2023-10-09T10:19:38.352 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:38.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:38.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:38.487 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:38.538 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:38.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 12035/1344 objects degraded (895.461%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:38.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 12035/1344 objects degraded (895.461%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:38.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 12035/1344 objects degraded (895.461%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:19:38.956 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:38.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:39.065 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:39.093 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:39.128 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:39.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:39.144 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:19:39.145 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:19:39.232 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:39.262 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:39.401 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:39 smithi110 ceph-mon[136032]: pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:39.401 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:39 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12035/1344 objects degraded (895.461%), 10 pgs degraded) 2023-10-09T10:19:39.541 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:39.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:39.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:39.580 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:19:39.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:39 smithi012 ceph-mon[147926]: pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:39 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12035/1344 objects degraded (895.461%), 10 pgs degraded) 2023-10-09T10:19:39.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:39 smithi012 ceph-mon[152263]: pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:39.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:39 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12035/1344 objects degraded (895.461%), 10 pgs degraded) 2023-10-09T10:19:39.692 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:39.732 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:39.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:39.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:39.836 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:39.869 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:40.057 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:40.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:40.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:40.189 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:40.283 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:40.949 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:19:40.970 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:19:40.991 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:19:41.010 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:19:41.027 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:41.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:41.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:41.039 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:19:41.059 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:19:41.082 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:19:41.100 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:19:41.102 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:19:41.102 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:19:41.102 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:19:41.102 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:19:41.102 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:19:41.102 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:19:41.103 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:19:41.103 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:19:41.134 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:41.165 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:41.284 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[147926]: pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:41.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2240043883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:41.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.285 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:19:41.268+0000 7f85a99d3700 -1 mon.a@0(leader).osd e423 definitely_dead 0 2023-10-09T10:19:41.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[152263]: pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:41.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2240043883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:41.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:41 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.353 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:41.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:41.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:41.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:41.491 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:41.534 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:41.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:41.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:41 smithi110 ceph-mon[136032]: pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:41 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:41 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2240043883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:41 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:41 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:41.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:41.668 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:41.752 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:19:41.753 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:19:41.753 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:19:41.753 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:19:41.806 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:41.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:41.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:41.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:41.939 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:41.973 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:41.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:41.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:42.077 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:42.097 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:19:42.105 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:42.138 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:42.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:42.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:42.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:42.266 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_4' created 2023-10-09T10:19:42.277 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.4. 2023-10-09T10:19:42.279 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:19:42.285 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:42.319 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:42.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:42.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3067329838' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1821197024' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/881790063' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:42.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3067329838' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1821197024' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/881790063' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:42 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:42.450 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:42.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3067329838' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:19:42.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T10:19:42.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1821197024' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T10:19:42.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T10:19:42.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:19:42.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:19:42.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/881790063' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:42.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:19:42.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:42 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:42.705 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:42.901 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:42.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:42.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:43.006 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:43.034 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:43.067 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:43.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:43.171 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:43.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:43.222 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T10:19:43.233 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:43.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:43.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:43.297 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T10:19:43.336 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:43.366 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:43.404 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:43.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:43.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:43.509 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:43.533 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:43 smithi110 ceph-mon[136032]: pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:43.533 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:43 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:43.533 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:43 smithi110 ceph-mon[136032]: failed to encode map e424 with expected crc 2023-10-09T10:19:43.533 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T10:19:43.534 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T10:19:43.534 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:19:43.534 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:43 smithi110 ceph-mon[136032]: osdmap e424: 8 total, 8 up, 8 in 2023-10-09T10:19:43.534 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:43 smithi110 ceph-mon[136032]: failed to encode map e424 with expected crc 2023-10-09T10:19:43.535 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:43.568 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:43.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:43.673 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[147926]: pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[147926]: failed to encode map e424 with expected crc 2023-10-09T10:19:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T10:19:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T10:19:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:19:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[147926]: osdmap e424: 8 total, 8 up, 8 in 2023-10-09T10:19:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[147926]: failed to encode map e424 with expected crc 2023-10-09T10:19:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[152263]: pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:19:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[152263]: failed to encode map e424 with expected crc 2023-10-09T10:19:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T10:19:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T10:19:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:19:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[152263]: osdmap e424: 8 total, 8 up, 8 in 2023-10-09T10:19:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:43 smithi012 ceph-mon[152263]: failed to encode map e424 with expected crc 2023-10-09T10:19:43.701 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:43.736 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:43.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:43.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:43.840 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:43.867 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:43.868 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:43.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:19:43.907 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:43.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:43.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:43.977 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:19:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:19:43.977 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:19:43.978 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:19:43.978 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:19:44.011 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:44.041 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:44.300 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:44.302 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:44 smithi110 ceph-mon[136032]: failed to encode map e425 with expected crc 2023-10-09T10:19:44.302 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:44 smithi110 ceph-mon[136032]: osdmap e425: 8 total, 8 up, 8 in 2023-10-09T10:19:44.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:44 smithi110 ceph-mon[136032]: failed to encode map e425 with expected crc 2023-10-09T10:19:44.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:44.303 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:44.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:44.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:44.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:19:44.405 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[147926]: failed to encode map e425 with expected crc 2023-10-09T10:19:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[147926]: osdmap e425: 8 total, 8 up, 8 in 2023-10-09T10:19:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[147926]: failed to encode map e425 with expected crc 2023-10-09T10:19:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:44.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:44.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[152263]: failed to encode map e425 with expected crc 2023-10-09T10:19:44.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[152263]: osdmap e425: 8 total, 8 up, 8 in 2023-10-09T10:19:44.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[152263]: failed to encode map e425 with expected crc 2023-10-09T10:19:44.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:44.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:44.466 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:44.501 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:44.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:44.606 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:44.638 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:44.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:44.675 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:44.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:44.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:44.812 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:44.846 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:44.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:44.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:44.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:19:44.952 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:44.972 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:44.994 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:44.994 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:45.081 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:45.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:45.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:45.187 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:45.214 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:45.250 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:45.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:45.338 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[147926]: pgmap v1185: 89 pgs: 3 peering, 2 creating+peering, 14 unknown, 70 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:45.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[147926]: osdmap e426: 8 total, 8 up, 8 in 2023-10-09T10:19:45.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[147926]: failed to encode map e426 with expected crc 2023-10-09T10:19:45.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[147926]: failed to encode map e426 with expected crc 2023-10-09T10:19:45.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:45.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[152263]: pgmap v1185: 89 pgs: 3 peering, 2 creating+peering, 14 unknown, 70 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:45.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[152263]: osdmap e426: 8 total, 8 up, 8 in 2023-10-09T10:19:45.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[152263]: failed to encode map e426 with expected crc 2023-10-09T10:19:45.339 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[152263]: failed to encode map e426 with expected crc 2023-10-09T10:19:45.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:45 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:45.354 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:45.389 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:45.434 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:45.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:45.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:45.511 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:45 smithi110 ceph-mon[136032]: pgmap v1185: 89 pgs: 3 peering, 2 creating+peering, 14 unknown, 70 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2023-10-09T10:19:45.511 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:45 smithi110 ceph-mon[136032]: osdmap e426: 8 total, 8 up, 8 in 2023-10-09T10:19:45.511 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:45 smithi110 ceph-mon[136032]: failed to encode map e426 with expected crc 2023-10-09T10:19:45.513 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:45 smithi110 ceph-mon[136032]: failed to encode map e426 with expected crc 2023-10-09T10:19:45.513 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:45 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:45.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:45.579 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:45.663 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:45.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:45.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:45.767 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:45.797 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:45.855 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:45.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:45.936 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:45.959 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:45.992 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:46.026 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:46.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:46.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:46.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:46.142 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:19:46.142 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:19:46.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:19:46.142 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:19:46.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.142 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGRecoveryDeleteReply(9.1 e357,353 9:85379d0d:::benchmark_data_smithi012_313365_object2478:head,346'7028)", 2023-10-09T10:19:46.142 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:08.372558+0000", 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 217.767276619, 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10804156400000001, 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372558+0000", 2023-10-09T10:19:46.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372558+0000", 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372556+0000", 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.062e-05 2023-10-09T10:19:46.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372567+0000", 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6056999999999999e-05 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372583+0000", 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0373e-05 2023-10-09T10:19:46.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372593+0000", 2023-10-09T10:19:46.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10772573100000001 2023-10-09T10:19:46.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.480319+0000", 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028045000000000001 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.480599+0000", 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10804156400000001 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.147 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156568 9.0 e376/338 9:001045d7:::benchmark_data_smithi012_313365_object44093:head v 376'8121, mlcod=376'8119)", 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.403086+0000", 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 157.736748277, 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043494592999999998, 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403086+0000", 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403086+0000", 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:19:46.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403084+0000", 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9354000000000001e-05 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403103+0000", 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.673e-05 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403130+0000", 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7969999999999995e-06 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403140+0000", 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021123579999999999 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405252+0000", 2023-10-09T10:19:46.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4102000000000001e-05 2023-10-09T10:19:46.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405276+0000", 2023-10-09T10:19:46.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041250728 2023-10-09T10:19:46.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.446527+0000", 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3520000000000003e-05 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.446581+0000", 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043494592999999998 2023-10-09T10:19:46.153 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156571 9.11 e377/365 9:88722a35:::benchmark_data_smithi012_313365_object44096:head v 377'8328, mlcod=376'8326)", 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.405148+0000", 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 157.734686759, 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040682307000000001, 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.154 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405148+0000", 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405148+0000", 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405147+0000", 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9359999999999999e-06 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405155+0000", 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5512e-05 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405171+0000", 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5290000000000008e-06 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405179+0000", 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025286290000000001 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.407708+0000", 2023-10-09T10:19:46.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4187000000000001e-05 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.407732+0000", 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038036767999999999 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.445769+0000", 2023-10-09T10:19:46.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1063999999999997e-05 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.445830+0000", 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040682307000000001 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.159 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156578 9.11 e377/365 9:8e383e1c:::benchmark_data_smithi012_313365_object44103:head v 377'8329, mlcod=376'8326)", 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 157.61426816700001, 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178583306, 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:19:46.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525564+0000", 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7572e-05 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525581+0000", 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3825999999999999e-05 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525605+0000", 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4969e-05 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525620+0000", 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13948444700000001 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.665105+0000", 2023-10-09T10:19:46.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8316000000000001e-05 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.665133+0000", 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038969654999999999 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.704103+0000", 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6774000000000003e-05 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.704149+0000", 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178583306 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:192494 9.b e398/378 9:d1959981:::benchmark_data_smithi012_313365_object80019:head v 398'10826, mlcod=340'7014)", 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:28.793077+0000", 2023-10-09T10:19:46.165 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 77.346757740000001, 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018067658, 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.793077+0000", 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.793077+0000", 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.793075+0000", 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2869e-05 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.793088+0000", 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2218000000000001e-05 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.793101+0000", 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6499999999999999e-06 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.793107+0000", 2023-10-09T10:19:46.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028039779999999999 2023-10-09T10:19:46.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.795911+0000", 2023-10-09T10:19:46.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3482999999999999e-05 2023-10-09T10:19:46.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.795935+0000", 2023-10-09T10:19:46.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015170596 2023-10-09T10:19:46.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.811105+0000", 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8942000000000001e-05 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:28.811144+0000", 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018067658 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.171 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:195795 9.1c e406/405 9:38ad7440:::benchmark_data_smithi012_313365_object83320:head v 406'11059, mlcod=0'0)", 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:40.336677+0000", 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 65.803157419000001, 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017647387, 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.336677+0000", 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.336677+0000", 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967296 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.336677+0000", 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5439999999999999e-06 2023-10-09T10:19:46.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.336680+0000", 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5210000000000004e-06 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.336685+0000", 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3299999999999999e-06 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.336688+0000", 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.004681859 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.341370+0000", 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2612999999999998e-05 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.341392+0000", 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012894048999999999 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.354287+0000", 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7706000000000001e-05 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.354324+0000", 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017647387 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.176 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:195959 9.20 e406/377 9:0620bfc0:::benchmark_data_smithi012_313365_object83484:head v 406'8719, mlcod=405'8717)", 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:40.366858+0000", 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 65.772976690999997, 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015718770999999999, 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366858+0000", 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366858+0000", 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366857+0000", 2023-10-09T10:19:46.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.64e-06 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366864+0000", 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0606e-05 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366875+0000", 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6290000000000004e-06 2023-10-09T10:19:46.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366881+0000", 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022010509999999999 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.369082+0000", 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8227999999999999e-05 2023-10-09T10:19:46.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.369100+0000", 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013435620000000001 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.382536+0000", 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0315000000000003e-05 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.382576+0000", 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015718770999999999 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:203507 9.21 e412/377 9:852db95d:::benchmark_data_smithi012_313365_object91032:head v 412'9079, mlcod=412'9077)", 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:53.841554+0000", 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 52.298279903999997, 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016023765999999998, 2023-10-09T10:19:46.182 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.841554+0000", 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.841554+0000", 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:46.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.841553+0000", 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3298999999999999e-05 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.841566+0000", 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0537e-05 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.841587+0000", 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2813000000000001e-05 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.841600+0000", 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029685649999999998 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.844568+0000", 2023-10-09T10:19:46.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1909999999999999e-05 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.844590+0000", 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012949239 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.857539+0000", 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8779999999999998e-05 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:53.857578+0000", 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016023765999999998 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:204604 9.11 e412/365 9:8ffef3b2:::benchmark_data_smithi012_313365_object92129:head v 412'9846, mlcod=412'9844)", 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:55.479527+0000", 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 50.660307363000001, 2023-10-09T10:19:46.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018110345999999999, 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.479527+0000", 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.479527+0000", 2023-10-09T10:19:46.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.479526+0000", 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4930000000000004e-06 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.479536+0000", 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1562e-05 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.479547+0000", 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8460000000000002e-06 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.479554+0000", 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3248999999999999e-05 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.479597+0000", 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9394000000000001e-05 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.479617+0000", 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017973365000000002 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.497590+0000", 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7286000000000001e-05 2023-10-09T10:19:46.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:55.497637+0000", 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018110345999999999 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:205812 9.1c e412/405 9:3f20d766:::benchmark_data_smithi012_313365_object93337:head v 412'11385, mlcod=412'11382)", 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:57.039874+0000", 2023-10-09T10:19:46.192 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 49.099960461999999, 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015779152000000001, 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.039874+0000", 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.039874+0000", 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.039873+0000", 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.943e-06 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.039881+0000", 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1106e-05 2023-10-09T10:19:46.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.039892+0000", 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8009999999999998e-06 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.039899+0000", 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010366597999999999 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.050265+0000", 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7373999999999999e-05 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.050293+0000", 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053154329999999996 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.055608+0000", 2023-10-09T10:19:46.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4713999999999999e-05 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:57.055653+0000", 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015779152000000001 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:210620 9.1 e413/377 9:83c40e45:::benchmark_data_smithi012_313365_object98145:head v 413'9223, mlcod=413'9221)", 2023-10-09T10:19:46.197 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:04.311722+0000", 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 41.828111956999997, 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018777104999999999, 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.311722+0000", 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.311722+0000", 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.311721+0000", 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0208e-05 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.311732+0000", 2023-10-09T10:19:46.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0321000000000001e-05 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.311752+0000", 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0180000000000001e-06 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.311760+0000", 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048113990000000001 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.316571+0000", 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2047000000000001e-05 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.316593+0000", 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013872101 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.330465+0000", 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3967000000000002e-05 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:04.330499+0000", 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018777104999999999 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.202 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:214552 9.0 e414/377 9:01f7c61c:::benchmark_data_smithi012_313365_object102077:head v 414'9040, mlcod=414'9038)", 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:09.911320+0000", 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 36.228514586999999, 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016247402000000001, 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.911320+0000", 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.911320+0000", 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.911318+0000", 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2850999999999999e-05 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.911331+0000", 2023-10-09T10:19:46.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0727e-05 2023-10-09T10:19:46.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.911352+0000", 2023-10-09T10:19:46.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4170000000000003e-06 2023-10-09T10:19:46.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.911359+0000", 2023-10-09T10:19:46.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9840999999999998e-05 2023-10-09T10:19:46.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.911419+0000", 2023-10-09T10:19:46.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6993000000000003e-05 2023-10-09T10:19:46.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.911456+0000", 2023-10-09T10:19:46.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016065554999999999 2023-10-09T10:19:46.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.927522+0000", 2023-10-09T10:19:46.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5275000000000003e-05 2023-10-09T10:19:46.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:09.927567+0000", 2023-10-09T10:19:46.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016247402000000001 2023-10-09T10:19:46.209 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.209 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.209 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.210 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:220912 9.c e418/416 9:36176ea3:::benchmark_data_smithi012_313365_object108437:head v 418'11844, mlcod=418'11841)", 2023-10-09T10:19:46.210 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:22.248883+0000", 2023-10-09T10:19:46.210 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 23.890951205, 2023-10-09T10:19:46.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015760737, 2023-10-09T10:19:46.210 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.211 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.211 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.248883+0000", 2023-10-09T10:19:46.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.248883+0000", 2023-10-09T10:19:46.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:46.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.248882+0000", 2023-10-09T10:19:46.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2179e-05 2023-10-09T10:19:46.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.248894+0000", 2023-10-09T10:19:46.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9219e-05 2023-10-09T10:19:46.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.248913+0000", 2023-10-09T10:19:46.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3489999999999992e-06 2023-10-09T10:19:46.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.248921+0000", 2023-10-09T10:19:46.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5442000000000002e-05 2023-10-09T10:19:46.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.248977+0000", 2023-10-09T10:19:46.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6971e-05 2023-10-09T10:19:46.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.249004+0000", 2023-10-09T10:19:46.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015587161 2023-10-09T10:19:46.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.264591+0000", 2023-10-09T10:19:46.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2760999999999998e-05 2023-10-09T10:19:46.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.264644+0000", 2023-10-09T10:19:46.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015760737 2023-10-09T10:19:46.224 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.224 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.224 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.225 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:220977 9.11 e418/365 9:8aa5514b:::benchmark_data_smithi012_313365_object108502:head v 418'10348, mlcod=418'10346)", 2023-10-09T10:19:46.225 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:22.327689+0000", 2023-10-09T10:19:46.225 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 23.812144882999998, 2023-10-09T10:19:46.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016397238000000001, 2023-10-09T10:19:46.226 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.226 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.226 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.327689+0000", 2023-10-09T10:19:46.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.327689+0000", 2023-10-09T10:19:46.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:19:46.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.327689+0000", 2023-10-09T10:19:46.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4190000000000002e-06 2023-10-09T10:19:46.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.327697+0000", 2023-10-09T10:19:46.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5302e-05 2023-10-09T10:19:46.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.327712+0000", 2023-10-09T10:19:46.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.511e-06 2023-10-09T10:19:46.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.327719+0000", 2023-10-09T10:19:46.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070591220000000001 2023-10-09T10:19:46.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.334778+0000", 2023-10-09T10:19:46.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0874999999999999e-05 2023-10-09T10:19:46.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.334799+0000", 2023-10-09T10:19:46.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0092412190000000002 2023-10-09T10:19:46.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.344040+0000", 2023-10-09T10:19:46.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6569000000000001e-05 2023-10-09T10:19:46.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:22.344087+0000", 2023-10-09T10:19:46.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016397238000000001 2023-10-09T10:19:46.239 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.240 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.240 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.241 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:221710 9.21 e418/377 9:86f33db6:::benchmark_data_smithi012_313365_object109235:head v 418'9361, mlcod=418'9359)", 2023-10-09T10:19:46.241 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:23.447808+0000", 2023-10-09T10:19:46.241 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 22.692026671000001, 2023-10-09T10:19:46.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020348725000000002, 2023-10-09T10:19:46.242 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.242 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.242 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.447808+0000", 2023-10-09T10:19:46.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.447808+0000", 2023-10-09T10:19:46.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:19:46.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.447807+0000", 2023-10-09T10:19:46.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0064999999999999e-05 2023-10-09T10:19:46.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.447817+0000", 2023-10-09T10:19:46.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3176000000000001e-05 2023-10-09T10:19:46.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.248 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.248 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.447830+0000", 2023-10-09T10:19:46.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0439999999999998e-06 2023-10-09T10:19:46.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.447837+0000", 2023-10-09T10:19:46.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5525000000000002e-05 2023-10-09T10:19:46.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.447882+0000", 2023-10-09T10:19:46.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5119000000000001e-05 2023-10-09T10:19:46.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.447907+0000", 2023-10-09T10:19:46.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020202793 2023-10-09T10:19:46.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.468110+0000", 2023-10-09T10:19:46.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6071e-05 2023-10-09T10:19:46.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:23.468156+0000", 2023-10-09T10:19:46.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020348725000000002 2023-10-09T10:19:46.254 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.255 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.255 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.256 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:222401 9.b e418/400 9:d2185d4a:::benchmark_data_smithi012_313365_object109926:head v 418'11746, mlcod=334'5174)", 2023-10-09T10:19:46.256 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:24.389990+0000", 2023-10-09T10:19:46.256 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 21.749844332999999, 2023-10-09T10:19:46.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021583564, 2023-10-09T10:19:46.257 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.257 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.257 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.389990+0000", 2023-10-09T10:19:46.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.389990+0000", 2023-10-09T10:19:46.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:19:46.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.389989+0000", 2023-10-09T10:19:46.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6120000000000001e-06 2023-10-09T10:19:46.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.389997+0000", 2023-10-09T10:19:46.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0369e-05 2023-10-09T10:19:46.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.390007+0000", 2023-10-09T10:19:46.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1059999999999996e-06 2023-10-09T10:19:46.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.390013+0000", 2023-10-09T10:19:46.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7930999999999998e-05 2023-10-09T10:19:46.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.390071+0000", 2023-10-09T10:19:46.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4382999999999998e-05 2023-10-09T10:19:46.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.390116+0000", 2023-10-09T10:19:46.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021407755000000001 2023-10-09T10:19:46.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.411523+0000", 2023-10-09T10:19:46.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0235000000000003e-05 2023-10-09T10:19:46.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:24.411573+0000", 2023-10-09T10:19:46.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021583564 2023-10-09T10:19:46.269 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.269 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.270 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.270 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:223702 9.c e418/416 9:343a3502:::benchmark_data_smithi012_313365_object111227:head v 418'11925, mlcod=418'11922)", 2023-10-09T10:19:46.271 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:26.165679+0000", 2023-10-09T10:19:46.271 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.974155183000001, 2023-10-09T10:19:46.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017221407000000001, 2023-10-09T10:19:46.271 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.271 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.272 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.165679+0000", 2023-10-09T10:19:46.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.165679+0000", 2023-10-09T10:19:46.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:19:46.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.165679+0000", 2023-10-09T10:19:46.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4900000000000002e-06 2023-10-09T10:19:46.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.165683+0000", 2023-10-09T10:19:46.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4099999999999999e-06 2023-10-09T10:19:46.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.165689+0000", 2023-10-09T10:19:46.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7000000000000002e-06 2023-10-09T10:19:46.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.165692+0000", 2023-10-09T10:19:46.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010113838 2023-10-09T10:19:46.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.175806+0000", 2023-10-09T10:19:46.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5559999999999999e-05 2023-10-09T10:19:46.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.175832+0000", 2023-10-09T10:19:46.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0070267309999999996 2023-10-09T10:19:46.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.182859+0000", 2023-10-09T10:19:46.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1977999999999997e-05 2023-10-09T10:19:46.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.182900+0000", 2023-10-09T10:19:46.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017221407000000001 2023-10-09T10:19:46.282 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.282 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.282 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.283 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:224290 9.11 e418/365 9:8846988a:::benchmark_data_smithi012_313365_object111815:head v 418'10455, mlcod=418'10453)", 2023-10-09T10:19:46.283 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:26.971223+0000", 2023-10-09T10:19:46.283 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.168611373000001, 2023-10-09T10:19:46.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018768274000000001, 2023-10-09T10:19:46.283 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.283 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.284 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.971223+0000", 2023-10-09T10:19:46.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.971223+0000", 2023-10-09T10:19:46.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:46.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.971222+0000", 2023-10-09T10:19:46.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2204e-05 2023-10-09T10:19:46.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.971234+0000", 2023-10-09T10:19:46.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9434999999999999e-05 2023-10-09T10:19:46.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.971253+0000", 2023-10-09T10:19:46.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3810000000000001e-06 2023-10-09T10:19:46.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.971261+0000", 2023-10-09T10:19:46.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7825999999999997e-05 2023-10-09T10:19:46.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.971328+0000", 2023-10-09T10:19:46.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5126e-05 2023-10-09T10:19:46.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.971354+0000", 2023-10-09T10:19:46.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018616988000000001 2023-10-09T10:19:46.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.989971+0000", 2023-10-09T10:19:46.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0588e-05 2023-10-09T10:19:46.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.989991+0000", 2023-10-09T10:19:46.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018768274000000001 2023-10-09T10:19:46.290 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.291 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.291 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.291 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:224292 9.21 e418/377 9:853f7223:::benchmark_data_smithi012_313365_object111817:head v 418'9401, mlcod=418'9399)", 2023-10-09T10:19:46.291 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:26.972718+0000", 2023-10-09T10:19:46.292 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.167116001, 2023-10-09T10:19:46.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017248332000000002, 2023-10-09T10:19:46.292 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.292 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.292 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.972718+0000", 2023-10-09T10:19:46.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.972718+0000", 2023-10-09T10:19:46.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:19:46.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.972718+0000", 2023-10-09T10:19:46.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7909999999999999e-06 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.972724+0000", 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2357e-05 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.972737+0000", 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2519999999999996e-06 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.972743+0000", 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0571e-05 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.972804+0000", 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8525999999999998e-05 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.972822+0000", 2023-10-09T10:19:46.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017102135000000001 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.989924+0000", 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2426999999999999e-05 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:26.989967+0000", 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017248332000000002 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.297 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:224321 9.1b e418/400 9:ddbda1cc:::benchmark_data_smithi012_313365_object111846:head v 418'11885, mlcod=334'5176)", 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:27.011225+0000", 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 19.128609605000001, 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015764011000000001, 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:19:46.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.011225+0000", 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.011225+0000", 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.011223+0000", 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2150000000000001e-05 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.011235+0000", 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0718999999999999e-05 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:19:46.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.011256+0000", 2023-10-09T10:19:46.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9359999999999999e-06 2023-10-09T10:19:46.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:19:46.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.011264+0000", 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9010000000000002e-05 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.011363+0000", 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0457e-05 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.011393+0000", 2023-10-09T10:19:46.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015542155 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.026936+0000", 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3047000000000002e-05 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:27.026989+0000", 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015764011000000001 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.303 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.304 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:19:46.304 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:19:46.304 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:19:46.304 INFO:teuthology.orchestra.run.smithi012.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T10:19:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:46.305 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:46.305 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:46.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:46.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:46 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3612260369' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:19:46.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:46 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:19:46.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:46 smithi012 ceph-mon[147926]: failed to encode map e426 with expected crc 2023-10-09T10:19:46.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:46 smithi012 ceph-mon[147926]: failed to encode map e426 with expected crc 2023-10-09T10:19:46.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:46 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3612260369' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:19:46.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:46 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:19:46.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:46 smithi012 ceph-mon[152263]: failed to encode map e426 with expected crc 2023-10-09T10:19:46.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:46 smithi012 ceph-mon[152263]: failed to encode map e426 with expected crc 2023-10-09T10:19:46.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:46.439 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:46.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:46 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3612260369' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:19:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:46 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:19:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:46 smithi110 ceph-mon[136032]: failed to encode map e426 with expected crc 2023-10-09T10:19:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:46 smithi110 ceph-mon[136032]: failed to encode map e426 with expected crc 2023-10-09T10:19:46.526 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:46.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:46.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:46.590 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:19:46.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:46.660 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:46.869 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:19:46.888 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:19:46.897 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:46.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:46.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:46.909 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:19:46.932 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:19:46.951 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:19:46.969 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:19:46.987 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:19:47.003 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:47.006 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:19:47.007 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:19:47.007 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:19:47.007 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:19:47.007 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:19:47.007 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:19:47.007 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:19:47.007 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:19:47.007 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:19:47.030 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:47.063 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:47.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:47.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:47.168 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:47.184 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:47.196 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:47.318 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:47.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:47.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:47.421 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:47.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[147926]: pgmap v1187: 89 pgs: 3 peering, 2 creating+peering, 84 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 105 B/s, 1 objects/s recovering 2023-10-09T10:19:47.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[147926]: failed to encode map e426 with expected crc 2023-10-09T10:19:47.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:19:47.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[147926]: osdmap e427: 8 total, 8 up, 8 in 2023-10-09T10:19:47.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[147926]: failed to encode map e427 with expected crc 2023-10-09T10:19:47.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2656605752' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:47.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[152263]: pgmap v1187: 89 pgs: 3 peering, 2 creating+peering, 84 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 105 B/s, 1 objects/s recovering 2023-10-09T10:19:47.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[152263]: failed to encode map e426 with expected crc 2023-10-09T10:19:47.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:19:47.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[152263]: osdmap e427: 8 total, 8 up, 8 in 2023-10-09T10:19:47.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[152263]: failed to encode map e427 with expected crc 2023-10-09T10:19:47.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2656605752' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:47.447 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:47.481 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:47.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:47.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:47.549 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:47 smithi110 ceph-mon[136032]: pgmap v1187: 89 pgs: 3 peering, 2 creating+peering, 84 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 105 B/s, 1 objects/s recovering 2023-10-09T10:19:47.549 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:47 smithi110 ceph-mon[136032]: failed to encode map e426 with expected crc 2023-10-09T10:19:47.549 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:47 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:19:47.550 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:47 smithi110 ceph-mon[136032]: osdmap e427: 8 total, 8 up, 8 in 2023-10-09T10:19:47.550 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:47 smithi110 ceph-mon[136032]: failed to encode map e427 with expected crc 2023-10-09T10:19:47.550 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2656605752' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:47.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:47.617 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:47.656 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:47.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:47.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:47.760 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:47.789 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:47.915 DEBUG:teuthology.orchestra.run.smithi012:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T10:19:47.918 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:19:47.918 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:19:47.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:47.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:47.974 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:hints = 1 2023-10-09T10:19:47.974 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:19:47.974 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object prefix: benchmark_data_smithi012_366312 2023-10-09T10:19:47.975 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:19:47.975 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:19:48.059 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:48.114 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:19:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:19:47] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-10-09T10:19:48.227 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:48.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:48.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:48.298 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:19:48.298 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:19:48.298 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-09T10:19:48.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:19:48.316 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:19:48.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:48.387 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:48.447 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:48.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:48.510 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:19:48.511 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:19:48.535 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:48 smithi110 ceph-mon[136032]: failed to encode map e427 with expected crc 2023-10-09T10:19:48.535 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:48 smithi110 ceph-mon[136032]: failed to encode map e427 with expected crc 2023-10-09T10:19:48.535 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/449804006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:48.535 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:48 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:48.553 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:48.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:48 smithi012 ceph-mon[147926]: failed to encode map e427 with expected crc 2023-10-09T10:19:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:48 smithi012 ceph-mon[147926]: failed to encode map e427 with expected crc 2023-10-09T10:19:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/449804006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:48.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:48 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:48 smithi012 ceph-mon[152263]: failed to encode map e427 with expected crc 2023-10-09T10:19:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:48 smithi012 ceph-mon[152263]: failed to encode map e427 with expected crc 2023-10-09T10:19:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/449804006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:48.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:48 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:19:48.843 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:19:48.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:48.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:19:48.922 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:19:48.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:48.975 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 1 16 3360 3344 208.896 209 0.00570466 0.00477464 2023-10-09T10:19:49.008 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:49.062 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:49.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:49.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:49.109 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:49.169 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:49.247 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:49.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:49.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:49.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:49.414 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:49.471 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:49.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:19:49.586 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[152263]: pgmap v1189: 89 pgs: 89 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 117 B/s, 1 objects/s recovering 2023-10-09T10:19:49.586 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:19:49.586 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:19:49.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[152263]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[152263]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:49.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[152263]: osdmap e428: 8 total, 8 up, 8 in 2023-10-09T10:19:49.587 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[152263]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.588 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[147926]: pgmap v1189: 89 pgs: 89 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 117 B/s, 1 objects/s recovering 2023-10-09T10:19:49.588 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:19:49.588 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:19:49.588 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[147926]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.588 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[147926]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.589 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:49.589 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[147926]: osdmap e428: 8 total, 8 up, 8 in 2023-10-09T10:19:49.589 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:49 smithi012 ceph-mon[147926]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:49 smithi110 ceph-mon[136032]: pgmap v1189: 89 pgs: 89 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 117 B/s, 1 objects/s recovering 2023-10-09T10:19:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:49 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:19:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:49 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:19:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:49 smithi110 ceph-mon[136032]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:49 smithi110 ceph-mon[136032]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:49 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:19:49.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:49 smithi110 ceph-mon[136032]: osdmap e428: 8 total, 8 up, 8 in 2023-10-09T10:19:49.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:49 smithi110 ceph-mon[136032]: failed to encode map e428 with expected crc 2023-10-09T10:19:49.746 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:49.874 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:49.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:49.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:49.975 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 2 16 6465 6449 201.472 194.062 0.00729348 0.00495675 2023-10-09T10:19:49.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:50.023 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:50.082 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:50.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:50.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:50.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:19:50.248 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:50.411 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:19:50.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:50.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:19:50.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:50.566 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:50.598 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:50.625 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:50.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:50.732 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:50.793 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:50.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:19:50.897 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:19:50.909 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:19:50.957 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:50.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:50.975 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 3 16 9273 9257 192.81 175.5 0.00408606 0.00517692 2023-10-09T10:19:51.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:19:51.120 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:51.251 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:19:51.251 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:19:51.385 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:19:51.390 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:51 smithi012 ceph-mon[147926]: pgmap v1191: 89 pgs: 89 active+clean; 57 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.4 MiB/s wr, 149 op/s; 105 B/s, 1 objects/s recovering 2023-10-09T10:19:51.390 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:51 smithi012 ceph-mon[152263]: pgmap v1191: 89 pgs: 89 active+clean; 57 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.4 MiB/s wr, 149 op/s; 105 B/s, 1 objects/s recovering 2023-10-09T10:19:51.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:51.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:19:51.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:51.529 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1191,"stamp":"2023-10-09T10:19:49.638083+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":59512736,"num_objects":923,"num_object_clones":0,"num_object_copies":2769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":923,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":1159,"num_write_kb":59981,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":1197394,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1093,"ondisk_log_size":1093,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6740676,"kb_used_data":1747908,"kb_used_omap":172,"kb_used_meta":4992531,"kb_avail":743220540,"statfs":{"total":767960285184,"available":761057832960,"internally_reserved":0,"allocated":1789857792,"data_stored":1780715227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":176164,"internal_metadata":5112352732},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":58916864,"num_objects":899,"num_object_clones":0,"num_object_copies":2697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":899,"num_write_kb":57536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":632,"num_keys_recovered":-10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001777"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.287175+0000","last_change":"2023-10-09T10:19:43.287175+0000","last_active":"2023-10-09T10:19:43.287175+0000","last_peered":"2023-10-09T10:19:43.287175+0000","last_clean":"2023-10-09T10:19:43.287175+0000","last_became_active":"2023-10-09T10:19:43.286833+0000","last_became_peered":"2023-10-09T10:19:43.286833+0000","last_unstale":"2023-10-09T10:19:43.287175+0000","last_undegraded":"2023-10-09T10:19:43.287175+0000","last_fullsized":"2023-10-09T10:19:43.287175+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":371,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312945+0000","last_change":"2023-10-09T10:19:43.279241+0000","last_active":"2023-10-09T10:19:47.312945+0000","last_peered":"2023-10-09T10:19:47.312945+0000","last_clean":"2023-10-09T10:19:47.312945+0000","last_became_active":"2023-10-09T10:19:43.278899+0000","last_became_peered":"2023-10-09T10:19:43.278899+0000","last_unstale":"2023-10-09T10:19:47.312945+0000","last_undegraded":"2023-10-09T10:19:47.312945+0000","last_fullsized":"2023-10-09T10:19:47.312945+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:45:14.209721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":467,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312458+0000","last_change":"2023-10-09T10:19:43.284958+0000","last_active":"2023-10-09T10:19:47.312458+0000","last_peered":"2023-10-09T10:19:47.312458+0000","last_clean":"2023-10-09T10:19:47.312458+0000","last_became_active":"2023-10-09T10:19:43.283799+0000","last_became_peered":"2023-10-09T10:19:43.283799+0000","last_unstale":"2023-10-09T10:19:47.312458+0000","last_undegraded":"2023-10-09T10:19:47.312458+0000","last_fullsized":"2023-10-09T10:19:47.312458+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:57.225333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":321,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.286027+0000","last_change":"2023-10-09T10:16:08.309799+0000","last_active":"2023-10-09T10:19:44.286027+0000","last_peered":"2023-10-09T10:19:44.286027+0000","last_clean":"2023-10-09T10:19:44.286027+0000","last_became_active":"2023-10-09T10:16:08.309448+0000","last_became_peered":"2023-10-09T10:16:08.309448+0000","last_unstale":"2023-10-09T10:19:44.286027+0000","last_undegraded":"2023-10-09T10:19:44.286027+0000","last_fullsized":"2023-10-09T10:19:44.286027+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:58.976233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":352,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312608+0000","last_change":"2023-10-09T10:18:34.268776+0000","last_active":"2023-10-09T10:19:47.312608+0000","last_peered":"2023-10-09T10:19:47.312608+0000","last_clean":"2023-10-09T10:19:47.312608+0000","last_became_active":"2023-10-09T10:18:34.268454+0000","last_became_peered":"2023-10-09T10:18:34.268454+0000","last_unstale":"2023-10-09T10:19:47.312608+0000","last_undegraded":"2023-10-09T10:19:47.312608+0000","last_fullsized":"2023-10-09T10:19:47.312608+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:40.225209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":222,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.313161+0000","last_change":"2023-10-09T10:16:41.473093+0000","last_active":"2023-10-09T10:19:47.313161+0000","last_peered":"2023-10-09T10:19:47.313161+0000","last_clean":"2023-10-09T10:19:47.313161+0000","last_became_active":"2023-10-09T10:16:41.472774+0000","last_became_peered":"2023-10-09T10:16:41.472774+0000","last_unstale":"2023-10-09T10:19:47.313161+0000","last_undegraded":"2023-10-09T10:19:47.313161+0000","last_fullsized":"2023-10-09T10:19:47.313161+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:07.117932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":356,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.285390+0000","last_change":"2023-10-09T10:16:18.273288+0000","last_active":"2023-10-09T10:19:44.285390+0000","last_peered":"2023-10-09T10:19:44.285390+0000","last_clean":"2023-10-09T10:19:44.285390+0000","last_became_active":"2023-10-09T10:16:18.272936+0000","last_became_peered":"2023-10-09T10:16:18.272936+0000","last_unstale":"2023-10-09T10:19:44.285390+0000","last_undegraded":"2023-10-09T10:19:44.285390+0000","last_fullsized":"2023-10-09T10:19:44.285390+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:39:34.847781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":392,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312817+0000","last_change":"2023-10-09T10:19:43.281639+0000","last_active":"2023-10-09T10:19:47.312817+0000","last_peered":"2023-10-09T10:19:47.312817+0000","last_clean":"2023-10-09T10:19:47.312817+0000","last_became_active":"2023-10-09T10:19:43.280745+0000","last_became_peered":"2023-10-09T10:19:43.280745+0000","last_unstale":"2023-10-09T10:19:47.312817+0000","last_undegraded":"2023-10-09T10:19:47.312817+0000","last_fullsized":"2023-10-09T10:19:47.312817+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:57.823383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":73,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.261785+0000","last_change":"2023-10-09T10:16:18.280500+0000","last_active":"2023-10-09T10:19:42.261785+0000","last_peered":"2023-10-09T10:19:42.261785+0000","last_clean":"2023-10-09T10:19:42.261785+0000","last_became_active":"2023-10-09T10:16:18.275348+0000","last_became_peered":"2023-10-09T10:16:18.275348+0000","last_unstale":"2023-10-09T10:19:42.261785+0000","last_undegraded":"2023-10-09T10:19:42.261785+0000","last_fullsized":"2023-10-09T10:19:42.261785+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":335,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.273884+0000","last_change":"2023-10-09T10:15:42.396546+0000","last_active":"2023-10-09T10:19:43.273884+0000","last_peered":"2023-10-09T10:19:43.273884+0000","last_clean":"2023-10-09T10:19:43.273884+0000","last_became_active":"2023-10-09T10:15:42.367055+0000","last_became_peered":"2023-10-09T10:15:42.367055+0000","last_unstale":"2023-10-09T10:19:43.273884+0000","last_undegraded":"2023-10-09T10:19:43.273884+0000","last_fullsized":"2023-10-09T10:19:43.273884+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:57.600046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":16,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.299477+0000","last_change":"2023-10-09T10:19:43.299477+0000","last_active":"2023-10-09T10:19:43.299477+0000","last_peered":"2023-10-09T10:19:43.299477+0000","last_clean":"2023-10-09T10:19:43.299477+0000","last_became_active":"2023-10-09T10:19:43.274448+0000","last_became_peered":"2023-10-09T10:19:43.274448+0000","last_unstale":"2023-10-09T10:19:43.299477+0000","last_undegraded":"2023-10-09T10:19:43.299477+0000","last_fullsized":"2023-10-09T10:19:43.299477+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":426,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312230+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:19:47.312230+0000","last_peered":"2023-10-09T10:19:47.312230+0000","last_clean":"2023-10-09T10:19:47.312230+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:19:47.312230+0000","last_undegraded":"2023-10-09T10:19:47.312230+0000","last_fullsized":"2023-10-09T10:19:47.312230+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":20,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.482921+0000","last_change":"2023-10-09T10:19:43.482921+0000","last_active":"2023-10-09T10:19:43.482921+0000","last_peered":"2023-10-09T10:19:43.482921+0000","last_clean":"2023-10-09T10:19:43.482921+0000","last_became_active":"2023-10-09T10:19:43.278592+0000","last_became_peered":"2023-10-09T10:19:43.278592+0000","last_unstale":"2023-10-09T10:19:43.482921+0000","last_undegraded":"2023-10-09T10:19:43.482921+0000","last_fullsized":"2023-10-09T10:19:43.482921+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.279159+0000","last_change":"2023-10-09T10:19:43.279159+0000","last_active":"2023-10-09T10:19:43.279159+0000","last_peered":"2023-10-09T10:19:43.279159+0000","last_clean":"2023-10-09T10:19:43.279159+0000","last_became_active":"2023-10-09T10:19:43.276994+0000","last_became_peered":"2023-10-09T10:19:43.276994+0000","last_unstale":"2023-10-09T10:19:43.279159+0000","last_undegraded":"2023-10-09T10:19:43.279159+0000","last_fullsized":"2023-10-09T10:19:43.279159+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.485535+0000","last_change":"2023-10-09T10:19:43.485535+0000","last_active":"2023-10-09T10:19:43.485535+0000","last_peered":"2023-10-09T10:19:43.485535+0000","last_clean":"2023-10-09T10:19:43.485535+0000","last_became_active":"2023-10-09T10:19:43.289115+0000","last_became_peered":"2023-10-09T10:19:43.289115+0000","last_unstale":"2023-10-09T10:19:43.485535+0000","last_undegraded":"2023-10-09T10:19:43.485535+0000","last_fullsized":"2023-10-09T10:19:43.485535+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":326,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312603+0000","last_change":"2023-10-09T10:18:45.342547+0000","last_active":"2023-10-09T10:19:47.312603+0000","last_peered":"2023-10-09T10:19:47.312603+0000","last_clean":"2023-10-09T10:19:47.312603+0000","last_became_active":"2023-10-09T10:18:45.342206+0000","last_became_peered":"2023-10-09T10:18:45.342206+0000","last_unstale":"2023-10-09T10:19:47.312603+0000","last_undegraded":"2023-10-09T10:19:47.312603+0000","last_fullsized":"2023-10-09T10:19:47.312603+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:32.932519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":320,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.284736+0000","last_change":"2023-10-09T10:16:08.455134+0000","last_active":"2023-10-09T10:19:44.284736+0000","last_peered":"2023-10-09T10:19:44.284736+0000","last_clean":"2023-10-09T10:19:44.284736+0000","last_became_active":"2023-10-09T10:16:08.454793+0000","last_became_peered":"2023-10-09T10:16:08.454793+0000","last_unstale":"2023-10-09T10:19:44.284736+0000","last_undegraded":"2023-10-09T10:19:44.284736+0000","last_fullsized":"2023-10-09T10:19:44.284736+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:25:33.883382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":73,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.261854+0000","last_change":"2023-10-09T10:16:18.255609+0000","last_active":"2023-10-09T10:19:42.261854+0000","last_peered":"2023-10-09T10:19:42.261854+0000","last_clean":"2023-10-09T10:19:42.261854+0000","last_became_active":"2023-10-09T10:16:18.244618+0000","last_became_peered":"2023-10-09T10:16:18.244618+0000","last_unstale":"2023-10-09T10:19:42.261854+0000","last_undegraded":"2023-10-09T10:19:42.261854+0000","last_fullsized":"2023-10-09T10:19:42.261854+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.282758+0000","last_change":"2023-10-09T10:19:43.282758+0000","last_active":"2023-10-09T10:19:43.282758+0000","last_peered":"2023-10-09T10:19:43.282758+0000","last_clean":"2023-10-09T10:19:43.282758+0000","last_became_active":"2023-10-09T10:19:43.282258+0000","last_became_peered":"2023-10-09T10:19:43.282258+0000","last_unstale":"2023-10-09T10:19:43.282758+0000","last_undegraded":"2023-10-09T10:19:43.282758+0000","last_fullsized":"2023-10-09T10:19:43.282758+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":277,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.267990+0000","last_change":"2023-10-09T10:18:34.264136+0000","last_active":"2023-10-09T10:19:43.267990+0000","last_peered":"2023-10-09T10:19:43.267990+0000","last_clean":"2023-10-09T10:19:43.267990+0000","last_became_active":"2023-10-09T10:18:34.263812+0000","last_became_peered":"2023-10-09T10:18:34.263812+0000","last_unstale":"2023-10-09T10:19:43.267990+0000","last_undegraded":"2023-10-09T10:19:43.267990+0000","last_fullsized":"2023-10-09T10:19:43.267990+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":340,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.258882+0000","last_change":"2023-10-09T10:18:34.286718+0000","last_active":"2023-10-09T10:19:42.258882+0000","last_peered":"2023-10-09T10:19:42.258882+0000","last_clean":"2023-10-09T10:19:42.258882+0000","last_became_active":"2023-10-09T10:18:34.271799+0000","last_became_peered":"2023-10-09T10:18:34.271799+0000","last_unstale":"2023-10-09T10:19:42.258882+0000","last_undegraded":"2023-10-09T10:19:42.258882+0000","last_fullsized":"2023-10-09T10:19:42.258882+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":374,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.313177+0000","last_change":"2023-10-09T10:16:08.618851+0000","last_active":"2023-10-09T10:19:47.313177+0000","last_peered":"2023-10-09T10:19:47.313177+0000","last_clean":"2023-10-09T10:19:47.313177+0000","last_became_active":"2023-10-09T10:16:08.618501+0000","last_became_peered":"2023-10-09T10:16:08.618501+0000","last_unstale":"2023-10-09T10:19:47.313177+0000","last_undegraded":"2023-10-09T10:19:47.313177+0000","last_fullsized":"2023-10-09T10:19:47.313177+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:45.133486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.277465+0000","last_change":"2023-10-09T10:19:43.277465+0000","last_active":"2023-10-09T10:19:43.277465+0000","last_peered":"2023-10-09T10:19:43.277465+0000","last_clean":"2023-10-09T10:19:43.277465+0000","last_became_active":"2023-10-09T10:19:43.277144+0000","last_became_peered":"2023-10-09T10:19:43.277144+0000","last_unstale":"2023-10-09T10:19:43.277465+0000","last_undegraded":"2023-10-09T10:19:43.277465+0000","last_fullsized":"2023-10-09T10:19:43.277465+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":14,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.477679+0000","last_change":"2023-10-09T10:19:43.477679+0000","last_active":"2023-10-09T10:19:43.477679+0000","last_peered":"2023-10-09T10:19:43.477679+0000","last_clean":"2023-10-09T10:19:43.477679+0000","last_became_active":"2023-10-09T10:19:43.274761+0000","last_became_peered":"2023-10-09T10:19:43.274761+0000","last_unstale":"2023-10-09T10:19:43.477679+0000","last_undegraded":"2023-10-09T10:19:43.477679+0000","last_fullsized":"2023-10-09T10:19:43.477679+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":73,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.258784+0000","last_change":"2023-10-09T10:16:18.237380+0000","last_active":"2023-10-09T10:19:42.258784+0000","last_peered":"2023-10-09T10:19:42.258784+0000","last_clean":"2023-10-09T10:19:42.258784+0000","last_became_active":"2023-10-09T10:16:18.236585+0000","last_became_peered":"2023-10-09T10:16:18.236585+0000","last_unstale":"2023-10-09T10:19:42.258784+0000","last_undegraded":"2023-10-09T10:19:42.258784+0000","last_fullsized":"2023-10-09T10:19:42.258784+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.284275+0000","last_change":"2023-10-09T10:19:43.284275+0000","last_active":"2023-10-09T10:19:43.284275+0000","last_peered":"2023-10-09T10:19:43.284275+0000","last_clean":"2023-10-09T10:19:43.284275+0000","last_became_active":"2023-10-09T10:19:43.283457+0000","last_became_peered":"2023-10-09T10:19:43.283457+0000","last_unstale":"2023-10-09T10:19:43.284275+0000","last_undegraded":"2023-10-09T10:19:43.284275+0000","last_fullsized":"2023-10-09T10:19:43.284275+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.388522+0000","last_change":"2023-10-09T10:19:43.388522+0000","last_active":"2023-10-09T10:19:43.388522+0000","last_peered":"2023-10-09T10:19:43.388522+0000","last_clean":"2023-10-09T10:19:43.388522+0000","last_became_active":"2023-10-09T10:19:43.274829+0000","last_became_peered":"2023-10-09T10:19:43.274829+0000","last_unstale":"2023-10-09T10:19:43.388522+0000","last_undegraded":"2023-10-09T10:19:43.388522+0000","last_fullsized":"2023-10-09T10:19:43.388522+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.282812+0000","last_change":"2023-10-09T10:19:43.282812+0000","last_active":"2023-10-09T10:19:43.282812+0000","last_peered":"2023-10-09T10:19:43.282812+0000","last_clean":"2023-10-09T10:19:43.282812+0000","last_became_active":"2023-10-09T10:19:43.282498+0000","last_became_peered":"2023-10-09T10:19:43.282498+0000","last_unstale":"2023-10-09T10:19:43.282812+0000","last_undegraded":"2023-10-09T10:19:43.282812+0000","last_fullsized":"2023-10-09T10:19:43.282812+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.291894+0000","last_change":"2023-10-09T10:19:43.291894+0000","last_active":"2023-10-09T10:19:43.291894+0000","last_peered":"2023-10-09T10:19:43.291894+0000","last_clean":"2023-10-09T10:19:43.291894+0000","last_became_active":"2023-10-09T10:19:43.291238+0000","last_became_peered":"2023-10-09T10:19:43.291238+0000","last_unstale":"2023-10-09T10:19:43.291894+0000","last_undegraded":"2023-10-09T10:19:43.291894+0000","last_fullsized":"2023-10-09T10:19:43.291894+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":77,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.261432+0000","last_change":"2023-10-09T10:16:18.362151+0000","last_active":"2023-10-09T10:19:42.261432+0000","last_peered":"2023-10-09T10:19:42.261432+0000","last_clean":"2023-10-09T10:19:42.261432+0000","last_became_active":"2023-10-09T10:16:18.323076+0000","last_became_peered":"2023-10-09T10:16:18.323076+0000","last_unstale":"2023-10-09T10:19:42.261432+0000","last_undegraded":"2023-10-09T10:19:42.261432+0000","last_fullsized":"2023-10-09T10:19:42.261432+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":268,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.267370+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:19:42.267370+0000","last_peered":"2023-10-09T10:19:42.267370+0000","last_clean":"2023-10-09T10:19:42.267370+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:19:42.267370+0000","last_undegraded":"2023-10-09T10:19:42.267370+0000","last_fullsized":"2023-10-09T10:19:42.267370+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":12,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.284634+0000","last_change":"2023-10-09T10:19:43.283774+0000","last_active":"2023-10-09T10:19:44.284634+0000","last_peered":"2023-10-09T10:19:44.284634+0000","last_clean":"2023-10-09T10:19:44.284634+0000","last_became_active":"2023-10-09T10:19:43.283468+0000","last_became_peered":"2023-10-09T10:19:43.283468+0000","last_unstale":"2023-10-09T10:19:44.284634+0000","last_undegraded":"2023-10-09T10:19:44.284634+0000","last_fullsized":"2023-10-09T10:19:44.284634+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:50.698088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.262404+0000","last_change":"2023-10-09T10:16:18.237242+0000","last_active":"2023-10-09T10:19:42.262404+0000","last_peered":"2023-10-09T10:19:42.262404+0000","last_clean":"2023-10-09T10:19:42.262404+0000","last_became_active":"2023-10-09T10:16:18.236617+0000","last_became_peered":"2023-10-09T10:16:18.236617+0000","last_unstale":"2023-10-09T10:19:42.262404+0000","last_undegraded":"2023-10-09T10:19:42.262404+0000","last_fullsized":"2023-10-09T10:19:42.262404+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":315,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.284958+0000","last_change":"2023-10-09T10:19:43.291654+0000","last_active":"2023-10-09T10:19:44.284958+0000","last_peered":"2023-10-09T10:19:44.284958+0000","last_clean":"2023-10-09T10:19:44.284958+0000","last_became_active":"2023-10-09T10:19:43.291302+0000","last_became_peered":"2023-10-09T10:19:43.291302+0000","last_unstale":"2023-10-09T10:19:44.284958+0000","last_undegraded":"2023-10-09T10:19:44.284958+0000","last_fullsized":"2023-10-09T10:19:44.284958+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:41.829196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"428'292","reported_seq":305,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T10:19:49.381900+0000","last_change":"2023-10-09T10:19:43.279466+0000","last_active":"2023-10-09T10:19:49.381900+0000","last_peered":"2023-10-09T10:19:49.381900+0000","last_clean":"2023-10-09T10:19:49.381900+0000","last_became_active":"2023-10-09T10:19:43.279157+0000","last_became_peered":"2023-10-09T10:19:43.279157+0000","last_unstale":"2023-10-09T10:19:49.381900+0000","last_undegraded":"2023-10-09T10:19:49.381900+0000","last_fullsized":"2023-10-09T10:19:49.381900+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":292,"log_dups_size":0,"ondisk_log_size":292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:55.938133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19136512,"num_objects":292,"num_object_clones":0,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":292,"num_write_kb":18688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.295184+0000","last_change":"2023-10-09T10:19:43.295184+0000","last_active":"2023-10-09T10:19:43.295184+0000","last_peered":"2023-10-09T10:19:43.295184+0000","last_clean":"2023-10-09T10:19:43.295184+0000","last_became_active":"2023-10-09T10:19:43.274495+0000","last_became_peered":"2023-10-09T10:19:43.274495+0000","last_unstale":"2023-10-09T10:19:43.295184+0000","last_undegraded":"2023-10-09T10:19:43.295184+0000","last_fullsized":"2023-10-09T10:19:43.295184+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.291696+0000","last_change":"2023-10-09T10:19:43.291696+0000","last_active":"2023-10-09T10:19:43.291696+0000","last_peered":"2023-10-09T10:19:43.291696+0000","last_clean":"2023-10-09T10:19:43.291696+0000","last_became_active":"2023-10-09T10:19:43.290863+0000","last_became_peered":"2023-10-09T10:19:43.290863+0000","last_unstale":"2023-10-09T10:19:43.291696+0000","last_undegraded":"2023-10-09T10:19:43.291696+0000","last_fullsized":"2023-10-09T10:19:43.291696+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.284340+0000","last_change":"2023-10-09T10:19:43.284340+0000","last_active":"2023-10-09T10:19:43.284340+0000","last_peered":"2023-10-09T10:19:43.284340+0000","last_clean":"2023-10-09T10:19:43.284340+0000","last_became_active":"2023-10-09T10:19:43.283218+0000","last_became_peered":"2023-10-09T10:19:43.283218+0000","last_unstale":"2023-10-09T10:19:43.284340+0000","last_undegraded":"2023-10-09T10:19:43.284340+0000","last_fullsized":"2023-10-09T10:19:43.284340+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.276421+0000","last_change":"2023-10-09T10:19:43.276421+0000","last_active":"2023-10-09T10:19:43.276421+0000","last_peered":"2023-10-09T10:19:43.276421+0000","last_clean":"2023-10-09T10:19:43.276421+0000","last_became_active":"2023-10-09T10:19:43.276070+0000","last_became_peered":"2023-10-09T10:19:43.276070+0000","last_unstale":"2023-10-09T10:19:43.276421+0000","last_undegraded":"2023-10-09T10:19:43.276421+0000","last_fullsized":"2023-10-09T10:19:43.276421+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.286865+0000","last_change":"2023-10-09T10:19:43.286865+0000","last_active":"2023-10-09T10:19:43.286865+0000","last_peered":"2023-10-09T10:19:43.286865+0000","last_clean":"2023-10-09T10:19:43.286865+0000","last_became_active":"2023-10-09T10:19:43.286530+0000","last_became_peered":"2023-10-09T10:19:43.286530+0000","last_unstale":"2023-10-09T10:19:43.286865+0000","last_undegraded":"2023-10-09T10:19:43.286865+0000","last_fullsized":"2023-10-09T10:19:43.286865+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"428'283","reported_seq":296,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T10:19:49.378303+0000","last_change":"2023-10-09T10:19:43.281938+0000","last_active":"2023-10-09T10:19:49.378303+0000","last_peered":"2023-10-09T10:19:49.378303+0000","last_clean":"2023-10-09T10:19:49.378303+0000","last_became_active":"2023-10-09T10:19:43.281098+0000","last_became_peered":"2023-10-09T10:19:43.281098+0000","last_unstale":"2023-10-09T10:19:49.378303+0000","last_undegraded":"2023-10-09T10:19:49.378303+0000","last_fullsized":"2023-10-09T10:19:49.378303+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":283,"log_dups_size":0,"ondisk_log_size":283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:53.762641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18546688,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":283,"num_write_kb":18112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.288603+0000","last_change":"2023-10-09T10:19:43.288603+0000","last_active":"2023-10-09T10:19:43.288603+0000","last_peered":"2023-10-09T10:19:43.288603+0000","last_clean":"2023-10-09T10:19:43.288603+0000","last_became_active":"2023-10-09T10:19:43.287994+0000","last_became_peered":"2023-10-09T10:19:43.287994+0000","last_unstale":"2023-10-09T10:19:43.288603+0000","last_undegraded":"2023-10-09T10:19:43.288603+0000","last_fullsized":"2023-10-09T10:19:43.288603+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.278989+0000","last_change":"2023-10-09T10:19:43.278989+0000","last_active":"2023-10-09T10:19:43.278989+0000","last_peered":"2023-10-09T10:19:43.278989+0000","last_clean":"2023-10-09T10:19:43.278989+0000","last_became_active":"2023-10-09T10:19:43.278658+0000","last_became_peered":"2023-10-09T10:19:43.278658+0000","last_unstale":"2023-10-09T10:19:43.278989+0000","last_undegraded":"2023-10-09T10:19:43.278989+0000","last_fullsized":"2023-10-09T10:19:43.278989+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.285977+0000","last_change":"2023-10-09T10:19:43.285977+0000","last_active":"2023-10-09T10:19:43.285977+0000","last_peered":"2023-10-09T10:19:43.285977+0000","last_clean":"2023-10-09T10:19:43.285977+0000","last_became_active":"2023-10-09T10:19:43.285666+0000","last_became_peered":"2023-10-09T10:19:43.285666+0000","last_unstale":"2023-10-09T10:19:43.285977+0000","last_undegraded":"2023-10-09T10:19:43.285977+0000","last_fullsized":"2023-10-09T10:19:43.285977+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":395,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312491+0000","last_change":"2023-10-09T10:19:43.281410+0000","last_active":"2023-10-09T10:19:47.312491+0000","last_peered":"2023-10-09T10:19:47.312491+0000","last_clean":"2023-10-09T10:19:47.312491+0000","last_became_active":"2023-10-09T10:19:43.280304+0000","last_became_peered":"2023-10-09T10:19:43.280304+0000","last_unstale":"2023-10-09T10:19:47.312491+0000","last_undegraded":"2023-10-09T10:19:47.312491+0000","last_fullsized":"2023-10-09T10:19:47.312491+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:11.948068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":468,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.313110+0000","last_change":"2023-10-09T10:19:43.284086+0000","last_active":"2023-10-09T10:19:47.313110+0000","last_peered":"2023-10-09T10:19:47.313110+0000","last_clean":"2023-10-09T10:19:47.313110+0000","last_became_active":"2023-10-09T10:19:43.283752+0000","last_became_peered":"2023-10-09T10:19:43.283752+0000","last_unstale":"2023-10-09T10:19:47.313110+0000","last_undegraded":"2023-10-09T10:19:47.313110+0000","last_fullsized":"2023-10-09T10:19:47.313110+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.817673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.291549+0000","last_change":"2023-10-09T10:19:43.291549+0000","last_active":"2023-10-09T10:19:43.291549+0000","last_peered":"2023-10-09T10:19:43.291549+0000","last_clean":"2023-10-09T10:19:43.291549+0000","last_became_active":"2023-10-09T10:19:43.290461+0000","last_became_peered":"2023-10-09T10:19:43.290461+0000","last_unstale":"2023-10-09T10:19:43.291549+0000","last_undegraded":"2023-10-09T10:19:43.291549+0000","last_fullsized":"2023-10-09T10:19:43.291549+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":435,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312792+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:19:47.312792+0000","last_peered":"2023-10-09T10:19:47.312792+0000","last_clean":"2023-10-09T10:19:47.312792+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:19:47.312792+0000","last_undegraded":"2023-10-09T10:19:47.312792+0000","last_fullsized":"2023-10-09T10:19:47.312792+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":73,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.261765+0000","last_change":"2023-10-09T10:16:18.323515+0000","last_active":"2023-10-09T10:19:42.261765+0000","last_peered":"2023-10-09T10:19:42.261765+0000","last_clean":"2023-10-09T10:19:42.261765+0000","last_became_active":"2023-10-09T10:16:18.321171+0000","last_became_peered":"2023-10-09T10:16:18.321171+0000","last_unstale":"2023-10-09T10:19:42.261765+0000","last_undegraded":"2023-10-09T10:19:42.261765+0000","last_fullsized":"2023-10-09T10:19:42.261765+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":427,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312848+0000","last_change":"2023-10-09T10:19:43.278853+0000","last_active":"2023-10-09T10:19:47.312848+0000","last_peered":"2023-10-09T10:19:47.312848+0000","last_clean":"2023-10-09T10:19:47.312848+0000","last_became_active":"2023-10-09T10:19:43.278351+0000","last_became_peered":"2023-10-09T10:19:43.278351+0000","last_unstale":"2023-10-09T10:19:47.312848+0000","last_undegraded":"2023-10-09T10:19:47.312848+0000","last_fullsized":"2023-10-09T10:19:47.312848+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:09.207569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.278170+0000","last_change":"2023-10-09T10:19:43.278170+0000","last_active":"2023-10-09T10:19:43.278170+0000","last_peered":"2023-10-09T10:19:43.278170+0000","last_clean":"2023-10-09T10:19:43.278170+0000","last_became_active":"2023-10-09T10:19:43.277881+0000","last_became_peered":"2023-10-09T10:19:43.277881+0000","last_unstale":"2023-10-09T10:19:43.278170+0000","last_undegraded":"2023-10-09T10:19:43.278170+0000","last_fullsized":"2023-10-09T10:19:43.278170+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":73,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.260149+0000","last_change":"2023-10-09T10:16:18.255781+0000","last_active":"2023-10-09T10:19:42.260149+0000","last_peered":"2023-10-09T10:19:42.260149+0000","last_clean":"2023-10-09T10:19:42.260149+0000","last_became_active":"2023-10-09T10:16:18.238531+0000","last_became_peered":"2023-10-09T10:16:18.238531+0000","last_unstale":"2023-10-09T10:19:42.260149+0000","last_undegraded":"2023-10-09T10:19:42.260149+0000","last_fullsized":"2023-10-09T10:19:42.260149+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.286169+0000","last_change":"2023-10-09T10:19:43.286169+0000","last_active":"2023-10-09T10:19:43.286169+0000","last_peered":"2023-10-09T10:19:43.286169+0000","last_clean":"2023-10-09T10:19:43.286169+0000","last_became_active":"2023-10-09T10:19:43.285440+0000","last_became_peered":"2023-10-09T10:19:43.285440+0000","last_unstale":"2023-10-09T10:19:43.286169+0000","last_undegraded":"2023-10-09T10:19:43.286169+0000","last_fullsized":"2023-10-09T10:19:43.286169+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":482,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312748+0000","last_change":"2023-10-09T10:16:08.230567+0000","last_active":"2023-10-09T10:19:47.312748+0000","last_peered":"2023-10-09T10:19:47.312748+0000","last_clean":"2023-10-09T10:19:47.312748+0000","last_became_active":"2023-10-09T10:16:08.229677+0000","last_became_peered":"2023-10-09T10:16:08.229677+0000","last_unstale":"2023-10-09T10:19:47.312748+0000","last_undegraded":"2023-10-09T10:19:47.312748+0000","last_fullsized":"2023-10-09T10:19:47.312748+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:52.866334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.276368+0000","last_change":"2023-10-09T10:19:43.276368+0000","last_active":"2023-10-09T10:19:43.276368+0000","last_peered":"2023-10-09T10:19:43.276368+0000","last_clean":"2023-10-09T10:19:43.276368+0000","last_became_active":"2023-10-09T10:19:43.276039+0000","last_became_peered":"2023-10-09T10:19:43.276039+0000","last_unstale":"2023-10-09T10:19:43.276368+0000","last_undegraded":"2023-10-09T10:19:43.276368+0000","last_fullsized":"2023-10-09T10:19:43.276368+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":338,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.285347+0000","last_change":"2023-10-09T10:16:08.269361+0000","last_active":"2023-10-09T10:19:44.285347+0000","last_peered":"2023-10-09T10:19:44.285347+0000","last_clean":"2023-10-09T10:19:44.285347+0000","last_became_active":"2023-10-09T10:16:08.266453+0000","last_became_peered":"2023-10-09T10:16:08.266453+0000","last_unstale":"2023-10-09T10:19:44.285347+0000","last_undegraded":"2023-10-09T10:19:44.285347+0000","last_fullsized":"2023-10-09T10:19:44.285347+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:07.633138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":427,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312233+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:19:47.312233+0000","last_peered":"2023-10-09T10:19:47.312233+0000","last_clean":"2023-10-09T10:19:47.312233+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:19:47.312233+0000","last_undegraded":"2023-10-09T10:19:47.312233+0000","last_fullsized":"2023-10-09T10:19:47.312233+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":474,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.313068+0000","last_change":"2023-10-09T10:18:34.270806+0000","last_active":"2023-10-09T10:19:47.313068+0000","last_peered":"2023-10-09T10:19:47.313068+0000","last_clean":"2023-10-09T10:19:47.313068+0000","last_became_active":"2023-10-09T10:18:34.270502+0000","last_became_peered":"2023-10-09T10:18:34.270502+0000","last_unstale":"2023-10-09T10:19:47.313068+0000","last_undegraded":"2023-10-09T10:19:47.313068+0000","last_fullsized":"2023-10-09T10:19:47.313068+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:44.839937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"428'324","reported_seq":337,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T10:19:49.385399+0000","last_change":"2023-10-09T10:19:43.279876+0000","last_active":"2023-10-09T10:19:49.385399+0000","last_peered":"2023-10-09T10:19:49.385399+0000","last_clean":"2023-10-09T10:19:49.385399+0000","last_became_active":"2023-10-09T10:19:43.279596+0000","last_became_peered":"2023-10-09T10:19:43.279596+0000","last_unstale":"2023-10-09T10:19:49.385399+0000","last_undegraded":"2023-10-09T10:19:49.385399+0000","last_fullsized":"2023-10-09T10:19:49.385399+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":324,"log_dups_size":0,"ondisk_log_size":324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:11.020851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21233664,"num_objects":324,"num_object_clones":0,"num_object_copies":972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":324,"num_write_kb":20736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":70,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.260769+0000","last_change":"2023-10-09T10:16:18.477748+0000","last_active":"2023-10-09T10:19:42.260769+0000","last_peered":"2023-10-09T10:19:42.260769+0000","last_clean":"2023-10-09T10:19:42.260769+0000","last_became_active":"2023-10-09T10:16:18.477398+0000","last_became_peered":"2023-10-09T10:16:18.477398+0000","last_unstale":"2023-10-09T10:19:42.260769+0000","last_undegraded":"2023-10-09T10:19:42.260769+0000","last_fullsized":"2023-10-09T10:19:42.260769+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":431,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312283+0000","last_change":"2023-10-09T10:19:14.282209+0000","last_active":"2023-10-09T10:19:47.312283+0000","last_peered":"2023-10-09T10:19:47.312283+0000","last_clean":"2023-10-09T10:19:47.312283+0000","last_became_active":"2023-10-09T10:19:14.281786+0000","last_became_peered":"2023-10-09T10:19:14.281786+0000","last_unstale":"2023-10-09T10:19:47.312283+0000","last_undegraded":"2023-10-09T10:19:47.312283+0000","last_fullsized":"2023-10-09T10:19:47.312283+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:28.589329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.283817+0000","last_change":"2023-10-09T10:19:43.283817+0000","last_active":"2023-10-09T10:19:43.283817+0000","last_peered":"2023-10-09T10:19:43.283817+0000","last_clean":"2023-10-09T10:19:43.283817+0000","last_became_active":"2023-10-09T10:19:43.283539+0000","last_became_peered":"2023-10-09T10:19:43.283539+0000","last_unstale":"2023-10-09T10:19:43.283817+0000","last_undegraded":"2023-10-09T10:19:43.283817+0000","last_fullsized":"2023-10-09T10:19:43.283817+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.281738+0000","last_change":"2023-10-09T10:19:43.281738+0000","last_active":"2023-10-09T10:19:43.281738+0000","last_peered":"2023-10-09T10:19:43.281738+0000","last_clean":"2023-10-09T10:19:43.281738+0000","last_became_active":"2023-10-09T10:19:43.280622+0000","last_became_peered":"2023-10-09T10:19:43.280622+0000","last_unstale":"2023-10-09T10:19:43.281738+0000","last_undegraded":"2023-10-09T10:19:43.281738+0000","last_fullsized":"2023-10-09T10:19:43.281738+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":12,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.285632+0000","last_change":"2023-10-09T10:19:43.281883+0000","last_active":"2023-10-09T10:19:44.285632+0000","last_peered":"2023-10-09T10:19:44.285632+0000","last_clean":"2023-10-09T10:19:44.285632+0000","last_became_active":"2023-10-09T10:19:43.281189+0000","last_became_peered":"2023-10-09T10:19:43.281189+0000","last_unstale":"2023-10-09T10:19:44.285632+0000","last_undegraded":"2023-10-09T10:19:44.285632+0000","last_fullsized":"2023-10-09T10:19:44.285632+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:56.958067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":407,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312719+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:19:47.312719+0000","last_peered":"2023-10-09T10:19:47.312719+0000","last_clean":"2023-10-09T10:19:47.312719+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:19:47.312719+0000","last_undegraded":"2023-10-09T10:19:47.312719+0000","last_fullsized":"2023-10-09T10:19:47.312719+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":359,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.285430+0000","last_change":"2023-10-09T10:19:43.292390+0000","last_active":"2023-10-09T10:19:44.285430+0000","last_peered":"2023-10-09T10:19:44.285430+0000","last_clean":"2023-10-09T10:19:44.285430+0000","last_became_active":"2023-10-09T10:19:43.290733+0000","last_became_peered":"2023-10-09T10:19:43.290733+0000","last_unstale":"2023-10-09T10:19:44.285430+0000","last_undegraded":"2023-10-09T10:19:44.285430+0000","last_fullsized":"2023-10-09T10:19:44.285430+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:53.407598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":288,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312886+0000","last_change":"2023-10-09T10:18:34.268478+0000","last_active":"2023-10-09T10:19:47.312886+0000","last_peered":"2023-10-09T10:19:47.312886+0000","last_clean":"2023-10-09T10:19:47.312886+0000","last_became_active":"2023-10-09T10:18:34.268098+0000","last_became_peered":"2023-10-09T10:18:34.268098+0000","last_unstale":"2023-10-09T10:19:47.312886+0000","last_undegraded":"2023-10-09T10:19:47.312886+0000","last_fullsized":"2023-10-09T10:19:47.312886+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.625264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.281214+0000","last_change":"2023-10-09T10:19:43.281214+0000","last_active":"2023-10-09T10:19:43.281214+0000","last_peered":"2023-10-09T10:19:43.281214+0000","last_clean":"2023-10-09T10:19:43.281214+0000","last_became_active":"2023-10-09T10:19:43.280958+0000","last_became_peered":"2023-10-09T10:19:43.280958+0000","last_unstale":"2023-10-09T10:19:43.281214+0000","last_undegraded":"2023-10-09T10:19:43.281214+0000","last_fullsized":"2023-10-09T10:19:43.281214+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.274868+0000","last_change":"2023-10-09T10:19:43.274868+0000","last_active":"2023-10-09T10:19:43.274868+0000","last_peered":"2023-10-09T10:19:43.274868+0000","last_clean":"2023-10-09T10:19:43.274868+0000","last_became_active":"2023-10-09T10:19:43.274441+0000","last_became_peered":"2023-10-09T10:19:43.274441+0000","last_unstale":"2023-10-09T10:19:43.274868+0000","last_undegraded":"2023-10-09T10:19:43.274868+0000","last_fullsized":"2023-10-09T10:19:43.274868+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":482,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312898+0000","last_change":"2023-10-09T10:16:18.261069+0000","last_active":"2023-10-09T10:19:47.312898+0000","last_peered":"2023-10-09T10:19:47.312898+0000","last_clean":"2023-10-09T10:19:47.312898+0000","last_became_active":"2023-10-09T10:16:18.260736+0000","last_became_peered":"2023-10-09T10:16:18.260736+0000","last_unstale":"2023-10-09T10:19:47.312898+0000","last_undegraded":"2023-10-09T10:19:47.312898+0000","last_fullsized":"2023-10-09T10:19:47.312898+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:18.217603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":318,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.285075+0000","last_change":"2023-10-09T10:16:18.477779+0000","last_active":"2023-10-09T10:19:44.285075+0000","last_peered":"2023-10-09T10:19:44.285075+0000","last_clean":"2023-10-09T10:19:44.285075+0000","last_became_active":"2023-10-09T10:16:18.477390+0000","last_became_peered":"2023-10-09T10:16:18.477390+0000","last_unstale":"2023-10-09T10:19:44.285075+0000","last_undegraded":"2023-10-09T10:19:44.285075+0000","last_fullsized":"2023-10-09T10:19:44.285075+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:06.576807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.287735+0000","last_change":"2023-10-09T10:19:43.287735+0000","last_active":"2023-10-09T10:19:43.287735+0000","last_peered":"2023-10-09T10:19:43.287735+0000","last_clean":"2023-10-09T10:19:43.287735+0000","last_became_active":"2023-10-09T10:19:43.287454+0000","last_became_peered":"2023-10-09T10:19:43.287454+0000","last_unstale":"2023-10-09T10:19:43.287735+0000","last_undegraded":"2023-10-09T10:19:43.287735+0000","last_fullsized":"2023-10-09T10:19:43.287735+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":211,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.285294+0000","last_change":"2023-10-09T10:18:45.347229+0000","last_active":"2023-10-09T10:19:44.285294+0000","last_peered":"2023-10-09T10:19:44.285294+0000","last_clean":"2023-10-09T10:19:44.285294+0000","last_became_active":"2023-10-09T10:18:45.346901+0000","last_became_peered":"2023-10-09T10:18:45.346901+0000","last_unstale":"2023-10-09T10:19:44.285294+0000","last_undegraded":"2023-10-09T10:19:44.285294+0000","last_fullsized":"2023-10-09T10:19:44.285294+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:53.628052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":426,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312973+0000","last_change":"2023-10-09T10:19:43.279293+0000","last_active":"2023-10-09T10:19:47.312973+0000","last_peered":"2023-10-09T10:19:47.312973+0000","last_clean":"2023-10-09T10:19:47.312973+0000","last_became_active":"2023-10-09T10:19:43.278798+0000","last_became_peered":"2023-10-09T10:19:43.278798+0000","last_unstale":"2023-10-09T10:19:47.312973+0000","last_undegraded":"2023-10-09T10:19:47.312973+0000","last_fullsized":"2023-10-09T10:19:47.312973+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:30.954175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":336,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312410+0000","last_change":"2023-10-09T10:18:34.274789+0000","last_active":"2023-10-09T10:19:47.312410+0000","last_peered":"2023-10-09T10:19:47.312410+0000","last_clean":"2023-10-09T10:19:47.312410+0000","last_became_active":"2023-10-09T10:18:34.274396+0000","last_became_peered":"2023-10-09T10:18:34.274396+0000","last_unstale":"2023-10-09T10:19:47.312410+0000","last_undegraded":"2023-10-09T10:19:47.312410+0000","last_fullsized":"2023-10-09T10:19:47.312410+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:37.788599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":9,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.280795+0000","last_change":"2023-10-09T10:19:43.280795+0000","last_active":"2023-10-09T10:19:43.280795+0000","last_peered":"2023-10-09T10:19:43.280795+0000","last_clean":"2023-10-09T10:19:43.280795+0000","last_became_active":"2023-10-09T10:19:43.280338+0000","last_became_peered":"2023-10-09T10:19:43.280338+0000","last_unstale":"2023-10-09T10:19:43.280795+0000","last_undegraded":"2023-10-09T10:19:43.280795+0000","last_fullsized":"2023-10-09T10:19:43.280795+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":333,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T10:19:42.272024+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:19:42.272024+0000","last_peered":"2023-10-09T10:19:42.272024+0000","last_clean":"2023-10-09T10:19:42.272024+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:19:42.272024+0000","last_undegraded":"2023-10-09T10:19:42.272024+0000","last_fullsized":"2023-10-09T10:19:42.272024+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":124,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.313081+0000","last_change":"2023-10-09T10:19:43.297049+0000","last_active":"2023-10-09T10:19:47.313081+0000","last_peered":"2023-10-09T10:19:47.313081+0000","last_clean":"2023-10-09T10:19:47.313081+0000","last_became_active":"2023-10-09T10:19:43.282272+0000","last_became_peered":"2023-10-09T10:19:43.282272+0000","last_unstale":"2023-10-09T10:19:47.313081+0000","last_undegraded":"2023-10-09T10:19:47.313081+0000","last_fullsized":"2023-10-09T10:19:47.313081+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:13:04.873466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.282576+0000","last_change":"2023-10-09T10:19:43.282576+0000","last_active":"2023-10-09T10:19:43.282576+0000","last_peered":"2023-10-09T10:19:43.282576+0000","last_clean":"2023-10-09T10:19:43.282576+0000","last_became_active":"2023-10-09T10:19:43.282091+0000","last_became_peered":"2023-10-09T10:19:43.282091+0000","last_unstale":"2023-10-09T10:19:43.282576+0000","last_undegraded":"2023-10-09T10:19:43.282576+0000","last_fullsized":"2023-10-09T10:19:43.282576+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.274922+0000","last_change":"2023-10-09T10:19:43.274922+0000","last_active":"2023-10-09T10:19:43.274922+0000","last_peered":"2023-10-09T10:19:43.274922+0000","last_clean":"2023-10-09T10:19:43.274922+0000","last_became_active":"2023-10-09T10:19:43.273984+0000","last_became_peered":"2023-10-09T10:19:43.273984+0000","last_unstale":"2023-10-09T10:19:43.274922+0000","last_undegraded":"2023-10-09T10:19:43.274922+0000","last_fullsized":"2023-10-09T10:19:43.274922+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":429,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312889+0000","last_change":"2023-10-09T10:19:43.278560+0000","last_active":"2023-10-09T10:19:47.312889+0000","last_peered":"2023-10-09T10:19:47.312889+0000","last_clean":"2023-10-09T10:19:47.312889+0000","last_became_active":"2023-10-09T10:19:43.277994+0000","last_became_peered":"2023-10-09T10:19:43.277994+0000","last_unstale":"2023-10-09T10:19:47.312889+0000","last_undegraded":"2023-10-09T10:19:47.312889+0000","last_fullsized":"2023-10-09T10:19:47.312889+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:54:56.214071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":103,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.298662+0000","last_change":"2023-10-09T10:19:43.298662+0000","last_active":"2023-10-09T10:19:43.298662+0000","last_peered":"2023-10-09T10:19:43.298662+0000","last_clean":"2023-10-09T10:19:43.298662+0000","last_became_active":"2023-10-09T10:19:43.282784+0000","last_became_peered":"2023-10-09T10:19:43.282784+0000","last_unstale":"2023-10-09T10:19:43.298662+0000","last_undegraded":"2023-10-09T10:19:43.298662+0000","last_fullsized":"2023-10-09T10:19:43.298662+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.286184+0000","last_change":"2023-10-09T10:19:43.286184+0000","last_active":"2023-10-09T10:19:43.286184+0000","last_peered":"2023-10-09T10:19:43.286184+0000","last_clean":"2023-10-09T10:19:43.286184+0000","last_became_active":"2023-10-09T10:19:43.285867+0000","last_became_peered":"2023-10-09T10:19:43.285867+0000","last_unstale":"2023-10-09T10:19:43.286184+0000","last_undegraded":"2023-10-09T10:19:43.286184+0000","last_fullsized":"2023-10-09T10:19:43.286184+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":99,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.300965+0000","last_change":"2023-10-09T10:19:43.300965+0000","last_active":"2023-10-09T10:19:43.300965+0000","last_peered":"2023-10-09T10:19:43.300965+0000","last_clean":"2023-10-09T10:19:43.300965+0000","last_became_active":"2023-10-09T10:19:43.283465+0000","last_became_peered":"2023-10-09T10:19:43.283465+0000","last_unstale":"2023-10-09T10:19:43.300965+0000","last_undegraded":"2023-10-09T10:19:43.300965+0000","last_fullsized":"2023-10-09T10:19:43.300965+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":247,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.284647+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:19:44.284647+0000","last_peered":"2023-10-09T10:19:44.284647+0000","last_clean":"2023-10-09T10:19:44.284647+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:19:44.284647+0000","last_undegraded":"2023-10-09T10:19:44.284647+0000","last_fullsized":"2023-10-09T10:19:44.284647+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":83,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-10-09T10:19:47.312784+0000","last_change":"2023-10-09T10:16:08.240555+0000","last_active":"2023-10-09T10:19:47.312784+0000","last_peered":"2023-10-09T10:19:47.312784+0000","last_clean":"2023-10-09T10:19:47.312784+0000","last_became_active":"2023-10-09T10:16:08.239692+0000","last_became_peered":"2023-10-09T10:16:08.239692+0000","last_unstale":"2023-10-09T10:19:47.312784+0000","last_undegraded":"2023-10-09T10:19:47.312784+0000","last_fullsized":"2023-10-09T10:19:47.312784+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:39.501717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":467,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T10:19:44.285258+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:19:44.285258+0000","last_peered":"2023-10-09T10:19:44.285258+0000","last_clean":"2023-10-09T10:19:44.285258+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:19:44.285258+0000","last_undegraded":"2023-10-09T10:19:44.285258+0000","last_fullsized":"2023-10-09T10:19:44.285258+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":399,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.388243+0000","last_change":"2023-10-09T10:19:43.388243+0000","last_active":"2023-10-09T10:19:43.388243+0000","last_peered":"2023-10-09T10:19:43.388243+0000","last_clean":"2023-10-09T10:19:43.388243+0000","last_became_active":"2023-10-09T10:19:43.283927+0000","last_became_peered":"2023-10-09T10:19:43.283927+0000","last_unstale":"2023-10-09T10:19:43.388243+0000","last_undegraded":"2023-10-09T10:19:43.388243+0000","last_fullsized":"2023-10-09T10:19:43.388243+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":10,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-10-09T10:19:43.282681+0000","last_change":"2023-10-09T10:19:43.282681+0000","last_active":"2023-10-09T10:19:43.282681+0000","last_peered":"2023-10-09T10:19:43.282681+0000","last_clean":"2023-10-09T10:19:43.282681+0000","last_became_active":"2023-10-09T10:19:43.282337+0000","last_became_peered":"2023-10-09T10:19:43.282337+0000","last_unstale":"2023-10-09T10:19:43.282681+0000","last_undegraded":"2023-10-09T10:19:43.282681+0000","last_fullsized":"2023-10-09T10:19:43.282681+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":58916864,"num_objects":899,"num_object_clones":0,"num_object_copies":2697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":899,"num_write_kb":57536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":78905344,"data_stored":78905344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":899,"ondisk_log_size":899,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":16658,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430601,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925036,"kb_used_data":4388,"kb_used_omap":0,"kb_used_meta":920640,"kb_avail":92820116,"statfs":{"total":95995035648,"available":95047798784,"internally_reserved":0,"allocated":4493312,"data_stored":3769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":942735360},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:18:59 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.145,"15min":2.0569999999999999},"min":{"1min":0.23599999999999999,"5min":0.223,"15min":0.217},"max":{"1min":2.4710000000000001,"5min":7.806,"15min":50.591999999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.2130000000000001,"15min":2.0800000000000001},"min":{"1min":0.222,"5min":0.22,"15min":0.22},"max":{"1min":8.8100000000000005,"5min":10.029,"15min":50.747},"last":0.996}]},{"osd":1,"last update":"Mon Oct 9 10:19:25 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.2709999999999999,"15min":1.968},"min":{"1min":0.246,"5min":0.23799999999999999,"15min":0.22500000000000001},"max":{"1min":3.6150000000000002,"5min":9.657,"15min":16.352},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.911,"5min":1.649,"15min":2.0459999999999998},"min":{"1min":0.309,"5min":0.249,"15min":0.249},"max":{"1min":8.8710000000000004,"5min":9.1110000000000007,"15min":18.27},"last":0.78900000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:19:15 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.363,"15min":1.9890000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":3.4089999999999998,"5min":7.8600000000000003,"15min":13.775},"last":0.82199999999999995},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.9930000000000001},"min":{"1min":0.34599999999999997,"5min":0.30099999999999999,"15min":0.28699999999999998},"max":{"1min":8.9339999999999993,"5min":12.845000000000001,"15min":12.845000000000001},"last":1.1830000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.492,"15min":1.536},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.7069999999999999,"5min":7.444,"15min":7.444},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.6639999999999999,"15min":1.625},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.7720000000000002,"5min":8.0779999999999994,"15min":8.0779999999999994},"last":0.873}]},{"osd":4,"last update":"Mon Oct 9 10:18:56 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.052,"15min":1.1679999999999999},"min":{"1min":0.33700000000000002,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":2.7309999999999999,"5min":7.8890000000000002,"15min":12.699999999999999},"last":0.316},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.82299999999999995,"15min":1.012},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.6640000000000001,"5min":5.3609999999999998,"15min":12.003},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 10:19:15 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.4370000000000001,"15min":1.399},"min":{"1min":0.376,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":6.6769999999999996,"5min":12.048,"15min":25.277999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.7639999999999998,"5min":1.4199999999999999,"15min":1.3620000000000001},"min":{"1min":0.41499999999999998,"5min":0.26800000000000002,"15min":0.22600000000000001},"max":{"1min":8.7989999999999995,"5min":13.083,"15min":25.754000000000001},"last":0.41799999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:19:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":1.1220000000000001,"15min":1.196},"min":{"1min":0.42599999999999999,"5min":0.28399999999999997,"15min":0.26100000000000001},"max":{"1min":8.9030000000000005,"5min":8.9030000000000005,"15min":8.9030000000000005},"last":0.439},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.052,"15min":1.1699999999999999},"min":{"1min":0.40000000000000002,"5min":0.307,"15min":0.27200000000000002},"max":{"1min":3.7440000000000002,"5min":6.702,"15min":10.625},"last":0.35999999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626829,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1111644,"kb_used_data":208276,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92633508,"statfs":{"total":95995035648,"available":94856712192,"internally_reserved":0,"allocated":213274624,"data_stored":212560702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.83399999999999996,"15min":1.6419999999999999},"min":{"1min":0.254,"5min":0.23100000000000001,"15min":0.23000000000000001},"max":{"1min":1.798,"5min":5.306,"15min":15.435},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.0070000000000001,"5min":1.619,"15min":1.8640000000000001},"min":{"1min":0.26800000000000002,"5min":0.26700000000000002,"15min":0.22},"max":{"1min":7.5700000000000003,"5min":10.539999999999999,"15min":15.308999999999999},"last":0.53200000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:19:20 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.365,"15min":2.0539999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":4.681,"5min":17.925000000000001,"15min":35.609999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.9299999999999999,"5min":1.7769999999999999,"15min":2.214},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":7.6420000000000003,"5min":10.58,"15min":32.396000000000001},"last":0.499}]},{"osd":2,"last update":"Mon Oct 9 10:19:03 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.827,"15min":1.764},"min":{"1min":0.307,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":7.3159999999999998,"5min":29.367999999999999,"15min":29.367999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.76600000000000001,"5min":1.306,"15min":2.1720000000000002},"min":{"1min":0.32200000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.0490000000000004,"5min":8.4480000000000004,"15min":14.542999999999999},"last":0.68300000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.3089999999999999,"15min":1.552},"min":{"1min":0.40400000000000003,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.9379999999999999,"5min":8.0229999999999997,"15min":8.0229999999999997},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.3700000000000001,"15min":1.454},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.5089999999999999,"5min":6.4189999999999996,"15min":6.4189999999999996},"last":0.59299999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":0.94899999999999995,"15min":0.79900000000000004},"min":{"1min":0.29599999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.1989999999999998,"5min":9.1989999999999998,"15min":10.113},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.72699999999999998,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":4.0469999999999997,"5min":4.0940000000000003,"15min":9.218},"last":0.46700000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":0.96099999999999997,"15min":0.94699999999999995},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.23699999999999999},"max":{"1min":3.5859999999999999,"5min":6.6210000000000004,"15min":7.9260000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.65000000000000002,"15min":0.96199999999999997},"min":{"1min":0.47199999999999998,"5min":0.28199999999999997,"15min":0.25800000000000001},"max":{"1min":1.9059999999999999,"5min":4.3360000000000003,"15min":14.465999999999999},"last":0.58099999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:19:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.083,"15min":1.0309999999999999},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.252},"max":{"1min":5.9950000000000001,"5min":6.0279999999999996,"15min":8.8330000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.1259999999999999,"15min":1.073},"min":{"1min":0.44800000000000001,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":6.0270000000000001,"5min":6.0629999999999997,"15min":8.8550000000000004},"last":0.61199999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253143,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":861028,"kb_used_data":138972,"kb_used_omap":9,"kb_used_meta":722038,"kb_avail":92884124,"statfs":{"total":95995035648,"available":95113342976,"internally_reserved":0,"allocated":142307328,"data_stored":140778946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":739367913},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":1.1459999999999999,"15min":2.3490000000000002},"min":{"1min":0.216,"5min":0.188,"15min":0.183},"max":{"1min":1.417,"5min":15.913,"15min":31.007000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.399,"15min":2.6349999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":3.4990000000000001,"5min":16.015000000000001,"15min":20.501999999999999},"last":0.30499999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:19:02 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.96,"15min":2.8759999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.217},"max":{"1min":4.3970000000000002,"5min":23.105,"15min":23.105},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.0740000000000001,"15min":2.532},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.22500000000000001},"max":{"1min":2.907,"5min":7.4939999999999998,"15min":28.024999999999999},"last":0.622}]},{"osd":3,"last update":"Mon Oct 9 10:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.96799999999999997,"15min":1.107},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.5529999999999999,"5min":6.673,"15min":6.673},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.495,"5min":1.194,"15min":1.54},"min":{"1min":0.32400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.93200000000000005,"5min":7.2160000000000002,"15min":7.2160000000000002},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.139,"15min":1.3959999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28699999999999998},"max":{"1min":2.7050000000000001,"5min":7.3789999999999996,"15min":21.620000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.4590000000000001,"15min":1.794},"min":{"1min":0.35599999999999998,"5min":0.314,"15min":0.308},"max":{"1min":5.1559999999999997,"5min":10.497999999999999,"15min":12.904},"last":0.59199999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8100000000000001,"5min":2.8610000000000002,"15min":2.5179999999999998},"min":{"1min":0.33600000000000002,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":13.147,"5min":92.177999999999997,"15min":92.177999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.081,"5min":2.9089999999999998,"15min":2.7759999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.29099999999999998},"max":{"1min":13.170999999999999,"5min":91.869,"15min":91.869},"last":0.93000000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":1.369,"15min":1.9279999999999999},"min":{"1min":0.41799999999999998,"5min":0.35599999999999998,"15min":0.34999999999999998},"max":{"1min":6.8540000000000001,"5min":7.4409999999999998,"15min":21.670999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.5389999999999999,"15min":2.1230000000000002},"min":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.35099999999999998},"max":{"1min":6.8390000000000004,"5min":9.1660000000000004,"15min":14.471},"last":0.55700000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.4990000000000001,"15min":2.3420000000000001},"min":{"1min":0.46899999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.7149999999999999,"5min":10.127000000000001,"15min":16.331},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.986,"5min":1.536,"15min":1.996},"min":{"1min":0.41699999999999998,"5min":0.371,"15min":0.371},"max":{"1min":10.627000000000001,"5min":10.627000000000001,"15min":21.593},"last":0.72099999999999997}]}]},{"osd":0,"up_from":79,"seq":339302416667,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":697184,"kb_used_data":158168,"kb_used_omap":28,"kb_used_meta":538979,"kb_avail":93047968,"statfs":{"total":95995035648,"available":95281119232,"internally_reserved":0,"allocated":161964032,"data_stored":160422325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":551915344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:19:33 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":1.8149999999999999,"15min":2.1880000000000002},"min":{"1min":0.22800000000000001,"5min":0.19800000000000001,"15min":0.16600000000000001},"max":{"1min":1.0389999999999999,"5min":54.518000000000001,"15min":54.518000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.72699999999999998,"5min":1.8600000000000001,"15min":3.327},"min":{"1min":0.27600000000000002,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.7269999999999999,"5min":56.950000000000003,"15min":56.950000000000003},"last":0.38900000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:19:12 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.8700000000000001,"15min":2.1800000000000002},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":2.8260000000000001,"5min":53.988999999999997,"15min":53.988999999999997},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.778,"15min":2.343},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":2.7589999999999999,"5min":54.119,"15min":54.119},"last":0.45500000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:18:47 2023","interfaces":[{"interface":"back","average":{"1min":3.8279999999999998,"5min":1.663,"15min":1.3380000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":56.984999999999999,"5min":56.984999999999999,"15min":56.984999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.5,"5min":1.6200000000000001,"15min":1.258},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":56.475999999999999,"5min":56.475999999999999,"15min":56.475999999999999},"last":0.36599999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:19:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":2.1360000000000001,"15min":2.1840000000000002},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.7539999999999996,"5min":57.719000000000001,"15min":57.719000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.286,"5min":2.0449999999999999,"15min":2.2469999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.7709999999999999,"5min":56.378,"15min":56.378},"last":0.52200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:19:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":4.4370000000000003,"15min":3.4260000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.9880000000000004,"5min":229.589,"15min":229.589},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.748,"5min":4.3550000000000004,"15min":2.7130000000000001},"min":{"1min":0.31900000000000001,"5min":0.30199999999999999,"15min":0.27000000000000002},"max":{"1min":6.9329999999999998,"5min":228.88900000000001,"15min":228.88900000000001},"last":0.51100000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:19:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":2.1579999999999999,"15min":2.085},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":5.5030000000000001,"5min":54.482999999999997,"15min":54.482999999999997},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.5269999999999999,"5min":2.3359999999999999,"15min":2.8860000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.3659999999999997,"5min":57.753,"15min":57.753},"last":0.496}]},{"osd":7,"last update":"Mon Oct 9 10:19:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":2.1989999999999998,"15min":2.7829999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":3.5569999999999999,"5min":56.411999999999999,"15min":56.411999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.6020000000000001,"5min":2.1019999999999999,"15min":2.1419999999999999},"min":{"1min":0.45000000000000001,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":7.2640000000000002,"5min":54.451999999999998,"15min":54.451999999999998},"last":0.56100000000000005}]}]},{"osd":2,"up_from":217,"seq":932007903402,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":636928,"kb_used_data":142840,"kb_used_omap":83,"kb_used_meta":493996,"kb_avail":93108224,"statfs":{"total":95995035648,"available":95342821376,"internally_reserved":0,"allocated":146268160,"data_stored":144870225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":505852399},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:19:07 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":1.361,"15min":1.907},"min":{"1min":0.222,"5min":0.20200000000000001,"15min":0.17899999999999999},"max":{"1min":33.289000000000001,"5min":33.289000000000001,"15min":33.289000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.859,"5min":1.899,"15min":2.8090000000000002},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":33.347999999999999,"5min":33.347999999999999,"15min":33.347999999999999},"last":0.42499999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:19:07 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":1.423,"15min":2.4119999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.8039999999999998,"5min":34.893999999999998,"15min":34.893999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.71899999999999997,"5min":1.9330000000000001,"15min":2.54},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":3.5659999999999998,"5min":34.962000000000003,"15min":34.962000000000003},"last":0.44900000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:19:44 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.1779999999999999,"15min":0.97399999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":2.0049999999999999,"5min":35.043999999999997,"15min":35.043999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.623,"15min":1.216},"min":{"1min":0.29599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.0169999999999999,"5min":50.406999999999996,"15min":50.406999999999996},"last":0.48099999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:18:56 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.3129999999999999,"15min":1.3580000000000001},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.26600000000000001},"max":{"1min":3.6440000000000001,"5min":10.996,"15min":18.873999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.569,"15min":1.496},"min":{"1min":0.35399999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.9089999999999998,"5min":24.352,"15min":24.352},"last":0.69899999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":2.2000000000000002,"15min":2.0150000000000001},"min":{"1min":0.32700000000000001,"5min":0.32100000000000001,"15min":0.27700000000000002},"max":{"1min":5.9400000000000004,"5min":57.298000000000002,"15min":57.298000000000002},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.75,"15min":1.885},"min":{"1min":0.34100000000000003,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":4.5510000000000002,"5min":57.828000000000003,"15min":57.828000000000003},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:19:07 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":1.4650000000000001,"15min":1.716},"min":{"1min":0.36799999999999999,"5min":0.36599999999999999,"15min":0.33100000000000002},"max":{"1min":4.0019999999999998,"5min":10.962999999999999,"15min":18.943999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.522,"5min":1.6759999999999999,"15min":1.9650000000000001},"min":{"1min":0.45800000000000002,"5min":0.40200000000000002,"15min":0.33300000000000002},"max":{"1min":5.4859999999999998,"5min":24.379000000000001,"15min":24.379000000000001},"last":0.66500000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.99,"15min":1.9410000000000001},"min":{"1min":0.441,"5min":0.308,"15min":0.308},"max":{"1min":4.9420000000000002,"5min":35.008000000000003,"15min":35.008000000000003},"last":0.627},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.9590000000000001,"15min":2.1779999999999999},"min":{"1min":0.44500000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.6539999999999999,"5min":24.323,"15min":24.323},"last":0.72999999999999998}]}]},{"osd":3,"up_from":352,"seq":1511828488242,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1247372,"kb_used_data":1057860,"kb_used_omap":15,"kb_used_meta":189488,"kb_avail":92497780,"statfs":{"total":95995035648,"available":94717726720,"internally_reserved":0,"allocated":1083248640,"data_stored":1081435512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":194036166},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:18:50 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.0149999999999999,"15min":1.0880000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":3.2490000000000001,"5min":7.4050000000000002,"15min":7.4050000000000002},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.157,"15min":1.004},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":3.1680000000000001,"5min":7.4850000000000003,"15min":7.4850000000000003},"last":0.377}]},{"osd":1,"last update":"Mon Oct 9 10:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.2869999999999999,"15min":1.169},"min":{"1min":0.31,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.415,"5min":15.970000000000001,"15min":15.970000000000001},"last":0.441},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.91900000000000004,"15min":0.76000000000000001},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.7450000000000001,"5min":7.516,"15min":7.516},"last":0.33500000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:18:50 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":1.8420000000000001,"15min":2.0720000000000001},"min":{"1min":0.26400000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.4590000000000001,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.29,"15min":1.244},"min":{"1min":0.30099999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.3690000000000002,"5min":16.018000000000001,"15min":16.018000000000001},"last":0.46300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:18:50 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":0.83999999999999997,"15min":0.71099999999999997},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.3799999999999999,"5min":7.3630000000000004,"15min":7.3630000000000004},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.3779999999999999,"15min":1.3109999999999999},"min":{"1min":0.35999999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1900000000000004,"5min":7.8689999999999998,"15min":7.8689999999999998},"last":0.64800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":0.99399999999999999,"15min":0.79400000000000004},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.1260000000000003,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.96499999999999997,"15min":0.79200000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.8630000000000004,"5min":7.8630000000000004,"15min":7.8630000000000004},"last":0.67900000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:18:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.431,"15min":1.3620000000000001},"min":{"1min":0.41599999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":2.423,"5min":7.9219999999999997,"15min":7.9219999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.619,"5min":1.129,"15min":0.97299999999999998},"min":{"1min":0.41999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.698,"5min":10.698,"15min":10.698},"last":0.71299999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:18:50 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.081,"15min":0.86099999999999999},"min":{"1min":0.40400000000000003,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":3.1339999999999999,"5min":10.459,"15min":10.459},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.7749999999999999,"5min":1.6180000000000001,"15min":1.4370000000000001},"min":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":8.827,"5min":8.827,"15min":8.827},"last":0.69999999999999996}]}]},{"osd":4,"up_from":266,"seq":1142461300871,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":326560,"kb_used_data":3800,"kb_used_omap":35,"kb_used_meta":322716,"kb_avail":93418592,"statfs":{"total":95995035648,"available":95660638208,"internally_reserved":0,"allocated":3891200,"data_stored":3171614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":330461422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:18:56 2023","interfaces":[{"interface":"back","average":{"1min":4.6189999999999998,"5min":1.6060000000000001,"15min":1.2490000000000001},"min":{"1min":0.247,"5min":0.222,"15min":0.21199999999999999},"max":{"1min":71.442999999999998,"5min":71.442999999999998,"15min":71.442999999999998},"last":0.64700000000000002},{"interface":"front","average":{"1min":4.915,"5min":1.9710000000000001,"15min":1.6930000000000001},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.224},"max":{"1min":73.015000000000001,"5min":73.015000000000001,"15min":73.015000000000001},"last":0.65600000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:18:56 2023","interfaces":[{"interface":"back","average":{"1min":4.8860000000000001,"5min":1.736,"15min":1.7509999999999999},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.25},"max":{"1min":71.457999999999998,"5min":71.457999999999998,"15min":71.457999999999998},"last":0.70699999999999996},{"interface":"front","average":{"1min":4.423,"5min":1.7090000000000001,"15min":1.7370000000000001},"min":{"1min":0.312,"5min":0.254,"15min":0.254},"max":{"1min":71.658000000000001,"5min":71.658000000000001,"15min":71.658000000000001},"last":0.63900000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.177,"15min":1.389},"min":{"1min":0.32500000000000001,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":4.4160000000000004,"5min":5.8259999999999996,"15min":9.2469999999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.3740000000000001,"5min":1.347,"15min":1.399},"min":{"1min":0.32000000000000001,"5min":0.28199999999999997,"15min":0.27700000000000002},"max":{"1min":4.6120000000000001,"5min":6.3029999999999999,"15min":11.778},"last":0.61699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:18:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.4470000000000001,"15min":1.601},"min":{"1min":0.34300000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.899,"5min":9.6170000000000009,"15min":9.6170000000000009},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.171,"5min":1.0369999999999999,"15min":1.0549999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.5830000000000002,"5min":3.5830000000000002,"15min":3.5830000000000002},"last":0.66400000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.9410000000000001,"15min":1.2889999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.23799999999999999},"max":{"1min":4.4569999999999999,"5min":99.585999999999999,"15min":99.585999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.569,"5min":1.984,"15min":1.119},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.5830000000000002,"5min":99.384,"15min":99.384},"last":0.57099999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":0.84799999999999998,"15min":0.81200000000000006},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.6870000000000003,"5min":4.6870000000000003,"15min":6.851},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.427,"5min":0.97099999999999997,"15min":0.90800000000000003},"min":{"1min":0.36099999999999999,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":7.96,"5min":7.96,"15min":12.255000000000001},"last":0.59099999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":0.997,"15min":1.0049999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":7.9850000000000003,"5min":7.9850000000000003,"15min":12.237},"last":0.627},{"interface":"front","average":{"1min":1.5309999999999999,"5min":1.0449999999999999,"15min":1.036},"min":{"1min":0.375,"5min":0.32100000000000001,"15min":0.30399999999999999},"max":{"1min":7.7130000000000001,"5min":7.7130000000000001,"15min":14.964},"last":0.60699999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823056,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":934924,"kb_used_data":33604,"kb_used_omap":0,"kb_used_meta":901312,"kb_avail":92810228,"statfs":{"total":95995035648,"available":95037673472,"internally_reserved":0,"allocated":34410496,"data_stored":33706580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922943488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:18:46 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.4099999999999999,"15min":1.661},"min":{"1min":0.251,"5min":0.219,"15min":0.21099999999999999},"max":{"1min":8.0180000000000007,"5min":16.079000000000001,"15min":16.079000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.355,"15min":1.569},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":7.0800000000000001,"5min":15.907,"15min":15.907},"last":0.44900000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:19:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.4430000000000001,"15min":1.7230000000000001},"min":{"1min":0.31900000000000001,"5min":0.246,"15min":0.23699999999999999},"max":{"1min":3.6339999999999999,"5min":16.056999999999999,"15min":16.056999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.363,"15min":1.756},"min":{"1min":0.34100000000000003,"5min":0.248,"15min":0.248},"max":{"1min":7.1280000000000001,"5min":7.1280000000000001,"15min":23.099},"last":0.32900000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:19:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.6200000000000001,"15min":1.9770000000000001},"min":{"1min":0.29099999999999998,"5min":0.28299999999999997,"15min":0.27200000000000002},"max":{"1min":2.1539999999999999,"5min":15.973000000000001,"15min":15.973000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.601,"5min":1.512,"15min":1.6779999999999999},"min":{"1min":0.33500000000000002,"5min":0.317,"15min":0.313},"max":{"1min":7.1449999999999996,"5min":15.929,"15min":15.929},"last":0.39600000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:18:46 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.389,"15min":1.446},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.851,"5min":8.8689999999999998,"15min":8.8689999999999998},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.546,"15min":1.4950000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":9.4480000000000004,"5min":9.4480000000000004,"15min":9.4480000000000004},"last":0.48299999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.0169999999999999,"15min":0.88600000000000001},"min":{"1min":0.27100000000000002,"5min":0.25800000000000001,"15min":0.24299999999999999},"max":{"1min":2.9300000000000002,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.375},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.0129999999999999,"15min":0.82599999999999996},"min":{"1min":0.36499999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.1100000000000003,"5min":7.1100000000000003,"15min":8.5559999999999992},"last":0.43099999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:18:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.077,"15min":1.0880000000000001},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.25},"max":{"1min":2.8820000000000001,"5min":10.222,"15min":13.483000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.425,"15min":1.202},"min":{"1min":0.33000000000000002,"5min":0.26700000000000002,"15min":0.22600000000000001},"max":{"1min":7.9539999999999997,"5min":15.997,"15min":18.600999999999999},"last":0.40999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.3640000000000001,"15min":1.206},"min":{"1min":0.35399999999999998,"5min":0.311,"15min":0.25600000000000001},"max":{"1min":4.2300000000000004,"5min":16.030999999999999,"15min":16.030999999999999},"last":0.437},{"interface":"front","average":{"1min":2.218,"5min":1.1699999999999999,"15min":0.92300000000000004},"min":{"1min":0.42299999999999999,"5min":0.27500000000000002,"15min":0.26800000000000002},"max":{"1min":8.4649999999999999,"5min":8.4649999999999999,"15min":8.5220000000000002},"last":0.46500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":78905344,"data_stored":78905344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:19:51.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-10-09T10:19:51.554 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:51.617 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:51.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:51.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:51.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:51 smithi110 ceph-mon[136032]: pgmap v1191: 89 pgs: 89 active+clean; 57 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.4 MiB/s wr, 149 op/s; 105 B/s, 1 objects/s recovering 2023-10-09T10:19:51.730 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:51.840 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:51.976 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 4 16 11879 11863 185.311 162.875 0.0034436 0.00539029 2023-10-09T10:19:52.020 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:52.024 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:52.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:52.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:52.132 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:19:52.187 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:52.263 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:19:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:52.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:19:52.325 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:52 smithi012 ceph-mon[152263]: from='client.56578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:19:52.367 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:52.385 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:19:52.422 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:19:52.454 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:19:52.465 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:52.487 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:19:52.524 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:19:52.562 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:19:52.598 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:19:52.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:19:52.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:19:52.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:19:52.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:19:52.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:19:52.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:19:52.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:19:52.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:19:52.654 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:19:52.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:52 smithi012 ceph-mon[147926]: from='client.56578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:19:52.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:52 smithi110 ceph-mon[136032]: from='client.56578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:19:52.976 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 5 16 14509 14493 181.121 164.375 0.00464318 0.00551408 2023-10-09T10:19:53.327 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:53 smithi012 ceph-mon[152263]: pgmap v1192: 89 pgs: 89 active+clean; 314 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 85 B/s, 1 objects/s recovering 2023-10-09T10:19:53.327 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/939641534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:53.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:53 smithi012 ceph-mon[147926]: pgmap v1192: 89 pgs: 89 active+clean; 314 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 85 B/s, 1 objects/s recovering 2023-10-09T10:19:53.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/939641534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:53.563 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:53.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:53.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:53.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:53 smithi110 ceph-mon[136032]: pgmap v1192: 89 pgs: 89 active+clean; 314 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 85 B/s, 1 objects/s recovering 2023-10-09T10:19:53.584 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/939641534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:19:53.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:19:53.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:53.716 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:53.976 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 6 16 17178 17162 178.722 166.812 0.00296898 0.00558868 2023-10-09T10:19:54.457 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:19:54.976 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 7 16 19581 19565 174.644 150.188 0.0025308 0.00571488 2023-10-09T10:19:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:55 smithi012 ceph-mon[147926]: pgmap v1193: 89 pgs: 89 active+clean; 415 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T10:19:55.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:55 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:55.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:55 smithi012 ceph-mon[152263]: pgmap v1193: 89 pgs: 89 active+clean; 415 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T10:19:55.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:55 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:55.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:55 smithi110 ceph-mon[136032]: pgmap v1193: 89 pgs: 89 active+clean; 415 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T10:19:55.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:55 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:55.730 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:55.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:55.833 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:19:55.888 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:55.976 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 8 16 21824 21808 170.336 140.188 0.00563443 0.00586531 2023-10-09T10:19:56.342 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:19:56.342 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:19:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:56.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:19:56.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:19:56.498 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:56.572 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:19:56.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:56.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:19:56.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:19:56.730 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:56.976 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 9 16 23844 23828 165.437 126.25 0.00678251 0.00603909 2023-10-09T10:19:57.065 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:19:57.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:57.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:19:57.173 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:19:57.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:19:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:57 smithi012 ceph-mon[147926]: pgmap v1194: 89 pgs: 89 active+clean; 1.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-10-09T10:19:57.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:57 smithi012 ceph-mon[152263]: pgmap v1194: 89 pgs: 89 active+clean; 1.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-10-09T10:19:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:57 smithi110 ceph-mon[136032]: pgmap v1194: 89 pgs: 89 active+clean; 1.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-10-09T10:19:57.957 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:19:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:19:57] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-10-09T10:19:57.977 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 10 16 25676 25660 160.326 114.5 0.00434108 0.0062306 2023-10-09T10:19:58.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:19:58.978 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 11 16 27252 27236 154.706 98.5 0.00605345 0.00645935 2023-10-09T10:19:59.369 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:19:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[147926]: pgmap v1195: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T10:19:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1349031556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[152263]: pgmap v1195: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T10:19:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1349031556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:59.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:19:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:59 smithi110 ceph-mon[136032]: pgmap v1195: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T10:19:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1349031556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:19:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:19:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:19:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:19:59.743 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:19:59.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:19:59.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:19:59.847 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:19:59.978 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 12 16 28972 28956 150.772 107.5 0.00283186 0.00662622 2023-10-09T10:19:59.990 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:00.535 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:00.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:20:00.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: osdmap e429: 8 total, 8 up, 8 in 2023-10-09T10:20:00.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.537 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.537 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T10:20:00.537 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T10:20:00.537 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]: dispatch 2023-10-09T10:20:00.537 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]: dispatch 2023-10-09T10:20:00.537 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2023-10-09T10:20:00.538 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]: dispatch 2023-10-09T10:20:00.538 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]: dispatch 2023-10-09T10:20:00.538 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2023-10-09T10:20:00.538 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:00.538 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:20:00.539 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: osdmap e429: 8 total, 8 up, 8 in 2023-10-09T10:20:00.539 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.539 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.539 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.539 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.539 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.540 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T10:20:00.540 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T10:20:00.540 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]: dispatch 2023-10-09T10:20:00.540 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]: dispatch 2023-10-09T10:20:00.540 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2023-10-09T10:20:00.541 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]: dispatch 2023-10-09T10:20:00.541 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]: dispatch 2023-10-09T10:20:00.541 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2023-10-09T10:20:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:20:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: osdmap e429: 8 total, 8 up, 8 in 2023-10-09T10:20:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: failed to encode map e429 with expected crc 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]: dispatch 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]: dispatch 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]: dispatch 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]: dispatch 2023-10-09T10:20:00.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2023-10-09T10:20:00.745 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:00.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:00.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:00.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:00.911 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:00.977 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:00.978 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 13 16 30619 30603 147.092 102.938 0.0102414 0.00679282 2023-10-09T10:20:00.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:00.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:01.015 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:01.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:01.147 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:01.212 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:01.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:01.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:01.383 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:01.407 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 10 pgp_num to 16 2023-10-09T10:20:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[152263]: pgmap v1197: 89 pgs: 89 active+clean; 1.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T10:20:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[152263]: failed to encode map e430 with expected crc 2023-10-09T10:20:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]': finished 2023-10-09T10:20:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]': finished 2023-10-09T10:20:01.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[152263]: osdmap e430: 8 total, 8 up, 8 in 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/481405942' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[147926]: pgmap v1197: 89 pgs: 89 active+clean; 1.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[147926]: failed to encode map e430 with expected crc 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]': finished 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]': finished 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[147926]: osdmap e430: 8 total, 8 up, 8 in 2023-10-09T10:20:01.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/481405942' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:20:01.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:20:01.553 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:01.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:01.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:01.658 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:01.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:01 smithi110 ceph-mon[136032]: pgmap v1197: 89 pgs: 89 active+clean; 1.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T10:20:01.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:01 smithi110 ceph-mon[136032]: failed to encode map e430 with expected crc 2023-10-09T10:20:01.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 0]}]': finished 2023-10-09T10:20:01.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 0]}]': finished 2023-10-09T10:20:01.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2023-10-09T10:20:01.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:01 smithi110 ceph-mon[136032]: osdmap e430: 8 total, 8 up, 8 in 2023-10-09T10:20:01.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/481405942' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:20:01.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:20:01.709 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:65159 9.3 9:cf47ceeb:::benchmark_data_smithi012_313365_object65158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 346.94939752699997, 2023-10-09T10:20:01.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19414071499999999, 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 65159 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:20:01.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008266+0000", 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008264+0000", 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8745000000000001e-05 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008323+0000", 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1172000000000001e-05 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008334+0000", 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4099999999999996e-06 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008341+0000", 2023-10-09T10:20:01.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6387999999999998e-05 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008377+0000", 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015405 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008531+0000", 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016722800000000001 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.008698+0000", 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023079260000000001 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.011006+0000", 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19127018400000001 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202277+0000", 2023-10-09T10:20:01.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9379999999999999e-05 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202316+0000", 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0695999999999997e-05 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202367+0000", 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9728000000000003e-05 2023-10-09T10:20:01.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:14:15.202406+0000", 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.19414071499999999 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114942 9.11 9:8d57c7a6:::benchmark_data_smithi012_313365_object2467:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:20:01.970 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.74371970800001, 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0078972580000001, 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114942 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999967 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213940+0000", 2023-10-09T10:20:01.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.791e-05 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213958+0000", 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5698999999999998e-05 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213994+0000", 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.436e-05 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214008+0000", 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122045 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214130+0000", 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3575999999999999e-05 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214154+0000", 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99465683800000004 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.208810+0000", 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042772410000000002 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213088+0000", 2023-10-09T10:20:01.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031397499999999997 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213402+0000", 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061402180000000002 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.219542+0000", 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021434869999999999 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221685+0000", 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7170999999999999e-05 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221743+0000", 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8197999999999997e-05 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:01.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221841+0000", 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0078972580000001 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114943 9.7 9:e38f5304:::benchmark_data_smithi012_313365_object2468:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.74363026600003, 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1890109880000002, 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:01.978 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114943 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214032+0000", 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9030000000000004e-06 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214040+0000", 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.774e-05 2023-10-09T10:20:01.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214057+0000", 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6160000000000008e-06 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214067+0000", 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013272499999999999 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214200+0000", 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2798000000000004e-05 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214262+0000", 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98051140199999998 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194774+0000", 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8810000000000008e-06 2023-10-09T10:20:01.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194784+0000", 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1642592760000001 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359043+0000", 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000316631 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:20:01.983 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359360+0000", 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026861052 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386221+0000", 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8226999999999994e-05 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386309+0000", 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030679600000000002 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386616+0000", 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060189639999999999 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392635+0000", 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1379999999999999e-05 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:01.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392646+0000", 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095982680000000001 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402244+0000", 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8916999999999999e-05 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402303+0000", 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00074067200000000005 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.403044+0000", 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1890109880000002 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114945 9.7 9:e1cd6305:::benchmark_data_smithi012_313365_object2470:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.74356926399997, 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.171928715, 2023-10-09T10:20:01.987 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114945 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:01.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214093+0000", 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3230000000000003e-06 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:01.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214099+0000", 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6458999999999999e-05 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214116+0000", 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1797000000000001e-05 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214128+0000", 2023-10-09T10:20:01.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015993 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214287+0000", 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214295+0000", 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98052440100000005 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194820+0000", 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0089999999999998e-06 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194825+0000", 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1645754820000001 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359400+0000", 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035076999999999998 2023-10-09T10:20:01.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359751+0000", 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015525427 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375276+0000", 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0332000000000004e-05 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375347+0000", 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032258599999999998 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375669+0000", 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095005880000000008 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385170+0000", 2023-10-09T10:20:01.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0859999999999997e-05 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385231+0000", 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5485999999999999e-05 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385266+0000", 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9339999999999999e-05 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385306+0000", 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00071704500000000003 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386023+0000", 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.171928715 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114953 9.1 9:85379d0d:::benchmark_data_smithi012_313365_object2478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.723124993, 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98747086900000003, 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114953 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.997 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.998 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234537+0000", 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234543+0000", 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3334e-05 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234557+0000", 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1240000000000004e-06 2023-10-09T10:20:01.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234562+0000", 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0877000000000002e-05 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234603+0000", 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5950000000000001e-06 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234610+0000", 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98059611300000005 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215206+0000", 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012311700000000001 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215329+0000", 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038846200000000001 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215718+0000", 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038298339999999998 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.219548+0000", 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023326010000000001 2023-10-09T10:20:02.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221880+0000", 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4744999999999999e-05 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221925+0000", 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3997000000000002e-05 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.222009+0000", 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98747086900000003 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114959 9.13 9:cf007434:::benchmark_data_smithi012_313365_object2484:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.68195713400002, 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96965389000000002, 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.004 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114959 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275705+0000", 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1540000000000002e-06 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275713+0000", 2023-10-09T10:20:02.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4321000000000001e-05 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275727+0000", 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4679999999999997e-06 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275733+0000", 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8029999999999999e-05 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275782+0000", 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0797e-05 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275792+0000", 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93510872899999997 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.210901+0000", 2023-10-09T10:20:02.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013108099999999999 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.211032+0000", 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035745900000000002 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1", 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.211390+0000", 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052776840000000004 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216667+0000", 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028541474000000001 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245209+0000", 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6895000000000003e-05 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245266+0000", 2023-10-09T10:20:02.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4310999999999999e-05 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245360+0000", 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96965389000000002 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114960 9.17 9:ee9c2079:::benchmark_data_smithi012_313365_object2485:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:20:02.011 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 287.67946502299998, 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1241943320000001, 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114960 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278197+0000", 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6739999999999996e-06 2023-10-09T10:20:02.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278202+0000", 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1857e-05 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278214+0000", 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4840000000000003e-06 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278219+0000", 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1851000000000002e-05 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278271+0000", 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6990000000000003e-06 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278279+0000", 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91653466100000003 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194814+0000", 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7139999999999996e-06 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194820+0000", 2023-10-09T10:20:02.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1641976199999999 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359018+0000", 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000340787 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359359+0000", 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026806301000000001 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386165+0000", 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1144999999999996e-05 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386246+0000", 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000216963 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386463+0000", 2023-10-09T10:20:02.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025360859999999999 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.388999+0000", 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003603794 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392603+0000", 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077717259999999996 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400375+0000", 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0828999999999999e-05 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400406+0000", 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019869430000000001 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402393+0000", 2023-10-09T10:20:02.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1241943320000001 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114968 9.7 9:e6edcf82:::benchmark_data_smithi012_313365_object2493:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 286.70610638800002, 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.148768977, 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.021 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114968 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251556+0000", 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1169999999999998e-06 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251563+0000", 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1269e-05 2023-10-09T10:20:02.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251574+0000", 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5419999999999996e-06 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251579+0000", 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049893740000000004 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.256569+0000", 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0169e-05 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.256579+0000", 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.103208593 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359788+0000", 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028333999999999998 2023-10-09T10:20:02.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.360071+0000", 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018386280000000001 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378457+0000", 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0075999999999998e-05 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378527+0000", 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023643399999999999 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378764+0000", 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064134869999999998 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385177+0000", 2023-10-09T10:20:02.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090589800000000003 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386083+0000", 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013396399 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.399480+0000", 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0691999999999993e-05 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.399550+0000", 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00077554999999999996 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400326+0000", 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.148768977 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.029 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128018 9.17 9:e84c3436:::benchmark_data_smithi012_313365_object15543:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 234.30437468, 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0015004270000001, 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128018 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653287+0000", 2023-10-09T10:20:02.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8730000000000004e-06 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653293+0000", 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6543999999999999e-05 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653310+0000", 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3600000000000001e-06 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653316+0000", 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0634999999999999e-05 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653367+0000", 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000173043 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653540+0000", 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036610800000000003 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653906+0000", 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023299039999999998 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.656236+0000", 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042009109999999999 2023-10-09T10:20:02.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.660437+0000", 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1856e-05 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.660448+0000", 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97237176999999997 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.632820+0000", 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013645000000000001 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.632957+0000", 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021710798 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.654667+0000", 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121478 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.654789+0000", 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0015004270000001 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.037 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128022 9.13 9:cae3607a:::benchmark_data_smithi012_313365_object15547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 234.16307895899999, 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.71981764000000004, 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128022 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.038 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.039 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794581+0000", 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7827999999999999e-05 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794599+0000", 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0599000000000003e-05 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794630+0000", 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2262000000000001e-05 2023-10-09T10:20:02.040 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794642+0000", 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7308e-05 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794709+0000", 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1711e-05 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:02.041 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794721+0000", 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.712568863 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507290+0000", 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013420899999999999 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507424+0000", 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036085199999999999 2023-10-09T10:20:02.042 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507785+0000", 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003524667 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.511310+0000", 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00080993999999999999 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.043 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512119+0000", 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00066952899999999998 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512789+0000", 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0370999999999997e-05 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512829+0000", 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015724840000000001 2023-10-09T10:20:02.044 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.514402+0000", 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.71981764000000004 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128023 9.17 9:e92c6749:::benchmark_data_smithi012_313365_object15548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 234.15273044400001, 2023-10-09T10:20:02.045 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86638305000000004, 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128023 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:20:02.046 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804931+0000", 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3567000000000001e-05 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.047 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804945+0000", 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9168e-05 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804964+0000", 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2880000000000004e-06 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804971+0000", 2023-10-09T10:20:02.048 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1679000000000004e-05 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.805033+0000", 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3217999999999999e-05 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.805046+0000", 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.85093918000000002 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.049 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.655985+0000", 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130085 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.656115+0000", 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031436700000000002 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.656430+0000", 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065475230000000004 2023-10-09T10:20:02.050 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.662977+0000", 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4881000000000004e-05 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.663052+0000", 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058596330000000004 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.051 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.668912+0000", 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9499999999999998e-05 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.668951+0000", 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023647130000000001 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.671316+0000", 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86638305000000004 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.052 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156571 9.11 e377/365 9:88722a35:::benchmark_data_smithi012_313365_object44096:head v 377'8328, mlcod=376'8326)", 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.412584+0000", 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 173.54507871499999, 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20419113999999999, 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.053 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412584+0000", 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412584+0000", 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412584+0000", 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3465999999999999e-05 2023-10-09T10:20:02.054 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412598+0000", 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9460999999999999e-05 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412617+0000", 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0070000000000003e-06 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.055 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412626+0000", 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0066720659999999999 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.419298+0000", 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8828999999999999e-05 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.419317+0000", 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.197365453 2023-10-09T10:20:02.056 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.616682+0000", 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3162e-05 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.616776+0000", 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20419113999999999 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.057 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156718 9.17 9:ea7220c0:::benchmark_data_smithi012_313365_object44243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:18.139525+0000", 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 163.81813862300001, 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.206501929, 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.058 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156718 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139525+0000", 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139525+0000", 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.059 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139523+0000", 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5899999999999996e-06 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139530+0000", 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2329e-05 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.060 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139542+0000", 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8539999999999996e-06 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139547+0000", 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1578000000000001e-05 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139588+0000", 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102212 2023-10-09T10:20:02.061 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139691+0000", 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037007000000000001 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.140061+0000", 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017259549999999999 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.062 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.141787+0000", 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00049306999999999997 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.142280+0000", 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20358306800000001 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.345863+0000", 2023-10-09T10:20:02.063 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9493000000000003e-05 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.345902+0000", 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012420000000000001 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.346027+0000", 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.206501929 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.064 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:157353 9.7 9:e2626f77:::benchmark_data_smithi012_313365_object44878:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:19.894438+0000", 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 162.063225582, 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081977412, 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.065 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 157353 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.894438+0000", 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.894438+0000", 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.066 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.894436+0000", 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6960000000000002e-06 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.894442+0000", 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1484999999999999e-05 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.067 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.894453+0000", 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7179999999999999e-06 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.894460+0000", 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00093777400000000005 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.895398+0000", 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015988800000000001 2023-10-09T10:20:02.068 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.895558+0000", 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00041040400000000001 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.895968+0000", 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041994659999999998 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.069 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.900168+0000", 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0739999999999998e-05 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.900228+0000", 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.075891400999999997 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.976120+0000", 2023-10-09T10:20:02.070 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5927999999999998e-05 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.976166+0000", 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024921200000000001 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:19.976415+0000", 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081977412 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.071 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:157975 9.7 9:e6a58e30:::benchmark_data_smithi012_313365_object45500:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:21.270084+0000", 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 160.68757886700001, 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082889518999999995, 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.072 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 157975 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270084+0000", 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270084+0000", 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:20:02.073 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270082+0000", 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0730999999999999e-05 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270092+0000", 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9011000000000001e-05 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.074 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270111+0000", 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7950000000000008e-06 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270119+0000", 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1427000000000007e-05 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270180+0000", 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016253 2023-10-09T10:20:02.075 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270343+0000", 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050332199999999995 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.270846+0000", 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002830361 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.076 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.273677+0000", 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1719999999999998e-06 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.273682+0000", 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.078769147999999997 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.352451+0000", 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3367000000000004e-05 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.077 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.352524+0000", 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044945500000000003 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:21.352974+0000", 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082889518999999995 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.078 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:160802 9.7 9:e6cc9245:::benchmark_data_smithi012_313365_object48327:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:28.202253+0000", 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 153.75541025499999, 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084130752000000003, 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 160802 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.079 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202253+0000", 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202253+0000", 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.080 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202252+0000", 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8490000000000006e-06 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202259+0000", 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3343000000000001e-05 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202273+0000", 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4140000000000002e-06 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.081 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202278+0000", 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3550999999999997e-05 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202322+0000", 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010911700000000001 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202431+0000", 2023-10-09T10:20:02.082 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000459889 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.202891+0000", 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023588340000000002 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.205250+0000", 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00097743799999999996 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.206227+0000", 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079921059000000003 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.286148+0000", 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1333000000000003e-05 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.286219+0000", 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016430399999999999 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.286384+0000", 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084130752000000003 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:160818 9.17 9:ed8f0618:::benchmark_data_smithi012_313365_object48343:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-10-09T10:20:02.085 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:28.230230+0000", 2023-10-09T10:20:02.086 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 153.727433193, 2023-10-09T10:20:02.086 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 14 16 32283 32267 143.994 104 0.0046168 0.00693975 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079688248000000003, 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 160818 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230230+0000", 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230230+0000", 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230229+0000", 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4229999999999999e-06 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230236+0000", 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4778e-05 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230251+0000", 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2490000000000001e-06 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230256+0000", 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6371e-05 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230313+0000", 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000163196 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230476+0000", 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00040233400000000001 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230878+0000", 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001622966 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.232501+0000", 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001059059 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.233560+0000", 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076115267 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.309675+0000", 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5072000000000003e-05 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.309740+0000", 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017779499999999999 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.309918+0000", 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079688248000000003 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:160819 9.7 9:e1686c0e:::benchmark_data_smithi012_313365_object48344:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:28.230375+0000", 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 153.727287781, 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079768740000000005, 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 160819 2023-10-09T10:20:02.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230375+0000", 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230375+0000", 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230375+0000", 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.535e-06 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230378+0000", 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0260000000000004e-06 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.098 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230384+0000", 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9490000000000004e-06 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230389+0000", 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7284000000000003e-05 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230437+0000", 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015339600000000001 2023-10-09T10:20:02.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.230590+0000", 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050493999999999995 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.231095+0000", 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001957196 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.233052+0000", 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00051011000000000003 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.233562+0000", 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.076398271000000004 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.309961+0000", 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4255999999999997e-05 2023-10-09T10:20:02.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.310015+0000", 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012930299999999999 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:28.310144+0000", 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.079768740000000005 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.102 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:162220 9.7 9:e5f6d980:::benchmark_data_smithi012_313365_object49745:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:31.208039+0000", 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 150.74962445599999, 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094084795999999998, 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.103 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 162220 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208039+0000", 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208039+0000", 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208037+0000", 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9409999999999998e-06 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208042+0000", 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9929999999999995e-06 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208050+0000", 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3590000000000001e-06 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208055+0000", 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1026999999999999e-05 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208096+0000", 2023-10-09T10:20:02.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000127219 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208223+0000", 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00049559599999999995 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.208719+0000", 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023899049999999999 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.211108+0000", 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001835555 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.212944+0000", 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.088972414999999999 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.301916+0000", 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5839000000000004e-05 2023-10-09T10:20:02.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.301982+0000", 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014123299999999999 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:31.302124+0000", 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.094084795999999998 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.109 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:163739 9.17 9:eb65235d:::benchmark_data_smithi012_313365_object51264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e386)", 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:34.284638+0000", 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 147.67302509800001, 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084795313999999997, 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:02.110 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 163739 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.284638+0000", 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.284638+0000", 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:02.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.284636+0000", 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7619999999999998e-06 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.284643+0000", 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2252000000000001e-05 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.284655+0000", 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.135e-06 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.284661+0000", 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7794999999999998e-05 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.284718+0000", 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017584400000000001 2023-10-09T10:20:02.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.284894+0000", 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00052829099999999996 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.285423+0000", 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002808287 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.288231+0000", 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016086099999999999 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.288392+0000", 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.080779058000000001 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.369171+0000", 2023-10-09T10:20:02.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0450000000000005e-05 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.369241+0000", 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000192239 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:34.369433+0000", 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084795313999999997 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.116 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.117 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.117 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:02.117 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:02.117 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:20:02.392 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:20:02.392 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:02 smithi110 ceph-mon[136032]: osdmap e431: 8 total, 8 up, 8 in 2023-10-09T10:20:02.392 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:02 smithi110 ceph-mon[136032]: failed to encode map e431 with expected crc 2023-10-09T10:20:02.392 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:02 smithi110 ceph-mon[136032]: failed to encode map e431 with expected crc 2023-10-09T10:20:02.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:20:02.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:02 smithi012 ceph-mon[147926]: osdmap e431: 8 total, 8 up, 8 in 2023-10-09T10:20:02.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:02 smithi012 ceph-mon[147926]: failed to encode map e431 with expected crc 2023-10-09T10:20:02.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:02 smithi012 ceph-mon[147926]: failed to encode map e431 with expected crc 2023-10-09T10:20:02.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:20:02.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:02 smithi012 ceph-mon[152263]: osdmap e431: 8 total, 8 up, 8 in 2023-10-09T10:20:02.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:02 smithi012 ceph-mon[152263]: failed to encode map e431 with expected crc 2023-10-09T10:20:02.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:02 smithi012 ceph-mon[152263]: failed to encode map e431 with expected crc 2023-10-09T10:20:02.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:20:02.980 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 15 16 34074 34058 141.857 111.938 0.00222083 0.00701601 2023-10-09T10:20:03.360 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:03.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:03.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:03.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:03.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:20:03.524 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:03.587 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:03 smithi012 ceph-mon[147926]: pgmap v1200: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T10:20:03.590 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:03 smithi012 ceph-mon[152263]: pgmap v1200: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T10:20:03.608 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:03.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:03.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:03 smithi110 ceph-mon[136032]: pgmap v1200: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T10:20:03.713 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:03.802 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:03.967 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:03.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:03.980 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 16 16 35439 35423 138.324 85.3125 0.00848305 0.00722329 2023-10-09T10:20:04.037 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:04.075 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:04.133 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:04.194 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:04.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:04.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:04.301 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:04.339 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:20:04.353 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:04.381 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:20:04.407 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4144991084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:04.414 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:04.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:20:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:04.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:04.447 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:20:04.476 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:20:04.508 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:20:04.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:04.559 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:20:04.579 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:04.580 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:20:04.580 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:20:04.580 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:20:04.580 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:20:04.580 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:20:04.580 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:20:04.581 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:20:04.581 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:20:04.581 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:20:04.616 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4144991084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:04.617 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4144991084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:04.981 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 17 16 37189 37173 136.616 109.375 0.00972308 0.00731247 2023-10-09T10:20:05.345 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:05.446 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:05.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:05.549 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:05.612 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:05.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:05 smithi012 ceph-mon[152263]: pgmap v1201: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T10:20:05.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:05 smithi012 ceph-mon[147926]: pgmap v1201: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T10:20:05.687 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:05.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:05 smithi110 ceph-mon[136032]: pgmap v1201: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T10:20:05.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:05.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:05.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:05.870 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:05.981 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 18 16 38909 38893 134.998 107.5 0.0179187 0.00740101 2023-10-09T10:20:06.193 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:06.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:06.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:06.299 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:06.363 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:06.417 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:06.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:06.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:06.603 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:06.671 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:06.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:06.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:06.861 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:06.980 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:06.981 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 19 16 40591 40575 133.426 105.125 0.00583722 0.00748966 2023-10-09T10:20:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:06.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:07.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:07.103 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:20:07.103 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:20:07.138 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:07.216 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:07.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:07.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:07.370 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:07.372 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:20:07.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:07 smithi110 ceph-mon[136032]: pgmap v1202: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-10-09T10:20:07.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1622785604' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:07.436 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:07.445 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:07.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:07.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:07.452 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:20:07.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:20:07.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:07.613 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:07.657 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:07.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:07.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:07.860 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:07 smithi012 ceph-mon[147926]: pgmap v1202: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-10-09T10:20:07.861 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1622785604' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:07.861 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:07.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:07 smithi012 ceph-mon[152263]: pgmap v1202: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-10-09T10:20:07.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1622785604' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:07.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:07.867 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:07.943 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:20:07.943 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:20:07.943 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T10:20:07.943 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:20:07.943 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:20:07.944 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T10:20:07.956 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:07.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:07.982 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:20:07.982776+0000 min lat: 0.00109621 max lat: 0.20453 avg lat: 0.00759054 2023-10-09T10:20:07.982 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:20:07.983 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 20 16 42157 42141 131.64 97.875 0.0110519 0.00759054 2023-10-09T10:20:08.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:08.117 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:20:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:20:07] "GET /metrics HTTP/1.1" 200 36637 "" "Prometheus/2.33.4" 2023-10-09T10:20:08.121 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:08.246 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:08.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:08.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:08.290 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:08.324 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:08.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:08.400 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:08.469 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:08.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:08.578 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:08.637 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:08.692 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:08 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:08 smithi110 ceph-mon[136032]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T10:20:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:08 smithi110 ceph-mon[136032]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:08 smithi110 ceph-mon[136032]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:08 smithi110 ceph-mon[136032]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:08.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:08.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:08.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:08.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[152263]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T10:20:08.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[152263]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[152263]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[152263]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:08.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:08.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[147926]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T10:20:08.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[147926]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[147926]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.737 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:08 smithi012 ceph-mon[147926]: failed to encode map e432 with expected crc 2023-10-09T10:20:08.764 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:20:08.802 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:08.869 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:08.936 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:08.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:08.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:08.982 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 21 16 43614 43598 129.708 91.0625 0.0048881 0.00770378 2023-10-09T10:20:09.043 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:09.136 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:09.745 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:09.749 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:09 smithi012 ceph-mon[147926]: pgmap v1204: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T10:20:09.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:09.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:09.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:09 smithi012 ceph-mon[152263]: pgmap v1204: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T10:20:09.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:09.906 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:09.908 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:09 smithi110 ceph-mon[136032]: pgmap v1204: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T10:20:09.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:20:09.984 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 22 16 45187 45171 128.275 98.3125 0.00488158 0.00779057 2023-10-09T10:20:10.074 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:10.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:10.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:10.228 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:10.823 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:20:10.858 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:20:10.877 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:20:10.878 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:20:10.894 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:20:10.951 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:20:10.987 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 23 16 46575 46559 126.451 86.75 0.0169724 0.00790271 2023-10-09T10:20:11.000 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:20:11.032 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:20:11.070 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:20:11.105 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:20:11.111 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:20:11.111 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:20:11.112 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:20:11.112 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:20:11.112 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:20:11.112 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:20:11.112 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:20:11.112 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:20:11.160 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:11.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:11.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:11.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:11.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:20:11.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:11.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:11 smithi012 ceph-mon[152263]: pgmap v1205: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T10:20:11.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/912280536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:11.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:11 smithi012 ceph-mon[147926]: pgmap v1205: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T10:20:11.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/912280536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:11.762 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:11.828 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:11.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:11.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:11.929 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:11 smithi110 ceph-mon[136032]: pgmap v1205: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T10:20:11.930 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/912280536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:11.935 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:11.994 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 24 16 47896 47880 124.588 82.5625 0.00933481 0.00802017 2023-10-09T10:20:12.003 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:12.110 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:12.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:12.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:12.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:12.265 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:20:12.265 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-10-09T10:20:12.265 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:20:12.276 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:12.505 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:12.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:12.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:12.611 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:12.681 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:12.753 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:12.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:12.792 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:12.866 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:12.918 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:12.963 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:12.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:12.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:12.994 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 25 16 49291 49275 123.092 87.1875 0.00241841 0.00810551 2023-10-09T10:20:13.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:13.096 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:13.186 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:13.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:13.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:13.290 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:13.362 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:13.440 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:13.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:13.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:13.550 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:13.600 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:13 smithi012 ceph-mon[152263]: pgmap v1206: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T10:20:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3493337455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:13.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:13 smithi012 ceph-mon[147926]: pgmap v1206: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T10:20:13.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3493337455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:13.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:13.707 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:13.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:13.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:13.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:20:13.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:13.925 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:13 smithi110 ceph-mon[136032]: pgmap v1206: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T10:20:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3493337455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:13.993 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:13.995 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 26 16 50319 50303 120.827 64.25 0.0136213 0.00826948 2023-10-09T10:20:14.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:14.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:14.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:14.145 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:14.207 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:14.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:14.298 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:20:14.320 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:14.372 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:14.430 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:14.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:14.484 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:14.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:14.563 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:20:14.563 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:20:14.563 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:20:14.563 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:20:14.563 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.563 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:108330 8.f e292/288 8:fca24286:::benchmark_data_smithi012_271002_object108329:head v 292'6808, mlcod=277'654)", 2023-10-09T10:20:14.563 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 593.48595208699999, 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068658946999999998, 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.564 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072322+0000", 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072351+0000", 2023-10-09T10:20:14.565 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9468e-05 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072361+0000", 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5270000000000004e-06 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.072368+0000", 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0310000000000001e-06 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.566 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.081510+0000", 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0091419859999999995 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.081536+0000", 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6675000000000001e-05 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.567 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.140944+0000", 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059407616000000003 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:21.140981+0000", 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7005999999999998e-05 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.568 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:110180 8.4 e293/288 8:2cd3d405:::benchmark_data_smithi012_271002_object1497:head v 293'6943, mlcod=0'0)", 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 590.68475288800005, 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068331423000000002, 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.569 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873521+0000", 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.570 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873525+0000", 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5710000000000002e-06 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873529+0000", 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4320000000000001e-06 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873533+0000", 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6629999999999999e-06 2023-10-09T10:20:14.571 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873743+0000", 2023-10-09T10:20:14.572 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000210688 2023-10-09T10:20:14.572 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.572 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.572 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.572 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.873766+0000", 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2617e-05 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.941812+0000", 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068045934000000002 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:10:23.941853+0000", 2023-10-09T10:20:14.573 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0711999999999999e-05 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45518.0:136718 8.4 e297/288 8:2f021f19:::benchmark_data_smithi012_271002_object28035:head v 297'8596, mlcod=0'0)", 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 550.02220172199998, 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.069669067000000001, 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.574 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536072+0000", 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.575 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536071+0000", 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536082+0000", 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1034e-05 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.576 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536091+0000", 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1370000000000008e-06 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.536097+0000", 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4620000000000004e-06 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.544788+0000", 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0086914880000000007 2023-10-09T10:20:14.577 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.578 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.578 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.578 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.544818+0000", 2023-10-09T10:20:14.578 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0079000000000001e-05 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.605693+0000", 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.06087447 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.579 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:11:04.605741+0000", 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8838000000000002e-05 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127554 9.b e355/342 9:d2a603fd:::benchmark_data_smithi012_313365_object15079:head v 355'7939, mlcod=0'0)", 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.362897+0000", 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 248.195376767, 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.056822035999999999, 2023-10-09T10:20:14.580 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.362897+0000", 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.362897+0000", 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.581 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.362896+0000", 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.362907+0000", 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1474000000000001e-05 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.582 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.362932+0000", 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4762000000000001e-05 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.362942+0000", 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5019999999999998e-06 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.363025+0000", 2023-10-09T10:20:14.583 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3510999999999994e-05 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.363055+0000", 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9935000000000001e-05 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.419672+0000", 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.056617153000000003 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.584 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.419719+0000", 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6984000000000003e-05 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127559 9.b e355/342 9:dad66366:::benchmark_data_smithi012_313365_object15084:head v 355'7940, mlcod=0'0)", 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.373657+0000", 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 248.184616723, 2023-10-09T10:20:14.585 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.046088121000000003, 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.373657+0000", 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.373657+0000", 2023-10-09T10:20:14.586 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.587 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.373656+0000", 2023-10-09T10:20:14.587 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.587 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.587 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.587 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.589 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.373667+0000", 2023-10-09T10:20:14.589 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0886999999999999e-05 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.373680+0000", 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3304e-05 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.373690+0000", 2023-10-09T10:20:14.590 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2099999999999999e-06 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.376255+0000", 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025656490000000001 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.376281+0000", 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6058999999999999e-05 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.591 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.419725+0000", 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.043443819000000002 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.419745+0000", 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0339e-05 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.592 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127564 9.b e355/342 9:d4b29209:::benchmark_data_smithi012_313365_object15089:head v 355'7941, mlcod=0'0)", 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.375393+0000", 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 248.18288076300001, 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044372250000000002, 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.593 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.375393+0000", 2023-10-09T10:20:14.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.594 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.594 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.375393+0000", 2023-10-09T10:20:14.594 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.594 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.594 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.595 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.595 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.375392+0000", 2023-10-09T10:20:14.595 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.595 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.595 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.375403+0000", 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0541e-05 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.375423+0000", 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0179000000000001e-05 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.596 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.375432+0000", 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3740000000000004e-06 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.390216+0000", 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014784209 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.390244+0000", 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.72e-05 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.597 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.419748+0000", 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029504186000000002 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.419766+0000", 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7731999999999999e-05 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.598 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127573 9.b e355/342 9:df63469b:::benchmark_data_smithi012_313365_object15098:head v 355'7943, mlcod=0'0)", 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.396874+0000", 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 248.16140003000001, 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.035791219999999999, 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.396874+0000", 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.599 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.396874+0000", 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.396873+0000", 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.600 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.396884+0000", 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0995000000000001e-05 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.396902+0000", 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7982000000000002e-05 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.601 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.396912+0000", 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6560000000000004e-06 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.410299+0000", 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013387209000000001 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.410335+0000", 2023-10-09T10:20:14.602 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6350999999999998e-05 2023-10-09T10:20:14.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.603 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.603 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.603 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.432607+0000", 2023-10-09T10:20:14.603 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022271616000000001 2023-10-09T10:20:14.603 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.432665+0000", 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8489999999999997e-05 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127762 9.b e355/342 9:d6753042:::benchmark_data_smithi012_313365_object15287:head v 355'7959, mlcod=0'0)", 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.868363+0000", 2023-10-09T10:20:14.604 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 247.68991136599999, 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.045153748, 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.868363+0000", 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.605 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.868363+0000", 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.868361+0000", 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.868374+0000", 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2319e-05 2023-10-09T10:20:14.606 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.868395+0000", 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1026000000000002e-05 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.868404+0000", 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7510000000000003e-06 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.607 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.868484+0000", 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9249e-05 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.868514+0000", 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0802000000000001e-05 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.913477+0000", 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.044962545999999999 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.608 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.913516+0000", 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9424000000000001e-05 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:127764 9.b e355/342 9:dfb36886:::benchmark_data_smithi012_313365_object15289:head v 355'7960, mlcod=0'0)", 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:06.874574+0000", 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 247.68369963399999, 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.038967649, 2023-10-09T10:20:14.609 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.874574+0000", 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.874574+0000", 2023-10-09T10:20:14.610 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.874573+0000", 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.874583+0000", 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1379999999999996e-06 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.874594+0000", 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1561e-05 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.874602+0000", 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1000000000000004e-06 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.886961+0000", 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012358656000000001 2023-10-09T10:20:14.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.886991+0000", 2023-10-09T10:20:14.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0536e-05 2023-10-09T10:20:14.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.913519+0000", 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026527882999999999 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:06.913542+0000", 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2724000000000001e-05 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.614 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:147572 9.f e374/362 9:ff6a51c7:::benchmark_data_smithi012_313365_object35097:head v 374'9270, mlcod=0'0)", 2023-10-09T10:20:14.615 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:56.071401+0000", 2023-10-09T10:20:14.615 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 198.48687303099999, 2023-10-09T10:20:14.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.038247574999999999, 2023-10-09T10:20:14.615 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.615 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.615 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.071401+0000", 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.071401+0000", 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.071400+0000", 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.071410+0000", 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6139999999999998e-06 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.071426+0000", 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6282000000000001e-05 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.071434+0000", 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8539999999999997e-06 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.072309+0000", 2023-10-09T10:20:14.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00087475400000000005 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.072352+0000", 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3074999999999997e-05 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.109592+0000", 2023-10-09T10:20:14.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.037240141999999997 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:56.109649+0000", 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6750999999999999e-05 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.f 375/362 [PushOp(9:f5241747:::benchmark_data_smithi012_313365_object39316:head, version: 324'2459, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:f5241747:::benchmark_data_smithi012_313365_object39316:head@324'2459, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:20:14.620 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:01.849700+0000", 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 192.70857447700001, 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.036946176999999997, 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.849700+0000", 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.849700+0000", 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.849699+0000", 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.849724+0000", 2023-10-09T10:20:14.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5007999999999998e-05 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.849737+0000", 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2829e-05 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.849752+0000", 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4378e-05 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.864245+0000", 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014493301 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.864253+0000", 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0649999999999999e-06 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:01.886646+0000", 2023-10-09T10:20:14.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022392863999999998 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:156579 9.7 e377/361 9:e10cf41e:::benchmark_data_smithi012_313365_object44104:head v 377'8119, mlcod=327'3621)", 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.633803+0000", 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 185.92447080100001, 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10126631799999999, 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.625 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633803+0000", 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633803+0000", 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633803+0000", 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633810+0000", 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5270000000000001e-06 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633818+0000", 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5259999999999996e-06 2023-10-09T10:20:14.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633826+0000", 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7139999999999999e-06 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.634044+0000", 2023-10-09T10:20:14.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021737500000000001 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.634073+0000", 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9030000000000002e-05 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.734986+0000", 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.100913221 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.735070+0000", 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3572000000000006e-05 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.1f 398/377 [PushOp(9:ff99a8f0:::benchmark_data_smithi012_313365_object80195:head, version: 333'5053, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:ff99a8f0:::benchmark_data_smithi012_313365_object80195:head@333'5053, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 105.507852762, 2023-10-09T10:20:14.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071748132000000006, 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050420+0000", 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050484+0000", 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3660999999999999e-05 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050499+0000", 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4959e-05 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050510+0000", 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1165999999999999e-05 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050573+0000", 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2452000000000004e-05 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050587+0000", 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4083000000000001e-05 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.634 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.122169+0000", 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071582693000000003 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:192723 9.16 e398/361 9:6c284d1c:::benchmark_data_smithi012_313365_object80248:head v 398'9382, mlcod=340'6834)", 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:29.054442+0000", 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 105.503832442, 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070307176999999998, 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.635 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.054442+0000", 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.054442+0000", 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.636 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.054441+0000", 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.054450+0000", 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9479999999999997e-06 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120912+0000", 2023-10-09T10:20:14.637 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066462038000000001 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120929+0000", 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7533e-05 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120982+0000", 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3090999999999997e-05 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.638 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.121010+0000", 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7526999999999999e-05 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.124712+0000", 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037020870000000002 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.639 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.124749+0000", 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7020999999999997e-05 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:192666 9.f e398/378 9:f5dbcf8b:::benchmark_data_smithi012_313365_object80191:head v 398'11046, mlcod=0'0)", 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:29.060753+0000", 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 105.497521273, 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.065306681000000005, 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.640 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.060753+0000", 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.060753+0000", 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.641 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.060752+0000", 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.060760+0000", 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4149999999999999e-06 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120924+0000", 2023-10-09T10:20:14.642 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060163258999999997 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120933+0000", 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9760000000000001e-06 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120975+0000", 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2363000000000001e-05 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.643 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120999+0000", 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4000000000000001e-05 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.126037+0000", 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050383399999999997 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.126059+0000", 2023-10-09T10:20:14.644 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2204e-05 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:20226 10.8 e428/424 10:1074ec9e:::benchmark_data_smithi012_366312_object20225:head v 428'1219, mlcod=428'1217)", 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:55.287430+0000", 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 19.270844116999999, 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054413136000000001, 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.645 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.287430+0000", 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.287430+0000", 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.646 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.287429+0000", 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.287452+0000", 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2515e-05 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.287462+0000", 2023-10-09T10:20:14.647 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0121e-05 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.287469+0000", 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7800000000000003e-06 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.287526+0000", 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7497999999999997e-05 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.648 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.287547+0000", 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0905000000000001e-05 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341796+0000", 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054248746 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.649 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341843+0000", 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7126999999999998e-05 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:20217 10.9 e428/424 10:95dea04f:::benchmark_data_smithi012_366312_object20216:head v 428'1292, mlcod=428'1289)", 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:55.288089+0000", 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 19.270184957000001, 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055461855999999997, 2023-10-09T10:20:14.650 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.288089+0000", 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.288089+0000", 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.651 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.288088+0000", 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.288138+0000", 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9291e-05 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.652 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.288148+0000", 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0749e-05 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.288156+0000", 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2799999999999998e-06 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.288197+0000", 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1300000000000001e-05 2023-10-09T10:20:14.653 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.288217+0000", 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9738999999999999e-05 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.343490+0000", 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.055273601999999998 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.654 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.343551+0000", 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0661000000000001e-05 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:20229 10.8 e428/424 10:1e5935f8:::benchmark_data_smithi012_366312_object20228:head v 428'1220, mlcod=428'1217)", 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:55.289137+0000", 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 19.269137318999999, 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057112417999999998, 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.655 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.289137+0000", 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.289137+0000", 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.656 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.289136+0000", 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.289161+0000", 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4559e-05 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.657 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341783+0000", 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052622865999999997 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341804+0000", 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0611e-05 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341871+0000", 2023-10-09T10:20:14.658 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7280000000000006e-05 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341890+0000", 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8766e-05 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.346207+0000", 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043169549999999999 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.659 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.346249+0000", 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2055000000000003e-05 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:20230 10.a e428/424 10:5dd392cd:::benchmark_data_smithi012_366312_object20229:head v 428'1221, mlcod=428'1219)", 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:55.289699+0000", 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 19.268575536, 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.053845863000000001, 2023-10-09T10:20:14.660 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.289699+0000", 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.289699+0000", 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.661 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.289698+0000", 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.289748+0000", 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0074999999999997e-05 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341790+0000", 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052042810000000002 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341804+0000", 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4044e-05 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341856+0000", 2023-10-09T10:20:14.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1576999999999998e-05 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.341883+0000", 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7248000000000001e-05 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.343506+0000", 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016231570000000001 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:55.343544+0000", 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7944000000000001e-05 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:51066 10.f e433/424 10:f6bad540:::benchmark_data_smithi012_366312_object51065:head v 433'3167, mlcod=433'3164)", 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:14.542518+0000", 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0157559, 2023-10-09T10:20:14.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014711046, 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.542518+0000", 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.542518+0000", 2023-10-09T10:20:14.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.542518+0000", 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.542550+0000", 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2480000000000001e-05 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.542559+0000", 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8319999999999995e-06 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.542565+0000", 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0349999999999998e-06 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.546115+0000", 2023-10-09T10:20:14.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035494239999999998 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.546141+0000", 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6369000000000001e-05 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.557183+0000", 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011042010999999999 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:14.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:14.557229+0000", 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6298000000000003e-05 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:14.670 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:20:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:14.672 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:14.672 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:14.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:14.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:14.830 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:14.916 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:14.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:14.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:14.995 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 27 16 51694 51678 119.535 85.9375 0.0142227 0.00836021 2023-10-09T10:20:15.020 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:15.086 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:14 smithi110 ceph-mon[136032]: pgmap v1207: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T10:20:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:14 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:14 smithi110 ceph-mon[136032]: failed to encode map e433 with expected crc 2023-10-09T10:20:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:20:15.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:14 smithi110 ceph-mon[136032]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T10:20:15.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[152263]: pgmap v1207: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T10:20:15.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:15.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:15.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:15.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[152263]: failed to encode map e433 with expected crc 2023-10-09T10:20:15.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:20:15.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[152263]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T10:20:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[147926]: pgmap v1207: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T10:20:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[147926]: failed to encode map e433 with expected crc 2023-10-09T10:20:15.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:20:15.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:14 smithi012 ceph-mon[147926]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T10:20:15.995 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 28 16 52873 52857 117.898 73.6875 0.013707 0.00847698 2023-10-09T10:20:16.548 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:16.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:16.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:16.653 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:16.737 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:16.774 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:16.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:16.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:16.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:16.947 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:16.995 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 29 16 54193 54177 116.678 82.5 0.00930705 0.00856569 2023-10-09T10:20:17.127 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:16 smithi012 ceph-mon[147926]: pgmap v1209: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-10-09T10:20:17.127 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:16 smithi012 ceph-mon[152263]: pgmap v1209: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-10-09T10:20:17.129 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:20:17.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:16 smithi110 ceph-mon[136032]: pgmap v1209: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-10-09T10:20:17.407 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:17.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:17.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:17.514 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:17.617 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:17.758 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:17.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:17.775 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:20:17.776 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:20:17.776 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T10:20:17.776 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:20:17.776 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:20:17.776 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T10:20:17.865 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:17.934 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:17.973 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:20:17.996 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 30 16 55548 55532 115.609 84.6875 0.006354 0.00864374 2023-10-09T10:20:18.043 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:18.050 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:20:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:20:17] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.33.4" 2023-10-09T10:20:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:18.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:18.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:18.216 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:18.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:18.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:18.440 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:18.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:18.587 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:18.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:18.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:18.694 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:18.728 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:18.769 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:18.797 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:20:18.824 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:20:18.855 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:20:18.886 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:20:18.904 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:20:18.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:20:18.996 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 31 16 56703 56687 114.209 72.1875 0.114295 0.00873174 2023-10-09T10:20:19.015 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:20:19.035 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:20:19.053 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:20:19.054 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:20:19.054 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:20:19.054 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:20:19.054 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:20:19.055 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:20:19.055 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:20:19.055 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:20:19.055 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:20:19.089 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:19.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:19.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:19.192 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:19.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:19 smithi012 ceph-mon[147926]: pgmap v1210: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T10:20:19.294 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2639799788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:19.330 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:19 smithi110 ceph-mon[136032]: pgmap v1210: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T10:20:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2639799788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:19.544 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:19.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:19.649 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:19.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:19 smithi012 ceph-mon[152263]: pgmap v1210: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T10:20:19.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2639799788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:19.718 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:19.804 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:19.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:19.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:19.912 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:19.986 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:19.996 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 32 16 57835 57819 112.851 70.75 0.00654683 0.00885562 2023-10-09T10:20:20.093 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:20.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:20.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:20.181 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:20:20.181 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:20:20.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:20.244 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:20.426 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:20.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:20.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:20.535 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:20.566 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:20.675 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:20.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:20.731 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:20.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:20.859 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:20.963 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:20.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:20.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:20.996 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 33 16 58999 58983 111.636 72.75 0.0120079 0.00895291 2023-10-09T10:20:21.069 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:21.146 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:21.252 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:21.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:21.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:21.299 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:21 smithi012 ceph-mon[147926]: pgmap v1211: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T10:20:21.306 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:20:21.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:21.426 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:21.515 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:21.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:21.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:21.524 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:21 smithi110 ceph-mon[136032]: pgmap v1211: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T10:20:21.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:21.633 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:20:21.634 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:20:21.634 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:20:21.641 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:21 smithi012 ceph-mon[152263]: pgmap v1211: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T10:20:21.699 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:21.760 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:21.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:21.867 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:21.927 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:21.996 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 34 16 60239 60223 110.633 77.5 0.00975299 0.00903341 2023-10-09T10:20:22.017 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:22.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:22.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:22.158 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:20:22.188 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:22.248 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:22.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:22.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:22.371 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:22.432 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:22.495 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:22.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:22.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:22.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:20:22.601 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:22.652 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:22.962 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:22.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:22.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:22.997 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 35 16 61488 61472 109.702 78.0625 0.00643211 0.00910989 2023-10-09T10:20:23.068 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:23.139 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:23.265 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:23.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:23.328 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:20:23.351 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:23.375 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:23.448 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:23.585 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:23.592 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:23 smithi110 ceph-mon[136032]: pgmap v1212: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T10:20:23.593 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:23 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2026723352' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:23.593 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:23.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:23.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:23 smithi012 ceph-mon[147926]: pgmap v1212: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T10:20:23.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:23 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2026723352' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:23.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:23.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:23 smithi012 ceph-mon[152263]: pgmap v1212: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T10:20:23.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:23 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2026723352' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:23.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:23.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:23.695 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:20:23.695 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:20:23.754 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:23.806 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:23.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:23.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:23.912 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:23.981 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:23.996 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 36 16 62693 62677 108.747 75.3125 0.00640967 0.00919095 2023-10-09T10:20:24.173 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:20:24.201 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:24.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:24.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:24.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:24.365 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:24.424 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:24.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:24.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:24.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:20:24.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:20:24.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:24.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T10:20:24.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:24 smithi110 ceph-mon[136032]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.530 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:24.625 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:24.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:20:24.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:20:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T10:20:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[147926]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:20:24.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:20:24.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:24.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T10:20:24.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:24 smithi012 ceph-mon[152263]: failed to encode map e434 with expected crc 2023-10-09T10:20:24.690 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:24.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:24.797 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:24.873 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:24.918 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:20:24.942 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:24.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:24.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:24.999 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 37 16 63897 63881 107.835 75.25 0.00979522 0.00926901 2023-10-09T10:20:25.013 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:20:25.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:25.049 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:20:25.082 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:20:25.119 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:20:25.119 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:25.145 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:20:25.170 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:25.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:25.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:25.181 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:20:25.225 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:20:25.264 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:20:25.268 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:20:25.268 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:20:25.269 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:20:25.269 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:20:25.269 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:20:25.269 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:20:25.269 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:20:25.269 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:20:25.275 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:25.346 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:25.406 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:25 smithi012 ceph-mon[147926]: pgmap v1214: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T10:20:25.406 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3877021841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:25.407 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:25 smithi012 ceph-mon[152263]: pgmap v1214: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T10:20:25.407 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3877021841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:25 smithi110 ceph-mon[136032]: pgmap v1214: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T10:20:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3877021841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:25.516 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:25.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:25.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:25.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:20:25.708 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:25.833 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:25.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:25.940 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:25.999 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 38 16 65095 65079 106.968 74.875 0.0125883 0.00934242 2023-10-09T10:20:26.023 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:26.146 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:26.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:26.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:26.254 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:26.364 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:26.439 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:26.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:26.479 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:26.545 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:26.582 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:20:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:20:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:20:26.582 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114937 9.c 9:3de29534:::benchmark_data_smithi012_313365_object2462:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 312.36228946900002, 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0079670510000001, 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.583 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114937 2023-10-09T10:20:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:20:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215214+0000", 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9450000000000005e-06 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215224+0000", 2023-10-09T10:20:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4849999999999992e-06 2023-10-09T10:20:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215232+0000", 2023-10-09T10:20:26.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3200000000000002e-06 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215239+0000", 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8306000000000002e-05 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215278+0000", 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3389999999999998e-06 2023-10-09T10:20:26.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215284+0000", 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98804788700000001 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.203332+0000", 2023-10-09T10:20:26.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00200256 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.205335+0000", 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015448999999999999 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.205489+0000", 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013383071999999999 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218872+0000", 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041514890000000004 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223024+0000", 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7812999999999997e-05 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223081+0000", 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100445 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223182+0000", 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0079670510000001 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.591 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114938 9.c 9:317dc30b:::benchmark_data_smithi012_313365_object2463:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:20:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:20:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 312.36225615299998, 2023-10-09T10:20:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0080620979999999, 2023-10-09T10:20:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.592 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114938 2023-10-09T10:20:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.593 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:20:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:20:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:20:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.979e-06 2023-10-09T10:20:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215250+0000", 2023-10-09T10:20:26.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5199999999999997e-06 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215258+0000", 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1250000000000001e-06 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215261+0000", 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8232000000000001e-05 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215320+0000", 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1710000000000002e-06 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215328+0000", 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99764452400000003 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.212972+0000", 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010863 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213081+0000", 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012440199999999999 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213205+0000", 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056800169999999999 2023-10-09T10:20:26.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218885+0000", 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043305829999999998 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223216+0000", 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1077000000000001e-05 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223247+0000", 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3231000000000005e-05 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223310+0000", 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0080620979999999 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114946 9.c 9:3c5b83c8:::benchmark_data_smithi012_313365_object2471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 312.36222295300001, 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0081399559999999, 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114946 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.601 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6449999999999999e-06 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215283+0000", 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7349999999999998e-06 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215286+0000", 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7470000000000001e-06 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215289+0000", 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2694000000000001e-05 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215342+0000", 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2049999999999998e-06 2023-10-09T10:20:26.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215345+0000", 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.000627545 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215973+0000", 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011626899999999999 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216089+0000", 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132232 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216221+0000", 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027664209999999998 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218988+0000", 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043527119999999999 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223340+0000", 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9572000000000001e-05 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223370+0000", 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1471000000000002e-05 2023-10-09T10:20:26.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223421+0000", 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0081399559999999 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128021 9.c 9:3ed78aba:::benchmark_data_smithi012_313365_object15546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:20:26.608 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 258.87428120300001, 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999, 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128021 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703222+0000", 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7700000000000004e-06 2023-10-09T10:20:26.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703229+0000", 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1178e-05 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703240+0000", 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0509999999999996e-06 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703245+0000", 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9240000000000002e-05 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703304+0000", 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.429e-05 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703318+0000", 2023-10-09T10:20:26.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.498177753 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201496+0000", 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2643e-05 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201519+0000", 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1325606320000001 2023-10-09T10:20:26.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334079+0000", 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016077600000000001 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334240+0000", 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016287699999999999 2023-10-09T10:20:26.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334403+0000", 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050087500000000002 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339412+0000", 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1050000000000007e-05 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339473+0000", 2023-10-09T10:20:26.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013735 2023-10-09T10:20:26.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339610+0000", 2023-10-09T10:20:26.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041248470000000001 2023-10-09T10:20:26.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343735+0000", 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9188999999999999e-05 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343784+0000", 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111032 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343895+0000", 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128030 9.c 9:39ef457f:::benchmark_data_smithi012_313365_object15555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:20:26.618 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 258.734270127, 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999, 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128030 2023-10-09T10:20:26.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:26.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843233+0000", 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843241+0000", 2023-10-09T10:20:26.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3162e-05 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843254+0000", 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5490000000000001e-06 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843261+0000", 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2747999999999997e-05 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843314+0000", 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7970000000000006e-06 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843323+0000", 2023-10-09T10:20:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35848033200000001 2023-10-09T10:20:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201803+0000", 2023-10-09T10:20:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6119999999999999e-06 2023-10-09T10:20:26.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201812+0000", 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1353121740000001 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337124+0000", 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8503000000000003e-05 2023-10-09T10:20:26.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337212+0000", 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0490999999999998e-05 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337303+0000", 2023-10-09T10:20:26.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002260742 2023-10-09T10:20:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339564+0000", 2023-10-09T10:20:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1705999999999994e-05 2023-10-09T10:20:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339635+0000", 2023-10-09T10:20:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002741145 2023-10-09T10:20:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.342376+0000", 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001560173 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343937+0000", 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6062000000000001e-05 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343973+0000", 2023-10-09T10:20:26.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4776000000000002e-05 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.344068+0000", 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195623 9.c 9:350660d8:::benchmark_data_smithi012_313365_object83148:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 113.32023587, 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.178670812, 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195623 2023-10-09T10:20:26.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257267+0000", 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8130000000000001e-06 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257273+0000", 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0166999999999999e-05 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257283+0000", 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2350000000000002e-06 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257290+0000", 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001555255 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.258845+0000", 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2446000000000001e-05 2023-10-09T10:20:26.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.258858+0000", 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99572670600000002 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254584+0000", 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2269999999999999e-06 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254594+0000", 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.157542818 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412136+0000", 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010587299999999999 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412242+0000", 2023-10-09T10:20:26.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000452194 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412695+0000", 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010428244999999999 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423123+0000", 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4022000000000001e-05 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423157+0000", 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012428642 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435585+0000", 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000185372 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435771+0000", 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5880000000000001e-05 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435817+0000", 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012248200000000001 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435939+0000", 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.178670812 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195633 9.1c 9:384521d8:::benchmark_data_smithi012_313365_object83158:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 113.300184625, 2023-10-09T10:20:26.640 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1456984170000002, 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195633 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:20:26.641 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277318+0000", 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4900000000000003e-06 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.642 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277326+0000", 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2858e-05 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277339+0000", 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1309999999999997e-06 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.643 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277344+0000", 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3996000000000002e-05 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277408+0000", 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0821e-05 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277419+0000", 2023-10-09T10:20:26.644 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97663378599999995 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254052+0000", 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1975e-05 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254064+0000", 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1520595250000001 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.645 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406124+0000", 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013994800000000001 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406264+0000", 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045309899999999999 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.646 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406717+0000", 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051173240000000004 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411834+0000", 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3628999999999999e-05 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411878+0000", 2023-10-09T10:20:26.647 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3909999999999998e-06 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411886+0000", 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010941673000000001 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.422828+0000", 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9455e-05 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.648 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.422877+0000", 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014060400000000001 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423018+0000", 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1456984170000002 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.649 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195634 9.c 9:36e83a2e:::benchmark_data_smithi012_313365_object83159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:20:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:20:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 113.299927428, 2023-10-09T10:20:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1586804549999998, 2023-10-09T10:20:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.650 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195634 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.651 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277576+0000", 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.331e-06 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.652 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277582+0000", 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2768e-05 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277595+0000", 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.653 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277601+0000", 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4212000000000003e-05 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277645+0000", 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1494999999999999e-05 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.654 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277657+0000", 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97695835200000003 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254615+0000", 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.798e-06 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.655 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254622+0000", 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1605278669999999 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415150+0000", 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011404399999999999 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.656 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415264+0000", 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039553599999999999 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415659+0000", 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075350449999999998 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.657 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423194+0000", 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01213207 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435326+0000", 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062050799999999998 2023-10-09T10:20:26.658 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435947+0000", 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013730799999999999 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436084+0000", 2023-10-09T10:20:26.659 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2330000000000002e-05 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436137+0000", 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120761 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436257+0000", 2023-10-09T10:20:26.660 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1586804549999998 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195652 9.1c 9:3e175140:::benchmark_data_smithi012_313365_object83177:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 113.26940718199999, 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1274454700000001, 2023-10-09T10:20:26.661 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195652 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:20:26.662 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308095+0000", 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.49e-06 2023-10-09T10:20:26.663 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308104+0000", 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7926000000000001e-05 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308122+0000", 2023-10-09T10:20:26.664 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6449999999999999e-06 2023-10-09T10:20:26.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.665 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308128+0000", 2023-10-09T10:20:26.665 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7748000000000003e-05 2023-10-09T10:20:26.665 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.665 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.665 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308196+0000", 2023-10-09T10:20:26.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3386e-05 2023-10-09T10:20:26.666 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.666 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.666 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.666 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308210+0000", 2023-10-09T10:20:26.666 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.945878781 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254088+0000", 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5209999999999997e-06 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254096+0000", 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.15516445 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.667 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409260+0000", 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000137047 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409397+0000", 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039575500000000002 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:20:26.668 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409793+0000", 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021158150000000001 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411909+0000", 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7689e-05 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411927+0000", 2023-10-09T10:20:26.669 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011128549999999999 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423055+0000", 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012286897999999999 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435342+0000", 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6456000000000003e-05 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.670 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435399+0000", 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014397600000000001 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435543+0000", 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1274454700000001 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.671 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195653 9.c 9:3443b6fc:::benchmark_data_smithi012_313365_object83178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 113.268934638, 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1279840910000001, 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.672 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195653 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:20:26.673 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308569+0000", 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1650000000000003e-06 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308575+0000", 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432000000000001e-05 2023-10-09T10:20:26.674 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308587+0000", 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.648e-06 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308593+0000", 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3375000000000003e-05 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.675 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308646+0000", 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3016e-05 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308659+0000", 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94597990200000004 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.676 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254639+0000", 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.489e-06 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254646+0000", 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.163105716 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417751+0000", 2023-10-09T10:20:26.677 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109895 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417861+0000", 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033677300000000001 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.418198+0000", 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017758396999999999 2023-10-09T10:20:26.678 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435956+0000", 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6774000000000002e-05 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436013+0000", 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5324000000000003e-05 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.679 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436049+0000", 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032668699999999998 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436375+0000", 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2172999999999997e-05 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436427+0000", 2023-10-09T10:20:26.680 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012633699999999999 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436554+0000", 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1279840910000001 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195656 9.c 9:310794b5:::benchmark_data_smithi012_313365_object83181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:20:26.681 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 113.254414677, 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1137697360000001, 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195656 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.682 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.683 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323088+0000", 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0759e-05 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323099+0000", 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7932999999999999e-05 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323116+0000", 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4930000000000003e-06 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.684 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323123+0000", 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0368000000000003e-05 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323173+0000", 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3370000000000006e-06 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.685 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323183+0000", 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93181647499999998 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254999+0000", 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0474e-05 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.255010+0000", 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.165299136 2023-10-09T10:20:26.686 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420309+0000", 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117821 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420427+0000", 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000380281 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.687 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420807+0000", 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015156678999999999 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435964+0000", 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033305500000000002 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.688 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436297+0000", 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0358999999999998e-05 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436337+0000", 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033388899999999997 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436671+0000", 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3112999999999998e-05 2023-10-09T10:20:26.689 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436724+0000", 2023-10-09T10:20:26.690 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000135403 2023-10-09T10:20:26.690 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.690 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.690 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.690 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436859+0000", 2023-10-09T10:20:26.691 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1137697360000001 2023-10-09T10:20:26.691 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.691 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.691 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.691 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.691 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.691 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195663 9.1c 9:3f09486e:::benchmark_data_smithi012_313365_object83188:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:20:26.691 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:20:26.692 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 113.23633526899999, 2023-10-09T10:20:26.692 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0945934909999999, 2023-10-09T10:20:26.692 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.692 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.692 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.692 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.692 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.692 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195663 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:20:26.693 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341167+0000", 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5290000000000001e-05 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.694 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341182+0000", 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1254e-05 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341204+0000", 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7949999999999996e-06 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.695 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341212+0000", 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6192000000000006e-05 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341279+0000", 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3284e-05 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.696 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341292+0000", 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91282471300000001 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254117+0000", 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.697 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254124+0000", 2023-10-09T10:20:26.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.157826198 2023-10-09T10:20:26.698 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.698 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.698 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.698 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411950+0000", 2023-10-09T10:20:26.698 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012947999999999999 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412079+0000", 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000355423 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:20:26.699 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412435+0000", 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002552646 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.414987+0000", 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027370139999999999 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.700 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417724+0000", 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053624290000000002 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423087+0000", 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012506992 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.701 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435594+0000", 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7438000000000002e-05 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435641+0000", 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012119 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435762+0000", 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0945934909999999 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.702 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195673 9.10 9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e400)", 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 112.319859769, 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6028094020000001, 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.703 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195673 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:20:26.704 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257643+0000", 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0550000000000002e-06 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257650+0000", 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8279999999999998e-06 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.705 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257660+0000", 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7660000000000001e-06 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257666+0000", 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032584879999999999 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.260925+0000", 2023-10-09T10:20:26.706 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0726000000000001e-05 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.260946+0000", 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9083686909999997 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.169314+0000", 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002054237 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.707 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.171369+0000", 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6018258909999998 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.773194+0000", 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011103770000000001 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:20:26.708 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.774305+0000", 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055754660999999997 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.830060+0000", 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001693766 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.831753+0000", 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034851140000000001 2023-10-09T10:20:26.709 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.835238+0000", 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019324406999999998 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.854563+0000", 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000167465 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.710 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.854730+0000", 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058527999999999998 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.855316+0000", 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00087701200000000004 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.856193+0000", 2023-10-09T10:20:26.711 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042613290000000003 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.860454+0000", 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6028094020000001 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195674 9.c 9:313b05bb:::benchmark_data_smithi012_313365_object83199:head [delete] snapc 0=[] ondisk+write+known_if_redirected e400)", 2023-10-09T10:20:26.712 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:34.315212+0000", 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 112.26229241999999, 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1218578619999999, 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195674 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.713 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315212+0000", 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315212+0000", 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.714 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315209+0000", 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2469999999999999e-05 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315222+0000", 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2121e-05 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315244+0000", 2023-10-09T10:20:26.715 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2990000000000002e-06 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315253+0000", 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2500000000000001e-05 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315316+0000", 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5967000000000001e-05 2023-10-09T10:20:26.716 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315332+0000", 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1078856859999999 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423217+0000", 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011951 2023-10-09T10:20:26.717 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423337+0000", 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035369399999999999 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423691+0000", 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012278888 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.718 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435970+0000", 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062949600000000001 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436599+0000", 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4643e-05 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.719 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436634+0000", 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000268072 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436902+0000", 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8890000000000001e-05 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.720 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436951+0000", 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119073 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.437070+0000", 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1218578619999999 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.721 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195693 9.10 9:0f588e8d:::benchmark_data_smithi012_313365_object83218:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 111.103949082, 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3933209699999995, 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195693 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.722 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:20:26.723 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473553+0000", 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0190000000000001e-05 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473563+0000", 2023-10-09T10:20:26.724 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2921999999999999e-05 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473576+0000", 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.994e-06 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473584+0000", 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.304343544 2023-10-09T10:20:26.725 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.777928+0000", 2023-10-09T10:20:26.726 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022209249999999999 2023-10-09T10:20:26.726 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.726 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.726 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:20:26.726 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.780149+0000", 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013458505000000001 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.793607+0000", 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001306698 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:20:26.727 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.794914+0000", 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051671473000000002 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.846585+0000", 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014171050000000001 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.728 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.848002+0000", 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003102222 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.851105+0000", 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0099542229999999999 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861059+0000", 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017274 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861231+0000", 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00060873299999999997 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861840+0000", 2023-10-09T10:20:26.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000843592 2023-10-09T10:20:26.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.862684+0000", 2023-10-09T10:20:26.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041923439999999998 2023-10-09T10:20:26.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.866876+0000", 2023-10-09T10:20:26.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3933209699999995 2023-10-09T10:20:26.732 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.732 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.732 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.733 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195716 9.10 9:0de169be:::benchmark_data_smithi012_313365_object83241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:20:26.733 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:20:26.733 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 111.067780064, 2023-10-09T10:20:26.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3902966980000002, 2023-10-09T10:20:26.733 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.733 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195716 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509723+0000", 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0609999999999998e-06 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509730+0000", 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3033e-05 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509743+0000", 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.613e-06 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509749+0000", 2023-10-09T10:20:26.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2706342480000004 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.780384+0000", 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023119730000000002 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.782696+0000", 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020697949 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.803393+0000", 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001386583 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.804780+0000", 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.078157095999999995 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.882937+0000", 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001764306 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.884701+0000", 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036488340000000001 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.888350+0000", 2023-10-09T10:20:26.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003904832 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.892255+0000", 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00076447400000000001 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.893020+0000", 2023-10-09T10:20:26.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022865630000000001 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.895306+0000", 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00084655199999999998 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.896153+0000", 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038682040000000001 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.900021+0000", 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3902966980000002 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195733 9.10 9:0c074ac9:::benchmark_data_smithi012_313365_object83258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:20:26.742 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:20:26.743 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 111.001917017, 2023-10-09T10:20:26.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3621552440000002, 2023-10-09T10:20:26.743 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.743 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.743 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.743 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195733 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575586+0000", 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5009999999999998e-06 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575593+0000", 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0510000000000002e-06 2023-10-09T10:20:26.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575602+0000", 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9590000000000003e-06 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575607+0000", 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2073396159999996 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.782947+0000", 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00182449 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.784772+0000", 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030197965 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.814970+0000", 2023-10-09T10:20:26.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001106353 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.816076+0000", 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.101857857 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.917934+0000", 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001720475 2023-10-09T10:20:26.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.919654+0000", 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003479429 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.923134+0000", 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041782599999999996 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.927312+0000", 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070182800000000004 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.928014+0000", 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043811029999999999 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.932395+0000", 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00094054900000000005 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.933335+0000", 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044071309999999999 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.937742+0000", 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3621552440000002 2023-10-09T10:20:26.751 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(9.10 403/400 [PushOp(9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head, version: 334'5146, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head@334'5146, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.405212792, 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6005742729999999, 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9161999999999997e-05 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172390+0000", 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8474999999999998e-05 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172408+0000", 2023-10-09T10:20:26.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0516000000000001e-05 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172419+0000", 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5926570120000001 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.765076+0000", 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017349800000000001 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.755 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.765249+0000", 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076164220000000003 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.772866+0000", 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6005742729999999 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195749 9.1c 9:3a233838:::benchmark_data_smithi012_313365_object83274:head [delete] snapc 0=[] ondisk+write+known_if_redirected e405)", 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:39.409149+0000", 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 107.16835489499999, 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94321279099999999, 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195749 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.757 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409149+0000", 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409149+0000", 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:20:26.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409147+0000", 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5019999999999998e-06 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409157+0000", 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6129000000000001e-05 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409173+0000", 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5889999999999997e-06 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409181+0000", 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9796000000000001e-05 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409240+0000", 2023-10-09T10:20:26.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9539999999999999e-06 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409250+0000", 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92036669000000004 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.329617+0000", 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011469299999999999 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.329732+0000", 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042454800000000002 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.330156+0000", 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068902920000000001 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.337047+0000", 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054030570000000002 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.342450+0000", 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098152039999999992 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.352265+0000", 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3492999999999998e-05 2023-10-09T10:20:26.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.352318+0000", 2023-10-09T10:20:26.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3871000000000001e-05 2023-10-09T10:20:26.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.352362+0000", 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94321279099999999 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195754 9.1c 9:3939aea6:::benchmark_data_smithi012_313365_object83279:head [delete] snapc 0=[] ondisk+write+known_if_redirected e405)", 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:39.448462+0000", 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 107.129041965, 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91816715999999998, 2023-10-09T10:20:26.765 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:20:26.766 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:20:26.766 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:20:26.766 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:20:26.766 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:20:26.766 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195754 2023-10-09T10:20:26.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.766 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448462+0000", 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448462+0000", 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:20:26.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448461+0000", 2023-10-09T10:20:26.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9729999999999997e-06 2023-10-09T10:20:26.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:20:26.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448469+0000", 2023-10-09T10:20:26.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5580999999999999e-05 2023-10-09T10:20:26.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448484+0000", 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.187e-06 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448492+0000", 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012920550000000001 2023-10-09T10:20:26.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.449784+0000", 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0560000000000005e-06 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.449792+0000", 2023-10-09T10:20:26.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88260967599999995 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.332401+0000", 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010365999999999999 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:20:26.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.332505+0000", 2023-10-09T10:20:26.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027790600000000002 2023-10-09T10:20:26.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:20:26.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.332783+0000", 2023-10-09T10:20:26.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098019160000000008 2023-10-09T10:20:26.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.342585+0000", 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098189569999999997 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.352404+0000", 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014127015999999999 2023-10-09T10:20:26.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366531+0000", 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4029000000000001e-05 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366585+0000", 2023-10-09T10:20:26.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4632000000000002e-05 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366629+0000", 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91816715999999998 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:20:26.775 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:20:26.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:26.776 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:26.777 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:26.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:26.878 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:26.927 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:26.986 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:26.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:26.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:26.999 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 39 16 66297 66281 106.152 75.125 0.0123946 0.00941534 2023-10-09T10:20:27.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:20:27.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:27.176 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:27.272 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:20:27.272 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:20:27.272 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T10:20:27.273 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T10:20:27.299 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:27.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:27.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:27.378 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:27 smithi012 ceph-mon[152263]: pgmap v1215: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:20:27.379 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:27 smithi012 ceph-mon[147926]: pgmap v1215: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:20:27.413 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:27.455 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:27.507 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:27.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:27.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:27.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:27.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:20:27.685 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:27.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:27 smithi110 ceph-mon[136032]: pgmap v1215: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:20:27.756 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:27.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:27.869 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:27.912 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:27.971 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:27.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:27.999 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:20:28.000114+0000 min lat: 0.00109621 max lat: 0.357108 avg lat: 0.00949689 2023-10-09T10:20:28.000 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:20:28.000 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 40 15 67404 67389 105.23 69.25 0.0097469 0.00949689 2023-10-09T10:20:28.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:28.129 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:28.145 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:20:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:20:27] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.33.4" 2023-10-09T10:20:28.151 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:28.403 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:28.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:28.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:28.512 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:28.568 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:28.572 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:28.635 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:28.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:28.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:28.744 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:28.810 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:28.864 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:28.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:28.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:28.971 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:29.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 41 16 68530 68514 104.378 70.3125 0.0261958 0.0095753 2023-10-09T10:20:29.026 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:29.087 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:29.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:29.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:29.251 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:29.280 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:29.591 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:29 smithi012 ceph-mon[147926]: pgmap v1216: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:20:29.591 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:29.591 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:29.591 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2874928803' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:20:29.591 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:29 smithi012 ceph-mon[152263]: pgmap v1216: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:20:29.591 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:29.591 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:29.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2874928803' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:20:29.687 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:29 smithi110 ceph-mon[136032]: pgmap v1216: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T10:20:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:29.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2874928803' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:20:29.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:29.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:29.806 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:29.881 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:29.987 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:20:30.000 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 42 16 69706 69690 103.643 73.5 0.028026 0.00964174 2023-10-09T10:20:30.006 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:30.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:30.116 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:30.175 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:30.241 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:30.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:30.347 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:30 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2874928803' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:20:30.390 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:30 smithi012 ceph-mon[152263]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T10:20:30.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:30 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:30.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2874928803' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:20:30.391 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:30 smithi012 ceph-mon[147926]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T10:20:30.439 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:30.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:30 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:30.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2874928803' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:20:30.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:30 smithi110 ceph-mon[136032]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T10:20:30.576 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:30.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:30.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:30.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:30.771 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:30.805 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:20:30.806 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:20:30.806 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:20:30.806 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:20:31.000 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 43 16 70824 70808 102.858 69.875 0.0097379 0.00971758 2023-10-09T10:20:31.030 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:31.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:31.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:31.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:31.220 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:31.369 INFO:teuthology.orchestra.run.smithi012.stdout:moved 2 / 267 (0.749064%) 2023-10-09T10:20:31.369 INFO:teuthology.orchestra.run.smithi012.stdout:avg 33.375 2023-10-09T10:20:31.369 INFO:teuthology.orchestra.run.smithi012.stdout:stddev 11.521 -> 10.8736 (expected baseline 5.40399) 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:min osd.3 with 13 -> 15 pgs (0.389513 -> 0.449438 * mean) 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:max osd.1 with 47 -> 47 pgs (1.40824 -> 1.40824 * mean) 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:oload 1000 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:max_change 0.05 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:max_change_osds 4 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:average_utilization 0.0248 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:overload_utilization 0.2482 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 weight 0.3955 -> 0.4455 2023-10-09T10:20:31.370 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 weight 0.7460 -> 0.7595 2023-10-09T10:20:31.375 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[152263]: pgmap v1217: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T10:20:31.376 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[152263]: failed to encode map e435 with expected crc 2023-10-09T10:20:31.376 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[152263]: failed to encode map e435 with expected crc 2023-10-09T10:20:31.376 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]: dispatch 2023-10-09T10:20:31.376 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]: dispatch 2023-10-09T10:20:31.431 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:31.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:31.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:31.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:31.606 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:31.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[147926]: pgmap v1217: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T10:20:31.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[147926]: failed to encode map e435 with expected crc 2023-10-09T10:20:31.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[147926]: failed to encode map e435 with expected crc 2023-10-09T10:20:31.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]: dispatch 2023-10-09T10:20:31.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:31 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]: dispatch 2023-10-09T10:20:31.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:20:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:31 smithi110 ceph-mon[136032]: pgmap v1217: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T10:20:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:31 smithi110 ceph-mon[136032]: failed to encode map e435 with expected crc 2023-10-09T10:20:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:31 smithi110 ceph-mon[136032]: failed to encode map e435 with expected crc 2023-10-09T10:20:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:31 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]: dispatch 2023-10-09T10:20:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:31 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]: dispatch 2023-10-09T10:20:31.955 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:31.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:31.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:32.000 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 44 16 72015 71999 102.212 74.4375 0.0112508 0.0097788 2023-10-09T10:20:32.003 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T10:20:32.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T10:20:32.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:32.151 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:20:32.193 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:32.202 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:20:32.242 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:20:32.288 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:32.289 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:20:32.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:32.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:32.332 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:20:32.380 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:20:32.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:32.424 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:20:32.465 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:20:32.484 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:32.503 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:32.503 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:20:32.503 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:20:32.503 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:20:32.503 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:20:32.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:20:32.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:20:32.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:20:32.504 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:20:32.561 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:32.563 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: from='client.46520 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]': finished 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2538715979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: from='client.46520 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]': finished 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2538715979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:32.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:32.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:32.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:32.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: from='client.46520 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'49772','3':'29198'}"}]': finished 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: failed to encode map e436 with expected crc 2023-10-09T10:20:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2538715979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:32.752 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:32.828 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:32.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:32.936 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:33.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 45 16 73055 73039 101.386 65 0.0156636 0.00985748 2023-10-09T10:20:33.004 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:33.068 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:33.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:33.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:33.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:33.242 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:33.400 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:33 smithi012 ceph-mon[147926]: pgmap v1220: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T10:20:33.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:33 smithi012 ceph-mon[152263]: pgmap v1220: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T10:20:33.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:33 smithi012 ceph-mon[152263]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T10:20:33.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:33 smithi012 ceph-mon[152263]: failed to encode map e437 with expected crc 2023-10-09T10:20:33.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:33 smithi012 ceph-mon[152263]: failed to encode map e437 with expected crc 2023-10-09T10:20:33.509 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:33.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:33.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:33.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:33.649 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:20:33.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:33 smithi012 ceph-mon[147926]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T10:20:33.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:33 smithi012 ceph-mon[147926]: failed to encode map e437 with expected crc 2023-10-09T10:20:33.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:33 smithi012 ceph-mon[147926]: failed to encode map e437 with expected crc 2023-10-09T10:20:33.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:33 smithi110 ceph-mon[136032]: pgmap v1220: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T10:20:33.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:33 smithi110 ceph-mon[136032]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T10:20:33.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:33 smithi110 ceph-mon[136032]: failed to encode map e437 with expected crc 2023-10-09T10:20:33.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:33 smithi110 ceph-mon[136032]: failed to encode map e437 with expected crc 2023-10-09T10:20:33.696 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:33.769 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:33.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:33.775 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:33.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:33.926 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:33.988 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:33.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:33.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:34.000 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 46 16 74189 74173 100.723 70.875 0.00803425 0.00992276 2023-10-09T10:20:34.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:34.163 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:34.253 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:20:34.253 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:20:34.254 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:34.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:34.367 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:34.452 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:34.502 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:34.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:34.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:34.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:34.776 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:34.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:34.824 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:20:34.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:34.941 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:35.000 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 47 16 75291 75275 100.044 68.875 0.0113125 0.00998937 2023-10-09T10:20:35.007 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:35.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:35.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:35.116 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:35.147 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:35.178 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:35.220 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:35.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:35.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:35.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:35.408 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:35.471 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:35.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:35.579 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:35.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:20:35.622 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:35 smithi110 ceph-mon[136032]: pgmap v1222: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:20:35.632 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:35.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:35 smithi012 ceph-mon[152263]: pgmap v1222: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:20:35.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:35 smithi012 ceph-mon[147926]: pgmap v1222: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:20:35.693 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:35.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:35.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:35.801 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:35.854 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:35.920 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:35.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:35.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:35.956 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:36.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 48 16 76308 76292 99.2843 63.5625 0.00540132 0.0100649 2023-10-09T10:20:36.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:36.104 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:36.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:36 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]: dispatch 2023-10-09T10:20:36.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:36 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]: dispatch 2023-10-09T10:20:36.438 INFO:teuthology.orchestra.run.smithi012.stdout:moved 1 / 267 (0.374532%) 2023-10-09T10:20:36.438 INFO:teuthology.orchestra.run.smithi012.stdout:avg 33.375 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:stddev 10.8736 -> 10.5823 (expected baseline 5.40399) 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:min osd.3 with 15 -> 16 pgs (0.449438 -> 0.479401 * mean) 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:max osd.1 with 47 -> 47 pgs (1.40824 -> 1.40824 * mean) 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:oload 1000 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:max_change 0.05 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:max_change_osds 4 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:average_utilization 0.0266 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:overload_utilization 0.2657 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 weight 0.4455 -> 0.4955 2023-10-09T10:20:36.439 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 weight 0.7595 -> 0.7913 2023-10-09T10:20:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:36 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]: dispatch 2023-10-09T10:20:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:36 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]: dispatch 2023-10-09T10:20:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:36 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]: dispatch 2023-10-09T10:20:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:36 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]: dispatch 2023-10-09T10:20:36.858 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:36.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:36.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:36.971 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:36.997 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:37.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 49 16 77426 77410 98.6842 69.875 0.00339938 0.010118 2023-10-09T10:20:37.028 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:37.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:37.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:37.132 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:37.187 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:37.275 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:37.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:37.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:37.381 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:37.444 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:37.498 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:37.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:37.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:37.605 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:37.668 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:37.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:37 smithi110 ceph-mon[136032]: pgmap v1223: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:37.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:37 smithi110 ceph-mon[136032]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:37.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:37 smithi110 ceph-mon[136032]: failed to encode map e438 with expected crc 2023-10-09T10:20:37.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:37 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]': finished 2023-10-09T10:20:37.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:37 smithi110 ceph-mon[136032]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T10:20:37.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:37 smithi110 ceph-mon[136032]: failed to encode map e438 with expected crc 2023-10-09T10:20:37.867 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:37.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[152263]: pgmap v1223: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:37.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[152263]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:37.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[152263]: failed to encode map e438 with expected crc 2023-10-09T10:20:37.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]': finished 2023-10-09T10:20:37.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[152263]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T10:20:37.872 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[152263]: failed to encode map e438 with expected crc 2023-10-09T10:20:37.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[147926]: pgmap v1223: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:37.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[147926]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:37.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[147926]: failed to encode map e438 with expected crc 2023-10-09T10:20:37.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'51859','3':'32474'}"}]': finished 2023-10-09T10:20:37.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[147926]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T10:20:37.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-mon[147926]: failed to encode map e438 with expected crc 2023-10-09T10:20:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:37.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:37.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:38.002 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 50 16 78539 78523 98.1019 69.5625 0.0203197 0.0101889 2023-10-09T10:20:38.026 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:20:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:20:37] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.33.4" 2023-10-09T10:20:38.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[152263]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T10:20:38.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[152263]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[152263]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[152263]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[152263]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.739 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[147926]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T10:20:38.739 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[147926]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.739 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[147926]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.740 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[147926]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.740 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:38 smithi012 ceph-mon[147926]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.894 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:38.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:38.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:38 smithi110 ceph-mon[136032]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T10:20:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:38 smithi110 ceph-mon[136032]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:38 smithi110 ceph-mon[136032]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:38 smithi110 ceph-mon[136032]: failed to encode map e439 with expected crc 2023-10-09T10:20:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:38 smithi110 ceph-mon[136032]: failed to encode map e439 with expected crc 2023-10-09T10:20:39.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 51 16 79640 79624 97.5277 68.8125 0.00477875 0.0102482 2023-10-09T10:20:39.004 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:39.062 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:39.135 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:39.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:39.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:39.212 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T10:20:39.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T10:20:39.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:39.248 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:39.340 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:39.596 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:39.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:39.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:39.706 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:39.715 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:39 smithi012 ceph-mon[147926]: pgmap v1226: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:39.715 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:39 smithi012 ceph-mon[152263]: pgmap v1226: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:39.751 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:39 smithi110 ceph-mon[136032]: pgmap v1226: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:39.760 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:39.831 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:39.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:39.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:39.999 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:40.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 52 16 80677 80661 96.8988 64.8125 0.0106854 0.0103151 2023-10-09T10:20:40.065 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:40.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:40.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:40.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:40.222 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:40.282 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:40.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:40.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:40.447 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:40.489 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:20:40.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2055071310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:40.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2055071310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:40.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2055071310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:41.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 53 16 81776 81760 96.3666 68.6875 0.0103009 0.010372 2023-10-09T10:20:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: pgmap v1227: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2055071310' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[147926]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: pgmap v1227: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2055071310' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T10:20:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:41 smithi012 ceph-mon[152263]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: pgmap v1227: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:20:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2055071310' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T10:20:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: failed to encode map e440 with expected crc 2023-10-09T10:20:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:41 smithi110 ceph-mon[136032]: failed to encode map e440 with expected crc 2023-10-09T10:20:42.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 54 16 82807 82791 95.7754 64.4375 0.00975478 0.0104347 2023-10-09T10:20:42.430 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:42.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:42.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:42.534 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:42.596 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:42.656 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:20:42.656 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:20:42.676 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:42.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:42.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:42.784 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:42.824 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:20:42.827 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:42.865 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:42.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:42.970 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:42.999 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:43.001 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 55 16 83505 83489 94.8275 43.625 0.00410487 0.0105021 2023-10-09T10:20:43.041 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:43.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:43.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:43.121 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:43.146 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:43.171 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:43.375 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:20:43.375 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:20:43.580 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:43 smithi012 ceph-mon[152263]: pgmap v1229: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T10:20:43.581 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:43 smithi012 ceph-mon[147926]: pgmap v1229: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T10:20:43.628 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:20:43.657 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:20:43.685 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:20:43.714 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:20:43.751 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:20:43.765 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:43.769 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:43 smithi110 ceph-mon[136032]: pgmap v1229: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T10:20:43.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:43.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:43.796 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:20:43.841 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:20:43.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:20:43.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:43.881 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:20:43.893 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:20:43.893 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:20:43.893 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:20:43.893 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:20:43.893 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:20:43.893 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:20:43.893 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:20:43.894 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:20:43.945 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:44.002 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 56 16 84271 84255 93.9879 47.875 0.0220157 0.0106326 2023-10-09T10:20:44.058 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:44.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:44.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:44.278 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:44.500 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:44.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:44.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:44.608 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:44.681 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:44.766 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:44.771 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/495316011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:44.771 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:44.771 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:44.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:44.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:44.809 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:20:44.850 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/495316011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:44.850 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:44.850 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:44.851 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/495316011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:44.851 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:44.851 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:44.873 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:44.922 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:44.990 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:44.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:45.002 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 57 16 85227 85211 93.3874 59.75 0.0266036 0.0107035 2023-10-09T10:20:45.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:45.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:45.221 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:45.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:45.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:45.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:45.387 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:45.403 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:45.607 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:45.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:45.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:45 smithi012 ceph-mon[147926]: pgmap v1230: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-09T10:20:45.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:45 smithi012 ceph-mon[152263]: pgmap v1230: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-09T10:20:45.718 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:45.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:45.825 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:45.831 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:45 smithi110 ceph-mon[136032]: pgmap v1230: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-09T10:20:45.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:45.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:45.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:45.999 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:46.003 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 58 16 86134 86118 92.7528 56.6875 0.0212778 0.0107759 2023-10-09T10:20:46.064 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:46.089 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:46.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:46.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout:moved 20 / 267 (7.49064%) 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout:avg 33.375 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout:stddev 10.5823 -> 7.56534 (expected baseline 5.40399) 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout:min osd.3 with 16 -> 36 pgs (0.479401 -> 1.07865 * mean) 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout:max osd.1 with 47 -> 43 pgs (1.40824 -> 1.28839 * mean) 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout:oload 1000 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout:max_change 1 2023-10-09T10:20:46.215 INFO:teuthology.orchestra.run.smithi012.stdout:max_change_osds 4 2023-10-09T10:20:46.216 INFO:teuthology.orchestra.run.smithi012.stdout:average_utilization 0.0291 2023-10-09T10:20:46.216 INFO:teuthology.orchestra.run.smithi012.stdout:overload_utilization 0.2907 2023-10-09T10:20:46.216 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 weight 0.4955 -> 0.9180 2023-10-09T10:20:46.216 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 weight 0.7913 -> 0.8272 2023-10-09T10:20:46.228 INFO:teuthology.orchestra.run.smithi012.stderr:no change 2023-10-09T10:20:46.275 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:46.347 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:46.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:46.428 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:20:46 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T10:20:46.209+0000 7f2cb4de3700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T10:20:46.459 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:46.514 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:46.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:20:46.677 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:46.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:46.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:46 smithi110 ceph-mon[136032]: pgmap v1231: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T10:20:46.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:46 smithi110 ceph-mon[136032]: from='client.36552 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:46.757 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:46 smithi012 ceph-mon[147926]: pgmap v1231: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T10:20:46.757 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:46 smithi012 ceph-mon[147926]: from='client.36552 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:46.758 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:46 smithi012 ceph-mon[152263]: pgmap v1231: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T10:20:46.758 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:46 smithi012 ceph-mon[152263]: from='client.36552 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:46.762 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T10:20:46.762 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-09T10:20:46.787 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:46.935 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:47.006 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 59 16 87023 87007 92.1196 55.5625 0.0117265 0.0108489 2023-10-09T10:20:47.018 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:47.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:47.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:47.134 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:47.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:47.371 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:47.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:47.483 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:47.532 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:20:47.532 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:20:47.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.532 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36474.0:87467 10.5 10:ab597c86:::benchmark_data_smithi012_366312_object87466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e440)", 2023-10-09T10:20:47.532 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:47.521212+0000", 2023-10-09T10:20:47.532 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0099522800000000008, 2023-10-09T10:20:47.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009989352, 2023-10-09T10:20:47.533 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:47.533 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:20:47.533 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:20:47.533 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36474", 2023-10-09T10:20:47.533 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:20:47.533 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 87467 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521212+0000", 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:47.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521212+0000", 2023-10-09T10:20:47.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:47.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521211+0000", 2023-10-09T10:20:47.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:20:47.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521255+0000", 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4409999999999997e-05 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521269+0000", 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4038e-05 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:47.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521276+0000", 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6050000000000003e-06 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521340+0000", 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4214000000000006e-05 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521505+0000", 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016560699999999999 2023-10-09T10:20:47.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521679+0000", 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00017371899999999999 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.527201+0000", 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055218960000000001 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.527209+0000", 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2320000000000001e-06 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36474.0:87468 10.5 10:ac6d30c6:::benchmark_data_smithi012_366312_object87467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e440)", 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:47.521290+0000", 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0098740010000000003, 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0099917589999999994, 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:47.539 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36474", 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 87468 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521290+0000", 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521290+0000", 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521290+0000", 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521311+0000", 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1348e-05 2023-10-09T10:20:47.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521333+0000", 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1994e-05 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.521357+0000", 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3947999999999998e-05 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.524137+0000", 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027799159999999999 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.524261+0000", 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000124449 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.524371+0000", 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109818 2023-10-09T10:20:47.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.528289+0000", 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039179030000000004 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:87471 10.3 e440/424 10:c68bf2ae:::benchmark_data_smithi012_366312_object87470:head v 440'5653, mlcod=440'5651)", 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:47.526091+0000", 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0050732360000000001, 2023-10-09T10:20:47.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052804009999999997, 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.526091+0000", 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.526091+0000", 2023-10-09T10:20:47.545 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.526091+0000", 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.526132+0000", 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1235000000000001e-05 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.546 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.526139+0000", 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5599999999999996e-06 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.526145+0000", 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2490000000000001e-06 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.526203+0000", 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8884000000000001e-05 2023-10-09T10:20:47.547 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.526222+0000", 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8612000000000001e-05 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36474.0:87478 10.8 10:12b6b44e:::benchmark_data_smithi012_366312_object87477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e440)", 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:47.529764+0000", 2023-10-09T10:20:47.548 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0014000130000000001, 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001646998, 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36474", 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 87478 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.529764+0000", 2023-10-09T10:20:47.549 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.529764+0000", 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.529762+0000", 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.550 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.529786+0000", 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3901999999999999e-05 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.529800+0000", 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3498e-05 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.529808+0000", 2023-10-09T10:20:47.551 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5739999999999996e-06 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.529857+0000", 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9086999999999999e-05 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.529999+0000", 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014203600000000001 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.552 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.530137+0000", 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013730900000000001 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:87474 10.b e440/424)", 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:47.530822+0000", 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00034160000000000001, 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062957599999999996, 2023-10-09T10:20:47.553 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.530822+0000", 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.530822+0000", 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:20:47.554 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.530822+0000", 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.530854+0000", 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2829e-05 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.555 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.530869+0000", 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4790000000000001e-05 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:47.530878+0000", 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2299999999999997e-06 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:20:47.556 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:20:47.557 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 5 2023-10-09T10:20:47.557 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:20:47.557 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:47.708 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:47.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:47.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:47.815 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:47.878 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:47.953 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:47.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:47.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:48.006 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:20:48.006491+0000 min lat: 0.00109621 max lat: 0.457669 avg lat: 0.0109191 2023-10-09T10:20:48.006 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:20:48.006 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 60 16 87942 87926 91.5417 57.4375 0.00743061 0.0109191 2023-10-09T10:20:48.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:48.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:20:48.158 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:20:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:20:47] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.33.4" 2023-10-09T10:20:48.161 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:48.418 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:48.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:48.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:48.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:48.606 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:48.721 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:48.763 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:48.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:48.874 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:48.940 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:49.006 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 61 15 88832 88817 90.9541 55.6875 0.013392 0.0109894 2023-10-09T10:20:49.008 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:49.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:49.120 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:49.171 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:49.288 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:49.308 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:49 smithi012 ceph-mon[147926]: pgmap v1232: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T10:20:49.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:49 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:49.309 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:49 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:49.491 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:49.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:49.599 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:49.617 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:49 smithi012 ceph-mon[152263]: pgmap v1232: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T10:20:49.617 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:49 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:49.617 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:49 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:49.652 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:49 smithi110 ceph-mon[136032]: pgmap v1232: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T10:20:49.652 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:49 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:49.652 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:49 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:49.658 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:49.693 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:49.719 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:49.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:49.823 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:49.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:49.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:49.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:49.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:50.006 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 62 16 89754 89738 90.4151 57.5625 0.00697672 0.0110556 2023-10-09T10:20:50.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:50.139 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:50.318 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/524990692' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:50.318 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:50 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:50.338 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:20:50.524 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:50.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:50.631 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:50.668 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/524990692' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:50.668 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:50 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:50.669 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/524990692' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:50.669 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:50 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:20:50.690 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:50.829 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:50.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:50.933 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:51.004 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:51.006 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 63 16 90739 90723 89.9573 61.5625 0.0177855 0.0111115 2023-10-09T10:20:51.322 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[147926]: pgmap v1233: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T10:20:51.322 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:51.322 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[147926]: failed to encode map e441 with expected crc 2023-10-09T10:20:51.322 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:20:51.322 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[147926]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T10:20:51.322 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[147926]: failed to encode map e441 with expected crc 2023-10-09T10:20:51.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[152263]: pgmap v1233: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T10:20:51.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:51.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[152263]: failed to encode map e441 with expected crc 2023-10-09T10:20:51.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:20:51.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[152263]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T10:20:51.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:51 smithi012 ceph-mon[152263]: failed to encode map e441 with expected crc 2023-10-09T10:20:51.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:51 smithi110 ceph-mon[136032]: pgmap v1233: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T10:20:51.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:51 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:20:51.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:51 smithi110 ceph-mon[136032]: failed to encode map e441 with expected crc 2023-10-09T10:20:51.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:20:51.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:51 smithi110 ceph-mon[136032]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T10:20:51.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:51 smithi110 ceph-mon[136032]: failed to encode map e441 with expected crc 2023-10-09T10:20:51.791 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:51.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:51.900 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:51.962 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:52.007 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 64 16 91716 91700 89.5059 61.0625 0.0163011 0.0111658 2023-10-09T10:20:52.020 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:52.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:52.125 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:52.185 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:20:52.186 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:52.225 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:20:52.248 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:52.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:52.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:52.274 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:20:52.340 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:20:52.354 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:52.386 INFO:teuthology.orchestra.run.smithi012.stdout:moved 7 / 267 (2.62172%) 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout:avg 33.375 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout:stddev 10.5823 -> 9.17793 (expected baseline 5.40399) 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout:min osd.3 with 16 -> 20 pgs (0.479401 -> 0.599251 * mean) 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout:max osd.1 with 47 -> 46 pgs (1.40824 -> 1.37828 * mean) 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout:oload 110 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout:max_change 0.05 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout:max_change_osds 4 2023-10-09T10:20:52.387 INFO:teuthology.orchestra.run.smithi012.stdout:average_utilization 0.0302 2023-10-09T10:20:52.388 INFO:teuthology.orchestra.run.smithi012.stdout:overload_utilization 0.0332 2023-10-09T10:20:52.388 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 weight 0.4955 -> 0.5455 2023-10-09T10:20:52.388 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 weight 1.0000 -> 0.9500 2023-10-09T10:20:52.388 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 weight 1.0000 -> 0.9500 2023-10-09T10:20:52.388 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 weight 0.6220 -> 0.5720 2023-10-09T10:20:52.388 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:20:52.409 INFO:teuthology.orchestra.run.smithi012.stderr:no change 2023-10-09T10:20:52.425 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:20:52.435 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:52.461 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:20:52.517 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:20:52.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:20:52.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:20:52.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:20:52.518 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:20:52.519 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:20:52.519 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:20:52.519 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:20:52.519 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:20:52.671 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T10:20:52.385+0000 7f2cb4de3700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-10-09T10:20:52.723 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:20:52.724 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:20:52.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[147926]: pgmap v1235: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-10-09T10:20:52.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:52.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:52.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/976164095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:52.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[147926]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:52.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:20:52.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:20:52.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:52.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[152263]: pgmap v1235: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-10-09T10:20:52.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:52.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:52.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/976164095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:52.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[152263]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:52.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:20:52.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:20:52.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:52 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:52.954 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T10:20:52.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T10:20:53.008 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 65 16 92564 92548 88.943 53 0.0208154 0.0112378 2023-10-09T10:20:53.029 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:53.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:53.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:53.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:52 smithi110 ceph-mon[136032]: pgmap v1235: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-10-09T10:20:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/976164095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:20:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:52 smithi110 ceph-mon[136032]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:20:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:52 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:20:53.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:52 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:20:53.209 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:53.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:20:53.522 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:53.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:53.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:20:53.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:53.755 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:53.850 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:53.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:53.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:54.008 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 66 16 93465 93449 88.4488 56.3125 0.0219885 0.0113005 2023-10-09T10:20:54.034 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:54.037 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:54.092 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:54.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:54.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:54.273 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:54.345 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:54.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:54.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:54.500 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:54.613 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:54.620 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:54.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:54.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:54.779 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:55.008 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 67 16 94436 94420 88.0345 60.6875 0.0235964 0.0113544 2023-10-09T10:20:55.089 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:55.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:55.131 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:55.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:20:55.270 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:55.371 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:20:55.374 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:55 smithi012 ceph-mon[152263]: pgmap v1236: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T10:20:55.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:55.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:20:55.376 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:55 smithi012 ceph-mon[147926]: pgmap v1236: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T10:20:55.476 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:55.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:20:55.531 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:55.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:55.583 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:55 smithi110 ceph-mon[136032]: pgmap v1236: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T10:20:55.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:55.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:55.685 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:55.725 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:55.797 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:55.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:55.909 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:55.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:56.008 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 68 16 95338 95322 87.569 56.375 0.014373 0.0114143 2023-10-09T10:20:56.054 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:56.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:56.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:56.159 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:56.224 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:56.292 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:56.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:56.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:56.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:20:56.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:56.562 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:20:56.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:56.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:20:56.672 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:56.735 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:56.800 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:56.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:56.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:20:56.963 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:57.015 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 69 16 96291 96275 87.1612 59.5625 0.0102269 0.0114679 2023-10-09T10:20:57.348 INFO:teuthology.orchestra.run.smithi012.stdout:moved 4 / 267 (1.49813%) 2023-10-09T10:20:57.348 INFO:teuthology.orchestra.run.smithi012.stdout:avg 33.375 2023-10-09T10:20:57.348 INFO:teuthology.orchestra.run.smithi012.stdout:stddev 10.5823 -> 9.45962 (expected baseline 5.40399) 2023-10-09T10:20:57.348 INFO:teuthology.orchestra.run.smithi012.stdout:min osd.3 with 16 -> 20 pgs (0.479401 -> 0.599251 * mean) 2023-10-09T10:20:57.348 INFO:teuthology.orchestra.run.smithi012.stdout:max osd.1 with 47 -> 46 pgs (1.40824 -> 1.37828 * mean) 2023-10-09T10:20:57.348 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:20:57.349 INFO:teuthology.orchestra.run.smithi012.stdout:oload 1000 2023-10-09T10:20:57.349 INFO:teuthology.orchestra.run.smithi012.stdout:max_change 0.05 2023-10-09T10:20:57.349 INFO:teuthology.orchestra.run.smithi012.stdout:max_change_osds 4 2023-10-09T10:20:57.349 INFO:teuthology.orchestra.run.smithi012.stdout:average_utilization 0.0313 2023-10-09T10:20:57.349 INFO:teuthology.orchestra.run.smithi012.stdout:overload_utilization 0.3130 2023-10-09T10:20:57.349 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 weight 0.4955 -> 0.5455 2023-10-09T10:20:57.349 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 weight 0.7913 -> 0.8281 2023-10-09T10:20:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:57 smithi012 ceph-mon[147926]: pgmap v1237: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T10:20:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:57 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]: dispatch 2023-10-09T10:20:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:57 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]: dispatch 2023-10-09T10:20:57.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:57 smithi012 ceph-mon[152263]: pgmap v1237: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T10:20:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:57 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]: dispatch 2023-10-09T10:20:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:57 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]: dispatch 2023-10-09T10:20:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:57 smithi110 ceph-mon[136032]: pgmap v1237: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T10:20:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:57 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]: dispatch 2023-10-09T10:20:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:57 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]: dispatch 2023-10-09T10:20:57.998 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:20:58.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:58.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:20:58.010 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 70 16 96845 96829 86.411 34.625 0.00423728 0.0114777 2023-10-09T10:20:58.097 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:20:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:20:57] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.33.4" 2023-10-09T10:20:58.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:58.137 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:58.284 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:58.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:58.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:20:58.388 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:20:58.389 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:20:58.407 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[147926]: from='client.46556 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:58.407 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[147926]: failed to encode map e442 with expected crc 2023-10-09T10:20:58.407 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]': finished 2023-10-09T10:20:58.407 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[147926]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T10:20:58.407 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[147926]: failed to encode map e442 with expected crc 2023-10-09T10:20:58.407 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[147926]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T10:20:58.408 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[152263]: from='client.46556 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:58.408 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[152263]: failed to encode map e442 with expected crc 2023-10-09T10:20:58.408 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]': finished 2023-10-09T10:20:58.408 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[152263]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T10:20:58.408 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[152263]: failed to encode map e442 with expected crc 2023-10-09T10:20:58.408 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:58 smithi012 ceph-mon[152263]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T10:20:58.416 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:58.450 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:20:58.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:58.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:20:58.554 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:20:58.580 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:58.612 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:20:58.614 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:58 smithi110 ceph-mon[136032]: from='client.46556 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:20:58.614 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:58 smithi110 ceph-mon[136032]: failed to encode map e442 with expected crc 2023-10-09T10:20:58.614 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:58 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'54271','3':'35750'}"}]': finished 2023-10-09T10:20:58.614 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:58 smithi110 ceph-mon[136032]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T10:20:58.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:58 smithi110 ceph-mon[136032]: failed to encode map e442 with expected crc 2023-10-09T10:20:58.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:58 smithi110 ceph-mon[136032]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T10:20:58.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:58.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:20:58.621 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:20:58.621 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:20:58.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:20:58.749 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:59.010 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 71 16 96845 96829 85.1944 0 - 0.0114777 2023-10-09T10:20:59.122 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:20:59.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:59.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:20:59.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:20:59.227 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:20:59.258 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:59.294 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:20:59.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:59.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:20:59.314 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:20:59.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:20:59.436 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:20:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: pgmap v1239: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1535452710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1535452710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[152263]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T10:20:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: pgmap v1239: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T10:20:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1535452710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:20:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:20:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1535452710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:59.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:20:59 smithi012 ceph-mon[147926]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: pgmap v1239: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: failed to encode map e443 with expected crc 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1535452710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:20:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1535452710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:20:59.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:20:59 smithi110 ceph-mon[136032]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T10:20:59.924 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:20:59.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:20:59.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:00.010 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 72 16 97245 97229 84.3582 12.5 0.0150737 0.0118489 2023-10-09T10:21:00.035 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:00.103 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:00.162 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:00.181 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:00.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:00.289 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:00.353 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:00.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:00 smithi012 ceph-mon[147926]: failed to encode map e444 with expected crc 2023-10-09T10:21:00.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:00 smithi012 ceph-mon[147926]: failed to encode map e444 with expected crc 2023-10-09T10:21:00.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:00 smithi012 ceph-mon[152263]: failed to encode map e444 with expected crc 2023-10-09T10:21:00.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:00 smithi012 ceph-mon[152263]: failed to encode map e444 with expected crc 2023-10-09T10:21:00.443 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:00.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:00.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:00.557 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:00.597 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:00.629 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:00.662 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:00 smithi110 ceph-mon[136032]: failed to encode map e444 with expected crc 2023-10-09T10:21:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:00 smithi110 ceph-mon[136032]: failed to encode map e444 with expected crc 2023-10-09T10:21:00.697 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:00.724 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:00.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:00.735 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:00.765 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:00.805 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:00.835 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:00.845 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:00.859 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:21:00.859 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:21:00.859 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:21:00.860 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:21:00.860 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:21:00.860 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:21:00.860 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:21:00.860 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:21:00.894 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:00.964 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:00.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:00.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:01.010 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 73 16 98082 98066 83.9193 52.3125 0.0130911 0.0119108 2023-10-09T10:21:01.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:01.140 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:01.248 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:01.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:01.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:01.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:01.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[147926]: pgmap v1242: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T10:21:01.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]: dispatch 2023-10-09T10:21:01.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T10:21:01.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]: dispatch 2023-10-09T10:21:01.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T10:21:01.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3132816076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:01.397 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[152263]: pgmap v1242: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T10:21:01.397 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]: dispatch 2023-10-09T10:21:01.397 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T10:21:01.397 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]: dispatch 2023-10-09T10:21:01.397 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T10:21:01.397 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3132816076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:01.401 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:01.527 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:01.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:01.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:01.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:01 smithi110 ceph-mon[136032]: pgmap v1242: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T10:21:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:01 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]: dispatch 2023-10-09T10:21:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:01 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T10:21:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]: dispatch 2023-10-09T10:21:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T10:21:01.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3132816076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:01.700 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:01.764 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:01.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:01.871 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:01.883 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:21:01.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:02.001 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:02.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:02.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:02.010 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 74 16 98947 98931 83.516 54.0625 0.0159954 0.0119681 2023-10-09T10:21:02.106 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:02.166 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:02.185 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:21:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:21:02.185 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:21:02.185 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:21:02.221 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:02.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:02.330 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:02.383 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:02.468 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:02.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:02.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:02.528 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]': finished 2023-10-09T10:21:02.528 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T10:21:02.528 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[152263]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T10:21:02.528 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[152263]: failed to encode map e445 with expected crc 2023-10-09T10:21:02.528 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[152263]: failed to encode map e445 with expected crc 2023-10-09T10:21:02.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]': finished 2023-10-09T10:21:02.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T10:21:02.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[147926]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T10:21:02.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[147926]: failed to encode map e445 with expected crc 2023-10-09T10:21:02.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:02 smithi012 ceph-mon[147926]: failed to encode map e445 with expected crc 2023-10-09T10:21:02.580 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:02.612 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:21:02.653 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:02.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 0]}]': finished 2023-10-09T10:21:02.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T10:21:02.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:02 smithi110 ceph-mon[136032]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T10:21:02.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:02 smithi110 ceph-mon[136032]: failed to encode map e445 with expected crc 2023-10-09T10:21:02.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:02 smithi110 ceph-mon[136032]: failed to encode map e445 with expected crc 2023-10-09T10:21:02.714 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:02.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:02.820 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:02.880 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:02.941 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:02.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:02.986 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:03.011 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 75 16 99760 99744 83.08 50.8125 0.0108539 0.0120312 2023-10-09T10:21:03.047 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:03.115 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:03.187 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:03.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:03.229 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:21:03.229 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:21:03.229 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:21:03.229 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-10-09T10:21:03.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-10-09T10:21:03.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:03.318 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:21:03.398 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:03.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 6063/291129 objects misplaced (2.083%); 59 KiB/s, 0 objects/s recovering 2023-10-09T10:21:03.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T10:21:03.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:03 smithi110 ceph-mon[136032]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 6063/291129 objects misplaced (2.083%); 59 KiB/s, 0 objects/s recovering 2023-10-09T10:21:03.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T10:21:03.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.499 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.500 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[147926]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 6063/291129 objects misplaced (2.083%); 59 KiB/s, 0 objects/s recovering 2023-10-09T10:21:03.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T10:21:03.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:03 smithi012 ceph-mon[152263]: failed to encode map e446 with expected crc 2023-10-09T10:21:03.596 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:03.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:03.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:03.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:03.765 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:03.902 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:03.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:03.936 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:04.011 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 76 16 100540 100524 82.6279 48.75 0.0104321 0.0120976 2023-10-09T10:21:04.014 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:04.084 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:04.168 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:04.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:04.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:04.274 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:04.379 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:04.463 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:04.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:04.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:04.568 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:04.618 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:04.700 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:04.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:04.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:04.862 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:05.011 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 77 16 101328 101312 82.1945 49.25 0.0197365 0.0121602 2023-10-09T10:21:05.127 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:05.132 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:05.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:05.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:05.135 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:05.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:05.295 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:05.421 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:05 smithi012 ceph-mon[147926]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 33 MiB/s wr, 523 op/s; 6063/292053 objects misplaced (2.076%); 48 KiB/s, 0 objects/s recovering 2023-10-09T10:21:05.421 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/117006743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:05.429 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:21:05.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:05 smithi012 ceph-mon[152263]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 33 MiB/s wr, 523 op/s; 6063/292053 objects misplaced (2.076%); 48 KiB/s, 0 objects/s recovering 2023-10-09T10:21:05.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/117006743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:05.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:05 smithi110 ceph-mon[136032]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 33 MiB/s wr, 523 op/s; 6063/292053 objects misplaced (2.076%); 48 KiB/s, 0 objects/s recovering 2023-10-09T10:21:05.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/117006743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:06.011 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 78 16 102182 102166 81.8251 53.375 0.0375056 0.0122161 2023-10-09T10:21:06.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:06 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:06.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:06 smithi110 ceph-mon[136032]: failed to encode map e447 with expected crc 2023-10-09T10:21:06.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/117006743' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:21:06.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:06 smithi110 ceph-mon[136032]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T10:21:06.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:06 smithi110 ceph-mon[136032]: failed to encode map e447 with expected crc 2023-10-09T10:21:06.931 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:06.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:06.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[147926]: failed to encode map e447 with expected crc 2023-10-09T10:21:06.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/117006743' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:21:06.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[147926]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T10:21:06.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[147926]: failed to encode map e447 with expected crc 2023-10-09T10:21:06.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:06.932 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[152263]: failed to encode map e447 with expected crc 2023-10-09T10:21:06.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/117006743' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:21:06.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[152263]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T10:21:06.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:06 smithi012 ceph-mon[152263]: failed to encode map e447 with expected crc 2023-10-09T10:21:06.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:06.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:07.011 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 79 16 103064 103048 81.4872 55.125 0.0131496 0.0122672 2023-10-09T10:21:07.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:07.091 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:07.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:07 smithi110 ceph-mon[136032]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 5962/302802 objects misplaced (1.969%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T10:21:07.824 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:07.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:07 smithi012 ceph-mon[152263]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 5962/302802 objects misplaced (1.969%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T10:21:07.828 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:07 smithi012 ceph-mon[147926]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 5962/302802 objects misplaced (1.969%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T10:21:07.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:07.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:07.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:08.009 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:08.012 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:21:08.012162+0000 min lat: 0.00109621 max lat: 2.16715 avg lat: 0.0123361 2023-10-09T10:21:08.012 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:21:08.012 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 80 16 103768 103752 81.0187 44 0.00623133 0.0123361 2023-10-09T10:21:08.082 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:21:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:21:07] "GET /metrics HTTP/1.1" 200 36637 "" "Prometheus/2.33.4" 2023-10-09T10:21:08.202 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:08.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:08.315 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:08.405 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:08.453 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:08.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:08.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:08.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:08.594 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:21:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:21:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:21:08.594 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:21:08.595 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:21:08.620 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:08.696 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:08.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:08.803 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:08.860 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:08.935 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:08.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:08.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:09.012 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 81 16 104540 104524 80.6142 48.25 0.00424283 0.0123991 2023-10-09T10:21:09.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:09.098 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:09.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:21:09.418 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:09.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:09.528 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:09.605 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:09.620 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:09 smithi012 ceph-mon[147926]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 5962/304326 objects misplaced (1.959%); 1.0 MiB/s, 15 objects/s recovering 2023-10-09T10:21:09.620 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:09 smithi012 ceph-mon[152263]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 5962/304326 objects misplaced (1.959%); 1.0 MiB/s, 15 objects/s recovering 2023-10-09T10:21:09.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:09.674 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:09.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:09.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:09.705 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:09.741 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:09.775 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:09.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:09.817 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:09.858 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:09.875 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:09.898 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:09.928 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:09.940 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:21:09.940 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:21:09.940 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:21:09.940 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:21:09.941 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:21:09.941 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:21:09.941 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:21:09.941 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:21:09.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:09 smithi110 ceph-mon[136032]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 5962/304326 objects misplaced (1.959%); 1.0 MiB/s, 15 objects/s recovering 2023-10-09T10:21:10.017 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 82 16 105332 105316 80.2345 49.5 0.0183158 0.0124574 2023-10-09T10:21:10.086 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:10.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:10.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:10.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:21:10.194 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:10.260 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:10.340 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:10.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:10.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:10.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:10.459 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3138463789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:10.493 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:10.540 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:10.542 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3138463789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:10.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:10.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:10.645 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:10.667 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:10.670 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:10.704 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:10.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:10.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:10.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:10.885 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3138463789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:10.898 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:10.973 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:10.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:10.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:11.012 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 83 16 106027 106011 79.7912 43.4375 0.0223735 0.0125281 2023-10-09T10:21:11.079 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:11.096 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:11.136 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:11.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:11.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:11.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:11.375 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:11.467 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:11 smithi012 ceph-mon[152263]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s; 5962/306606 objects misplaced (1.945%); 808 KiB/s, 12 objects/s recovering 2023-10-09T10:21:11.467 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/161424295' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:21:11.467 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:11 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:21:11.487 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.2 primary-affinity to 0 (802) 2023-10-09T10:21:11.488 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:11.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:11.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:11.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:11.670 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:11.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:11 smithi012 ceph-mon[147926]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s; 5962/306606 objects misplaced (1.945%); 808 KiB/s, 12 objects/s recovering 2023-10-09T10:21:11.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/161424295' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:21:11.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:11 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:21:11.801 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:11.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:11.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:11.914 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:11.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:11 smithi110 ceph-mon[136032]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s; 5962/306606 objects misplaced (1.945%); 808 KiB/s, 12 objects/s recovering 2023-10-09T10:21:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/161424295' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:21:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:11 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T10:21:11.966 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:12.013 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 84 16 106819 106803 79.4301 49.5 0.0211431 0.0125837 2023-10-09T10:21:12.028 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:12.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:12.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:12.135 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:12.222 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:12.333 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:12.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:12.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:12.435 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:21:12.435 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:21:12.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:12.518 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:12.532 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:21:12.602 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:12 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T10:21:12.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:12 smithi012 ceph-mon[147926]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T10:21:12.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:12 smithi012 ceph-mon[147926]: failed to encode map e448 with expected crc 2023-10-09T10:21:12.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:12 smithi012 ceph-mon[147926]: failed to encode map e448 with expected crc 2023-10-09T10:21:12.603 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:12 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T10:21:12.607 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:12 smithi012 ceph-mon[152263]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T10:21:12.607 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:12 smithi012 ceph-mon[152263]: failed to encode map e448 with expected crc 2023-10-09T10:21:12.607 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:12 smithi012 ceph-mon[152263]: failed to encode map e448 with expected crc 2023-10-09T10:21:12.916 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:12.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:12.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:12 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T10:21:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:12 smithi110 ceph-mon[136032]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T10:21:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:12 smithi110 ceph-mon[136032]: failed to encode map e448 with expected crc 2023-10-09T10:21:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:12 smithi110 ceph-mon[136032]: failed to encode map e448 with expected crc 2023-10-09T10:21:12.963 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:21:13.013 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 85 16 107586 107570 79.0597 47.9375 0.0198599 0.0126427 2023-10-09T10:21:13.032 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:13.122 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:13.221 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:13.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:13.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:13.333 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:13.406 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:13.465 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:13.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:13.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:13.497 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:13 smithi012 ceph-mon[147926]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 5857/314883 objects misplaced (1.860%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:13.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:13 smithi012 ceph-mon[152263]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 5857/314883 objects misplaced (1.860%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:13.566 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:13.574 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:13.629 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:13 smithi110 ceph-mon[136032]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 5857/314883 objects misplaced (1.860%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:13.761 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:13.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:13.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:13.822 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:13.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:13.912 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:13.980 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:13.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:14.013 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 86 16 108331 108315 78.6819 46.5625 0.0242411 0.0127045 2023-10-09T10:21:14.087 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:14.182 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:14.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:14.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:14.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:14.439 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:14.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:14.536 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:21:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1950604542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:14.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1950604542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:14.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:14.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1950604542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:14.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:14.739 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:14.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:14.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:14.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:14.905 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:14.948 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:14.978 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:14.991 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:14.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:15.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 87 16 109084 109068 78.3186 47.0625 0.0161645 0.0127633 2023-10-09T10:21:15.014 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:15.036 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:15.063 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:15.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:15.102 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:15.136 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:15.166 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:15.171 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:15.174 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:21:15.174 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:21:15.174 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:21:15.174 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:21:15.174 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:21:15.174 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:21:15.174 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:21:15.174 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:21:15.248 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:15.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:15.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:15.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:15.421 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:15.429 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:21:15.429 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:21:15.429 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.429 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109368 10.7 10:e500aca4:::benchmark_data_smithi012_366312_object109367:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.429 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.396280+0000", 2023-10-09T10:21:15.430 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.022709189000000001, 2023-10-09T10:21:15.430 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.022735376000000002, 2023-10-09T10:21:15.430 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.430 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.430 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.430 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.431 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.431 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109368 2023-10-09T10:21:15.431 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.431 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.431 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.431 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.432 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396280+0000", 2023-10-09T10:21:15.432 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.432 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.432 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.432 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396280+0000", 2023-10-09T10:21:15.433 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:21:15.433 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.433 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.433 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.433 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396277+0000", 2023-10-09T10:21:15.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8381999999999998e-05 2023-10-09T10:21:15.434 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.434 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.434 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.434 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396336+0000", 2023-10-09T10:21:15.434 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2901e-05 2023-10-09T10:21:15.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.435 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.435 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.435 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396359+0000", 2023-10-09T10:21:15.435 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4800000000000001e-06 2023-10-09T10:21:15.435 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.436 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.436 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.436 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396367+0000", 2023-10-09T10:21:15.436 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5201000000000002e-05 2023-10-09T10:21:15.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.437 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396422+0000", 2023-10-09T10:21:15.437 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015470499999999999 2023-10-09T10:21:15.437 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.437 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.437 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396577+0000", 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000166992 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.396744+0000", 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010805761000000001 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407550+0000", 2023-10-09T10:21:15.438 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011270017 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109370 10.3 10:c00281d0:::benchmark_data_smithi012_366312_object109369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.407570+0000", 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.011418792000000001, 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011503485000000001, 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.439 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109370 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407570+0000", 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.440 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407570+0000", 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407568+0000", 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4984000000000003e-05 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407603+0000", 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3365e-05 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.441 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407617+0000", 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2609999999999993e-06 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407625+0000", 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5936000000000001e-05 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.442 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407681+0000", 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000191869 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407873+0000", 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018253599999999999 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.408055+0000", 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0096696899999999999 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.443 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.417725+0000", 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010154761999999999 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109371 10.6 10:6197875a:::benchmark_data_smithi012_366312_object109370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.407640+0000", 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.011349512000000001, 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011483327999999999, 2023-10-09T10:21:15.444 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109371 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407640+0000", 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.445 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407640+0000", 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407639+0000", 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8067000000000001e-05 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.446 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.407667+0000", 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044292020000000001 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412096+0000", 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0737000000000001e-05 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412107+0000", 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047175 2023-10-09T10:21:15.447 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412579+0000", 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017148700000000001 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412750+0000", 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000143814 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:21:15.448 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412894+0000", 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052546040000000004 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109374 10.9 10:9869afab:::benchmark_data_smithi012_366312_object109373:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.412132+0000", 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00685719, 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.007035978, 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.449 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109374 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412132+0000", 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.450 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412132+0000", 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412130+0000", 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9255999999999998e-05 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412170+0000", 2023-10-09T10:21:15.451 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0291e-05 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412180+0000", 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.685e-06 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412185+0000", 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8523000000000003e-05 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.452 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412233+0000", 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019218699999999999 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412425+0000", 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000197529 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.412623+0000", 2023-10-09T10:21:15.453 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061088000000000002 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.418732+0000", 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065998519999999998 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109377 10.6 10:6890df5d:::benchmark_data_smithi012_366312_object109376:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.454 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.415258+0000", 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0037307249999999998, 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039563829999999999, 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109377 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.455 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.415258+0000", 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.415258+0000", 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.415257+0000", 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2305000000000002e-05 2023-10-09T10:21:15.456 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.415299+0000", 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6266999999999999e-05 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.415315+0000", 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4569999999999999e-06 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.457 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.415323+0000", 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001243594 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.416566+0000", 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012230699999999999 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.458 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.416689+0000", 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000144943 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.416834+0000", 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00157515 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.459 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109355 10.6 10:626ed558:::benchmark_data_smithi012_366312_object109354:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.377538+0000", 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.041450737000000001, 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041720643000000002, 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109355 2023-10-09T10:21:15.460 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377538+0000", 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377538+0000", 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.461 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377537+0000", 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010278 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377640+0000", 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4772e-05 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.462 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377655+0000", 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7340000000000008e-06 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377662+0000", 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4221999999999996e-05 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377727+0000", 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015799600000000001 2023-10-09T10:21:15.463 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377885+0000", 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000179777 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.378064+0000", 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076592429999999996 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.464 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.385724+0000", 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011795053 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.397519+0000", 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019980250000000001 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.465 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109356 10.6 10:64893c23:::benchmark_data_smithi012_366312_object109355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.377676+0000", 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.041313322, 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041653839999999998, 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109356 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.466 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377676+0000", 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377676+0000", 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.467 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377675+0000", 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3355e-05 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377728+0000", 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7099999999999996e-06 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377737+0000", 2023-10-09T10:21:15.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5419999999999996e-06 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.377743+0000", 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038956059999999998 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.381638+0000", 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015254 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.469 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.381791+0000", 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018590300000000001 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.381977+0000", 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077169059999999999 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389694+0000", 2023-10-09T10:21:15.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078324799999999993 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.397526+0000", 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019850359000000001 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109359 10.6 10:6bb8bcb1:::benchmark_data_smithi012_366312_object109358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.383318+0000", 2023-10-09T10:21:15.471 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.035671451999999999, 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036082298999999998, 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109359 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.383318+0000", 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.383318+0000", 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.383316+0000", 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3930999999999997e-05 2023-10-09T10:21:15.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.383360+0000", 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5807e-05 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.383376+0000", 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7599999999999997e-06 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.383382+0000", 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023608430000000001 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.385743+0000", 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012726099999999999 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.385870+0000", 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015974399999999999 2023-10-09T10:21:15.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.386030+0000", 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011458607000000001 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.397489+0000", 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2415e-05 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.397531+0000", 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014213549000000001 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:109361 10.6 10:6568040f:::benchmark_data_smithi012_366312_object109360:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:15.389567+0000", 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.029421607999999998, 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029910839000000002, 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:15.477 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 109361 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389567+0000", 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389567+0000", 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389566+0000", 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9644999999999997e-05 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389605+0000", 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1705999999999999e-05 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389617+0000", 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8089999999999998e-06 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389623+0000", 2023-10-09T10:21:15.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1248000000000005e-05 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389714+0000", 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011312800000000001 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389827+0000", 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139698 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.389967+0000", 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012235482000000001 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.402202+0000", 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0031338059999999998 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:15.405336+0000", 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015768691000000001 2023-10-09T10:21:15.482 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.483 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:15.483 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.483 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:15.483 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:21:15.483 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 9 2023-10-09T10:21:15.483 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:21:15.488 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:15.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:15.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:15.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:15.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 34 MiB/s wr, 544 op/s; 5857/316299 objects misplaced (1.852%); 813 KiB/s, 12 objects/s recovering 2023-10-09T10:21:15.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:15.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:15.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T10:21:15.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.603 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.604 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.604 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.604 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.604 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.604 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/808463376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:15.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 34 MiB/s wr, 544 op/s; 5857/316299 objects misplaced (1.852%); 813 KiB/s, 12 objects/s recovering 2023-10-09T10:21:15.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:15.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:15.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T10:21:15.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.605 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.606 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.606 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.606 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.606 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/808463376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:15.655 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:15.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 34 MiB/s wr, 544 op/s; 5857/316299 objects misplaced (1.852%); 813 KiB/s, 12 objects/s recovering 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: failed to encode map e449 with expected crc 2023-10-09T10:21:15.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/808463376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:15.702 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:15.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:15.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:15.852 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:21:15.857 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:15.919 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:15.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:16.013 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 88 16 109840 109824 77.9656 47.25 0.0157332 0.0128203 2023-10-09T10:21:16.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:16.076 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:16.125 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:16.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:16.235 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:16.315 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:16.384 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:16.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:16.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:16.494 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:16.595 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:16.684 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:16.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:16.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:16.740 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:16.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:16.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:16 smithi110 ceph-mon[136032]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 5761/326133 objects misplaced (1.766%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:16.856 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:16.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:16 smithi012 ceph-mon[152263]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 5761/326133 objects misplaced (1.766%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:16.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:16 smithi012 ceph-mon[147926]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 5761/326133 objects misplaced (1.766%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:16.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:16.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:16.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:17.013 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 89 16 110630 110614 77.6444 49.375 0.0212667 0.0128742 2023-10-09T10:21:17.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:17.121 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:17.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:21:17.322 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:17.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:17.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:17.373 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:21:17.374 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:21:17.374 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-09T10:21:17.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:21:17.426 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:17.499 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:17.606 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:21:17.651 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:17.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:17.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:17.834 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:17.898 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:17.928 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:21:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:21:17] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.33.4" 2023-10-09T10:21:17.928 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:17.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:17.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total time run: 90.0213 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total writes made: 111326 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Write size: 65536 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object size: 65536 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Bandwidth (MB/sec): 77.2915 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Bandwidth: 36.7485 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max bandwidth (MB/sec): 209 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min bandwidth (MB/sec): 0 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average IOPS: 1236 2023-10-09T10:21:18.014 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev IOPS: 587.977 2023-10-09T10:21:18.015 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max IOPS: 3344 2023-10-09T10:21:18.015 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min IOPS: 0 2023-10-09T10:21:18.015 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average Latency(s): 0.0129329 2023-10-09T10:21:18.015 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Latency(s): 0.0280057 2023-10-09T10:21:18.015 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max latency(s): 2.16715 2023-10-09T10:21:18.015 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min latency(s): 0.00109621 2023-10-09T10:21:18.035 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:21:18.041 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:18.121 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:18.213 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:18.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:18.288 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:21:18.288 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:21:18.288 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:21:18.288 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:21:18.288 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:21:18.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:18.388 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:18.488 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:18.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:18.509 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:18.600 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:18.674 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:18.741 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:21:18.760 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:18.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:18.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:18.870 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:18.962 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:18.987 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:19.041 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:19.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:19.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:19.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:19.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:19.292 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:19.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:19.402 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:19.455 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:19 smithi110 ceph-mon[136032]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 5761/327585 objects misplaced (1.759%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:19.467 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:19.537 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:19.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:19.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:19.553 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:19 smithi012 ceph-mon[147926]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 5761/327585 objects misplaced (1.759%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:19.554 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:19 smithi012 ceph-mon[152263]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 5761/327585 objects misplaced (1.759%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:19.632 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:19.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:19.695 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:19.817 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:19.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:19.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:19.922 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:19.978 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:20.038 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:20.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:20.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:20.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:20.212 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:20.339 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:20.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:20.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:20.405 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:21:20.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:20 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3282313897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:20.448 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:20.509 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:20.573 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:20.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:20.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:20.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:20 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3282313897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:20.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:20 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3282313897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:20.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:20.744 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:20.932 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:20.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:20.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:21.014 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:21:21.022 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:21:21.038 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:21.128 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:21.157 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:21:21.157 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:21:21.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156576 9.1f 9:fa882c35:::benchmark_data_smithi012_313365_object44101:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.513695+0000", 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 252.63199522799999, 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8210460340000001, 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.158 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156576 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513695+0000", 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513695+0000", 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513693+0000", 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1532000000000001e-05 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513734+0000", 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9835000000000001e-05 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513774+0000", 2023-10-09T10:21:21.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.024e-05 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.513784+0000", 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014760687999999999 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.528545+0000", 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1260000000000001e-05 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.528556+0000", 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0299076679999999 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.558464+0000", 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083750999999999999 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.566839+0000", 2023-10-09T10:21:21.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.574253793 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.141093+0000", 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024537180000000001 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.143546+0000", 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056110070000000003 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:21:21.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.149157+0000", 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019102211000000001 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.168260+0000", 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15842210500000001 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.326682+0000", 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085343699999999997 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.327535+0000", 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001518244 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.329053+0000", 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056875099999999998 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.334741+0000", 2023-10-09T10:21:21.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8210460340000001 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156587 9.1f 9:fdc5c06c:::benchmark_data_smithi012_313365_object44112:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.741326+0000", 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 252.40436392500001, 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6031445180000006, 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.166 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156587 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741326+0000", 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741326+0000", 2023-10-09T10:21:21.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741325+0000", 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7740000000000004e-06 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741332+0000", 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6189999999999999e-05 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741349+0000", 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.879e-06 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741355+0000", 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4529e-05 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741399+0000", 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6150000000000002e-06 2023-10-09T10:21:21.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.741406+0000", 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8257677169999997 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.567173+0000", 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0073666579999999999 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:21:21.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.574540+0000", 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.59405215 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.168592+0000", 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029038190000000002 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.171496+0000", 2023-10-09T10:21:21.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057499700000000001 2023-10-09T10:21:21.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:21:21.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.177246+0000", 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.024581421999999999 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.201827+0000", 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13390748299999999 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.335735+0000", 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00092294899999999995 2023-10-09T10:21:21.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.336658+0000", 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001351713 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.338010+0000", 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064611529999999999 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.344471+0000", 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6031445180000006 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156590 9.f 9:f095e657:::benchmark_data_smithi012_313365_object44115:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e378)", 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.392835+0000", 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.75285490499999, 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9880701189999996, 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.175 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156590 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392835+0000", 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392835+0000", 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392835+0000", 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8289999999999998e-06 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392840+0000", 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0960000000000001e-06 2023-10-09T10:21:21.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392843+0000", 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.277e-06 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.392845+0000", 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003385271 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.396230+0000", 2023-10-09T10:21:21.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0803e-05 2023-10-09T10:21:21.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.396241+0000", 2023-10-09T10:21:21.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2029989060000004 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.599240+0000", 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084562230000000006 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.607696+0000", 2023-10-09T10:21:21.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6506387739999999 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.258335+0000", 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027081309999999999 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.261043+0000", 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0055769770000000003 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.266620+0000", 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057250374999999999 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.323871+0000", 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.045280169000000002 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.369151+0000", 2023-10-09T10:21:21.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014548790000000001 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.370606+0000", 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001960318 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.372566+0000", 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0083394309999999996 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.380905+0000", 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9880701189999996 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156617 9.1f 9:ff63f14a:::benchmark_data_smithi012_313365_object44142:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.540544+0000", 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.605145808, 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8147314209999994, 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.185 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156617 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540544+0000", 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540544+0000", 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540542+0000", 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5820000000000004e-06 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540550+0000", 2023-10-09T10:21:21.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4589999999999999e-05 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540565+0000", 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9290000000000003e-06 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540571+0000", 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1217000000000001e-05 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540622+0000", 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0083e-05 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.540632+0000", 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.034325011 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.574957+0000", 2023-10-09T10:21:21.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075580600000000001 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.582515+0000", 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6093602599999999 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.191875+0000", 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021833450000000002 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.194059+0000", 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045656330000000004 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.198624+0000", 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.033429538000000002 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.232054+0000", 2023-10-09T10:21:21.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.113659181 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.345713+0000", 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010585200000000001 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.346771+0000", 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001606884 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.348378+0000", 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068974500000000003 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.355276+0000", 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8147314209999994 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156618 9.1f 9:fc73e5f2:::benchmark_data_smithi012_313365_object44143:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.543841+0000", 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.601848679, 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8224182819999992, 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156618 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.194 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543841+0000", 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543841+0000", 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543840+0000", 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9709999999999998e-06 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543847+0000", 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3609e-05 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543860+0000", 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.032e-06 2023-10-09T10:21:21.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543866+0000", 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4990000000000001e-05 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543911+0000", 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4530000000000002e-06 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.543920+0000", 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0390108969999998 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.582931+0000", 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0081182119999999997 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591049+0000", 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6286248130000001 2023-10-09T10:21:21.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.219674+0000", 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033205679999999999 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.222994+0000", 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057524799999999999 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.228747+0000", 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029478516 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.258225+0000", 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098204816 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.356430+0000", 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099631200000000007 2023-10-09T10:21:21.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.357426+0000", 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018869570000000001 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.359313+0000", 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0069463579999999997 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.366260+0000", 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8224182819999992 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156620 9.1f 9:fa83bd3e:::benchmark_data_smithi012_313365_object44145:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.545391+0000", 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.600298995, 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8318410220000008, 2023-10-09T10:21:21.202 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156620 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545391+0000", 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545391+0000", 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545390+0000", 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1890000000000001e-06 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545397+0000", 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4773000000000001e-05 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545412+0000", 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1689999999999997e-06 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545418+0000", 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0813000000000003e-05 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545469+0000", 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1269999999999992e-06 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.545478+0000", 2023-10-09T10:21:21.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0459541999999997 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.591432+0000", 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080512090000000001 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.599483+0000", 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6481732660000001 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.247657+0000", 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022613690000000001 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.249918+0000", 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047491729999999998 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.254667+0000", 2023-10-09T10:21:21.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.042271461000000003 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.296939+0000", 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.070329569999999994 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.367268+0000", 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010517650000000001 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.368320+0000", 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001726287 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.370046+0000", 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071859460000000003 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.377232+0000", 2023-10-09T10:21:21.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8318410220000008 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156621 9.f 9:f03746ff:::benchmark_data_smithi012_313365_object44146:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.546166+0000", 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.599524253, 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8468432579999998, 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.211 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156621 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546166+0000", 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546166+0000", 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546165+0000", 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8250000000000001e-06 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546171+0000", 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2918999999999999e-05 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546184+0000", 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0870000000000003e-06 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546190+0000", 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0841999999999998e-05 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546251+0000", 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4081e-05 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.546265+0000", 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.061941139 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.608206+0000", 2023-10-09T10:21:21.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0090726560000000001 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.617279+0000", 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6674236599999999 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.284702+0000", 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002600265 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.287302+0000", 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053856440000000002 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.292688+0000", 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.031507306999999998 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.324195+0000", 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057879049000000002 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.382074+0000", 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011540610000000001 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.383229+0000", 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001782921 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.385011+0000", 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0079976449999999994 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.393009+0000", 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8468432579999998 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156626 9.f 9:f552e3bf:::benchmark_data_smithi012_313365_object44151:head [delete] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:09.557031+0000", 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.58865886500001, 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8484499169999999, 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156626 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.220 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557031+0000", 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557031+0000", 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557030+0000", 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4510000000000003e-06 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557037+0000", 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4916000000000001e-05 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557052+0000", 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1659999999999997e-06 2023-10-09T10:21:21.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557059+0000", 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4586000000000002e-05 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557114+0000", 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7219999999999994e-06 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:09.557124+0000", 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0606237529999998 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.617747+0000", 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084044659999999993 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:16.626152+0000", 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.684881012 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.311033+0000", 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002790294 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.313823+0000", 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058745819999999997 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.319698+0000", 2023-10-09T10:21:21.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052440439999999998 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.324942+0000", 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.069203928999999997 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.394146+0000", 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001441814 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.395587+0000", 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021981000000000001 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.397786+0000", 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076955579999999999 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.405481+0000", 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8484499169999999 2023-10-09T10:21:21.227 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195608 9.b 9:d2c8e661:::benchmark_data_smithi012_313365_object83133:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e399)", 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.250976+0000", 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.89471386299999, 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2440204069999998, 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.228 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195608 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.250976+0000", 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.250976+0000", 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:21.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.250974+0000", 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.875e-06 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.250982+0000", 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1048e-05 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.250993+0000", 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5750000000000003e-06 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251001+0000", 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037774710000000001 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.254778+0000", 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0434e-05 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.254789+0000", 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0040196429999999 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.258808+0000", 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3020000000000001e-05 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.258821+0000", 2023-10-09T10:21:21.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1955982299999999 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.454420+0000", 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044134179999999997 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.458833+0000", 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.034268377000000003 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.493102+0000", 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018951250000000001 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.494997+0000", 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2440204069999998 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195611 9.1b 9:d942d15f:::benchmark_data_smithi012_313365_object83136:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e399)", 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.251013+0000", 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.89467674400001, 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1024443719999999, 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195611 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.235 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251013+0000", 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251013+0000", 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251013+0000", 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3179999999999998e-06 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251016+0000", 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0419999999999997e-06 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251023+0000", 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6639999999999997e-06 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251028+0000", 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037848740000000001 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.254813+0000", 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8800000000000008e-06 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.254821+0000", 2023-10-09T10:21:21.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.004947314 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259768+0000", 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7409999999999992e-06 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259776+0000", 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0621693189999997 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.321945+0000", 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028123430000000001 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.324757+0000", 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026793898999999999 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.351551+0000", 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001906551 2023-10-09T10:21:21.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.353458+0000", 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1024443719999999 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195614 9.1b 9:dc4d3a07:::benchmark_data_smithi012_313365_object83139:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e399)", 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.251032+0000", 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.894657954, 2023-10-09T10:21:21.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1433925169999997, 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195614 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251032+0000", 2023-10-09T10:21:21.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251032+0000", 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251032+0000", 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3470000000000001e-06 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251034+0000", 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3619999999999997e-06 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251039+0000", 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5990000000000004e-06 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251044+0000", 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003796701 2023-10-09T10:21:21.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.254841+0000", 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8730000000000001e-06 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.254848+0000", 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.004947909 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259796+0000", 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1380000000000004e-06 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259802+0000", 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0938960270000004 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.353698+0000", 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026606009999999999 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.356358+0000", 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.03503473 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.391393+0000", 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030316380000000001 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.394425+0000", 2023-10-09T10:21:21.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1433925169999997 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195619 9.1b 9:dde1902d:::benchmark_data_smithi012_313365_object83144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.251048+0000", 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.89464183000001, 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1511079439999996, 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195619 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251048+0000", 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251048+0000", 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:21.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251048+0000", 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7460000000000001e-06 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251051+0000", 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7529999999999998e-06 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251056+0000", 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6449999999999998e-06 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.251060+0000", 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038100679999999998 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.254870+0000", 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5340000000000002e-06 2023-10-09T10:21:21.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.254878+0000", 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.004942682 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259820+0000", 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9220000000000002e-06 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259826+0000", 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0967841959999998 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.356610+0000", 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029430469999999999 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.359553+0000", 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048787359999999998 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.364432+0000", 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029718715 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.394151+0000", 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010966940000000001 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.395248+0000", 2023-10-09T10:21:21.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085678399999999997 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.396104+0000", 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012298070000000001 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.397334+0000", 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048220490000000001 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.255 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.402156+0000", 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1511079439999996 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195648 9.1b 9:d89b8125:::benchmark_data_smithi012_313365_object83173:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.299774+0000", 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.845915985, 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1109046359999999, 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.256 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195648 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.299774+0000", 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.299774+0000", 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.299773+0000", 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2840000000000006e-06 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.299781+0000", 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3294e-05 2023-10-09T10:21:21.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.299794+0000", 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1684e-05 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.299806+0000", 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000219275 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.300025+0000", 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1748e-05 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.300037+0000", 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.95980764299999999 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259845+0000", 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9229999999999999e-06 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259850+0000", 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1085887080000001 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.368439+0000", 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003592002 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.372031+0000", 2023-10-09T10:21:21.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054187539999999996 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.377450+0000", 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016970676000000001 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.394421+0000", 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085412950000000008 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.262 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.402962+0000", 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00088241700000000003 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.403844+0000", 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001416863 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.405261+0000", 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054175430000000004 2023-10-09T10:21:21.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.410679+0000", 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1109046359999999 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195669 9.1b 9:de8b7910:::benchmark_data_smithi012_313365_object83194:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.353010+0000", 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.792679937, 2023-10-09T10:21:21.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0929318630000004, 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195669 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.353010+0000", 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.353010+0000", 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.353009+0000", 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4300000000000003e-06 2023-10-09T10:21:21.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.353015+0000", 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1059e-05 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.353026+0000", 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6890000000000003e-06 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.353031+0000", 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8760999999999997e-05 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.353070+0000", 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9530000000000002e-06 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.353077+0000", 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90679488699999999 2023-10-09T10:21:21.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259872+0000", 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.002e-06 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259879+0000", 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1522734320000003 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.412152+0000", 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003116761 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.415269+0000", 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048815079999999997 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.420150+0000", 2023-10-09T10:21:21.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018070145999999999 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.438220+0000", 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00087823899999999995 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.439099+0000", 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00098434799999999991 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.440083+0000", 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001078421 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.441161+0000", 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047806510000000003 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.445942+0000", 2023-10-09T10:21:21.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0929318630000004 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195671 9.1b 9:de29b7fd:::benchmark_data_smithi012_313365_object83196:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.359298+0000", 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 167.78639254999999, 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0944784749999998, 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.273 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195671 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.359298+0000", 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.359298+0000", 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.359296+0000", 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0800000000000006e-06 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.359304+0000", 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9199999999999999e-06 2023-10-09T10:21:21.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.359314+0000", 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7350000000000004e-06 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.359319+0000", 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8816000000000002e-05 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.359358+0000", 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3709999999999996e-06 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.359364+0000", 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.90053613499999996 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259900+0000", 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3869999999999997e-06 2023-10-09T10:21:21.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.259908+0000", 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1645912880000004 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.424499+0000", 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034303839999999999 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.427929+0000", 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051887089999999997 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.433118+0000", 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0067362979999999999 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.439854+0000", 2023-10-09T10:21:21.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063550819999999997 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.446209+0000", 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00075495499999999997 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.446964+0000", 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013540690000000001 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448318+0000", 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.005457534 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.453776+0000", 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0944784749999998 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(9.3 401/391 [PushOp(9:c12d9ed3:::benchmark_data_smithi012_313365_object83204:head, version: 334'5111, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:c12d9ed3:::benchmark_data_smithi012_313365_object83204:head@334'5111, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.438556+0000", 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 165.70713375099999, 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8993445580000001, 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.438556+0000", 2023-10-09T10:21:21.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.438556+0000", 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.438556+0000", 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1539000000000003e-05 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.438598+0000", 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3822000000000001e-05 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.438611+0000", 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1139999999999995e-06 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.438620+0000", 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8978298250000001 2023-10-09T10:21:21.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.336450+0000", 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0090000000000007e-06 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.336458+0000", 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014426020000000001 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.337901+0000", 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8993445580000001 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(9.13 401/392 [PushOp(9:c8f1253c:::benchmark_data_smithi012_313365_object83208:head, version: 334'5112, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:c8f1253c:::benchmark_data_smithi012_313365_object83208:head@334'5112, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.442231+0000", 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 165.70345868800001, 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9018889789999998, 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:21:21.286 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.442231+0000", 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.442231+0000", 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.442230+0000", 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7086999999999997e-05 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.442288+0000", 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8618999999999999e-05 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.442306+0000", 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1104000000000001e-05 2023-10-09T10:21:21.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.442317+0000", 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8994497209999999 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.341767+0000", 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9309999999999997e-06 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.341776+0000", 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002344512 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.344120+0000", 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9018889789999998 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195710 9.1b 9:dfad9c1a:::benchmark_data_smithi012_313365_object83235:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.500606+0000", 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 165.64508367100001, 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9908939820000002, 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195710 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.291 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.500606+0000", 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.500606+0000", 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.500605+0000", 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8467000000000001e-05 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.500623+0000", 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.36e-05 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.500637+0000", 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8800000000000002e-06 2023-10-09T10:21:21.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.500644+0000", 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9564821979999998 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.457126+0000", 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0033479690000000001 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.460474+0000", 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0045132929999999998 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.464987+0000", 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018585266 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.483572+0000", 2023-10-09T10:21:21.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000293319 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.483866+0000", 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00086814900000000003 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.484734+0000", 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001406137 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.486140+0000", 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053605509999999999 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.491500+0000", 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9908939820000002 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195735 9.1b 9:d847f26c:::benchmark_data_smithi012_313365_object83260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.575921+0000", 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 165.56976911000001, 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.92342543, 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195735 2023-10-09T10:21:21.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575921+0000", 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575921+0000", 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575920+0000", 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2040000000000003e-06 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575929+0000", 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0675e-05 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575940+0000", 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3999999999999997e-06 2023-10-09T10:21:21.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575946+0000", 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.892932262 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.468879+0000", 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003604585 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.472483+0000", 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057378819999999997 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.478221+0000", 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013998246000000001 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.492219+0000", 2023-10-09T10:21:21.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027322499999999999 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.492493+0000", 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00082177699999999999 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.493314+0000", 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001390032 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.494704+0000", 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046420810000000002 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.499346+0000", 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.92342543 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.304 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195740 9.1b 9:da330dd4:::benchmark_data_smithi012_313365_object83265:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.621398+0000", 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 165.524292481, 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9161633509999998, 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195740 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.305 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.621398+0000", 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.621398+0000", 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.621396+0000", 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1820000000000006e-06 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.621404+0000", 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0977e-05 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.621415+0000", 2023-10-09T10:21:21.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3410000000000002e-06 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.621420+0000", 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8782163970000001 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.499637+0000", 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035939079999999998 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.503231+0000", 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056746909999999999 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.508905+0000", 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020507210000000001 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.529412+0000", 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027952499999999998 2023-10-09T10:21:21.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.529692+0000", 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000876541 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.530568+0000", 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012925720000000001 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:21.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.531861+0000", 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056999099999999999 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.537561+0000", 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9161633509999998 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:21.311 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:21:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:21.313 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:21.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:21.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 36 MiB/s wr, 601 op/s; 5761/328521 objects misplaced (1.754%); 748 KiB/s, 11 objects/s recovering 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3282313897' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:21 smithi110 ceph-mon[136032]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.479 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:21.506 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 36 MiB/s wr, 601 op/s; 5761/328521 objects misplaced (1.754%); 748 KiB/s, 11 objects/s recovering 2023-10-09T10:21:21.507 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:21.507 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3282313897' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:21:21.507 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T10:21:21.507 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.507 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.507 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.507 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.507 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[152263]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 36 MiB/s wr, 601 op/s; 5761/328521 objects misplaced (1.754%); 748 KiB/s, 11 objects/s recovering 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3282313897' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.508 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:21 smithi012 ceph-mon[147926]: failed to encode map e450 with expected crc 2023-10-09T10:21:21.559 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1258,"stamp":"2023-10-09T10:21:20.353879+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7215650744,"num_objects":110118,"num_object_clones":0,"num_object_copies":330354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5660,"num_objects_unfound":0,"num_objects_dirty":110118,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":112720,"num_write_kb":7124110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":27870546,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110054,"ondisk_log_size":110054,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":26598012,"kb_used_data":21303932,"kb_used_omap":172,"kb_used_meta":5293587,"kb_avail":723363204,"statfs":{"total":767960285184,"available":740723920896,"internally_reserved":0,"allocated":21815226368,"data_stored":21805820757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":176164,"internal_metadata":5420634076},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,4,5,2,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":337903640,"num_objects":5157,"num_object_clones":0,"num_object_copies":15471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-197,"num_objects_unfound":0,"num_objects_dirty":5157,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7523,"num_write_kb":405697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":197,"num_bytes_recovered":12910592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.683269"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470277+0000","last_change":"2023-10-09T10:19:43.287175+0000","last_active":"2023-10-09T10:21:11.470277+0000","last_peered":"2023-10-09T10:21:11.470277+0000","last_clean":"2023-10-09T10:21:11.470277+0000","last_became_active":"2023-10-09T10:19:43.286833+0000","last_became_peered":"2023-10-09T10:19:43.286833+0000","last_unstale":"2023-10-09T10:21:11.470277+0000","last_undegraded":"2023-10-09T10:21:11.470277+0000","last_fullsized":"2023-10-09T10:21:11.470277+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":402,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476245+0000","last_change":"2023-10-09T10:20:58.311336+0000","last_active":"2023-10-09T10:21:11.476245+0000","last_peered":"2023-10-09T10:21:11.476245+0000","last_clean":"2023-10-09T10:21:11.476245+0000","last_became_active":"2023-10-09T10:20:58.310941+0000","last_became_peered":"2023-10-09T10:20:58.310941+0000","last_unstale":"2023-10-09T10:21:11.476245+0000","last_undegraded":"2023-10-09T10:21:11.476245+0000","last_fullsized":"2023-10-09T10:21:11.476245+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:01.468077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":489,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476071+0000","last_change":"2023-10-09T10:19:43.284958+0000","last_active":"2023-10-09T10:21:11.476071+0000","last_peered":"2023-10-09T10:21:11.476071+0000","last_clean":"2023-10-09T10:21:11.476071+0000","last_became_active":"2023-10-09T10:19:43.283799+0000","last_became_peered":"2023-10-09T10:19:43.283799+0000","last_unstale":"2023-10-09T10:21:11.476071+0000","last_undegraded":"2023-10-09T10:21:11.476071+0000","last_fullsized":"2023-10-09T10:21:11.476071+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:57.225333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":344,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477577+0000","last_change":"2023-10-09T10:16:08.309799+0000","last_active":"2023-10-09T10:21:11.477577+0000","last_peered":"2023-10-09T10:21:11.477577+0000","last_clean":"2023-10-09T10:21:11.477577+0000","last_became_active":"2023-10-09T10:16:08.309448+0000","last_became_peered":"2023-10-09T10:16:08.309448+0000","last_unstale":"2023-10-09T10:21:11.477577+0000","last_undegraded":"2023-10-09T10:21:11.477577+0000","last_fullsized":"2023-10-09T10:21:11.477577+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:58.976233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":383,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.478171+0000","last_change":"2023-10-09T10:20:58.354386+0000","last_active":"2023-10-09T10:21:11.478171+0000","last_peered":"2023-10-09T10:21:11.478171+0000","last_clean":"2023-10-09T10:21:11.478171+0000","last_became_active":"2023-10-09T10:20:58.354039+0000","last_became_peered":"2023-10-09T10:20:58.354039+0000","last_unstale":"2023-10-09T10:21:11.478171+0000","last_undegraded":"2023-10-09T10:21:11.478171+0000","last_fullsized":"2023-10-09T10:21:11.478171+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:09.055455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":243,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476772+0000","last_change":"2023-10-09T10:16:41.473093+0000","last_active":"2023-10-09T10:21:11.476772+0000","last_peered":"2023-10-09T10:21:11.476772+0000","last_clean":"2023-10-09T10:21:11.476772+0000","last_became_active":"2023-10-09T10:16:41.472774+0000","last_became_peered":"2023-10-09T10:16:41.472774+0000","last_unstale":"2023-10-09T10:21:11.476772+0000","last_undegraded":"2023-10-09T10:21:11.476772+0000","last_fullsized":"2023-10-09T10:21:11.476772+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:07.117932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":378,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477661+0000","last_change":"2023-10-09T10:16:18.273288+0000","last_active":"2023-10-09T10:21:11.477661+0000","last_peered":"2023-10-09T10:21:11.477661+0000","last_clean":"2023-10-09T10:21:11.477661+0000","last_became_active":"2023-10-09T10:16:18.272936+0000","last_became_peered":"2023-10-09T10:16:18.272936+0000","last_unstale":"2023-10-09T10:21:11.477661+0000","last_undegraded":"2023-10-09T10:21:11.477661+0000","last_fullsized":"2023-10-09T10:21:11.477661+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:39:34.847781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":423,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.478325+0000","last_change":"2023-10-09T10:20:32.431422+0000","last_active":"2023-10-09T10:21:11.478325+0000","last_peered":"2023-10-09T10:21:11.478325+0000","last_clean":"2023-10-09T10:21:11.478325+0000","last_became_active":"2023-10-09T10:20:32.431084+0000","last_became_peered":"2023-10-09T10:20:32.431084+0000","last_unstale":"2023-10-09T10:21:11.478325+0000","last_undegraded":"2023-10-09T10:21:11.478325+0000","last_fullsized":"2023-10-09T10:21:11.478325+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:56.876614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":97,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471309+0000","last_change":"2023-10-09T10:16:18.280500+0000","last_active":"2023-10-09T10:21:11.471309+0000","last_peered":"2023-10-09T10:21:11.471309+0000","last_clean":"2023-10-09T10:21:11.471309+0000","last_became_active":"2023-10-09T10:16:18.275348+0000","last_became_peered":"2023-10-09T10:16:18.275348+0000","last_unstale":"2023-10-09T10:21:11.471309+0000","last_undegraded":"2023-10-09T10:21:11.471309+0000","last_fullsized":"2023-10-09T10:21:11.471309+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":359,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.461158+0000","last_change":"2023-10-09T10:15:42.396546+0000","last_active":"2023-10-09T10:21:11.461158+0000","last_peered":"2023-10-09T10:21:11.461158+0000","last_clean":"2023-10-09T10:21:11.461158+0000","last_became_active":"2023-10-09T10:15:42.367055+0000","last_became_peered":"2023-10-09T10:15:42.367055+0000","last_unstale":"2023-10-09T10:21:11.461158+0000","last_undegraded":"2023-10-09T10:21:11.461158+0000","last_fullsized":"2023-10-09T10:21:11.461158+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:57.600046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":39,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.482949+0000","last_change":"2023-10-09T10:19:43.299477+0000","last_active":"2023-10-09T10:21:11.482949+0000","last_peered":"2023-10-09T10:21:11.482949+0000","last_clean":"2023-10-09T10:21:11.482949+0000","last_became_active":"2023-10-09T10:19:43.274448+0000","last_became_peered":"2023-10-09T10:19:43.274448+0000","last_unstale":"2023-10-09T10:21:11.482949+0000","last_undegraded":"2023-10-09T10:21:11.482949+0000","last_fullsized":"2023-10-09T10:21:11.482949+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":448,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476911+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:21:11.476911+0000","last_peered":"2023-10-09T10:21:11.476911+0000","last_clean":"2023-10-09T10:21:11.476911+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:21:11.476911+0000","last_undegraded":"2023-10-09T10:21:11.476911+0000","last_fullsized":"2023-10-09T10:21:11.476911+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":43,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471771+0000","last_change":"2023-10-09T10:19:43.482921+0000","last_active":"2023-10-09T10:21:11.471771+0000","last_peered":"2023-10-09T10:21:11.471771+0000","last_clean":"2023-10-09T10:21:11.471771+0000","last_became_active":"2023-10-09T10:19:43.278592+0000","last_became_peered":"2023-10-09T10:19:43.278592+0000","last_unstale":"2023-10-09T10:21:11.471771+0000","last_undegraded":"2023-10-09T10:21:11.471771+0000","last_fullsized":"2023-10-09T10:21:11.471771+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":42,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471064+0000","last_change":"2023-10-09T10:20:01.433412+0000","last_active":"2023-10-09T10:21:11.471064+0000","last_peered":"2023-10-09T10:21:11.471064+0000","last_clean":"2023-10-09T10:21:11.471064+0000","last_became_active":"2023-10-09T10:20:01.433114+0000","last_became_peered":"2023-10-09T10:20:01.433114+0000","last_unstale":"2023-10-09T10:21:11.471064+0000","last_undegraded":"2023-10-09T10:21:11.471064+0000","last_fullsized":"2023-10-09T10:21:11.471064+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":39,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470991+0000","last_change":"2023-10-09T10:19:43.485535+0000","last_active":"2023-10-09T10:21:11.470991+0000","last_peered":"2023-10-09T10:21:11.470991+0000","last_clean":"2023-10-09T10:21:11.470991+0000","last_became_active":"2023-10-09T10:19:43.289115+0000","last_became_peered":"2023-10-09T10:19:43.289115+0000","last_unstale":"2023-10-09T10:21:11.470991+0000","last_undegraded":"2023-10-09T10:21:11.470991+0000","last_fullsized":"2023-10-09T10:21:11.470991+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":348,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476048+0000","last_change":"2023-10-09T10:18:45.342547+0000","last_active":"2023-10-09T10:21:11.476048+0000","last_peered":"2023-10-09T10:21:11.476048+0000","last_clean":"2023-10-09T10:21:11.476048+0000","last_became_active":"2023-10-09T10:18:45.342206+0000","last_became_peered":"2023-10-09T10:18:45.342206+0000","last_unstale":"2023-10-09T10:21:11.476048+0000","last_undegraded":"2023-10-09T10:21:11.476048+0000","last_fullsized":"2023-10-09T10:21:11.476048+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:32.932519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":343,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:12.452915+0000","last_change":"2023-10-09T10:16:08.455134+0000","last_active":"2023-10-09T10:21:12.452915+0000","last_peered":"2023-10-09T10:21:12.452915+0000","last_clean":"2023-10-09T10:21:12.452915+0000","last_became_active":"2023-10-09T10:16:08.454793+0000","last_became_peered":"2023-10-09T10:16:08.454793+0000","last_unstale":"2023-10-09T10:21:12.452915+0000","last_undegraded":"2023-10-09T10:21:12.452915+0000","last_fullsized":"2023-10-09T10:21:12.452915+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:25:33.883382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471221+0000","last_change":"2023-10-09T10:16:18.255609+0000","last_active":"2023-10-09T10:21:11.471221+0000","last_peered":"2023-10-09T10:21:11.471221+0000","last_clean":"2023-10-09T10:21:11.471221+0000","last_became_active":"2023-10-09T10:16:18.244618+0000","last_became_peered":"2023-10-09T10:16:18.244618+0000","last_unstale":"2023-10-09T10:21:11.471221+0000","last_undegraded":"2023-10-09T10:21:11.471221+0000","last_fullsized":"2023-10-09T10:21:11.471221+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470253+0000","last_change":"2023-10-09T10:19:43.282758+0000","last_active":"2023-10-09T10:21:11.470253+0000","last_peered":"2023-10-09T10:21:11.470253+0000","last_clean":"2023-10-09T10:21:11.470253+0000","last_became_active":"2023-10-09T10:19:43.282258+0000","last_became_peered":"2023-10-09T10:19:43.282258+0000","last_unstale":"2023-10-09T10:21:11.470253+0000","last_undegraded":"2023-10-09T10:21:11.470253+0000","last_fullsized":"2023-10-09T10:21:11.470253+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.461004+0000","last_change":"2023-10-09T10:20:37.462345+0000","last_active":"2023-10-09T10:21:11.461004+0000","last_peered":"2023-10-09T10:21:11.461004+0000","last_clean":"2023-10-09T10:21:11.461004+0000","last_became_active":"2023-10-09T10:20:37.462057+0000","last_became_peered":"2023-10-09T10:20:37.462057+0000","last_unstale":"2023-10-09T10:21:11.461004+0000","last_undegraded":"2023-10-09T10:21:11.461004+0000","last_fullsized":"2023-10-09T10:21:11.461004+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:35.024832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":364,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470551+0000","last_change":"2023-10-09T10:18:34.286718+0000","last_active":"2023-10-09T10:21:11.470551+0000","last_peered":"2023-10-09T10:21:11.470551+0000","last_clean":"2023-10-09T10:21:11.470551+0000","last_became_active":"2023-10-09T10:18:34.271799+0000","last_became_peered":"2023-10-09T10:18:34.271799+0000","last_unstale":"2023-10-09T10:21:11.470551+0000","last_undegraded":"2023-10-09T10:21:11.470551+0000","last_fullsized":"2023-10-09T10:21:11.470551+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":396,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476703+0000","last_change":"2023-10-09T10:16:08.618851+0000","last_active":"2023-10-09T10:21:11.476703+0000","last_peered":"2023-10-09T10:21:11.476703+0000","last_clean":"2023-10-09T10:21:11.476703+0000","last_became_active":"2023-10-09T10:16:08.618501+0000","last_became_peered":"2023-10-09T10:16:08.618501+0000","last_unstale":"2023-10-09T10:21:11.476703+0000","last_undegraded":"2023-10-09T10:21:11.476703+0000","last_fullsized":"2023-10-09T10:21:11.476703+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:45.133486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"449'7079","reported_seq":7112,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.770729+0000","last_change":"2023-10-09T10:19:43.277465+0000","last_active":"2023-10-09T10:21:19.770729+0000","last_peered":"2023-10-09T10:21:19.770729+0000","last_clean":"2023-10-09T10:21:19.770729+0000","last_became_active":"2023-10-09T10:19:43.277144+0000","last_became_peered":"2023-10-09T10:19:43.277144+0000","last_unstale":"2023-10-09T10:21:19.770729+0000","last_undegraded":"2023-10-09T10:21:19.770729+0000","last_fullsized":"2023-10-09T10:21:19.770729+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7079,"log_dups_size":0,"ondisk_log_size":7079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":450822144,"num_objects":6879,"num_object_clones":0,"num_object_copies":20637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7079,"num_write_kb":446656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477365+0000","last_change":"2023-10-09T10:20:01.411305+0000","last_active":"2023-10-09T10:21:11.477365+0000","last_peered":"2023-10-09T10:21:11.477365+0000","last_clean":"2023-10-09T10:21:11.477365+0000","last_became_active":"2023-10-09T10:20:01.400821+0000","last_became_peered":"2023-10-09T10:20:01.400821+0000","last_unstale":"2023-10-09T10:21:11.477365+0000","last_undegraded":"2023-10-09T10:21:11.477365+0000","last_fullsized":"2023-10-09T10:21:11.477365+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:23.524079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":97,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470242+0000","last_change":"2023-10-09T10:16:18.237380+0000","last_active":"2023-10-09T10:21:11.470242+0000","last_peered":"2023-10-09T10:21:11.470242+0000","last_clean":"2023-10-09T10:21:11.470242+0000","last_became_active":"2023-10-09T10:16:18.236585+0000","last_became_peered":"2023-10-09T10:16:18.236585+0000","last_unstale":"2023-10-09T10:21:11.470242+0000","last_undegraded":"2023-10-09T10:21:11.470242+0000","last_fullsized":"2023-10-09T10:21:11.470242+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"449'6872","reported_seq":6905,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.733026+0000","last_change":"2023-10-09T10:19:43.284275+0000","last_active":"2023-10-09T10:21:19.733026+0000","last_peered":"2023-10-09T10:21:19.733026+0000","last_clean":"2023-10-09T10:21:19.733026+0000","last_became_active":"2023-10-09T10:19:43.283457+0000","last_became_peered":"2023-10-09T10:19:43.283457+0000","last_unstale":"2023-10-09T10:21:19.733026+0000","last_undegraded":"2023-10-09T10:21:19.733026+0000","last_fullsized":"2023-10-09T10:21:19.733026+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6872,"log_dups_size":0,"ondisk_log_size":6872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439877632,"num_objects":6712,"num_object_clones":0,"num_object_copies":20136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6872,"num_write_kb":434688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":37,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.472862+0000","last_change":"2023-10-09T10:19:43.388522+0000","last_active":"2023-10-09T10:21:11.472862+0000","last_peered":"2023-10-09T10:21:11.472862+0000","last_clean":"2023-10-09T10:21:11.472862+0000","last_became_active":"2023-10-09T10:19:43.274829+0000","last_became_peered":"2023-10-09T10:19:43.274829+0000","last_unstale":"2023-10-09T10:21:11.472862+0000","last_undegraded":"2023-10-09T10:21:11.472862+0000","last_fullsized":"2023-10-09T10:21:11.472862+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470330+0000","last_change":"2023-10-09T10:19:43.282812+0000","last_active":"2023-10-09T10:21:11.470330+0000","last_peered":"2023-10-09T10:21:11.470330+0000","last_clean":"2023-10-09T10:21:11.470330+0000","last_became_active":"2023-10-09T10:19:43.282498+0000","last_became_peered":"2023-10-09T10:19:43.282498+0000","last_unstale":"2023-10-09T10:21:11.470330+0000","last_undegraded":"2023-10-09T10:21:11.470330+0000","last_fullsized":"2023-10-09T10:21:11.470330+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"449'7031","reported_seq":7063,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.895609+0000","last_change":"2023-10-09T10:19:43.291894+0000","last_active":"2023-10-09T10:21:19.895609+0000","last_peered":"2023-10-09T10:21:19.895609+0000","last_clean":"2023-10-09T10:21:19.895609+0000","last_became_active":"2023-10-09T10:19:43.291238+0000","last_became_peered":"2023-10-09T10:19:43.291238+0000","last_unstale":"2023-10-09T10:21:19.895609+0000","last_undegraded":"2023-10-09T10:21:19.895609+0000","last_fullsized":"2023-10-09T10:21:19.895609+0000","mapping_epoch":424,"log_start":"428'300","ondisk_log_start":"428'300","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448724992,"num_objects":6847,"num_object_clones":0,"num_object_copies":20541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7031,"num_write_kb":444096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":101,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470975+0000","last_change":"2023-10-09T10:16:18.362151+0000","last_active":"2023-10-09T10:21:11.470975+0000","last_peered":"2023-10-09T10:21:11.470975+0000","last_clean":"2023-10-09T10:21:11.470975+0000","last_became_active":"2023-10-09T10:16:18.323076+0000","last_became_peered":"2023-10-09T10:16:18.323076+0000","last_unstale":"2023-10-09T10:21:11.470975+0000","last_undegraded":"2023-10-09T10:21:11.470975+0000","last_fullsized":"2023-10-09T10:21:11.470975+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":292,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.474784+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:21:11.474784+0000","last_peered":"2023-10-09T10:21:11.474784+0000","last_clean":"2023-10-09T10:21:11.474784+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:21:11.474784+0000","last_undegraded":"2023-10-09T10:21:11.474784+0000","last_fullsized":"2023-10-09T10:21:11.474784+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"449'7021","reported_seq":7055,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:17.344256+0000","last_change":"2023-10-09T10:19:43.283774+0000","last_active":"2023-10-09T10:21:17.344256+0000","last_peered":"2023-10-09T10:21:17.344256+0000","last_clean":"2023-10-09T10:21:17.344256+0000","last_became_active":"2023-10-09T10:19:43.283468+0000","last_became_peered":"2023-10-09T10:19:43.283468+0000","last_unstale":"2023-10-09T10:21:17.344256+0000","last_undegraded":"2023-10-09T10:21:17.344256+0000","last_fullsized":"2023-10-09T10:21:17.344256+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:50.698088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":460128256,"num_objects":7021,"num_object_clones":0,"num_object_copies":21063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7021,"num_write_kb":449344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470892+0000","last_change":"2023-10-09T10:20:58.316346+0000","last_active":"2023-10-09T10:21:11.470892+0000","last_peered":"2023-10-09T10:21:11.470892+0000","last_clean":"2023-10-09T10:21:11.470892+0000","last_became_active":"2023-10-09T10:20:58.316000+0000","last_became_peered":"2023-10-09T10:20:58.316000+0000","last_unstale":"2023-10-09T10:21:11.470892+0000","last_undegraded":"2023-10-09T10:21:11.470892+0000","last_fullsized":"2023-10-09T10:21:11.470892+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":338,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.478501+0000","last_change":"2023-10-09T10:19:43.291654+0000","last_active":"2023-10-09T10:21:11.478501+0000","last_peered":"2023-10-09T10:21:11.478501+0000","last_clean":"2023-10-09T10:21:11.478501+0000","last_became_active":"2023-10-09T10:19:43.291302+0000","last_became_peered":"2023-10-09T10:19:43.291302+0000","last_unstale":"2023-10-09T10:21:11.478501+0000","last_undegraded":"2023-10-09T10:21:11.478501+0000","last_fullsized":"2023-10-09T10:21:11.478501+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:41.829196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"449'7020","reported_seq":7054,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.405348+0000","last_change":"2023-10-09T10:19:43.279466+0000","last_active":"2023-10-09T10:21:19.405348+0000","last_peered":"2023-10-09T10:21:19.405348+0000","last_clean":"2023-10-09T10:21:19.405348+0000","last_became_active":"2023-10-09T10:19:43.279157+0000","last_became_peered":"2023-10-09T10:19:43.279157+0000","last_unstale":"2023-10-09T10:21:19.405348+0000","last_undegraded":"2023-10-09T10:21:19.405348+0000","last_fullsized":"2023-10-09T10:21:19.405348+0000","mapping_epoch":424,"log_start":"428'300","ondisk_log_start":"428'300","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:55.938133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":451936256,"num_objects":6896,"num_object_clones":0,"num_object_copies":20688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7020,"num_write_kb":445312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":18,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.478276+0000","last_change":"2023-10-09T10:21:02.424185+0000","last_active":"2023-10-09T10:21:11.478276+0000","last_peered":"2023-10-09T10:21:11.478276+0000","last_clean":"2023-10-09T10:21:11.478276+0000","last_became_active":"2023-10-09T10:21:02.415689+0000","last_became_peered":"2023-10-09T10:21:02.415689+0000","last_unstale":"2023-10-09T10:21:11.478276+0000","last_undegraded":"2023-10-09T10:21:11.478276+0000","last_fullsized":"2023-10-09T10:21:11.478276+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:09.473233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471680+0000","last_change":"2023-10-09T10:19:43.291696+0000","last_active":"2023-10-09T10:21:11.471680+0000","last_peered":"2023-10-09T10:21:11.471680+0000","last_clean":"2023-10-09T10:21:11.471680+0000","last_became_active":"2023-10-09T10:19:43.290863+0000","last_became_peered":"2023-10-09T10:19:43.290863+0000","last_unstale":"2023-10-09T10:21:11.471680+0000","last_undegraded":"2023-10-09T10:21:11.471680+0000","last_fullsized":"2023-10-09T10:21:11.471680+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"449'7185","reported_seq":7218,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.807685+0000","last_change":"2023-10-09T10:19:43.284340+0000","last_active":"2023-10-09T10:21:19.807685+0000","last_peered":"2023-10-09T10:21:19.807685+0000","last_clean":"2023-10-09T10:21:19.807685+0000","last_became_active":"2023-10-09T10:19:43.283218+0000","last_became_peered":"2023-10-09T10:19:43.283218+0000","last_unstale":"2023-10-09T10:21:19.807685+0000","last_undegraded":"2023-10-09T10:21:19.807685+0000","last_fullsized":"2023-10-09T10:21:19.807685+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7185,"log_dups_size":0,"ondisk_log_size":7185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458817536,"num_objects":7001,"num_object_clones":0,"num_object_copies":21003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7185,"num_write_kb":453952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470390+0000","last_change":"2023-10-09T10:19:43.276421+0000","last_active":"2023-10-09T10:21:11.470390+0000","last_peered":"2023-10-09T10:21:11.470390+0000","last_clean":"2023-10-09T10:21:11.470390+0000","last_became_active":"2023-10-09T10:19:43.276070+0000","last_became_peered":"2023-10-09T10:19:43.276070+0000","last_unstale":"2023-10-09T10:21:11.470390+0000","last_undegraded":"2023-10-09T10:21:11.470390+0000","last_fullsized":"2023-10-09T10:21:11.470390+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.482781+0000","last_change":"2023-10-09T10:19:43.286865+0000","last_active":"2023-10-09T10:21:11.482781+0000","last_peered":"2023-10-09T10:21:11.482781+0000","last_clean":"2023-10-09T10:21:11.482781+0000","last_became_active":"2023-10-09T10:19:43.286530+0000","last_became_peered":"2023-10-09T10:19:43.286530+0000","last_unstale":"2023-10-09T10:21:11.482781+0000","last_undegraded":"2023-10-09T10:21:11.482781+0000","last_fullsized":"2023-10-09T10:21:11.482781+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"449'6973","reported_seq":7007,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.392940+0000","last_change":"2023-10-09T10:19:43.281938+0000","last_active":"2023-10-09T10:21:19.392940+0000","last_peered":"2023-10-09T10:21:19.392940+0000","last_clean":"2023-10-09T10:21:19.392940+0000","last_became_active":"2023-10-09T10:19:43.281098+0000","last_became_peered":"2023-10-09T10:19:43.281098+0000","last_unstale":"2023-10-09T10:21:19.392940+0000","last_undegraded":"2023-10-09T10:21:19.392940+0000","last_fullsized":"2023-10-09T10:21:19.392940+0000","mapping_epoch":424,"log_start":"428'300","ondisk_log_start":"428'300","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:53.762641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":449249280,"num_objects":6855,"num_object_clones":0,"num_object_copies":20565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6973,"num_write_kb":442496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470410+0000","last_change":"2023-10-09T10:19:43.288603+0000","last_active":"2023-10-09T10:21:11.470410+0000","last_peered":"2023-10-09T10:21:11.470410+0000","last_clean":"2023-10-09T10:21:11.470410+0000","last_became_active":"2023-10-09T10:19:43.287994+0000","last_became_peered":"2023-10-09T10:19:43.287994+0000","last_unstale":"2023-10-09T10:21:11.470410+0000","last_undegraded":"2023-10-09T10:21:11.470410+0000","last_fullsized":"2023-10-09T10:21:11.470410+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":42,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.472853+0000","last_change":"2023-10-09T10:21:02.448313+0000","last_active":"2023-10-09T10:21:11.472853+0000","last_peered":"2023-10-09T10:21:11.472853+0000","last_clean":"2023-10-09T10:21:11.472853+0000","last_became_active":"2023-10-09T10:21:02.447928+0000","last_became_peered":"2023-10-09T10:21:02.447928+0000","last_unstale":"2023-10-09T10:21:11.472853+0000","last_undegraded":"2023-10-09T10:21:11.472853+0000","last_fullsized":"2023-10-09T10:21:11.472853+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"449'6974","reported_seq":7006,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.906646+0000","last_change":"2023-10-09T10:19:43.285977+0000","last_active":"2023-10-09T10:21:19.906646+0000","last_peered":"2023-10-09T10:21:19.906646+0000","last_clean":"2023-10-09T10:21:19.906646+0000","last_became_active":"2023-10-09T10:19:43.285666+0000","last_became_peered":"2023-10-09T10:19:43.285666+0000","last_unstale":"2023-10-09T10:21:19.906646+0000","last_undegraded":"2023-10-09T10:21:19.906646+0000","last_fullsized":"2023-10-09T10:21:19.906646+0000","mapping_epoch":424,"log_start":"428'300","ondisk_log_start":"428'300","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444727296,"num_objects":6786,"num_object_clones":0,"num_object_copies":20358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6974,"num_write_kb":440320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":416,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477957+0000","last_change":"2023-10-09T10:19:43.281410+0000","last_active":"2023-10-09T10:21:11.477957+0000","last_peered":"2023-10-09T10:21:11.477957+0000","last_clean":"2023-10-09T10:21:11.477957+0000","last_became_active":"2023-10-09T10:19:43.280304+0000","last_became_peered":"2023-10-09T10:19:43.280304+0000","last_unstale":"2023-10-09T10:21:11.477957+0000","last_undegraded":"2023-10-09T10:21:11.477957+0000","last_fullsized":"2023-10-09T10:21:11.477957+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:11.948068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":489,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476637+0000","last_change":"2023-10-09T10:19:43.284086+0000","last_active":"2023-10-09T10:21:11.476637+0000","last_peered":"2023-10-09T10:21:11.476637+0000","last_clean":"2023-10-09T10:21:11.476637+0000","last_became_active":"2023-10-09T10:19:43.283752+0000","last_became_peered":"2023-10-09T10:19:43.283752+0000","last_unstale":"2023-10-09T10:21:11.476637+0000","last_undegraded":"2023-10-09T10:21:11.476637+0000","last_fullsized":"2023-10-09T10:21:11.476637+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.817673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"449'6999","reported_seq":7031,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.931199+0000","last_change":"2023-10-09T10:19:43.291549+0000","last_active":"2023-10-09T10:21:19.931199+0000","last_peered":"2023-10-09T10:21:19.931199+0000","last_clean":"2023-10-09T10:21:19.931199+0000","last_became_active":"2023-10-09T10:19:43.290461+0000","last_became_peered":"2023-10-09T10:19:43.290461+0000","last_unstale":"2023-10-09T10:21:19.931199+0000","last_undegraded":"2023-10-09T10:21:19.931199+0000","last_fullsized":"2023-10-09T10:21:19.931199+0000","mapping_epoch":424,"log_start":"428'300","ondisk_log_start":"428'300","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":447283200,"num_objects":6825,"num_object_clones":0,"num_object_copies":20475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6999,"num_write_kb":442368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":456,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476137+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:21:11.476137+0000","last_peered":"2023-10-09T10:21:11.476137+0000","last_clean":"2023-10-09T10:21:11.476137+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:21:11.476137+0000","last_undegraded":"2023-10-09T10:21:11.476137+0000","last_fullsized":"2023-10-09T10:21:11.476137+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":97,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471140+0000","last_change":"2023-10-09T10:16:18.323515+0000","last_active":"2023-10-09T10:21:11.471140+0000","last_peered":"2023-10-09T10:21:11.471140+0000","last_clean":"2023-10-09T10:21:11.471140+0000","last_became_active":"2023-10-09T10:16:18.321171+0000","last_became_peered":"2023-10-09T10:16:18.321171+0000","last_unstale":"2023-10-09T10:21:11.471140+0000","last_undegraded":"2023-10-09T10:21:11.471140+0000","last_fullsized":"2023-10-09T10:21:11.471140+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":449,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477246+0000","last_change":"2023-10-09T10:19:43.278853+0000","last_active":"2023-10-09T10:21:11.477246+0000","last_peered":"2023-10-09T10:21:11.477246+0000","last_clean":"2023-10-09T10:21:11.477246+0000","last_became_active":"2023-10-09T10:19:43.278351+0000","last_became_peered":"2023-10-09T10:19:43.278351+0000","last_unstale":"2023-10-09T10:21:11.477246+0000","last_undegraded":"2023-10-09T10:21:11.477246+0000","last_fullsized":"2023-10-09T10:21:11.477246+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:09.207569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"449'6975","reported_seq":7008,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.794704+0000","last_change":"2023-10-09T10:19:43.278170+0000","last_active":"2023-10-09T10:21:19.794704+0000","last_peered":"2023-10-09T10:21:19.794704+0000","last_clean":"2023-10-09T10:21:19.794704+0000","last_became_active":"2023-10-09T10:19:43.277881+0000","last_became_peered":"2023-10-09T10:19:43.277881+0000","last_unstale":"2023-10-09T10:21:19.794704+0000","last_undegraded":"2023-10-09T10:21:19.794704+0000","last_fullsized":"2023-10-09T10:21:19.794704+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6975,"log_dups_size":0,"ondisk_log_size":6975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445054976,"num_objects":6791,"num_object_clones":0,"num_object_copies":20373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6975,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":97,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470735+0000","last_change":"2023-10-09T10:16:18.255781+0000","last_active":"2023-10-09T10:21:11.470735+0000","last_peered":"2023-10-09T10:21:11.470735+0000","last_clean":"2023-10-09T10:21:11.470735+0000","last_became_active":"2023-10-09T10:16:18.238531+0000","last_became_peered":"2023-10-09T10:16:18.238531+0000","last_unstale":"2023-10-09T10:21:11.470735+0000","last_undegraded":"2023-10-09T10:21:11.470735+0000","last_fullsized":"2023-10-09T10:21:11.470735+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471049+0000","last_change":"2023-10-09T10:19:43.286169+0000","last_active":"2023-10-09T10:21:11.471049+0000","last_peered":"2023-10-09T10:21:11.471049+0000","last_clean":"2023-10-09T10:21:11.471049+0000","last_became_active":"2023-10-09T10:19:43.285440+0000","last_became_peered":"2023-10-09T10:19:43.285440+0000","last_unstale":"2023-10-09T10:21:11.471049+0000","last_undegraded":"2023-10-09T10:21:11.471049+0000","last_fullsized":"2023-10-09T10:21:11.471049+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":503,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476586+0000","last_change":"2023-10-09T10:16:08.230567+0000","last_active":"2023-10-09T10:21:11.476586+0000","last_peered":"2023-10-09T10:21:11.476586+0000","last_clean":"2023-10-09T10:21:11.476586+0000","last_became_active":"2023-10-09T10:16:08.229677+0000","last_became_peered":"2023-10-09T10:16:08.229677+0000","last_unstale":"2023-10-09T10:21:11.476586+0000","last_undegraded":"2023-10-09T10:21:11.476586+0000","last_fullsized":"2023-10-09T10:21:11.476586+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:52.866334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"449'7064","reported_seq":7096,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.807720+0000","last_change":"2023-10-09T10:19:43.276368+0000","last_active":"2023-10-09T10:21:19.807720+0000","last_peered":"2023-10-09T10:21:19.807720+0000","last_clean":"2023-10-09T10:21:19.807720+0000","last_became_active":"2023-10-09T10:19:43.276039+0000","last_became_peered":"2023-10-09T10:19:43.276039+0000","last_unstale":"2023-10-09T10:21:19.807720+0000","last_undegraded":"2023-10-09T10:21:19.807720+0000","last_fullsized":"2023-10-09T10:21:19.807720+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":452460544,"num_objects":6904,"num_object_clones":0,"num_object_copies":20712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7064,"num_write_kb":446976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":360,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477570+0000","last_change":"2023-10-09T10:16:08.269361+0000","last_active":"2023-10-09T10:21:11.477570+0000","last_peered":"2023-10-09T10:21:11.477570+0000","last_clean":"2023-10-09T10:21:11.477570+0000","last_became_active":"2023-10-09T10:16:08.266453+0000","last_became_peered":"2023-10-09T10:16:08.266453+0000","last_unstale":"2023-10-09T10:21:11.477570+0000","last_undegraded":"2023-10-09T10:21:11.477570+0000","last_fullsized":"2023-10-09T10:21:11.477570+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:07.633138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":449,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.478244+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:21:11.478244+0000","last_peered":"2023-10-09T10:21:11.478244+0000","last_clean":"2023-10-09T10:21:11.478244+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:21:11.478244+0000","last_undegraded":"2023-10-09T10:21:11.478244+0000","last_fullsized":"2023-10-09T10:21:11.478244+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":29,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477568+0000","last_change":"2023-10-09T10:20:01.441278+0000","last_active":"2023-10-09T10:21:11.477568+0000","last_peered":"2023-10-09T10:21:11.477568+0000","last_clean":"2023-10-09T10:21:11.477568+0000","last_became_active":"2023-10-09T10:20:01.440907+0000","last_became_peered":"2023-10-09T10:20:01.440907+0000","last_unstale":"2023-10-09T10:21:11.477568+0000","last_undegraded":"2023-10-09T10:21:11.477568+0000","last_fullsized":"2023-10-09T10:21:11.477568+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:24.452698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"449'7157","reported_seq":7190,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.395184+0000","last_change":"2023-10-09T10:19:43.279876+0000","last_active":"2023-10-09T10:21:19.395184+0000","last_peered":"2023-10-09T10:21:19.395184+0000","last_clean":"2023-10-09T10:21:19.395184+0000","last_became_active":"2023-10-09T10:19:43.279596+0000","last_became_peered":"2023-10-09T10:19:43.279596+0000","last_unstale":"2023-10-09T10:21:19.395184+0000","last_undegraded":"2023-10-09T10:21:19.395184+0000","last_fullsized":"2023-10-09T10:21:19.395184+0000","mapping_epoch":424,"log_start":"428'400","ondisk_log_start":"428'400","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:11.020851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458686464,"num_objects":6999,"num_object_clones":0,"num_object_copies":20997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7157,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":94,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471327+0000","last_change":"2023-10-09T10:16:18.477748+0000","last_active":"2023-10-09T10:21:11.471327+0000","last_peered":"2023-10-09T10:21:11.471327+0000","last_clean":"2023-10-09T10:21:11.471327+0000","last_became_active":"2023-10-09T10:16:18.477398+0000","last_became_peered":"2023-10-09T10:16:18.477398+0000","last_unstale":"2023-10-09T10:21:11.471327+0000","last_undegraded":"2023-10-09T10:21:11.471327+0000","last_fullsized":"2023-10-09T10:21:11.471327+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":453,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:14.422159+0000","last_change":"2023-10-09T10:19:14.282209+0000","last_active":"2023-10-09T10:21:14.422159+0000","last_peered":"2023-10-09T10:21:14.422159+0000","last_clean":"2023-10-09T10:21:14.422159+0000","last_became_active":"2023-10-09T10:19:14.281786+0000","last_became_peered":"2023-10-09T10:19:14.281786+0000","last_unstale":"2023-10-09T10:21:14.422159+0000","last_undegraded":"2023-10-09T10:21:14.422159+0000","last_fullsized":"2023-10-09T10:21:14.422159+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:28.589329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":115,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470815+0000","last_change":"2023-10-09T10:19:43.283817+0000","last_active":"2023-10-09T10:21:11.470815+0000","last_peered":"2023-10-09T10:21:11.470815+0000","last_clean":"2023-10-09T10:21:11.470815+0000","last_became_active":"2023-10-09T10:19:43.283539+0000","last_became_peered":"2023-10-09T10:19:43.283539+0000","last_unstale":"2023-10-09T10:21:11.470815+0000","last_undegraded":"2023-10-09T10:21:11.470815+0000","last_fullsized":"2023-10-09T10:21:11.470815+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.482858+0000","last_change":"2023-10-09T10:19:43.281738+0000","last_active":"2023-10-09T10:21:11.482858+0000","last_peered":"2023-10-09T10:21:11.482858+0000","last_clean":"2023-10-09T10:21:11.482858+0000","last_became_active":"2023-10-09T10:19:43.280622+0000","last_became_peered":"2023-10-09T10:19:43.280622+0000","last_unstale":"2023-10-09T10:21:11.482858+0000","last_undegraded":"2023-10-09T10:21:11.482858+0000","last_fullsized":"2023-10-09T10:21:11.482858+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"449'6945","reported_seq":6979,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:17.407339+0000","last_change":"2023-10-09T10:19:43.281883+0000","last_active":"2023-10-09T10:21:17.407339+0000","last_peered":"2023-10-09T10:21:17.407339+0000","last_clean":"2023-10-09T10:21:17.407339+0000","last_became_active":"2023-10-09T10:19:43.281189+0000","last_became_peered":"2023-10-09T10:19:43.281189+0000","last_unstale":"2023-10-09T10:21:17.407339+0000","last_undegraded":"2023-10-09T10:21:17.407339+0000","last_fullsized":"2023-10-09T10:21:17.407339+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6945,"log_dups_size":0,"ondisk_log_size":6945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:56.958067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":455147520,"num_objects":6945,"num_object_clones":0,"num_object_copies":20835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6945,"num_write_kb":444480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":429,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.478402+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:21:11.478402+0000","last_peered":"2023-10-09T10:21:11.478402+0000","last_clean":"2023-10-09T10:21:11.478402+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:21:11.478402+0000","last_undegraded":"2023-10-09T10:21:11.478402+0000","last_fullsized":"2023-10-09T10:21:11.478402+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":382,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:14.521441+0000","last_change":"2023-10-09T10:19:43.292390+0000","last_active":"2023-10-09T10:21:14.521441+0000","last_peered":"2023-10-09T10:21:14.521441+0000","last_clean":"2023-10-09T10:21:14.521441+0000","last_became_active":"2023-10-09T10:19:43.290733+0000","last_became_peered":"2023-10-09T10:19:43.290733+0000","last_unstale":"2023-10-09T10:21:14.521441+0000","last_undegraded":"2023-10-09T10:21:14.521441+0000","last_fullsized":"2023-10-09T10:21:14.521441+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:53.407598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":310,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476363+0000","last_change":"2023-10-09T10:18:34.268478+0000","last_active":"2023-10-09T10:21:11.476363+0000","last_peered":"2023-10-09T10:21:11.476363+0000","last_clean":"2023-10-09T10:21:11.476363+0000","last_became_active":"2023-10-09T10:18:34.268098+0000","last_became_peered":"2023-10-09T10:18:34.268098+0000","last_unstale":"2023-10-09T10:21:11.476363+0000","last_undegraded":"2023-10-09T10:21:11.476363+0000","last_fullsized":"2023-10-09T10:21:11.476363+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.625264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"449'7107","reported_seq":7966,"reported_epoch":449,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:21:19.969586+0000","last_change":"2023-10-09T10:21:02.506752+0000","last_active":"2023-10-09T10:21:19.969586+0000","last_peered":"2023-10-09T10:21:19.969586+0000","last_clean":"2023-10-09T10:20:57.308413+0000","last_became_active":"2023-10-09T10:20:59.487335+0000","last_became_peered":"2023-10-09T10:20:59.487335+0000","last_unstale":"2023-10-09T10:21:19.969586+0000","last_undegraded":"2023-10-09T10:21:19.969586+0000","last_fullsized":"2023-10-09T10:20:59.313493+0000","mapping_epoch":443,"log_start":"428'400","ondisk_log_start":"428'400","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6707,"log_dups_size":0,"ondisk_log_size":6707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":454557720,"num_objects":6937,"num_object_clones":0,"num_object_copies":20811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5660,"num_objects_unfound":0,"num_objects_dirty":6937,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7107,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":407,"num_bytes_recovered":26673152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6","objects":1277},{"shards":"4,6,7","objects":5660}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.483033+0000","last_change":"2023-10-09T10:19:43.274868+0000","last_active":"2023-10-09T10:21:11.483033+0000","last_peered":"2023-10-09T10:21:11.483033+0000","last_clean":"2023-10-09T10:21:11.483033+0000","last_became_active":"2023-10-09T10:19:43.274441+0000","last_became_peered":"2023-10-09T10:19:43.274441+0000","last_unstale":"2023-10-09T10:21:11.483033+0000","last_undegraded":"2023-10-09T10:21:11.483033+0000","last_fullsized":"2023-10-09T10:21:11.483033+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":503,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476228+0000","last_change":"2023-10-09T10:16:18.261069+0000","last_active":"2023-10-09T10:21:11.476228+0000","last_peered":"2023-10-09T10:21:11.476228+0000","last_clean":"2023-10-09T10:21:11.476228+0000","last_became_active":"2023-10-09T10:16:18.260736+0000","last_became_peered":"2023-10-09T10:16:18.260736+0000","last_unstale":"2023-10-09T10:21:11.476228+0000","last_undegraded":"2023-10-09T10:21:11.476228+0000","last_fullsized":"2023-10-09T10:21:11.476228+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:18.217603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":341,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:14.523206+0000","last_change":"2023-10-09T10:16:18.477779+0000","last_active":"2023-10-09T10:21:14.523206+0000","last_peered":"2023-10-09T10:21:14.523206+0000","last_clean":"2023-10-09T10:21:14.523206+0000","last_became_active":"2023-10-09T10:16:18.477390+0000","last_became_peered":"2023-10-09T10:16:18.477390+0000","last_unstale":"2023-10-09T10:21:14.523206+0000","last_undegraded":"2023-10-09T10:21:14.523206+0000","last_fullsized":"2023-10-09T10:21:14.523206+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:06.576807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"449'6973","reported_seq":7005,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.817939+0000","last_change":"2023-10-09T10:19:43.287735+0000","last_active":"2023-10-09T10:21:19.817939+0000","last_peered":"2023-10-09T10:21:19.817939+0000","last_clean":"2023-10-09T10:21:19.817939+0000","last_became_active":"2023-10-09T10:19:43.287454+0000","last_became_peered":"2023-10-09T10:19:43.287454+0000","last_unstale":"2023-10-09T10:21:19.817939+0000","last_undegraded":"2023-10-09T10:21:19.817939+0000","last_fullsized":"2023-10-09T10:21:19.817939+0000","mapping_epoch":424,"log_start":"428'300","ondisk_log_start":"428'300","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445448192,"num_objects":6797,"num_object_clones":0,"num_object_copies":20391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6973,"num_write_kb":440640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":234,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477465+0000","last_change":"2023-10-09T10:18:45.347229+0000","last_active":"2023-10-09T10:21:11.477465+0000","last_peered":"2023-10-09T10:21:11.477465+0000","last_clean":"2023-10-09T10:21:11.477465+0000","last_became_active":"2023-10-09T10:18:45.346901+0000","last_became_peered":"2023-10-09T10:18:45.346901+0000","last_unstale":"2023-10-09T10:21:11.477465+0000","last_undegraded":"2023-10-09T10:21:11.477465+0000","last_fullsized":"2023-10-09T10:21:11.477465+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:53.628052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":448,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476457+0000","last_change":"2023-10-09T10:19:43.279293+0000","last_active":"2023-10-09T10:21:11.476457+0000","last_peered":"2023-10-09T10:21:11.476457+0000","last_clean":"2023-10-09T10:21:11.476457+0000","last_became_active":"2023-10-09T10:19:43.278798+0000","last_became_peered":"2023-10-09T10:19:43.278798+0000","last_unstale":"2023-10-09T10:21:11.476457+0000","last_undegraded":"2023-10-09T10:21:11.476457+0000","last_fullsized":"2023-10-09T10:21:11.476457+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:30.954175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":358,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477804+0000","last_change":"2023-10-09T10:18:34.274789+0000","last_active":"2023-10-09T10:21:11.477804+0000","last_peered":"2023-10-09T10:21:11.477804+0000","last_clean":"2023-10-09T10:21:11.477804+0000","last_became_active":"2023-10-09T10:18:34.274396+0000","last_became_peered":"2023-10-09T10:18:34.274396+0000","last_unstale":"2023-10-09T10:21:11.477804+0000","last_undegraded":"2023-10-09T10:21:11.477804+0000","last_fullsized":"2023-10-09T10:21:11.477804+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:37.788599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.f","version":"449'7085","reported_seq":7118,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:19.955739+0000","last_change":"2023-10-09T10:19:43.280795+0000","last_active":"2023-10-09T10:21:19.955739+0000","last_peered":"2023-10-09T10:21:19.955739+0000","last_clean":"2023-10-09T10:21:19.955739+0000","last_became_active":"2023-10-09T10:19:43.280338+0000","last_became_peered":"2023-10-09T10:19:43.280338+0000","last_unstale":"2023-10-09T10:21:19.955739+0000","last_undegraded":"2023-10-09T10:21:19.955739+0000","last_fullsized":"2023-10-09T10:21:19.955739+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7085,"log_dups_size":0,"ondisk_log_size":7085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":452132864,"num_objects":6899,"num_object_clones":0,"num_object_copies":20697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7085,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":357,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.474421+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:21:11.474421+0000","last_peered":"2023-10-09T10:21:11.474421+0000","last_clean":"2023-10-09T10:21:11.474421+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:21:11.474421+0000","last_undegraded":"2023-10-09T10:21:11.474421+0000","last_fullsized":"2023-10-09T10:21:11.474421+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":146,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476563+0000","last_change":"2023-10-09T10:19:43.297049+0000","last_active":"2023-10-09T10:21:11.476563+0000","last_peered":"2023-10-09T10:21:11.476563+0000","last_clean":"2023-10-09T10:21:11.476563+0000","last_became_active":"2023-10-09T10:19:43.282272+0000","last_became_peered":"2023-10-09T10:19:43.282272+0000","last_unstale":"2023-10-09T10:21:11.476563+0000","last_undegraded":"2023-10-09T10:21:11.476563+0000","last_fullsized":"2023-10-09T10:21:11.476563+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:13:04.873466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470167+0000","last_change":"2023-10-09T10:19:43.282576+0000","last_active":"2023-10-09T10:21:11.470167+0000","last_peered":"2023-10-09T10:21:11.470167+0000","last_clean":"2023-10-09T10:21:11.470167+0000","last_became_active":"2023-10-09T10:19:43.282091+0000","last_became_peered":"2023-10-09T10:19:43.282091+0000","last_unstale":"2023-10-09T10:21:11.470167+0000","last_undegraded":"2023-10-09T10:21:11.470167+0000","last_fullsized":"2023-10-09T10:21:11.470167+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470962+0000","last_change":"2023-10-09T10:19:43.274922+0000","last_active":"2023-10-09T10:21:11.470962+0000","last_peered":"2023-10-09T10:21:11.470962+0000","last_clean":"2023-10-09T10:21:11.470962+0000","last_became_active":"2023-10-09T10:19:43.273984+0000","last_became_peered":"2023-10-09T10:19:43.273984+0000","last_unstale":"2023-10-09T10:21:11.470962+0000","last_undegraded":"2023-10-09T10:21:11.470962+0000","last_fullsized":"2023-10-09T10:21:11.470962+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":451,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476325+0000","last_change":"2023-10-09T10:19:43.278560+0000","last_active":"2023-10-09T10:21:11.476325+0000","last_peered":"2023-10-09T10:21:11.476325+0000","last_clean":"2023-10-09T10:21:11.476325+0000","last_became_active":"2023-10-09T10:19:43.277994+0000","last_became_peered":"2023-10-09T10:19:43.277994+0000","last_unstale":"2023-10-09T10:21:11.476325+0000","last_undegraded":"2023-10-09T10:21:11.476325+0000","last_fullsized":"2023-10-09T10:21:11.476325+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:54:56.214071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":127,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470871+0000","last_change":"2023-10-09T10:19:43.298662+0000","last_active":"2023-10-09T10:21:11.470871+0000","last_peered":"2023-10-09T10:21:11.470871+0000","last_clean":"2023-10-09T10:21:11.470871+0000","last_became_active":"2023-10-09T10:19:43.282784+0000","last_became_peered":"2023-10-09T10:19:43.282784+0000","last_unstale":"2023-10-09T10:21:11.470871+0000","last_undegraded":"2023-10-09T10:21:11.470871+0000","last_fullsized":"2023-10-09T10:21:11.470871+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471125+0000","last_change":"2023-10-09T10:19:43.286184+0000","last_active":"2023-10-09T10:21:11.471125+0000","last_peered":"2023-10-09T10:21:11.471125+0000","last_clean":"2023-10-09T10:21:11.471125+0000","last_became_active":"2023-10-09T10:19:43.285867+0000","last_became_peered":"2023-10-09T10:19:43.285867+0000","last_unstale":"2023-10-09T10:21:11.471125+0000","last_undegraded":"2023-10-09T10:21:11.471125+0000","last_fullsized":"2023-10-09T10:21:11.471125+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":123,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.471061+0000","last_change":"2023-10-09T10:19:43.300965+0000","last_active":"2023-10-09T10:21:11.471061+0000","last_peered":"2023-10-09T10:21:11.471061+0000","last_clean":"2023-10-09T10:21:11.471061+0000","last_became_active":"2023-10-09T10:19:43.283465+0000","last_became_peered":"2023-10-09T10:19:43.283465+0000","last_unstale":"2023-10-09T10:21:11.471061+0000","last_undegraded":"2023-10-09T10:21:11.471061+0000","last_fullsized":"2023-10-09T10:21:11.471061+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":270,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T10:21:14.521191+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:21:14.521191+0000","last_peered":"2023-10-09T10:21:14.521191+0000","last_clean":"2023-10-09T10:21:14.521191+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:21:14.521191+0000","last_undegraded":"2023-10-09T10:21:14.521191+0000","last_fullsized":"2023-10-09T10:21:14.521191+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":105,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.476370+0000","last_change":"2023-10-09T10:16:08.240555+0000","last_active":"2023-10-09T10:21:11.476370+0000","last_peered":"2023-10-09T10:21:11.476370+0000","last_clean":"2023-10-09T10:21:11.476370+0000","last_became_active":"2023-10-09T10:16:08.239692+0000","last_became_peered":"2023-10-09T10:16:08.239692+0000","last_unstale":"2023-10-09T10:21:11.476370+0000","last_undegraded":"2023-10-09T10:21:11.476370+0000","last_fullsized":"2023-10-09T10:21:11.476370+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:39.501717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":490,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.477487+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:21:11.477487+0000","last_peered":"2023-10-09T10:21:11.477487+0000","last_clean":"2023-10-09T10:21:11.477487+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:21:11.477487+0000","last_undegraded":"2023-10-09T10:21:11.477487+0000","last_fullsized":"2023-10-09T10:21:11.477487+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":423,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470913+0000","last_change":"2023-10-09T10:19:43.388243+0000","last_active":"2023-10-09T10:21:11.470913+0000","last_peered":"2023-10-09T10:21:11.470913+0000","last_clean":"2023-10-09T10:21:11.470913+0000","last_became_active":"2023-10-09T10:19:43.283927+0000","last_became_peered":"2023-10-09T10:19:43.283927+0000","last_unstale":"2023-10-09T10:21:11.470913+0000","last_undegraded":"2023-10-09T10:21:11.470913+0000","last_fullsized":"2023-10-09T10:21:11.470913+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":33,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T10:21:11.470069+0000","last_change":"2023-10-09T10:19:43.282681+0000","last_active":"2023-10-09T10:21:11.470069+0000","last_peered":"2023-10-09T10:21:11.470069+0000","last_clean":"2023-10-09T10:21:11.470069+0000","last_became_active":"2023-10-09T10:19:43.282337+0000","last_became_peered":"2023-10-09T10:19:43.282337+0000","last_unstale":"2023-10-09T10:21:11.470069+0000","last_undegraded":"2023-10-09T10:21:11.470069+0000","last_fullsized":"2023-10-09T10:21:11.470069+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":7215054872,"num_objects":110094,"num_object_clones":0,"num_object_copies":330282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5660,"num_objects_unfound":0,"num_objects_dirty":110094,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112460,"num_write_kb":7121665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":407,"num_bytes_recovered":26673152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21654544384,"data_stored":21654536240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109860,"ondisk_log_size":109860,"up":48,"acting":47,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":16658,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430619,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3048612,"kb_used_data":2132060,"kb_used_omap":0,"kb_used_meta":916544,"kb_avail":90696540,"statfs":{"total":95995035648,"available":92873256960,"internally_reserved":0,"allocated":2183229440,"data_stored":2182499542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":938541056},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.1919999999999999,"15min":1.6899999999999999},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":7.5389999999999997,"5min":7.798,"15min":11.164},"last":4.4080000000000004},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.2450000000000001,"15min":1.6619999999999999},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":8.577,"5min":10.029,"15min":15.974},"last":0.61199999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":1.0169999999999999,"15min":1.861},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.22500000000000001},"max":{"1min":2.8319999999999999,"5min":9.657,"15min":16.352},"last":0.315},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.4179999999999999,"15min":2.0110000000000001},"min":{"1min":0.313,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":5.0899999999999999,"5min":8.8710000000000004,"15min":18.27},"last":0.66300000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:20:16 2023","interfaces":[{"interface":"back","average":{"1min":3.2120000000000002,"5min":1.6890000000000001,"15min":2.0329999999999999},"min":{"1min":0.311,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":42.725999999999999,"5min":42.725999999999999,"15min":42.725999999999999},"last":0.78800000000000003},{"interface":"front","average":{"1min":2.762,"5min":1.754,"15min":2.028},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":42.752000000000002,"5min":42.752000000000002,"15min":42.752000000000002},"last":1.1299999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":1.542,"15min":1.5529999999999999},"min":{"1min":0.33100000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.436,"5min":12.436,"15min":12.436},"last":0.71499999999999997},{"interface":"front","average":{"1min":2.0710000000000002,"5min":1.6120000000000001,"15min":1.6080000000000001},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":12.161,"5min":12.161,"15min":12.161},"last":0.75700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":1.1899999999999999,"15min":1.2090000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":7.5140000000000002,"5min":7.8890000000000002,"15min":12.699999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.425,"15min":1.1870000000000001},"min":{"1min":0.36799999999999999,"5min":0.313,"15min":0.25700000000000001},"max":{"1min":8.6259999999999994,"5min":8.8550000000000004,"15min":12.003},"last":0.57399999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.421,"15min":1.472},"min":{"1min":0.29499999999999998,"5min":0.23999999999999999,"15min":0.219},"max":{"1min":21.712,"5min":21.712,"15min":25.277999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.502,"15min":1.4570000000000001},"min":{"1min":0.39600000000000002,"5min":0.26800000000000002,"15min":0.22600000000000001},"max":{"1min":22.824999999999999,"5min":22.824999999999999,"15min":25.754000000000001},"last":0.69099999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.1419999999999999,"15min":1.2150000000000001},"min":{"1min":0.40000000000000002,"5min":0.28399999999999997,"15min":0.26100000000000001},"max":{"1min":3.298,"5min":8.9030000000000005,"15min":8.9030000000000005},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.0840000000000001,"15min":1.1899999999999999},"min":{"1min":0.35999999999999999,"5min":0.307,"15min":0.27200000000000002},"max":{"1min":4.8520000000000003,"5min":5.806,"15min":10.625},"last":0.52500000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626848,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3997600,"kb_used_data":3098264,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":89747552,"statfs":{"total":95995035648,"available":91901493248,"internally_reserved":0,"allocated":3172622336,"data_stored":3171898066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:07 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.639,"15min":1.7629999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.9359999999999999,"5min":44.055,"15min":44.055},"last":0.30499999999999999},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.1150000000000002,"15min":2.0249999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.22},"max":{"1min":8.7330000000000005,"5min":43.869999999999997,"15min":43.869999999999997},"last":1.393}]},{"osd":1,"last update":"Mon Oct 9 10:20:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":1.4610000000000001,"15min":2.0790000000000002},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":19.613,"5min":19.613,"15min":35.609999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.4790000000000001,"5min":1.8360000000000001,"15min":2.2090000000000001},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":19.585000000000001,"5min":19.585000000000001,"15min":32.396000000000001},"last":1.4299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:21:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":1.79,"15min":1.7350000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.9500000000000002,"5min":29.367999999999999,"15min":29.367999999999999},"last":1.508},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.2889999999999999,"15min":1.9259999999999999},"min":{"1min":0.32400000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3700000000000001,"5min":8.4480000000000004,"15min":14.542999999999999},"last":0.46400000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:20:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.2869999999999999,"15min":1.5449999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":7.4889999999999999,"5min":8.0229999999999997,"15min":8.0229999999999997},"last":0.438},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.585,"15min":1.526},"min":{"1min":0.316,"5min":0.31,"15min":0.31},"max":{"1min":7.4630000000000001,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":1.4590000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:20:55 2023","interfaces":[{"interface":"back","average":{"1min":7.4290000000000003,"5min":2.4100000000000001,"15min":1.3240000000000001},"min":{"1min":0.29199999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":122.041,"5min":122.041,"15min":122.041},"last":1.667},{"interface":"front","average":{"1min":7.843,"5min":2.3650000000000002,"15min":1.302},"min":{"1min":0.34999999999999998,"5min":0.33900000000000002,"15min":0.28699999999999998},"max":{"1min":122.13200000000001,"5min":122.13200000000001,"15min":122.13200000000001},"last":0.48999999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:20:52 2023","interfaces":[{"interface":"back","average":{"1min":7.2939999999999996,"5min":2.5379999999999998,"15min":1.4770000000000001},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.23699999999999999},"max":{"1min":121.98999999999999,"5min":121.98999999999999,"15min":121.98999999999999},"last":7.3419999999999996},{"interface":"front","average":{"1min":7.0640000000000001,"5min":2.1869999999999998,"15min":1.466},"min":{"1min":0.33400000000000002,"5min":0.32900000000000001,"15min":0.25800000000000001},"max":{"1min":122.346,"5min":122.346,"15min":122.346},"last":0.50800000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:20:21 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.052,"15min":1.052},"min":{"1min":0.33900000000000002,"5min":0.29199999999999998,"15min":0.252},"max":{"1min":6.1429999999999998,"5min":6.1429999999999998,"15min":8.8330000000000002},"last":3.5670000000000002},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.1000000000000001,"15min":1.0960000000000001},"min":{"1min":0.38400000000000001,"5min":0.35699999999999998,"15min":0.30099999999999999},"max":{"1min":6.1900000000000004,"5min":6.1900000000000004,"15min":8.8550000000000004},"last":3.5960000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253161,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2555064,"kb_used_data":1780528,"kb_used_omap":9,"kb_used_meta":774518,"kb_avail":91190088,"statfs":{"total":95995035648,"available":93378650112,"internally_reserved":0,"allocated":1823260672,"data_stored":1821686173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":793107433},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:20:38 2023","interfaces":[{"interface":"back","average":{"1min":6.7750000000000004,"5min":2.125,"15min":2.472},"min":{"1min":0.20599999999999999,"5min":0.188,"15min":0.183},"max":{"1min":108.87,"5min":108.87,"15min":108.87},"last":5.4409999999999998},{"interface":"front","average":{"1min":7.8109999999999999,"5min":2.702,"15min":2.9129999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":107.82899999999999,"5min":107.82899999999999,"15min":107.82899999999999},"last":0.52000000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:21:05 2023","interfaces":[{"interface":"back","average":{"1min":2.181,"5min":1.6240000000000001,"15min":2.5230000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":6.1319999999999997,"5min":14.223000000000001,"15min":23.105},"last":3.4750000000000001},{"interface":"front","average":{"1min":1.496,"5min":1.024,"15min":2.109},"min":{"1min":0.26400000000000001,"5min":0.26000000000000001,"15min":0.22500000000000001},"max":{"1min":5.1159999999999997,"5min":6.6470000000000002,"15min":28.024999999999999},"last":5.3970000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:20:50 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.046,"15min":1.133},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1580000000000004,"5min":6.673,"15min":6.673},"last":5.4660000000000002},{"interface":"front","average":{"1min":2.024,"5min":1.256,"15min":1.5609999999999999},"min":{"1min":0.29599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.2859999999999996,"5min":7.2160000000000002,"15min":7.2160000000000002},"last":0.54200000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.1379999999999999,"15min":1.3799999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":5.2389999999999999,"5min":6.7300000000000004,"15min":21.620000000000001},"last":10.797000000000001},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.5960000000000001,"15min":1.8520000000000001},"min":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.308},"max":{"1min":5.1109999999999998,"5min":10.497999999999999,"15min":12.904},"last":3.2959999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:20:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.5029999999999999,"15min":2.476},"min":{"1min":0.51000000000000001,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":5.0620000000000003,"5min":13.147,"15min":92.177999999999997},"last":0.623},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.524,"15min":2.5750000000000002},"min":{"1min":0.35099999999999998,"5min":0.32000000000000001,"15min":0.29099999999999998},"max":{"1min":4.8360000000000003,"5min":13.170999999999999,"15min":91.869},"last":5.9989999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.2070000000000001,"15min":1.7869999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.2060000000000004,"5min":6.8540000000000001,"15min":21.670999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.4870000000000001,"15min":2.0169999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35099999999999998},"max":{"1min":3.8380000000000001,"5min":9.1660000000000004,"15min":14.471},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 10:20:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.486,"15min":2.1099999999999999},"min":{"1min":0.38200000000000001,"5min":0.35899999999999999,"15min":0.32300000000000001},"max":{"1min":4.8959999999999999,"5min":7.8010000000000002,"15min":16.331},"last":3.5299999999999998},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.391,"15min":1.8939999999999999},"min":{"1min":0.42799999999999999,"5min":0.38500000000000001,"15min":0.371},"max":{"1min":5.2649999999999997,"5min":10.627000000000001,"15min":21.593},"last":5.4900000000000002}]}]},{"osd":0,"up_from":79,"seq":339302416685,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4232472,"kb_used_data":3567120,"kb_used_omap":28,"kb_used_meta":665315,"kb_avail":89512680,"statfs":{"total":95995035648,"available":91660984320,"internally_reserved":0,"allocated":3652730880,"data_stored":3651123741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":681283408},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.3979999999999999,"15min":2.081},"min":{"1min":0.23100000000000001,"5min":0.19800000000000001,"15min":0.16600000000000001},"max":{"1min":1.8700000000000001,"5min":54.518000000000001,"15min":54.518000000000001},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.4610000000000001,"15min":3.093},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":2.6579999999999999,"5min":56.950000000000003,"15min":56.950000000000003},"last":0.71099999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:21:15 2023","interfaces":[{"interface":"back","average":{"1min":4.8840000000000003,"5min":2.0659999999999998,"15min":2.2320000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":47.215000000000003,"5min":53.988999999999997,"15min":53.988999999999997},"last":0.51300000000000001},{"interface":"front","average":{"1min":5.2249999999999996,"5min":2.1419999999999999,"15min":2.2610000000000001},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":47.000999999999998,"5min":54.119,"15min":54.119},"last":0.63200000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:20:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.669,"15min":1.3400000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":16.879999999999999,"5min":56.984999999999999,"15min":56.984999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.385,"5min":1.6379999999999999,"15min":1.264},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.2469999999999999,"5min":56.475999999999999,"15min":56.475999999999999},"last":0.745}]},{"osd":4,"last update":"Mon Oct 9 10:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.157,"15min":2.2330000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":16.997,"5min":57.719000000000001,"15min":57.719000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.9259999999999999,"15min":2.1659999999999999},"min":{"1min":0.35199999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.3689999999999998,"5min":56.378,"15min":56.378},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.9630000000000001,"15min":3.3039999999999998},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.4770000000000003,"5min":56.445999999999998,"15min":229.589},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.198,"5min":2.0499999999999998,"15min":2.758},"min":{"1min":0.43099999999999999,"5min":0.31900000000000001,"15min":0.27000000000000002},"max":{"1min":8.7530000000000001,"5min":54.411000000000001,"15min":228.88900000000001},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2.548,"5min":2.2679999999999998,"15min":2.2170000000000001},"min":{"1min":0.47099999999999997,"5min":0.34000000000000002,"15min":0.33100000000000002},"max":{"1min":8.7810000000000006,"5min":54.482999999999997,"15min":54.482999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.6179999999999999,"5min":2.3679999999999999,"15min":2.6930000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.954999999999998,"5min":57.753,"15min":57.753},"last":2.3260000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":2.1190000000000002,"15min":2.6070000000000002},"min":{"1min":0.38600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.4020000000000001,"5min":56.411999999999999,"15min":56.411999999999999},"last":6.5419999999999998},{"interface":"front","average":{"1min":2.4180000000000001,"5min":2.2370000000000001,"15min":2.1970000000000001},"min":{"1min":0.438,"5min":0.438,"15min":0.36499999999999999},"max":{"1min":8.8010000000000002,"5min":54.451999999999998,"15min":54.451999999999998},"last":3.266}]}]},{"osd":2,"up_from":217,"seq":932007903421,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3241968,"kb_used_data":2661736,"kb_used_omap":83,"kb_used_meta":580140,"kb_avail":90503184,"statfs":{"total":95995035648,"available":92675260416,"internally_reserved":0,"allocated":2725617664,"data_stored":2724169278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":594063855},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8009999999999999,"5min":1.454,"15min":1.8959999999999999},"min":{"1min":0.219,"5min":0.19,"15min":0.17899999999999999},"max":{"1min":5.1779999999999999,"5min":33.289000000000001,"15min":33.289000000000001},"last":1.716},{"interface":"front","average":{"1min":2.2490000000000001,"5min":1.609,"15min":2.6120000000000001},"min":{"1min":0.24199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":7.5949999999999998,"5min":33.347999999999999,"15min":33.347999999999999},"last":0.56000000000000005}]},{"osd":1,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.3560000000000001,"15min":2.2210000000000001},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.6909999999999998,"5min":34.893999999999998,"15min":34.893999999999998},"last":1.929},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.4370000000000001,"15min":2.3490000000000002},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":5.492,"5min":34.962000000000003,"15min":34.962000000000003},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.577,"5min":1.319,"15min":1.0209999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.54,"5min":35.043999999999997,"15min":35.043999999999997},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.6040000000000001,"15min":1.21},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.0339999999999998,"5min":50.406999999999996,"15min":50.406999999999996},"last":0.45000000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:21:04 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.87,"15min":1.3640000000000001},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.26600000000000001},"max":{"1min":3.6619999999999999,"5min":4.9279999999999999,"15min":18.873999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.534,"5min":1.3520000000000001,"15min":1.5389999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0810000000000004,"5min":24.352,"15min":24.352},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":1.3859999999999999,"15min":1.9650000000000001},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.27700000000000002},"max":{"1min":3.7490000000000001,"5min":24.23,"15min":57.298000000000002},"last":6.3040000000000003},{"interface":"front","average":{"1min":1.175,"5min":1.226,"15min":1.8500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.315},"max":{"1min":3.3199999999999998,"5min":6.5449999999999999,"15min":57.828000000000003},"last":2.2429999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3610000000000002,"5min":1.282,"15min":1.8029999999999999},"min":{"1min":0.40100000000000002,"5min":0.36599999999999999,"15min":0.33100000000000002},"max":{"1min":7.8600000000000003,"5min":7.8600000000000003,"15min":18.943999999999999},"last":4.4909999999999997},{"interface":"front","average":{"1min":1.929,"5min":1.554,"15min":1.931},"min":{"1min":0.46100000000000002,"5min":0.41799999999999998,"15min":0.33300000000000002},"max":{"1min":9.8399999999999999,"5min":24.379000000000001,"15min":24.379000000000001},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.875,"15min":2.0600000000000001},"min":{"1min":0.435,"5min":0.41099999999999998,"15min":0.308},"max":{"1min":13.821,"5min":35.008000000000003,"15min":35.008000000000003},"last":0.628},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.4630000000000001,"15min":2.1150000000000002},"min":{"1min":0.44900000000000001,"5min":0.39300000000000002,"15min":0.35899999999999999},"max":{"1min":3.6909999999999998,"5min":24.323,"15min":24.323},"last":0.50900000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488260,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1680652,"kb_used_data":1455044,"kb_used_omap":15,"kb_used_meta":225584,"kb_avail":92064500,"statfs":{"total":95995035648,"available":94274048000,"internally_reserved":0,"allocated":1489965056,"data_stored":1488079689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":230998470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.84399999999999997,"15min":1.0309999999999999},"min":{"1min":0.224,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":3.3140000000000001,"5min":7.4050000000000002,"15min":7.4050000000000002},"last":5.0830000000000002},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.2789999999999999,"15min":1.0449999999999999},"min":{"1min":0.23899999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.3070000000000004,"5min":8.3070000000000004,"15min":8.3070000000000004},"last":1.393}]},{"osd":1,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":8.6440000000000001,"5min":2.6869999999999998,"15min":1.6359999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":105.401,"5min":105.401,"15min":105.401},"last":4.1740000000000004},{"interface":"front","average":{"1min":9.5510000000000002,"5min":2.6709999999999998,"15min":1.3440000000000001},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":104.914,"5min":104.914,"15min":104.914},"last":5.1550000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":1.359,"15min":1.911},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.0919999999999996,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":4.9489999999999998},{"interface":"front","average":{"1min":0.90200000000000002,"5min":1.107,"15min":1.1830000000000001},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1379999999999999,"5min":16.018000000000001,"15min":16.018000000000001},"last":2.0649999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2349999999999999,"5min":1.4059999999999999,"15min":0.90000000000000002},"min":{"1min":0.29799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":21.831,"5min":21.831,"15min":21.831},"last":1.4399999999999999},{"interface":"front","average":{"1min":3.3700000000000001,"5min":1.7150000000000001,"15min":1.423},"min":{"1min":0.39400000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":22.097999999999999,"5min":22.097999999999999,"15min":22.097999999999999},"last":4.992}]},{"osd":5,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.097,"15min":0.82899999999999996},"min":{"1min":0.373,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.3109999999999999,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":5.1260000000000003},{"interface":"front","average":{"1min":2.4220000000000002,"5min":1.5109999999999999,"15min":0.97399999999999998},"min":{"1min":0.433,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.2859999999999996,"5min":8.2859999999999996,"15min":8.2859999999999996},"last":1.8720000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.3700000000000001,"15min":1.3420000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.218,"5min":7.9219999999999997,"15min":7.9219999999999997},"last":0.378},{"interface":"front","average":{"1min":1.502,"5min":1.387,"15min":1.0589999999999999},"min":{"1min":0.45300000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.5789999999999997,"5min":10.698,"15min":10.698},"last":2.407}]},{"osd":7,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7690000000000001,"5min":1.7150000000000001,"15min":1.0720000000000001},"min":{"1min":0.39900000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":8.2680000000000007,"5min":10.459,"15min":10.459},"last":1.544},{"interface":"front","average":{"1min":1.635,"5min":1.6180000000000001,"15min":1.4370000000000001},"min":{"1min":0.41299999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":4.5300000000000002,"5min":8.827,"15min":8.827},"last":0.73299999999999998}]}]},{"osd":4,"up_from":266,"seq":1142461300890,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4718880,"kb_used_data":4395928,"kb_used_omap":35,"kb_used_meta":322716,"kb_avail":89026272,"statfs":{"total":95995035648,"available":91162902528,"internally_reserved":0,"allocated":4501430272,"data_stored":4500704412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":330461422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,4,1,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":9.2140000000000004,"5min":3.3290000000000002,"15min":1.8440000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":97.721999999999994,"5min":97.721999999999994,"15min":97.721999999999994},"last":2.1440000000000001},{"interface":"front","average":{"1min":10.239000000000001,"5min":3.8610000000000002,"15min":2.3599999999999999},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":97.915999999999997,"5min":97.915999999999997,"15min":97.915999999999997},"last":3.1629999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":2.2160000000000002,"15min":1.944},"min":{"1min":0.34399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.281000000000001,"5min":71.457999999999998,"15min":71.457999999999998},"last":3.2029999999999998},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.8440000000000001,"15min":1.776},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":6.7599999999999998,"5min":71.658000000000001,"15min":71.658000000000001},"last":5.2039999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.254,"15min":1.462},"min":{"1min":0.33800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.7279999999999998,"5min":6.7279999999999998,"15min":9.2469999999999999},"last":5.1820000000000004},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.2370000000000001,"15min":1.48},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":5.4950000000000001,"5min":5.4950000000000001,"15min":11.778},"last":2.1230000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.476,"15min":1.6100000000000001},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.3070000000000004,"5min":9.6170000000000009,"15min":9.6170000000000009},"last":3.226},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.246,"15min":1.125},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.6929999999999996,"5min":6.6929999999999996,"15min":6.6929999999999996},"last":5.1500000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":1.3939999999999999,"15min":1.472},"min":{"1min":0.25700000000000001,"5min":0.252,"15min":0.23799999999999999},"max":{"1min":9.8510000000000009,"5min":11.913,"15min":99.585999999999999},"last":5.298},{"interface":"front","average":{"1min":1.478,"5min":1.05,"15min":1.1990000000000001},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.632,"5min":10.632,"15min":99.384},"last":7.5339999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.151,"15min":0.94899999999999995},"min":{"1min":0.32200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.662000000000001,"5min":10.662000000000001,"15min":10.662000000000001},"last":7.6479999999999997},{"interface":"front","average":{"1min":2.165,"5min":1.284,"15min":1.0580000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27000000000000002},"max":{"1min":7.1340000000000003,"5min":7.96,"15min":12.255000000000001},"last":3.109}]},{"osd":7,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.2669999999999999,"15min":1.115},"min":{"1min":0.315,"5min":0.307,"15min":0.307},"max":{"1min":7.1779999999999999,"5min":7.9850000000000003,"15min":12.237},"last":10.358000000000001},{"interface":"front","average":{"1min":1.9830000000000001,"5min":1.1950000000000001,"15min":1.125},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":9.9220000000000006,"5min":9.9220000000000006,"15min":14.964},"last":5.1180000000000003}]}]},{"osd":5,"up_from":36,"seq":154618823075,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3122764,"kb_used_data":2213252,"kb_used_omap":0,"kb_used_meta":909504,"kb_avail":90622388,"statfs":{"total":95995035648,"available":92797325312,"internally_reserved":0,"allocated":2266370048,"data_stored":2265659856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931332096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:20:48 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":1.1120000000000001,"15min":1.536},"min":{"1min":0.24199999999999999,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":3.7050000000000001,"5min":8.0180000000000007,"15min":16.079000000000001},"last":4.4699999999999998},{"interface":"front","average":{"1min":1.359,"5min":1.133,"15min":1.5489999999999999},"min":{"1min":0.247,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":9.657,"5min":9.657,"15min":15.907},"last":3.706}]},{"osd":1,"last update":"Mon Oct 9 10:20:31 2023","interfaces":[{"interface":"back","average":{"1min":3.2000000000000002,"5min":1.5409999999999999,"15min":1.8560000000000001},"min":{"1min":0.33700000000000002,"5min":0.254,"15min":0.23699999999999999},"max":{"1min":29.370000000000001,"5min":29.370000000000001,"15min":29.370000000000001},"last":4.5049999999999999},{"interface":"front","average":{"1min":3.536,"5min":1.7150000000000001,"15min":1.853},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":29.065000000000001,"5min":29.065000000000001,"15min":29.065000000000001},"last":3.7330000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:21:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.5,"15min":1.835},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27200000000000002},"max":{"1min":5.9249999999999998,"5min":12.553000000000001,"15min":15.973000000000001},"last":4.5750000000000002},{"interface":"front","average":{"1min":3.044,"5min":1.655,"15min":1.6060000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":12.026,"5min":12.026,"15min":15.929},"last":8.2590000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":1.675,"15min":1.5409999999999999},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.029,"5min":10.029,"15min":10.029},"last":4.1509999999999998},{"interface":"front","average":{"1min":1.306,"5min":1.627,"15min":1.522},"min":{"1min":0.39400000000000002,"5min":0.315,"15min":0.315},"max":{"1min":9.7439999999999998,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":3.7770000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.3420000000000001,"15min":1.0680000000000001},"min":{"1min":0.308,"5min":0.26400000000000001,"15min":0.24299999999999999},"max":{"1min":5.5190000000000001,"5min":8.468,"15min":9.9819999999999993},"last":4.2910000000000004},{"interface":"front","average":{"1min":1.877,"5min":1.498,"15min":0.97099999999999997},"min":{"1min":0.371,"5min":0.29299999999999998,"15min":0.23599999999999999},"max":{"1min":9.7970000000000006,"5min":9.7970000000000006,"15min":9.7970000000000006},"last":8.2189999999999994}]},{"osd":6,"last update":"Mon Oct 9 10:20:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.3720000000000001,"15min":1.2589999999999999},"min":{"1min":0.36699999999999999,"5min":0.318,"15min":0.25},"max":{"1min":5.6740000000000004,"5min":12.772,"15min":13.483000000000001},"last":9.0050000000000008},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.4850000000000001,"15min":1.2829999999999999},"min":{"1min":0.38800000000000001,"5min":0.28399999999999997,"15min":0.22600000000000001},"max":{"1min":6.399,"5min":9.4160000000000004,"15min":18.600999999999999},"last":0.34899999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:20:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.2350000000000001,"15min":1.27},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.25600000000000001},"max":{"1min":4.9050000000000002,"5min":8.093,"15min":16.030999999999999},"last":4.5460000000000003},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.28,"15min":0.95599999999999996},"min":{"1min":0.32600000000000001,"5min":0.27500000000000002,"15min":0.26800000000000002},"max":{"1min":9.6950000000000003,"5min":9.6950000000000003,"15min":9.6950000000000003},"last":3.7549999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3652190208,"data_stored":3652190208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1810104320,"data_stored":1810104320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2712862720,"data_stored":2712862720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1425670144,"data_stored":1425670144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4485156864,"data_stored":4485152792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2246967296,"data_stored":2246967296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3140030464,"data_stored":3140026392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2181562368,"data_stored":2181562368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:21:21.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-10-09T10:21:21.567 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:21:21.573 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:21.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:21.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:21.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:21.775 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:21.836 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:21.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:21.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:21.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:22.031 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:22.043 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:22.067 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:22.089 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:22.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:22.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:22.096 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:22.136 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:22.163 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:22.194 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:22.201 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:22.236 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:22.255 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:22.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:22.278 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:21:22.278 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:21:22.278 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:21:22.278 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:21:22.278 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:21:22.278 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:21:22.278 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:21:22.279 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:21:22.336 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:22.338 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:22.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:22.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:22.413 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:22.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:22.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:22 smithi110 ceph-mon[136032]: from='client.36603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:21:22.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2496740444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:22.531 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:22.632 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:22.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:22.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:22.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:22 smithi012 ceph-mon[147926]: from='client.36603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:21:22.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2496740444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:22.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:22 smithi012 ceph-mon[152263]: from='client.36603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:21:22.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2496740444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:22.743 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:22.815 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:22.886 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:22.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:22.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:22.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:21:22.990 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:23.103 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:23.327 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:23.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:23.436 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:23.499 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:23.553 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:23.554 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:23 smithi110 ceph-mon[136032]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 46 MiB/s wr, 880 op/s; 5660/330354 objects misplaced (1.713%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:23.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:23.668 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:23.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:23 smithi012 ceph-mon[147926]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 46 MiB/s wr, 880 op/s; 5660/330354 objects misplaced (1.713%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:23.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:23 smithi012 ceph-mon[152263]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 46 MiB/s wr, 880 op/s; 5660/330354 objects misplaced (1.713%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:23.737 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:23.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:23.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:23.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:24.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:24.031 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:24.086 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:24.181 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:21:24.181 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:21:24.305 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:24.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:24.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:24.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:24.486 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:24.582 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:21:24.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:21:24.870 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:24.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:24.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:25.051 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:25.129 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:25.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:25.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:25.246 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:25.339 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:25.405 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:25.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:25.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:25.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:25 smithi012 ceph-mon[147926]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 111 B/s rd, 36 MiB/s wr, 745 op/s; 5660/329193 objects misplaced (1.719%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:21:25.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2074527354' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:21:25.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:21:25.420 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 10 pgp_num to 16 2023-10-09T10:21:25.454 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:25.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:25.567 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:25.645 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:25.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:25.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:25 smithi012 ceph-mon[152263]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 111 B/s rd, 36 MiB/s wr, 745 op/s; 5660/329193 objects misplaced (1.719%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:21:25.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2074527354' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:21:25.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:21:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:25 smithi110 ceph-mon[136032]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 111 B/s rd, 36 MiB/s wr, 745 op/s; 5660/329193 objects misplaced (1.719%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:21:25.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2074527354' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:21:25.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:21:25.713 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:25.754 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:25.813 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:25.902 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:25.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:25.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:26.010 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:26.085 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:26.421 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:26.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:26.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:26.441 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:21:26.530 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:26.585 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:26.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:21:26.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T10:21:26.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/252473713' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:26.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:26.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:21:26.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T10:21:26.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/252473713' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:26.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:26.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:21:26.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T10:21:26.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: failed to encode map e451 with expected crc 2023-10-09T10:21:26.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/252473713' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:26.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:26.835 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:26.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:26.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:26.940 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:26.984 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:27.105 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:27.111 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:21:27.111 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:21:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:27.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:27.213 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:27.254 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:27.313 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:27.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:27.419 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:27.495 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:27.549 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:27.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:27.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:27.590 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:21:27.659 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:27.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 893 op/s; 5564/319068 objects misplaced (1.744%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:27.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:21:27.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:21:27.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T10:21:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[152263]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 893 op/s; 5564/319068 objects misplaced (1.744%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-mon[147926]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 893 op/s; 5564/319068 objects misplaced (1.744%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:27 smithi110 ceph-mon[136032]: failed to encode map e452 with expected crc 2023-10-09T10:21:27.723 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:27.724 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:27.749 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:27.781 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:27.808 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:27.826 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:27.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:27.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:27.852 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:27.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:27.928 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:27.936 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:27.967 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:27.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:21:27.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:21:27.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:21:27.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:21:27.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:21:27.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:21:27.985 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:21:27.985 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:21:28.016 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:28.090 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:28.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:21:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:21:27] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.33.4" 2023-10-09T10:21:28.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:28.243 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:28.303 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:28.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:28.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:28.458 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:28.469 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:28.524 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:28.527 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2300230508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:28.527 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2300230508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:28.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:28.629 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:28.692 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:28.731 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2300230508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:28.775 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:28.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:28.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:28.889 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:28.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:21:28.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:29.094 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:29.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:29.207 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:29.283 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:29.371 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:29.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:29.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:29.479 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:29.552 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:29 smithi110 ceph-mon[136032]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 437 KiB/s wr, 590 op/s; 5564/317670 objects misplaced (1.752%); 838 KiB/s, 13 objects/s recovering 2023-10-09T10:21:29.552 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:29.552 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:29.562 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:29.636 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:29.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:29.720 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:29.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:29.848 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:29.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:29 smithi012 ceph-mon[147926]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 437 KiB/s wr, 590 op/s; 5564/317670 objects misplaced (1.752%); 838 KiB/s, 13 objects/s recovering 2023-10-09T10:21:29.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:29.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:29.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:29 smithi012 ceph-mon[152263]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 437 KiB/s wr, 590 op/s; 5564/317670 objects misplaced (1.752%); 838 KiB/s, 13 objects/s recovering 2023-10-09T10:21:29.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:29.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:29.974 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:29.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:30.082 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGRecoveryDeleteReply(9.1 e357,353 9:85379d0d:::benchmark_data_smithi012_313365_object2478:head,346'7028)", 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:08.372558+0000", 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 321.70507016099998, 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10804156400000001, 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.083 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372558+0000", 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372558+0000", 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:30.084 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372556+0000", 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.062e-05 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372567+0000", 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6056999999999999e-05 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.085 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372583+0000", 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0373e-05 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.372593+0000", 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10772573100000001 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.086 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.480319+0000", 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028045000000000001 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.480599+0000", 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.10804156400000001 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.087 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156568 9.0 e376/338 9:001045d7:::benchmark_data_smithi012_313365_object44093:head v 376'8121, mlcod=376'8119)", 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.403086+0000", 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 261.67454181900001, 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043494592999999998, 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403086+0000", 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.088 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403086+0000", 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403084+0000", 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9354000000000001e-05 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.089 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403103+0000", 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.673e-05 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403130+0000", 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7969999999999995e-06 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.403140+0000", 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021123579999999999 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.090 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405252+0000", 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4102000000000001e-05 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405276+0000", 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041250728 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.091 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.446527+0000", 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3520000000000003e-05 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.446581+0000", 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043494592999999998 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.092 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156571 9.11 e377/365 9:88722a35:::benchmark_data_smithi012_313365_object44096:head v 377'8328, mlcod=376'8326)", 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.405148+0000", 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 261.67248030100001, 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040682307000000001, 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405148+0000", 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.093 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405148+0000", 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405147+0000", 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9359999999999999e-06 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.094 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405155+0000", 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5512e-05 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405171+0000", 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5290000000000008e-06 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.405179+0000", 2023-10-09T10:21:30.095 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025286290000000001 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.407708+0000", 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4187000000000001e-05 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.407732+0000", 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038036767999999999 2023-10-09T10:21:30.096 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.445769+0000", 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1063999999999997e-05 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.445830+0000", 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040682307000000001 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.097 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156578 9.11 e377/365 9:8e383e1c:::benchmark_data_smithi012_313365_object44103:head v 377'8329, mlcod=376'8326)", 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 261.55206170899999, 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178583306, 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.098 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.099 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525564+0000", 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7572e-05 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525581+0000", 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3825999999999999e-05 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525605+0000", 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4969e-05 2023-10-09T10:21:30.100 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525620+0000", 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13948444700000001 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.665105+0000", 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8316000000000001e-05 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.101 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.665133+0000", 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038969654999999999 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.704103+0000", 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6774000000000003e-05 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.102 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.704149+0000", 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178583306 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:20240 10.f 10:fc5c1488:::benchmark_data_smithi012_366312_object20239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:55.292633+0000", 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 94.784995156999997, 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051599318999999998, 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.103 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 20240 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.292633+0000", 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.104 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.292633+0000", 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.292631+0000", 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7596000000000002e-05 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.105 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.292688+0000", 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2676e-05 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.292701+0000", 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1759999999999998e-06 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.292707+0000", 2023-10-09T10:21:30.106 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00073842299999999999 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.293446+0000", 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000133048 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.293579+0000", 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000145246 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.107 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.293724+0000", 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015061860000000001 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.295230+0000", 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013009600000000001 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.108 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.295360+0000", 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.048771676999999999 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.344132+0000", 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9032e-05 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.344191+0000", 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0945999999999997e-05 2023-10-09T10:21:30.109 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:55.344232+0000", 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051599318999999998 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:26119 10.d e428/424 10:b0671fbd:::benchmark_data_smithi012_366312_object26118:head v 428'1626, mlcod=428'1624)", 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.185505+0000", 2023-10-09T10:21:30.110 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.892122596999997, 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047286468999999998, 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185505+0000", 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.111 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185505+0000", 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185504+0000", 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9275e-05 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185534+0000", 2023-10-09T10:21:30.112 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0369e-05 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185544+0000", 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8889999999999999e-06 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185550+0000", 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9332000000000001e-05 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.113 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185599+0000", 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1362999999999999e-05 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185621+0000", 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047136300999999999 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.114 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232757+0000", 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4730000000000001e-05 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232792+0000", 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047286468999999998 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.115 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:26116 10.b e428/424 10:d751c43a:::benchmark_data_smithi012_366312_object26115:head v 428'1631, mlcod=428'1628)", 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.185592+0000", 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.892035563999997, 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047189480999999998, 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185592+0000", 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185592+0000", 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185592+0000", 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2787999999999998e-05 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185624+0000", 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2060000000000001e-06 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185632+0000", 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.425e-06 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185636+0000", 2023-10-09T10:21:30.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1220999999999997e-05 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185677+0000", 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8595999999999999e-05 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.185696+0000", 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047051016000000001 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232747+0000", 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4768000000000003e-05 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232782+0000", 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047189480999999998 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:26124 10.0 e428/424 10:0095af71:::benchmark_data_smithi012_366312_object26123:head v 428'1631, mlcod=428'1629)", 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.187784+0000", 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.889844246999999, 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047898141999999998, 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.187784+0000", 2023-10-09T10:21:30.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.187784+0000", 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.187783+0000", 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5605000000000002e-05 2023-10-09T10:21:30.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.187818+0000", 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044905992999999998 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232724+0000", 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1263e-05 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.123 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232735+0000", 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4265999999999999e-05 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232780+0000", 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5941000000000001e-05 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232806+0000", 2023-10-09T10:21:30.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0028448369999999998 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.235650+0000", 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1225e-05 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.235682+0000", 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047898141999999998 2023-10-09T10:21:30.125 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:26126 10.7 e428/424 10:eac49d4a:::benchmark_data_smithi012_366312_object26125:head v 428'1616, mlcod=428'1613)", 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.188990+0000", 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.888638075000003, 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047351621000000003, 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.126 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.188990+0000", 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.188990+0000", 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.188989+0000", 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.932e-05 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.189018+0000", 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.043727859000000001 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232746+0000", 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0772e-05 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232757+0000", 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7848e-05 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232804+0000", 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3759e-05 2023-10-09T10:21:30.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.232818+0000", 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003497792 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.236316+0000", 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5340000000000001e-05 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.236341+0000", 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.047351621000000003 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:27078 10.f 10:f852e76d:::benchmark_data_smithi012_366312_object27077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.686967+0000", 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.390660804999996, 2023-10-09T10:21:30.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16440834700000001, 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27078 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.686967+0000", 2023-10-09T10:21:30.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.686967+0000", 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.686965+0000", 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4931999999999997e-05 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.687000+0000", 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3113e-05 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.687013+0000", 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8889999999999999e-06 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.687019+0000", 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001864283 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.688884+0000", 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000125807 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.689009+0000", 2023-10-09T10:21:30.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8033000000000001e-05 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.689107+0000", 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002756816 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.691864+0000", 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000158092 2023-10-09T10:21:30.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.692022+0000", 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15928362700000001 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851306+0000", 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9937000000000001e-05 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851346+0000", 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9476e-05 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851375+0000", 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16440834700000001 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:27099 10.f 10:f4e4b7e3:::benchmark_data_smithi012_366312_object27098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.698261+0000", 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.379366348999994, 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15328217499999999, 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27099 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.139 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698261+0000", 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698261+0000", 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698260+0000", 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8139e-05 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698318+0000", 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2685e-05 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698331+0000", 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0840000000000003e-06 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698338+0000", 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4275000000000004e-05 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698402+0000", 2023-10-09T10:21:30.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146921 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698549+0000", 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013566899999999999 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698685+0000", 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011869719999999999 2023-10-09T10:21:30.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.699872+0000", 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015024600000000001 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.700022+0000", 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.151410247 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851432+0000", 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5823999999999999e-05 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851488+0000", 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5804999999999998e-05 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851544+0000", 2023-10-09T10:21:30.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15328217499999999 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:27106 10.f 10:f3b8aa96:::benchmark_data_smithi012_366312_object27105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.703416+0000", 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.374212120999999, 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.148246932, 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.146 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27106 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703416+0000", 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703416+0000", 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703414+0000", 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101159 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703515+0000", 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.649e-05 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703531+0000", 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6210000000000004e-06 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703539+0000", 2023-10-09T10:21:30.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1502999999999998e-05 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703590+0000", 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000145067 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703736+0000", 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014309699999999999 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703879+0000", 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021440740000000002 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.706023+0000", 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029154099999999998 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.706314+0000", 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14527401100000001 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851588+0000", 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2505999999999997e-05 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851621+0000", 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1841999999999997e-05 2023-10-09T10:21:30.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851663+0000", 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.148246932 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:27110 10.f 10:f11365c8:::benchmark_data_smithi012_366312_object27109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.703981+0000", 2023-10-09T10:21:30.153 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 91.373646746000006, 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14781381499999999, 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27110 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703981+0000", 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703981+0000", 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703980+0000", 2023-10-09T10:21:30.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016381699999999999 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.704144+0000", 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9469999999999992e-06 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.704153+0000", 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3970000000000002e-06 2023-10-09T10:21:30.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.704158+0000", 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00079055900000000003 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.704949+0000", 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010478 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.157 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.705053+0000", 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013406799999999999 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.705187+0000", 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024943090000000001 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.707682+0000", 2023-10-09T10:21:30.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085955199999999997 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.708541+0000", 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14316135599999999 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851703+0000", 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9663999999999997e-05 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851752+0000", 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2533000000000001e-05 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851795+0000", 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14781381499999999 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.160 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:51601 10.0 e433/424 10:01339ec6:::benchmark_data_smithi012_366312_object51600:head v 433'3194, mlcod=433'3192)", 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:20:14.918676+0000", 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 75.158952283000005, 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041739199999999997, 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918676+0000", 2023-10-09T10:21:30.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918676+0000", 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918674+0000", 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0760000000000003e-05 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918714+0000", 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5858e-05 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918730+0000", 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8939999999999993e-06 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918738+0000", 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2815999999999997e-05 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918791+0000", 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1370999999999999e-05 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918812+0000", 2023-10-09T10:21:30.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041493724000000003 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.960306+0000", 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010871599999999999 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.960415+0000", 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041739199999999997 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.165 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:51595 10.e e433/424 10:77d8313d:::benchmark_data_smithi012_366312_object51594:head v 433'3185, mlcod=433'3183)", 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:20:14.918772+0000", 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 75.158855329999994, 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040612132000000002, 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918772+0000", 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918772+0000", 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.167 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918772+0000", 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3030000000000001e-05 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918795+0000", 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5610000000000001e-06 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918802+0000", 2023-10-09T10:21:30.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2699999999999998e-06 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918807+0000", 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3392000000000001e-05 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.918850+0000", 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036657230999999998 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.955507+0000", 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003842411 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.959350+0000", 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4981000000000003e-05 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:14.959385+0000", 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.040612132000000002 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:52056 10.f 10:fe1370cb:::benchmark_data_smithi012_366312_object52055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e433)", 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:20:15.367475+0000", 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 74.710152358000002, 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04715014, 2023-10-09T10:21:30.171 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 52056 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367475+0000", 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367475+0000", 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367474+0000", 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6279000000000002e-05 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367520+0000", 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1078e-05 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367531+0000", 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3579999999999999e-06 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367538+0000", 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6121000000000001e-05 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367584+0000", 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012741099999999999 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367711+0000", 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000154136 2023-10-09T10:21:30.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.367865+0000", 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001980271 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.369845+0000", 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030600459999999999 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.176 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.372905+0000", 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041656994000000003 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.414562+0000", 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3878000000000003e-05 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.414596+0000", 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9224999999999999e-05 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.414626+0000", 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.04715014 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.178 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:52070 10.f 10:f1759f14:::benchmark_data_smithi012_366312_object52069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e433)", 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:20:15.372901+0000", 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 74.704726937000004, 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041834462000000003, 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 52070 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.372901+0000", 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.372901+0000", 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.372899+0000", 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0991999999999997e-05 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.372930+0000", 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2448e-05 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.372943+0000", 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4849999999999997e-06 2023-10-09T10:21:30.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.372949+0000", 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062330600000000001 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.373572+0000", 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013574600000000001 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.373708+0000", 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000148126 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.373856+0000", 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030570810000000001 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.376913+0000", 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012058590000000001 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.378119+0000", 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.036551126000000003 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.414670+0000", 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6956000000000003e-05 2023-10-09T10:21:30.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.414707+0000", 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8034e-05 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:15.414735+0000", 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.041834462000000003 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.185 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:53370 10.f 10:f9fec699:::benchmark_data_smithi012_366312_object53369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e433)", 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:20:16.349214+0000", 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 73.728414178999998, 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098253160000000006, 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.186 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 53370 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349214+0000", 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349214+0000", 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349212+0000", 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9627000000000003e-05 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349281+0000", 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7509000000000001e-05 2023-10-09T10:21:30.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349299+0000", 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9070000000000001e-06 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349306+0000", 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0889e-05 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349357+0000", 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012534300000000001 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349482+0000", 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000175348 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349657+0000", 2023-10-09T10:21:30.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021320929999999998 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.351790+0000", 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019880399999999999 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.351988+0000", 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095375489999999993 2023-10-09T10:21:30.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447364+0000", 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9750999999999998e-05 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447424+0000", 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3198999999999998e-05 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447467+0000", 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098253160000000006 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:53401 10.f 10:fc91b071:::benchmark_data_smithi012_366312_object53400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e433)", 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:20:16.396324+0000", 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 73.681303667999998, 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051479672999999997, 2023-10-09T10:21:30.193 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 53401 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396324+0000", 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396324+0000", 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396323+0000", 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5451000000000001e-05 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396348+0000", 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2466000000000001e-05 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396361+0000", 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3459999999999996e-06 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396368+0000", 2023-10-09T10:21:30.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4066999999999997e-05 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396412+0000", 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108896 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396521+0000", 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011168 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.396633+0000", 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016474560000000001 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.398280+0000", 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029826400000000002 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.398578+0000", 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.049140141999999998 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447718+0000", 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9798e-05 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447768+0000", 2023-10-09T10:21:30.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5571000000000002e-05 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447804+0000", 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051479672999999997 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:113692 10.f 10:f2ca5024:::benchmark_data_smithi012_366312_object2363:head [delete] snapc 0=[] ondisk+write+known_if_redirected e450)", 2023-10-09T10:21:30.200 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:21.100257+0000", 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 8.977371218, 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044612062000000001, 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 113692 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.100257+0000", 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.100257+0000", 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.100256+0000", 2023-10-09T10:21:30.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4229999999999999e-06 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.100259+0000", 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0740000000000004e-06 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.100266+0000", 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6739999999999999e-06 2023-10-09T10:21:30.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.100269+0000", 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020453298000000002 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.120723+0000", 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0883000000000004e-05 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:30.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.120814+0000", 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010102000000000001 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.120915+0000", 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0080439359999999998 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.128959+0000", 2023-10-09T10:21:30.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058981629999999997 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.134857+0000", 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0099169510000000002 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.144774+0000", 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3677000000000001e-05 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.144827+0000", 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1356999999999997e-05 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:21.144869+0000", 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044612062000000001 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.207 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:30.208 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:30.208 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:21:30.208 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:30.314 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:30.352 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:21:30.474 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:30.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:30.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:21:30.586 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:30.668 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:30.742 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:30.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:30.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:30.851 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:30.917 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:31.009 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:31.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:31.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:31.118 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:31.166 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:21:31.166 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:21:31.167 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T10:21:31.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:21:31.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:31.246 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:31.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:31.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:31.299 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:31.300 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:31.355 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:31.388 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:31.458 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:31.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:31.487 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:31 smithi012 ceph-mon[147926]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 400 KiB/s wr, 629 op/s; 5564/315537 objects misplaced (1.763%); 768 KiB/s, 11 objects/s recovering 2023-10-09T10:21:31.562 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:31.654 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:31 smithi110 ceph-mon[136032]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 400 KiB/s wr, 629 op/s; 5564/315537 objects misplaced (1.763%); 768 KiB/s, 11 objects/s recovering 2023-10-09T10:21:31.661 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:31.747 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:31.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:31.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:31.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:31.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:31 smithi012 ceph-mon[152263]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 400 KiB/s wr, 629 op/s; 5564/315537 objects misplaced (1.763%); 768 KiB/s, 11 objects/s recovering 2023-10-09T10:21:31.957 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:32.031 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:32.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:32.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:32.202 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:32.364 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:32.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:32.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:32.516 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:32 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1341167688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:32.516 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:32 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:32.533 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:32.565 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:21:32.651 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:32 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1341167688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:32.651 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:32 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:32.687 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:32.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:32.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:32.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:32.828 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:32 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1341167688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:32.828 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:32 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:21:32.829 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:32.943 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:21:32.943 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:21:32.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.943 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:122732 10.1 e453/424 10:89612b62:::benchmark_data_smithi012_366312_object11403:head v 453'7581, mlcod=453'7579)", 2023-10-09T10:21:32.943 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:32.934216+0000", 2023-10-09T10:21:32.943 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.008555511, 2023-10-09T10:21:32.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085904499999999995, 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.934216+0000", 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.934216+0000", 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:32.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.934215+0000", 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1134e-05 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.934226+0000", 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6735000000000002e-05 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.934242+0000", 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1300000000000003e-06 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.934250+0000", 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3232999999999997e-05 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.934303+0000", 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0650999999999999e-05 2023-10-09T10:21:32.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.934324+0000", 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107793 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:122736 10.8 e453/424 10:12355301:::benchmark_data_smithi012_366312_object11407:head v 453'7655, mlcod=453'7653)", 2023-10-09T10:21:32.947 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:32.935806+0000", 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0069648949999999996, 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00707129, 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.935806+0000", 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:21:32.948 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.935806+0000", 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.935805+0000", 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1719000000000001e-05 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.935817+0000", 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2194e-05 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.935829+0000", 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0700000000000003e-06 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.935835+0000", 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5851000000000004e-05 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:21:32.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.935901+0000", 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9074999999999999e-05 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:32.935920+0000", 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011402 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 2 2023-10-09T10:21:32.951 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:21:33.247 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:33.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:33.355 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:33.414 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:33.488 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:33.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:33.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:33.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:21:33.604 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:33.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:33 smithi110 ceph-mon[136032]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 5462/307467 objects misplaced (1.776%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:33.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:33 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:33.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:33 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:21:33.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:33 smithi110 ceph-mon[136032]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T10:21:33.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:33 smithi110 ceph-mon[136032]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:33 smithi110 ceph-mon[136032]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.674 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:33 smithi110 ceph-mon[136032]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:33.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[147926]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 5462/307467 objects misplaced (1.776%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:33.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:33.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:21:33.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[147926]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T10:21:33.751 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[147926]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.751 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[147926]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.751 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[147926]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[152263]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 5462/307467 objects misplaced (1.776%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:21:33.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:33.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:21:33.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[152263]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T10:21:33.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[152263]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.752 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[152263]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.752 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:33 smithi012 ceph-mon[152263]: failed to encode map e453 with expected crc 2023-10-09T10:21:33.773 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:33.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:33.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:33.926 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:34.457 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:35.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:35 smithi110 ceph-mon[136032]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 5462/306078 objects misplaced (1.785%); 816 KiB/s, 12 objects/s recovering 2023-10-09T10:21:35.814 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:35 smithi012 ceph-mon[152263]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 5462/306078 objects misplaced (1.785%); 816 KiB/s, 12 objects/s recovering 2023-10-09T10:21:35.817 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:35 smithi012 ceph-mon[147926]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 5462/306078 objects misplaced (1.785%); 816 KiB/s, 12 objects/s recovering 2023-10-09T10:21:36.157 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:21:36.157 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:21:36.157 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:21:36.157 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:21:36.157 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:21:36.250 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:36.255 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:36.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:36.360 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:36.451 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:36.590 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:36.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:36.697 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:36.700 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:21:36.701 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:36 smithi110 ceph-mon[136032]: pgmap v1269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 102/296109 objects degraded (0.034%); 5254/296109 objects misplaced (1.774%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:21:36.751 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:36.808 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:36 smithi012 ceph-mon[147926]: pgmap v1269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 102/296109 objects degraded (0.034%); 5254/296109 objects misplaced (1.774%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:21:36.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:36 smithi012 ceph-mon[152263]: pgmap v1269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 102/296109 objects degraded (0.034%); 5254/296109 objects misplaced (1.774%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:21:36.871 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:36.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:36.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:37.030 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:37.080 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:37.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:37.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:37.137 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:37.184 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:37.188 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:37.217 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:37.253 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:37.280 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:37.299 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:37.304 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:37.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:37.357 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:37.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:37.369 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:37.382 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:21:37.382 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:21:37.382 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:21:37.382 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:21:37.382 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:21:37.382 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:21:37.382 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:21:37.383 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:21:37.468 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:37.528 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:37.596 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:37.597 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:37.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:37 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 102/296109 objects degraded (0.034%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3671500630' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:37.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:37 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 102/296109 objects degraded (0.034%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:37.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3671500630' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:37.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:37.776 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:37.851 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:37.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:37.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:37.881 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:37 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 102/296109 objects degraded (0.034%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:37.881 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3671500630' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:37.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:37.966 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:21:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:21:37] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.33.4" 2023-10-09T10:21:38.030 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:38.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:21:38.136 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:38.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:38.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:38.347 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:38.432 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:38.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:38.541 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:38.593 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:21:38.603 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:21:38.642 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:38.711 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:38.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:38.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:38.820 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:38.938 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:38.991 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1270,"stamp":"2023-10-09T10:21:37.685648+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6435510200,"num_objects":98214,"num_object_clones":0,"num_object_copies":294642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":102,"num_objects_misplaced":5254,"num_objects_unfound":0,"num_objects_dirty":98214,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":124724,"num_write_kb":7127310,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":807,"num_bytes_recovered":47793490,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116258,"ondisk_log_size":116258,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24289144,"kb_used_data":18985400,"kb_used_omap":172,"kb_used_meta":5303443,"kb_avail":725672072,"statfs":{"total":767960285184,"available":743088201728,"internally_reserved":0,"allocated":19441049600,"data_stored":19431583345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":176164,"internal_metadata":5430726620},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,8,3,4,6,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-503054336,"num_objects":-7676,"num_object_clones":0,"num_object_copies":-23028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":102,"num_objects_misplaced":-310,"num_objects_unfound":0,"num_objects_dirty":-7676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7676,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":208,"num_bytes_recovered":13631488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002706"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.455505+0000","last_change":"2023-10-09T10:19:43.287175+0000","last_active":"2023-10-09T10:21:26.455505+0000","last_peered":"2023-10-09T10:21:26.455505+0000","last_clean":"2023-10-09T10:21:26.455505+0000","last_became_active":"2023-10-09T10:19:43.286833+0000","last_became_peered":"2023-10-09T10:19:43.286833+0000","last_unstale":"2023-10-09T10:21:26.455505+0000","last_undegraded":"2023-10-09T10:21:26.455505+0000","last_fullsized":"2023-10-09T10:21:26.455505+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":406,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448543+0000","last_change":"2023-10-09T10:20:58.311336+0000","last_active":"2023-10-09T10:21:26.448543+0000","last_peered":"2023-10-09T10:21:26.448543+0000","last_clean":"2023-10-09T10:21:26.448543+0000","last_became_active":"2023-10-09T10:20:58.310941+0000","last_became_peered":"2023-10-09T10:20:58.310941+0000","last_unstale":"2023-10-09T10:21:26.448543+0000","last_undegraded":"2023-10-09T10:21:26.448543+0000","last_fullsized":"2023-10-09T10:21:26.448543+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:01.468077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":493,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448395+0000","last_change":"2023-10-09T10:19:43.284958+0000","last_active":"2023-10-09T10:21:26.448395+0000","last_peered":"2023-10-09T10:21:26.448395+0000","last_clean":"2023-10-09T10:21:26.448395+0000","last_became_active":"2023-10-09T10:19:43.283799+0000","last_became_peered":"2023-10-09T10:19:43.283799+0000","last_unstale":"2023-10-09T10:21:26.448395+0000","last_undegraded":"2023-10-09T10:21:26.448395+0000","last_fullsized":"2023-10-09T10:21:26.448395+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:57.225333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":348,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.439019+0000","last_change":"2023-10-09T10:16:08.309799+0000","last_active":"2023-10-09T10:21:26.439019+0000","last_peered":"2023-10-09T10:21:26.439019+0000","last_clean":"2023-10-09T10:21:26.439019+0000","last_became_active":"2023-10-09T10:16:08.309448+0000","last_became_peered":"2023-10-09T10:16:08.309448+0000","last_unstale":"2023-10-09T10:21:26.439019+0000","last_undegraded":"2023-10-09T10:21:26.439019+0000","last_fullsized":"2023-10-09T10:21:26.439019+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:58.976233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":387,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448029+0000","last_change":"2023-10-09T10:20:58.354386+0000","last_active":"2023-10-09T10:21:26.448029+0000","last_peered":"2023-10-09T10:21:26.448029+0000","last_clean":"2023-10-09T10:21:26.448029+0000","last_became_active":"2023-10-09T10:20:58.354039+0000","last_became_peered":"2023-10-09T10:20:58.354039+0000","last_unstale":"2023-10-09T10:21:26.448029+0000","last_undegraded":"2023-10-09T10:21:26.448029+0000","last_fullsized":"2023-10-09T10:21:26.448029+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:09.055455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":248,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.449472+0000","last_change":"2023-10-09T10:16:41.473093+0000","last_active":"2023-10-09T10:21:26.449472+0000","last_peered":"2023-10-09T10:21:26.449472+0000","last_clean":"2023-10-09T10:21:26.449472+0000","last_became_active":"2023-10-09T10:16:41.472774+0000","last_became_peered":"2023-10-09T10:16:41.472774+0000","last_unstale":"2023-10-09T10:21:26.449472+0000","last_undegraded":"2023-10-09T10:21:26.449472+0000","last_fullsized":"2023-10-09T10:21:26.449472+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:07.117932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":382,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.438930+0000","last_change":"2023-10-09T10:16:18.273288+0000","last_active":"2023-10-09T10:21:26.438930+0000","last_peered":"2023-10-09T10:21:26.438930+0000","last_clean":"2023-10-09T10:21:26.438930+0000","last_became_active":"2023-10-09T10:16:18.272936+0000","last_became_peered":"2023-10-09T10:16:18.272936+0000","last_unstale":"2023-10-09T10:21:26.438930+0000","last_undegraded":"2023-10-09T10:21:26.438930+0000","last_fullsized":"2023-10-09T10:21:26.438930+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:39:34.847781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":427,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448204+0000","last_change":"2023-10-09T10:20:32.431422+0000","last_active":"2023-10-09T10:21:26.448204+0000","last_peered":"2023-10-09T10:21:26.448204+0000","last_clean":"2023-10-09T10:21:26.448204+0000","last_became_active":"2023-10-09T10:20:32.431084+0000","last_became_peered":"2023-10-09T10:20:32.431084+0000","last_unstale":"2023-10-09T10:21:26.448204+0000","last_undegraded":"2023-10-09T10:21:26.448204+0000","last_fullsized":"2023-10-09T10:21:26.448204+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:56.876614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":101,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.444122+0000","last_change":"2023-10-09T10:16:18.280500+0000","last_active":"2023-10-09T10:21:26.444122+0000","last_peered":"2023-10-09T10:21:26.444122+0000","last_clean":"2023-10-09T10:21:26.444122+0000","last_became_active":"2023-10-09T10:16:18.275348+0000","last_became_peered":"2023-10-09T10:16:18.275348+0000","last_unstale":"2023-10-09T10:21:26.444122+0000","last_undegraded":"2023-10-09T10:21:26.444122+0000","last_fullsized":"2023-10-09T10:21:26.444122+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":363,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.435832+0000","last_change":"2023-10-09T10:15:42.396546+0000","last_active":"2023-10-09T10:21:26.435832+0000","last_peered":"2023-10-09T10:21:26.435832+0000","last_clean":"2023-10-09T10:21:26.435832+0000","last_became_active":"2023-10-09T10:15:42.367055+0000","last_became_peered":"2023-10-09T10:15:42.367055+0000","last_unstale":"2023-10-09T10:21:26.435832+0000","last_undegraded":"2023-10-09T10:21:26.435832+0000","last_fullsized":"2023-10-09T10:21:26.435832+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:57.600046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":43,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.457993+0000","last_change":"2023-10-09T10:19:43.299477+0000","last_active":"2023-10-09T10:21:26.457993+0000","last_peered":"2023-10-09T10:21:26.457993+0000","last_clean":"2023-10-09T10:21:26.457993+0000","last_became_active":"2023-10-09T10:19:43.274448+0000","last_became_peered":"2023-10-09T10:19:43.274448+0000","last_unstale":"2023-10-09T10:21:26.457993+0000","last_undegraded":"2023-10-09T10:21:26.457993+0000","last_fullsized":"2023-10-09T10:21:26.457993+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":452,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448420+0000","last_change":"2023-10-09T10:06:02.843933+0000","last_active":"2023-10-09T10:21:26.448420+0000","last_peered":"2023-10-09T10:21:26.448420+0000","last_clean":"2023-10-09T10:21:26.448420+0000","last_became_active":"2023-10-09T10:06:02.843487+0000","last_became_peered":"2023-10-09T10:06:02.843487+0000","last_unstale":"2023-10-09T10:21:26.448420+0000","last_undegraded":"2023-10-09T10:21:26.448420+0000","last_fullsized":"2023-10-09T10:21:26.448420+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:36.531592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":47,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.455579+0000","last_change":"2023-10-09T10:19:43.482921+0000","last_active":"2023-10-09T10:21:26.455579+0000","last_peered":"2023-10-09T10:21:26.455579+0000","last_clean":"2023-10-09T10:21:26.455579+0000","last_became_active":"2023-10-09T10:19:43.278592+0000","last_became_peered":"2023-10-09T10:19:43.278592+0000","last_unstale":"2023-10-09T10:21:26.455579+0000","last_undegraded":"2023-10-09T10:21:26.455579+0000","last_fullsized":"2023-10-09T10:21:26.455579+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":46,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.458760+0000","last_change":"2023-10-09T10:20:01.433412+0000","last_active":"2023-10-09T10:21:26.458760+0000","last_peered":"2023-10-09T10:21:26.458760+0000","last_clean":"2023-10-09T10:21:26.458760+0000","last_became_active":"2023-10-09T10:20:01.433114+0000","last_became_peered":"2023-10-09T10:20:01.433114+0000","last_unstale":"2023-10-09T10:21:26.458760+0000","last_undegraded":"2023-10-09T10:21:26.458760+0000","last_fullsized":"2023-10-09T10:21:26.458760+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":43,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.458628+0000","last_change":"2023-10-09T10:19:43.485535+0000","last_active":"2023-10-09T10:21:26.458628+0000","last_peered":"2023-10-09T10:21:26.458628+0000","last_clean":"2023-10-09T10:21:26.458628+0000","last_became_active":"2023-10-09T10:19:43.289115+0000","last_became_peered":"2023-10-09T10:19:43.289115+0000","last_unstale":"2023-10-09T10:21:26.458628+0000","last_undegraded":"2023-10-09T10:21:26.458628+0000","last_fullsized":"2023-10-09T10:21:26.458628+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":352,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448834+0000","last_change":"2023-10-09T10:18:45.342547+0000","last_active":"2023-10-09T10:21:26.448834+0000","last_peered":"2023-10-09T10:21:26.448834+0000","last_clean":"2023-10-09T10:21:26.448834+0000","last_became_active":"2023-10-09T10:18:45.342206+0000","last_became_peered":"2023-10-09T10:18:45.342206+0000","last_unstale":"2023-10-09T10:21:26.448834+0000","last_undegraded":"2023-10-09T10:21:26.448834+0000","last_fullsized":"2023-10-09T10:21:26.448834+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:32.932519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":347,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.438736+0000","last_change":"2023-10-09T10:16:08.455134+0000","last_active":"2023-10-09T10:21:26.438736+0000","last_peered":"2023-10-09T10:21:26.438736+0000","last_clean":"2023-10-09T10:21:26.438736+0000","last_became_active":"2023-10-09T10:16:08.454793+0000","last_became_peered":"2023-10-09T10:16:08.454793+0000","last_unstale":"2023-10-09T10:21:26.438736+0000","last_undegraded":"2023-10-09T10:21:26.438736+0000","last_fullsized":"2023-10-09T10:21:26.438736+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:25:33.883382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":101,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.444238+0000","last_change":"2023-10-09T10:16:18.255609+0000","last_active":"2023-10-09T10:21:26.444238+0000","last_peered":"2023-10-09T10:21:26.444238+0000","last_clean":"2023-10-09T10:21:26.444238+0000","last_became_active":"2023-10-09T10:16:18.244618+0000","last_became_peered":"2023-10-09T10:16:18.244618+0000","last_unstale":"2023-10-09T10:21:26.444238+0000","last_undegraded":"2023-10-09T10:21:26.444238+0000","last_fullsized":"2023-10-09T10:21:26.444238+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.457057+0000","last_change":"2023-10-09T10:19:43.282758+0000","last_active":"2023-10-09T10:21:26.457057+0000","last_peered":"2023-10-09T10:21:26.457057+0000","last_clean":"2023-10-09T10:21:26.457057+0000","last_became_active":"2023-10-09T10:19:43.282258+0000","last_became_peered":"2023-10-09T10:19:43.282258+0000","last_unstale":"2023-10-09T10:21:26.457057+0000","last_undegraded":"2023-10-09T10:21:26.457057+0000","last_fullsized":"2023-10-09T10:21:26.457057+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":25,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.436035+0000","last_change":"2023-10-09T10:20:37.462345+0000","last_active":"2023-10-09T10:21:26.436035+0000","last_peered":"2023-10-09T10:21:26.436035+0000","last_clean":"2023-10-09T10:21:26.436035+0000","last_became_active":"2023-10-09T10:20:37.462057+0000","last_became_peered":"2023-10-09T10:20:37.462057+0000","last_unstale":"2023-10-09T10:21:26.436035+0000","last_undegraded":"2023-10-09T10:21:26.436035+0000","last_fullsized":"2023-10-09T10:21:26.436035+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:35.024832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":368,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.443642+0000","last_change":"2023-10-09T10:18:34.286718+0000","last_active":"2023-10-09T10:21:26.443642+0000","last_peered":"2023-10-09T10:21:26.443642+0000","last_clean":"2023-10-09T10:21:26.443642+0000","last_became_active":"2023-10-09T10:18:34.271799+0000","last_became_peered":"2023-10-09T10:18:34.271799+0000","last_unstale":"2023-10-09T10:21:26.443642+0000","last_undegraded":"2023-10-09T10:21:26.443642+0000","last_fullsized":"2023-10-09T10:21:26.443642+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":400,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.449392+0000","last_change":"2023-10-09T10:16:08.618851+0000","last_active":"2023-10-09T10:21:26.449392+0000","last_peered":"2023-10-09T10:21:26.449392+0000","last_clean":"2023-10-09T10:21:26.449392+0000","last_became_active":"2023-10-09T10:16:08.618501+0000","last_became_peered":"2023-10-09T10:16:08.618501+0000","last_unstale":"2023-10-09T10:21:26.449392+0000","last_undegraded":"2023-10-09T10:21:26.449392+0000","last_fullsized":"2023-10-09T10:21:26.449392+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:45.133486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.6","version":"453'7798","reported_seq":7835,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.800115+0000","last_change":"2023-10-09T10:19:43.277465+0000","last_active":"2023-10-09T10:21:34.800115+0000","last_peered":"2023-10-09T10:21:34.800115+0000","last_clean":"2023-10-09T10:21:34.800115+0000","last_became_active":"2023-10-09T10:19:43.277144+0000","last_became_peered":"2023-10-09T10:19:43.277144+0000","last_unstale":"2023-10-09T10:21:34.800115+0000","last_undegraded":"2023-10-09T10:21:34.800115+0000","last_fullsized":"2023-10-09T10:21:34.800115+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7798,"log_dups_size":0,"ondisk_log_size":7798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403701760,"num_objects":6160,"num_object_clones":0,"num_object_copies":18480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7798,"num_write_kb":446656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.438740+0000","last_change":"2023-10-09T10:20:01.411305+0000","last_active":"2023-10-09T10:21:26.438740+0000","last_peered":"2023-10-09T10:21:26.438740+0000","last_clean":"2023-10-09T10:21:26.438740+0000","last_became_active":"2023-10-09T10:20:01.400821+0000","last_became_peered":"2023-10-09T10:20:01.400821+0000","last_unstale":"2023-10-09T10:21:26.438740+0000","last_undegraded":"2023-10-09T10:21:26.438740+0000","last_fullsized":"2023-10-09T10:21:26.438740+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:23.524079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":101,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.443543+0000","last_change":"2023-10-09T10:16:18.237380+0000","last_active":"2023-10-09T10:21:26.443543+0000","last_peered":"2023-10-09T10:21:26.443543+0000","last_clean":"2023-10-09T10:21:26.443543+0000","last_became_active":"2023-10-09T10:16:18.236585+0000","last_became_peered":"2023-10-09T10:16:18.236585+0000","last_unstale":"2023-10-09T10:21:26.443543+0000","last_undegraded":"2023-10-09T10:21:26.443543+0000","last_fullsized":"2023-10-09T10:21:26.443543+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"453'7558","reported_seq":7595,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.811930+0000","last_change":"2023-10-09T10:19:43.284275+0000","last_active":"2023-10-09T10:21:34.811930+0000","last_peered":"2023-10-09T10:21:34.811930+0000","last_clean":"2023-10-09T10:21:34.811930+0000","last_became_active":"2023-10-09T10:19:43.283457+0000","last_became_peered":"2023-10-09T10:19:43.283457+0000","last_unstale":"2023-10-09T10:21:34.811930+0000","last_undegraded":"2023-10-09T10:21:34.811930+0000","last_fullsized":"2023-10-09T10:21:34.811930+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394919936,"num_objects":6026,"num_object_clones":0,"num_object_copies":18078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7558,"num_write_kb":434688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":41,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.463861+0000","last_change":"2023-10-09T10:19:43.388522+0000","last_active":"2023-10-09T10:21:26.463861+0000","last_peered":"2023-10-09T10:21:26.463861+0000","last_clean":"2023-10-09T10:21:26.463861+0000","last_became_active":"2023-10-09T10:19:43.274829+0000","last_became_peered":"2023-10-09T10:19:43.274829+0000","last_unstale":"2023-10-09T10:21:26.463861+0000","last_undegraded":"2023-10-09T10:21:26.463861+0000","last_fullsized":"2023-10-09T10:21:26.463861+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.457123+0000","last_change":"2023-10-09T10:19:43.282812+0000","last_active":"2023-10-09T10:21:26.457123+0000","last_peered":"2023-10-09T10:21:26.457123+0000","last_clean":"2023-10-09T10:21:26.457123+0000","last_became_active":"2023-10-09T10:19:43.282498+0000","last_became_peered":"2023-10-09T10:19:43.282498+0000","last_unstale":"2023-10-09T10:21:26.457123+0000","last_undegraded":"2023-10-09T10:21:26.457123+0000","last_fullsized":"2023-10-09T10:21:26.457123+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"453'7825","reported_seq":7861,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.989447+0000","last_change":"2023-10-09T10:19:43.291894+0000","last_active":"2023-10-09T10:21:34.989447+0000","last_peered":"2023-10-09T10:21:34.989447+0000","last_clean":"2023-10-09T10:21:34.989447+0000","last_became_active":"2023-10-09T10:19:43.291238+0000","last_became_peered":"2023-10-09T10:19:43.291238+0000","last_unstale":"2023-10-09T10:21:34.989447+0000","last_undegraded":"2023-10-09T10:21:34.989447+0000","last_fullsized":"2023-10-09T10:21:34.989447+0000","mapping_epoch":424,"log_start":"428'1100","ondisk_log_start":"428'1100","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396689408,"num_objects":6053,"num_object_clones":0,"num_object_copies":18159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7825,"num_write_kb":444096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":105,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.444021+0000","last_change":"2023-10-09T10:16:18.362151+0000","last_active":"2023-10-09T10:21:26.444021+0000","last_peered":"2023-10-09T10:21:26.444021+0000","last_clean":"2023-10-09T10:21:26.444021+0000","last_became_active":"2023-10-09T10:16:18.323076+0000","last_became_peered":"2023-10-09T10:16:18.323076+0000","last_unstale":"2023-10-09T10:21:26.444021+0000","last_undegraded":"2023-10-09T10:21:26.444021+0000","last_fullsized":"2023-10-09T10:21:26.444021+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":296,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.440265+0000","last_change":"2023-10-09T10:09:01.253046+0000","last_active":"2023-10-09T10:21:26.440265+0000","last_peered":"2023-10-09T10:21:26.440265+0000","last_clean":"2023-10-09T10:21:26.440265+0000","last_became_active":"2023-10-09T10:09:01.252710+0000","last_became_peered":"2023-10-09T10:09:01.252710+0000","last_unstale":"2023-10-09T10:21:26.440265+0000","last_undegraded":"2023-10-09T10:21:26.440265+0000","last_fullsized":"2023-10-09T10:21:26.440265+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"453'8004","reported_seq":8042,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:37.476571+0000","last_change":"2023-10-09T10:19:43.283774+0000","last_active":"2023-10-09T10:21:37.476571+0000","last_peered":"2023-10-09T10:21:37.476571+0000","last_clean":"2023-10-09T10:21:37.476571+0000","last_became_active":"2023-10-09T10:19:43.283468+0000","last_became_peered":"2023-10-09T10:19:43.283468+0000","last_unstale":"2023-10-09T10:21:37.476571+0000","last_undegraded":"2023-10-09T10:21:37.476571+0000","last_fullsized":"2023-10-09T10:21:37.476571+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8004,"log_dups_size":0,"ondisk_log_size":8004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:50.698088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399376384,"num_objects":6094,"num_object_clones":0,"num_object_copies":18282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8004,"num_write_kb":451136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.443922+0000","last_change":"2023-10-09T10:20:58.316346+0000","last_active":"2023-10-09T10:21:26.443922+0000","last_peered":"2023-10-09T10:21:26.443922+0000","last_clean":"2023-10-09T10:21:26.443922+0000","last_became_active":"2023-10-09T10:20:58.316000+0000","last_became_peered":"2023-10-09T10:20:58.316000+0000","last_unstale":"2023-10-09T10:21:26.443922+0000","last_undegraded":"2023-10-09T10:21:26.443922+0000","last_fullsized":"2023-10-09T10:21:26.443922+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":342,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.441180+0000","last_change":"2023-10-09T10:19:43.291654+0000","last_active":"2023-10-09T10:21:26.441180+0000","last_peered":"2023-10-09T10:21:26.441180+0000","last_clean":"2023-10-09T10:21:26.441180+0000","last_became_active":"2023-10-09T10:19:43.291302+0000","last_became_peered":"2023-10-09T10:19:43.291302+0000","last_unstale":"2023-10-09T10:21:26.441180+0000","last_undegraded":"2023-10-09T10:21:26.441180+0000","last_fullsized":"2023-10-09T10:21:26.441180+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:41.829196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"453'7712","reported_seq":7750,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.391307+0000","last_change":"2023-10-09T10:19:43.279466+0000","last_active":"2023-10-09T10:21:34.391307+0000","last_peered":"2023-10-09T10:21:34.391307+0000","last_clean":"2023-10-09T10:21:34.391307+0000","last_became_active":"2023-10-09T10:19:43.279157+0000","last_became_peered":"2023-10-09T10:19:43.279157+0000","last_unstale":"2023-10-09T10:21:34.391307+0000","last_undegraded":"2023-10-09T10:21:34.391307+0000","last_fullsized":"2023-10-09T10:21:34.391307+0000","mapping_epoch":424,"log_start":"428'1000","ondisk_log_start":"428'1000","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:55.938133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406585344,"num_objects":6204,"num_object_clones":0,"num_object_copies":18612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7712,"num_write_kb":445312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":22,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.440949+0000","last_change":"2023-10-09T10:21:02.424185+0000","last_active":"2023-10-09T10:21:26.440949+0000","last_peered":"2023-10-09T10:21:26.440949+0000","last_clean":"2023-10-09T10:21:26.440949+0000","last_became_active":"2023-10-09T10:21:02.415689+0000","last_became_peered":"2023-10-09T10:21:02.415689+0000","last_unstale":"2023-10-09T10:21:26.440949+0000","last_undegraded":"2023-10-09T10:21:26.440949+0000","last_fullsized":"2023-10-09T10:21:26.440949+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:09.473233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.455469+0000","last_change":"2023-10-09T10:19:43.291696+0000","last_active":"2023-10-09T10:21:26.455469+0000","last_peered":"2023-10-09T10:21:26.455469+0000","last_clean":"2023-10-09T10:21:26.455469+0000","last_became_active":"2023-10-09T10:19:43.290863+0000","last_became_peered":"2023-10-09T10:19:43.290863+0000","last_unstale":"2023-10-09T10:21:26.455469+0000","last_undegraded":"2023-10-09T10:21:26.455469+0000","last_fullsized":"2023-10-09T10:21:26.455469+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"453'7933","reported_seq":7970,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.813396+0000","last_change":"2023-10-09T10:19:43.284340+0000","last_active":"2023-10-09T10:21:34.813396+0000","last_peered":"2023-10-09T10:21:34.813396+0000","last_clean":"2023-10-09T10:21:34.813396+0000","last_became_active":"2023-10-09T10:19:43.283218+0000","last_became_peered":"2023-10-09T10:19:43.283218+0000","last_unstale":"2023-10-09T10:21:34.813396+0000","last_undegraded":"2023-10-09T10:21:34.813396+0000","last_fullsized":"2023-10-09T10:21:34.813396+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7933,"log_dups_size":0,"ondisk_log_size":7933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409796608,"num_objects":6253,"num_object_clones":0,"num_object_copies":18759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7933,"num_write_kb":453952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.455645+0000","last_change":"2023-10-09T10:19:43.276421+0000","last_active":"2023-10-09T10:21:26.455645+0000","last_peered":"2023-10-09T10:21:26.455645+0000","last_clean":"2023-10-09T10:21:26.455645+0000","last_became_active":"2023-10-09T10:19:43.276070+0000","last_became_peered":"2023-10-09T10:19:43.276070+0000","last_unstale":"2023-10-09T10:21:26.455645+0000","last_undegraded":"2023-10-09T10:21:26.455645+0000","last_fullsized":"2023-10-09T10:21:26.455645+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.457847+0000","last_change":"2023-10-09T10:19:43.286865+0000","last_active":"2023-10-09T10:21:26.457847+0000","last_peered":"2023-10-09T10:21:26.457847+0000","last_clean":"2023-10-09T10:21:26.457847+0000","last_became_active":"2023-10-09T10:19:43.286530+0000","last_became_peered":"2023-10-09T10:19:43.286530+0000","last_unstale":"2023-10-09T10:21:26.457847+0000","last_undegraded":"2023-10-09T10:21:26.457847+0000","last_fullsized":"2023-10-09T10:21:26.457847+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"453'7721","reported_seq":7759,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.391333+0000","last_change":"2023-10-09T10:19:43.281938+0000","last_active":"2023-10-09T10:21:34.391333+0000","last_peered":"2023-10-09T10:21:34.391333+0000","last_clean":"2023-10-09T10:21:34.391333+0000","last_became_active":"2023-10-09T10:19:43.281098+0000","last_became_peered":"2023-10-09T10:19:43.281098+0000","last_unstale":"2023-10-09T10:21:34.391333+0000","last_undegraded":"2023-10-09T10:21:34.391333+0000","last_fullsized":"2023-10-09T10:21:34.391333+0000","mapping_epoch":424,"log_start":"428'1000","ondisk_log_start":"428'1000","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:53.762641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400228352,"num_objects":6107,"num_object_clones":0,"num_object_copies":18321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7721,"num_write_kb":442496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.457192+0000","last_change":"2023-10-09T10:19:43.288603+0000","last_active":"2023-10-09T10:21:26.457192+0000","last_peered":"2023-10-09T10:21:26.457192+0000","last_clean":"2023-10-09T10:21:26.457192+0000","last_became_active":"2023-10-09T10:19:43.287994+0000","last_became_peered":"2023-10-09T10:19:43.287994+0000","last_unstale":"2023-10-09T10:21:26.457192+0000","last_undegraded":"2023-10-09T10:21:26.457192+0000","last_fullsized":"2023-10-09T10:21:26.457192+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":46,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.455220+0000","last_change":"2023-10-09T10:21:02.448313+0000","last_active":"2023-10-09T10:21:26.455220+0000","last_peered":"2023-10-09T10:21:26.455220+0000","last_clean":"2023-10-09T10:21:26.455220+0000","last_became_active":"2023-10-09T10:21:02.447928+0000","last_became_peered":"2023-10-09T10:21:02.447928+0000","last_unstale":"2023-10-09T10:21:26.455220+0000","last_undegraded":"2023-10-09T10:21:26.455220+0000","last_fullsized":"2023-10-09T10:21:26.455220+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"453'7691","reported_seq":7727,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:35.012861+0000","last_change":"2023-10-09T10:19:43.285977+0000","last_active":"2023-10-09T10:21:35.012861+0000","last_peered":"2023-10-09T10:21:35.012861+0000","last_clean":"2023-10-09T10:21:35.012861+0000","last_became_active":"2023-10-09T10:19:43.285666+0000","last_became_peered":"2023-10-09T10:19:43.285666+0000","last_unstale":"2023-10-09T10:21:35.012861+0000","last_undegraded":"2023-10-09T10:21:35.012861+0000","last_fullsized":"2023-10-09T10:21:35.012861+0000","mapping_epoch":424,"log_start":"428'1000","ondisk_log_start":"428'1000","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397737984,"num_objects":6069,"num_object_clones":0,"num_object_copies":18207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7691,"num_write_kb":440320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":420,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.447925+0000","last_change":"2023-10-09T10:19:43.281410+0000","last_active":"2023-10-09T10:21:26.447925+0000","last_peered":"2023-10-09T10:21:26.447925+0000","last_clean":"2023-10-09T10:21:26.447925+0000","last_became_active":"2023-10-09T10:19:43.280304+0000","last_became_peered":"2023-10-09T10:19:43.280304+0000","last_unstale":"2023-10-09T10:21:26.447925+0000","last_undegraded":"2023-10-09T10:21:26.447925+0000","last_fullsized":"2023-10-09T10:21:26.447925+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:11.948068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":494,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:32.538747+0000","last_change":"2023-10-09T10:19:43.284086+0000","last_active":"2023-10-09T10:21:32.538747+0000","last_peered":"2023-10-09T10:21:32.538747+0000","last_clean":"2023-10-09T10:21:32.538747+0000","last_became_active":"2023-10-09T10:19:43.283752+0000","last_became_peered":"2023-10-09T10:19:43.283752+0000","last_unstale":"2023-10-09T10:21:32.538747+0000","last_undegraded":"2023-10-09T10:21:32.538747+0000","last_fullsized":"2023-10-09T10:21:32.538747+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.817673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"453'7693","reported_seq":7728,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.999064+0000","last_change":"2023-10-09T10:19:43.291549+0000","last_active":"2023-10-09T10:21:34.999064+0000","last_peered":"2023-10-09T10:21:34.999064+0000","last_clean":"2023-10-09T10:21:34.999064+0000","last_became_active":"2023-10-09T10:19:43.290461+0000","last_became_peered":"2023-10-09T10:19:43.290461+0000","last_unstale":"2023-10-09T10:21:34.999064+0000","last_undegraded":"2023-10-09T10:21:34.999064+0000","last_fullsized":"2023-10-09T10:21:34.999064+0000","mapping_epoch":424,"log_start":"428'1000","ondisk_log_start":"428'1000","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6693,"log_dups_size":0,"ondisk_log_size":6693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401801216,"num_objects":6131,"num_object_clones":0,"num_object_copies":18393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7693,"num_write_kb":442368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":460,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448990+0000","last_change":"2023-10-09T10:02:29.882386+0000","last_active":"2023-10-09T10:21:26.448990+0000","last_peered":"2023-10-09T10:21:26.448990+0000","last_clean":"2023-10-09T10:21:26.448990+0000","last_became_active":"2023-10-09T10:02:29.882020+0000","last_became_peered":"2023-10-09T10:02:29.882020+0000","last_unstale":"2023-10-09T10:21:26.448990+0000","last_undegraded":"2023-10-09T10:21:26.448990+0000","last_fullsized":"2023-10-09T10:21:26.448990+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:08:42.575360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":101,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.444175+0000","last_change":"2023-10-09T10:16:18.323515+0000","last_active":"2023-10-09T10:21:26.444175+0000","last_peered":"2023-10-09T10:21:26.444175+0000","last_clean":"2023-10-09T10:21:26.444175+0000","last_became_active":"2023-10-09T10:16:18.321171+0000","last_became_peered":"2023-10-09T10:16:18.321171+0000","last_unstale":"2023-10-09T10:21:26.444175+0000","last_undegraded":"2023-10-09T10:21:26.444175+0000","last_fullsized":"2023-10-09T10:21:26.444175+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":453,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448766+0000","last_change":"2023-10-09T10:19:43.278853+0000","last_active":"2023-10-09T10:21:26.448766+0000","last_peered":"2023-10-09T10:21:26.448766+0000","last_clean":"2023-10-09T10:21:26.448766+0000","last_became_active":"2023-10-09T10:19:43.278351+0000","last_became_peered":"2023-10-09T10:19:43.278351+0000","last_unstale":"2023-10-09T10:21:26.448766+0000","last_undegraded":"2023-10-09T10:21:26.448766+0000","last_fullsized":"2023-10-09T10:21:26.448766+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:09.207569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"453'7666","reported_seq":7703,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.718440+0000","last_change":"2023-10-09T10:19:43.278170+0000","last_active":"2023-10-09T10:21:34.718440+0000","last_peered":"2023-10-09T10:21:34.718440+0000","last_clean":"2023-10-09T10:21:34.718440+0000","last_became_active":"2023-10-09T10:19:43.277881+0000","last_became_peered":"2023-10-09T10:19:43.277881+0000","last_unstale":"2023-10-09T10:21:34.718440+0000","last_undegraded":"2023-10-09T10:21:34.718440+0000","last_fullsized":"2023-10-09T10:21:34.718440+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7666,"log_dups_size":0,"ondisk_log_size":7666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399769600,"num_objects":6100,"num_object_clones":0,"num_object_copies":18300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7666,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":101,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.445201+0000","last_change":"2023-10-09T10:16:18.255781+0000","last_active":"2023-10-09T10:21:26.445201+0000","last_peered":"2023-10-09T10:21:26.445201+0000","last_clean":"2023-10-09T10:21:26.445201+0000","last_became_active":"2023-10-09T10:16:18.238531+0000","last_became_peered":"2023-10-09T10:16:18.238531+0000","last_unstale":"2023-10-09T10:21:26.445201+0000","last_undegraded":"2023-10-09T10:21:26.445201+0000","last_fullsized":"2023-10-09T10:21:26.445201+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.458696+0000","last_change":"2023-10-09T10:19:43.286169+0000","last_active":"2023-10-09T10:21:26.458696+0000","last_peered":"2023-10-09T10:21:26.458696+0000","last_clean":"2023-10-09T10:21:26.458696+0000","last_became_active":"2023-10-09T10:19:43.285440+0000","last_became_peered":"2023-10-09T10:19:43.285440+0000","last_unstale":"2023-10-09T10:21:26.458696+0000","last_undegraded":"2023-10-09T10:21:26.458696+0000","last_fullsized":"2023-10-09T10:21:26.458696+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":507,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448914+0000","last_change":"2023-10-09T10:16:08.230567+0000","last_active":"2023-10-09T10:21:26.448914+0000","last_peered":"2023-10-09T10:21:26.448914+0000","last_clean":"2023-10-09T10:21:26.448914+0000","last_became_active":"2023-10-09T10:16:08.229677+0000","last_became_peered":"2023-10-09T10:16:08.229677+0000","last_unstale":"2023-10-09T10:21:26.448914+0000","last_undegraded":"2023-10-09T10:21:26.448914+0000","last_fullsized":"2023-10-09T10:21:26.448914+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:52.866334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"453'7788","reported_seq":7825,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.801030+0000","last_change":"2023-10-09T10:19:43.276368+0000","last_active":"2023-10-09T10:21:34.801030+0000","last_peered":"2023-10-09T10:21:34.801030+0000","last_clean":"2023-10-09T10:21:34.801030+0000","last_became_active":"2023-10-09T10:19:43.276039+0000","last_became_peered":"2023-10-09T10:19:43.276039+0000","last_unstale":"2023-10-09T10:21:34.801030+0000","last_undegraded":"2023-10-09T10:21:34.801030+0000","last_fullsized":"2023-10-09T10:21:34.801030+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7788,"log_dups_size":0,"ondisk_log_size":7788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405012480,"num_objects":6180,"num_object_clones":0,"num_object_copies":18540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7788,"num_write_kb":446976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":365,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:32.483047+0000","last_change":"2023-10-09T10:16:08.269361+0000","last_active":"2023-10-09T10:21:32.483047+0000","last_peered":"2023-10-09T10:21:32.483047+0000","last_clean":"2023-10-09T10:21:32.483047+0000","last_became_active":"2023-10-09T10:16:08.266453+0000","last_became_peered":"2023-10-09T10:16:08.266453+0000","last_unstale":"2023-10-09T10:21:32.483047+0000","last_undegraded":"2023-10-09T10:21:32.483047+0000","last_fullsized":"2023-10-09T10:21:32.483047+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:07.633138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":453,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448121+0000","last_change":"2023-10-09T10:06:02.845070+0000","last_active":"2023-10-09T10:21:26.448121+0000","last_peered":"2023-10-09T10:21:26.448121+0000","last_clean":"2023-10-09T10:21:26.448121+0000","last_became_active":"2023-10-09T10:06:02.844739+0000","last_became_peered":"2023-10-09T10:06:02.844739+0000","last_unstale":"2023-10-09T10:21:26.448121+0000","last_undegraded":"2023-10-09T10:21:26.448121+0000","last_fullsized":"2023-10-09T10:21:26.448121+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:53.597578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.438886+0000","last_change":"2023-10-09T10:20:01.441278+0000","last_active":"2023-10-09T10:21:26.438886+0000","last_peered":"2023-10-09T10:21:26.438886+0000","last_clean":"2023-10-09T10:21:26.438886+0000","last_became_active":"2023-10-09T10:20:01.440907+0000","last_became_peered":"2023-10-09T10:20:01.440907+0000","last_unstale":"2023-10-09T10:21:26.438886+0000","last_undegraded":"2023-10-09T10:21:26.438886+0000","last_fullsized":"2023-10-09T10:21:26.438886+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:24.452698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"453'7864","reported_seq":7902,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.414874+0000","last_change":"2023-10-09T10:19:43.279876+0000","last_active":"2023-10-09T10:21:34.414874+0000","last_peered":"2023-10-09T10:21:34.414874+0000","last_clean":"2023-10-09T10:21:34.414874+0000","last_became_active":"2023-10-09T10:19:43.279596+0000","last_became_peered":"2023-10-09T10:19:43.279596+0000","last_unstale":"2023-10-09T10:21:34.414874+0000","last_undegraded":"2023-10-09T10:21:34.414874+0000","last_fullsized":"2023-10-09T10:21:34.414874+0000","mapping_epoch":424,"log_start":"428'1100","ondisk_log_start":"428'1100","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:11.020851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412352512,"num_objects":6292,"num_object_clones":0,"num_object_copies":18876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7864,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":98,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.443956+0000","last_change":"2023-10-09T10:16:18.477748+0000","last_active":"2023-10-09T10:21:26.443956+0000","last_peered":"2023-10-09T10:21:26.443956+0000","last_clean":"2023-10-09T10:21:26.443956+0000","last_became_active":"2023-10-09T10:16:18.477398+0000","last_became_peered":"2023-10-09T10:16:18.477398+0000","last_unstale":"2023-10-09T10:21:26.443956+0000","last_undegraded":"2023-10-09T10:21:26.443956+0000","last_fullsized":"2023-10-09T10:21:26.443956+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":457,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.447707+0000","last_change":"2023-10-09T10:19:14.282209+0000","last_active":"2023-10-09T10:21:26.447707+0000","last_peered":"2023-10-09T10:21:26.447707+0000","last_clean":"2023-10-09T10:21:26.447707+0000","last_became_active":"2023-10-09T10:19:14.281786+0000","last_became_peered":"2023-10-09T10:19:14.281786+0000","last_unstale":"2023-10-09T10:21:26.447707+0000","last_undegraded":"2023-10-09T10:21:26.447707+0000","last_fullsized":"2023-10-09T10:21:26.447707+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:28.589329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.445277+0000","last_change":"2023-10-09T10:19:43.283817+0000","last_active":"2023-10-09T10:21:26.445277+0000","last_peered":"2023-10-09T10:21:26.445277+0000","last_clean":"2023-10-09T10:21:26.445277+0000","last_became_active":"2023-10-09T10:19:43.283539+0000","last_became_peered":"2023-10-09T10:19:43.283539+0000","last_unstale":"2023-10-09T10:21:26.445277+0000","last_undegraded":"2023-10-09T10:21:26.445277+0000","last_fullsized":"2023-10-09T10:21:26.445277+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.457916+0000","last_change":"2023-10-09T10:19:43.281738+0000","last_active":"2023-10-09T10:21:26.457916+0000","last_peered":"2023-10-09T10:21:26.457916+0000","last_clean":"2023-10-09T10:21:26.457916+0000","last_became_active":"2023-10-09T10:19:43.280622+0000","last_became_peered":"2023-10-09T10:19:43.280622+0000","last_unstale":"2023-10-09T10:21:26.457916+0000","last_undegraded":"2023-10-09T10:21:26.457916+0000","last_fullsized":"2023-10-09T10:21:26.457916+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"453'7867","reported_seq":7905,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:37.260477+0000","last_change":"2023-10-09T10:19:43.281883+0000","last_active":"2023-10-09T10:21:37.260477+0000","last_peered":"2023-10-09T10:21:37.260477+0000","last_clean":"2023-10-09T10:21:37.260477+0000","last_became_active":"2023-10-09T10:19:43.281189+0000","last_became_peered":"2023-10-09T10:19:43.281189+0000","last_unstale":"2023-10-09T10:21:37.260477+0000","last_undegraded":"2023-10-09T10:21:37.260477+0000","last_fullsized":"2023-10-09T10:21:37.260477+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7867,"log_dups_size":0,"ondisk_log_size":7867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:56.958067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397606912,"num_objects":6067,"num_object_clones":0,"num_object_copies":18201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7867,"num_write_kb":445888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":433,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448290+0000","last_change":"2023-10-09T10:02:29.881607+0000","last_active":"2023-10-09T10:21:26.448290+0000","last_peered":"2023-10-09T10:21:26.448290+0000","last_clean":"2023-10-09T10:21:26.448290+0000","last_became_active":"2023-10-09T10:02:29.881209+0000","last_became_peered":"2023-10-09T10:02:29.881209+0000","last_unstale":"2023-10-09T10:21:26.448290+0000","last_undegraded":"2023-10-09T10:21:26.448290+0000","last_fullsized":"2023-10-09T10:21:26.448290+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:02.623427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":386,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.438811+0000","last_change":"2023-10-09T10:19:43.292390+0000","last_active":"2023-10-09T10:21:26.438811+0000","last_peered":"2023-10-09T10:21:26.438811+0000","last_clean":"2023-10-09T10:21:26.438811+0000","last_became_active":"2023-10-09T10:19:43.290733+0000","last_became_peered":"2023-10-09T10:19:43.290733+0000","last_unstale":"2023-10-09T10:21:26.438811+0000","last_undegraded":"2023-10-09T10:21:26.438811+0000","last_fullsized":"2023-10-09T10:21:26.438811+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:53.407598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":314,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448846+0000","last_change":"2023-10-09T10:18:34.268478+0000","last_active":"2023-10-09T10:21:26.448846+0000","last_peered":"2023-10-09T10:21:26.448846+0000","last_clean":"2023-10-09T10:21:26.448846+0000","last_became_active":"2023-10-09T10:18:34.268098+0000","last_became_peered":"2023-10-09T10:18:34.268098+0000","last_unstale":"2023-10-09T10:21:26.448846+0000","last_undegraded":"2023-10-09T10:21:26.448846+0000","last_fullsized":"2023-10-09T10:21:26.448846+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.625264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"453'7834","reported_seq":9305,"reported_epoch":453,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:21:35.011836+0000","last_change":"2023-10-09T10:21:31.037758+0000","last_active":"2023-10-09T10:21:35.011836+0000","last_peered":"2023-10-09T10:21:35.011836+0000","last_clean":"2023-10-09T10:20:57.308413+0000","last_became_active":"2023-10-09T10:20:59.487335+0000","last_became_peered":"2023-10-09T10:20:59.487335+0000","last_unstale":"2023-10-09T10:21:35.011836+0000","last_undegraded":"2023-10-09T10:21:31.033854+0000","last_fullsized":"2023-10-09T10:20:59.313493+0000","mapping_epoch":443,"log_start":"428'1100","ondisk_log_start":"428'1100","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6734,"log_dups_size":0,"ondisk_log_size":6734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406913048,"num_objects":6210,"num_object_clones":0,"num_object_copies":18630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":102,"num_objects_misplaced":5254,"num_objects_unfound":0,"num_objects_dirty":6210,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7834,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":711,"num_bytes_recovered":46596096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6","objects":854},{"shards":"4,6,7","objects":5254}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.458075+0000","last_change":"2023-10-09T10:19:43.274868+0000","last_active":"2023-10-09T10:21:26.458075+0000","last_peered":"2023-10-09T10:21:26.458075+0000","last_clean":"2023-10-09T10:21:26.458075+0000","last_became_active":"2023-10-09T10:19:43.274441+0000","last_became_peered":"2023-10-09T10:19:43.274441+0000","last_unstale":"2023-10-09T10:21:26.458075+0000","last_undegraded":"2023-10-09T10:21:26.458075+0000","last_fullsized":"2023-10-09T10:21:26.458075+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":507,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.449068+0000","last_change":"2023-10-09T10:16:18.261069+0000","last_active":"2023-10-09T10:21:26.449068+0000","last_peered":"2023-10-09T10:21:26.449068+0000","last_clean":"2023-10-09T10:21:26.449068+0000","last_became_active":"2023-10-09T10:16:18.260736+0000","last_became_peered":"2023-10-09T10:16:18.260736+0000","last_unstale":"2023-10-09T10:21:26.449068+0000","last_undegraded":"2023-10-09T10:21:26.449068+0000","last_fullsized":"2023-10-09T10:21:26.449068+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:18.217603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":345,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.441318+0000","last_change":"2023-10-09T10:16:18.477779+0000","last_active":"2023-10-09T10:21:26.441318+0000","last_peered":"2023-10-09T10:21:26.441318+0000","last_clean":"2023-10-09T10:21:26.441318+0000","last_became_active":"2023-10-09T10:16:18.477390+0000","last_became_peered":"2023-10-09T10:16:18.477390+0000","last_unstale":"2023-10-09T10:21:26.441318+0000","last_undegraded":"2023-10-09T10:21:26.441318+0000","last_fullsized":"2023-10-09T10:21:26.441318+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:06.576807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"453'7695","reported_seq":7731,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:34.975915+0000","last_change":"2023-10-09T10:19:43.287735+0000","last_active":"2023-10-09T10:21:34.975915+0000","last_peered":"2023-10-09T10:21:34.975915+0000","last_clean":"2023-10-09T10:21:34.975915+0000","last_became_active":"2023-10-09T10:19:43.287454+0000","last_became_peered":"2023-10-09T10:19:43.287454+0000","last_unstale":"2023-10-09T10:21:34.975915+0000","last_undegraded":"2023-10-09T10:21:34.975915+0000","last_fullsized":"2023-10-09T10:21:34.975915+0000","mapping_epoch":424,"log_start":"428'1000","ondisk_log_start":"428'1000","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6695,"log_dups_size":0,"ondisk_log_size":6695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398131200,"num_objects":6075,"num_object_clones":0,"num_object_copies":18225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7695,"num_write_kb":440640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":238,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.438830+0000","last_change":"2023-10-09T10:18:45.347229+0000","last_active":"2023-10-09T10:21:26.438830+0000","last_peered":"2023-10-09T10:21:26.438830+0000","last_clean":"2023-10-09T10:21:26.438830+0000","last_became_active":"2023-10-09T10:18:45.346901+0000","last_became_peered":"2023-10-09T10:18:45.346901+0000","last_unstale":"2023-10-09T10:21:26.438830+0000","last_undegraded":"2023-10-09T10:21:26.438830+0000","last_fullsized":"2023-10-09T10:21:26.438830+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:53.628052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":452,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448919+0000","last_change":"2023-10-09T10:19:43.279293+0000","last_active":"2023-10-09T10:21:26.448919+0000","last_peered":"2023-10-09T10:21:26.448919+0000","last_clean":"2023-10-09T10:21:26.448919+0000","last_became_active":"2023-10-09T10:19:43.278798+0000","last_became_peered":"2023-10-09T10:19:43.278798+0000","last_unstale":"2023-10-09T10:21:26.448919+0000","last_undegraded":"2023-10-09T10:21:26.448919+0000","last_fullsized":"2023-10-09T10:21:26.448919+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:30.954175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":362,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.447806+0000","last_change":"2023-10-09T10:18:34.274789+0000","last_active":"2023-10-09T10:21:26.447806+0000","last_peered":"2023-10-09T10:21:26.447806+0000","last_clean":"2023-10-09T10:21:26.447806+0000","last_became_active":"2023-10-09T10:18:34.274396+0000","last_became_peered":"2023-10-09T10:18:34.274396+0000","last_unstale":"2023-10-09T10:21:26.447806+0000","last_undegraded":"2023-10-09T10:21:26.447806+0000","last_fullsized":"2023-10-09T10:21:26.447806+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:37.788599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.f","version":"453'7815","reported_seq":7852,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T10:21:35.012057+0000","last_change":"2023-10-09T10:19:43.280795+0000","last_active":"2023-10-09T10:21:35.012057+0000","last_peered":"2023-10-09T10:21:35.012057+0000","last_clean":"2023-10-09T10:21:35.012057+0000","last_became_active":"2023-10-09T10:19:43.280338+0000","last_became_peered":"2023-10-09T10:19:43.280338+0000","last_unstale":"2023-10-09T10:21:35.012057+0000","last_undegraded":"2023-10-09T10:21:35.012057+0000","last_fullsized":"2023-10-09T10:21:35.012057+0000","mapping_epoch":424,"log_start":"428'100","ondisk_log_start":"428'100","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7715,"log_dups_size":0,"ondisk_log_size":7715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404291584,"num_objects":6169,"num_object_clones":0,"num_object_copies":18507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7815,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":361,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.439770+0000","last_change":"2023-10-09T10:07:02.359130+0000","last_active":"2023-10-09T10:21:26.439770+0000","last_peered":"2023-10-09T10:21:26.439770+0000","last_clean":"2023-10-09T10:21:26.439770+0000","last_became_active":"2023-10-09T10:07:02.358737+0000","last_became_peered":"2023-10-09T10:07:02.358737+0000","last_unstale":"2023-10-09T10:21:26.439770+0000","last_undegraded":"2023-10-09T10:21:26.439770+0000","last_fullsized":"2023-10-09T10:21:26.439770+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":150,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.449070+0000","last_change":"2023-10-09T10:19:43.297049+0000","last_active":"2023-10-09T10:21:26.449070+0000","last_peered":"2023-10-09T10:21:26.449070+0000","last_clean":"2023-10-09T10:21:26.449070+0000","last_became_active":"2023-10-09T10:19:43.282272+0000","last_became_peered":"2023-10-09T10:19:43.282272+0000","last_unstale":"2023-10-09T10:21:26.449070+0000","last_undegraded":"2023-10-09T10:21:26.449070+0000","last_fullsized":"2023-10-09T10:21:26.449070+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:13:04.873466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.456985+0000","last_change":"2023-10-09T10:19:43.282576+0000","last_active":"2023-10-09T10:21:26.456985+0000","last_peered":"2023-10-09T10:21:26.456985+0000","last_clean":"2023-10-09T10:21:26.456985+0000","last_became_active":"2023-10-09T10:19:43.282091+0000","last_became_peered":"2023-10-09T10:19:43.282091+0000","last_unstale":"2023-10-09T10:21:26.456985+0000","last_undegraded":"2023-10-09T10:21:26.456985+0000","last_fullsized":"2023-10-09T10:21:26.456985+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.458550+0000","last_change":"2023-10-09T10:19:43.274922+0000","last_active":"2023-10-09T10:21:26.458550+0000","last_peered":"2023-10-09T10:21:26.458550+0000","last_clean":"2023-10-09T10:21:26.458550+0000","last_became_active":"2023-10-09T10:19:43.273984+0000","last_became_peered":"2023-10-09T10:19:43.273984+0000","last_unstale":"2023-10-09T10:21:26.458550+0000","last_undegraded":"2023-10-09T10:21:26.458550+0000","last_fullsized":"2023-10-09T10:21:26.458550+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":455,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.449155+0000","last_change":"2023-10-09T10:19:43.278560+0000","last_active":"2023-10-09T10:21:26.449155+0000","last_peered":"2023-10-09T10:21:26.449155+0000","last_clean":"2023-10-09T10:21:26.449155+0000","last_became_active":"2023-10-09T10:19:43.277994+0000","last_became_peered":"2023-10-09T10:19:43.277994+0000","last_unstale":"2023-10-09T10:21:26.449155+0000","last_undegraded":"2023-10-09T10:21:26.449155+0000","last_fullsized":"2023-10-09T10:21:26.449155+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:54:56.214071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":131,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.443846+0000","last_change":"2023-10-09T10:19:43.298662+0000","last_active":"2023-10-09T10:21:26.443846+0000","last_peered":"2023-10-09T10:21:26.443846+0000","last_clean":"2023-10-09T10:21:26.443846+0000","last_became_active":"2023-10-09T10:19:43.282784+0000","last_became_peered":"2023-10-09T10:19:43.282784+0000","last_unstale":"2023-10-09T10:21:26.443846+0000","last_undegraded":"2023-10-09T10:21:26.443846+0000","last_fullsized":"2023-10-09T10:21:26.443846+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.458830+0000","last_change":"2023-10-09T10:19:43.286184+0000","last_active":"2023-10-09T10:21:26.458830+0000","last_peered":"2023-10-09T10:21:26.458830+0000","last_clean":"2023-10-09T10:21:26.458830+0000","last_became_active":"2023-10-09T10:19:43.285867+0000","last_became_peered":"2023-10-09T10:19:43.285867+0000","last_unstale":"2023-10-09T10:21:26.458830+0000","last_undegraded":"2023-10-09T10:21:26.458830+0000","last_fullsized":"2023-10-09T10:21:26.458830+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":127,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.444102+0000","last_change":"2023-10-09T10:19:43.300965+0000","last_active":"2023-10-09T10:21:26.444102+0000","last_peered":"2023-10-09T10:21:26.444102+0000","last_clean":"2023-10-09T10:21:26.444102+0000","last_became_active":"2023-10-09T10:19:43.283465+0000","last_became_peered":"2023-10-09T10:19:43.283465+0000","last_unstale":"2023-10-09T10:21:26.444102+0000","last_undegraded":"2023-10-09T10:21:26.444102+0000","last_fullsized":"2023-10-09T10:21:26.444102+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":274,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.438650+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:21:26.438650+0000","last_peered":"2023-10-09T10:21:26.438650+0000","last_clean":"2023-10-09T10:21:26.438650+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:21:26.438650+0000","last_undegraded":"2023-10-09T10:21:26.438650+0000","last_fullsized":"2023-10-09T10:21:26.438650+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":109,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.448750+0000","last_change":"2023-10-09T10:16:08.240555+0000","last_active":"2023-10-09T10:21:26.448750+0000","last_peered":"2023-10-09T10:21:26.448750+0000","last_clean":"2023-10-09T10:21:26.448750+0000","last_became_active":"2023-10-09T10:16:08.239692+0000","last_became_peered":"2023-10-09T10:16:08.239692+0000","last_unstale":"2023-10-09T10:21:26.448750+0000","last_undegraded":"2023-10-09T10:21:26.448750+0000","last_fullsized":"2023-10-09T10:21:26.448750+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:39.501717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":494,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.438637+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:21:26.438637+0000","last_peered":"2023-10-09T10:21:26.438637+0000","last_clean":"2023-10-09T10:21:26.438637+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:21:26.438637+0000","last_undegraded":"2023-10-09T10:21:26.438637+0000","last_fullsized":"2023-10-09T10:21:26.438637+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":427,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.444341+0000","last_change":"2023-10-09T10:19:43.388243+0000","last_active":"2023-10-09T10:21:26.444341+0000","last_peered":"2023-10-09T10:21:26.444341+0000","last_clean":"2023-10-09T10:21:26.444341+0000","last_became_active":"2023-10-09T10:19:43.283927+0000","last_became_peered":"2023-10-09T10:19:43.283927+0000","last_unstale":"2023-10-09T10:21:26.444341+0000","last_undegraded":"2023-10-09T10:21:26.444341+0000","last_fullsized":"2023-10-09T10:21:26.444341+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":37,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T10:21:26.457533+0000","last_change":"2023-10-09T10:19:43.282681+0000","last_active":"2023-10-09T10:21:26.457533+0000","last_peered":"2023-10-09T10:21:26.457533+0000","last_clean":"2023-10-09T10:21:26.457533+0000","last_became_active":"2023-10-09T10:19:43.282337+0000","last_became_peered":"2023-10-09T10:19:43.282337+0000","last_unstale":"2023-10-09T10:21:26.457533+0000","last_undegraded":"2023-10-09T10:21:26.457533+0000","last_fullsized":"2023-10-09T10:21:26.457533+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6434914328,"num_objects":98190,"num_object_clones":0,"num_object_copies":294570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":102,"num_objects_misplaced":5254,"num_objects_unfound":0,"num_objects_dirty":98190,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124464,"num_write_kb":7124865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":711,"num_bytes_recovered":46596096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19328872448,"data_stored":19328860232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116064,"ondisk_log_size":116064,"up":48,"acting":47,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":16658,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430623,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2879172,"kb_used_data":1962620,"kb_used_omap":0,"kb_used_meta":916544,"kb_avail":90865980,"statfs":{"total":95995035648,"available":93046763520,"internally_reserved":0,"allocated":2009722880,"data_stored":2008992262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":938541056},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.1919999999999999,"15min":1.6899999999999999},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":7.5389999999999997,"5min":7.798,"15min":11.164},"last":0.26200000000000001},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.2450000000000001,"15min":1.6619999999999999},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":8.577,"5min":10.029,"15min":15.974},"last":0.53200000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:21:30 2023","interfaces":[{"interface":"back","average":{"1min":9.1649999999999991,"5min":2.6619999999999999,"15min":2.3159999999999998},"min":{"1min":0.28399999999999997,"5min":0.246,"15min":0.23799999999999999},"max":{"1min":129.53700000000001,"5min":129.53700000000001,"15min":129.53700000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":8.7300000000000004,"5min":2.927,"15min":2.4870000000000001},"min":{"1min":0.30499999999999999,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":128.85599999999999,"5min":128.85599999999999,"15min":128.85599999999999},"last":0.56599999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:21:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5630000000000002,"5min":1.966,"15min":2.0350000000000001},"min":{"1min":0.376,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":10.532999999999999,"5min":42.725999999999999,"15min":42.725999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.0489999999999999,"5min":1.6950000000000001,"15min":2.016},"min":{"1min":0.372,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":8.6739999999999995,"5min":42.752000000000002,"15min":42.752000000000002},"last":0.59099999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":1.542,"15min":1.5529999999999999},"min":{"1min":0.33100000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.436,"5min":12.436,"15min":12.436},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.0710000000000002,"5min":1.6120000000000001,"15min":1.6080000000000001},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":12.161,"5min":12.161,"15min":12.161},"last":0.66500000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":1.1899999999999999,"15min":1.2090000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":7.5140000000000002,"5min":7.8890000000000002,"15min":12.699999999999999},"last":8.3759999999999994},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.425,"15min":1.1870000000000001},"min":{"1min":0.36799999999999999,"5min":0.313,"15min":0.25700000000000001},"max":{"1min":8.6259999999999994,"5min":8.8550000000000004,"15min":12.003},"last":1.3089999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:21:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.631,"15min":1.476},"min":{"1min":0.318,"5min":0.29499999999999998,"15min":0.219},"max":{"1min":10.619,"5min":21.712,"15min":25.277999999999999},"last":4.1399999999999997},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.712,"15min":1.4530000000000001},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.22600000000000001},"max":{"1min":9.3059999999999992,"5min":22.824999999999999,"15min":25.754000000000001},"last":14.983000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:21:16 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.4019999999999999,"15min":1.208},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.26100000000000001},"max":{"1min":8.7140000000000004,"5min":8.9030000000000005,"15min":8.9030000000000005},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.4399999999999999,"5min":1.286,"15min":1.1950000000000001},"min":{"1min":0.35599999999999998,"5min":0.33400000000000002,"15min":0.27200000000000002},"max":{"1min":6.0510000000000002,"5min":6.0510000000000002,"15min":10.625},"last":0.625}]}]},{"osd":6,"up_from":42,"seq":180388626851,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3655872,"kb_used_data":2756600,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":90089280,"statfs":{"total":95995035648,"available":92251422720,"internally_reserved":0,"allocated":2822758400,"data_stored":2822033410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:07 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.639,"15min":1.7629999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.9359999999999999,"5min":44.055,"15min":44.055},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.1150000000000002,"15min":2.0249999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.22},"max":{"1min":8.7330000000000005,"5min":43.869999999999997,"15min":43.869999999999997},"last":0.746}]},{"osd":1,"last update":"Mon Oct 9 10:21:22 2023","interfaces":[{"interface":"back","average":{"1min":8.9260000000000002,"5min":2.8130000000000002,"15min":2.5680000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":155.01400000000001,"5min":155.01400000000001,"15min":155.01400000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":8.5649999999999995,"5min":3.1299999999999999,"15min":2.5459999999999998},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":154.71100000000001,"5min":154.71100000000001,"15min":154.71100000000001},"last":0.77900000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:21:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":1.79,"15min":1.7350000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.9500000000000002,"5min":29.367999999999999,"15min":29.367999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.2889999999999999,"15min":1.9259999999999999},"min":{"1min":0.32400000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3700000000000001,"5min":8.4480000000000004,"15min":14.542999999999999},"last":0.55400000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:20:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.2869999999999999,"15min":1.5449999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":7.4889999999999999,"5min":8.0229999999999997,"15min":8.0229999999999997},"last":0.373},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.585,"15min":1.526},"min":{"1min":0.316,"5min":0.31,"15min":0.31},"max":{"1min":7.4630000000000001,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":0.80800000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:20:55 2023","interfaces":[{"interface":"back","average":{"1min":7.4290000000000003,"5min":2.4100000000000001,"15min":1.3240000000000001},"min":{"1min":0.29199999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":122.041,"5min":122.041,"15min":122.041},"last":1.2689999999999999},{"interface":"front","average":{"1min":7.843,"5min":2.3650000000000002,"15min":1.302},"min":{"1min":0.34999999999999998,"5min":0.33900000000000002,"15min":0.28699999999999998},"max":{"1min":122.13200000000001,"5min":122.13200000000001,"15min":122.13200000000001},"last":9.8320000000000007}]},{"osd":5,"last update":"Mon Oct 9 10:20:52 2023","interfaces":[{"interface":"back","average":{"1min":7.2939999999999996,"5min":2.5379999999999998,"15min":1.4770000000000001},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.23699999999999999},"max":{"1min":121.98999999999999,"5min":121.98999999999999,"15min":121.98999999999999},"last":1.879},{"interface":"front","average":{"1min":7.0640000000000001,"5min":2.1869999999999998,"15min":1.466},"min":{"1min":0.33400000000000002,"5min":0.32900000000000001,"15min":0.25800000000000001},"max":{"1min":122.346,"5min":122.346,"15min":122.346},"last":0.39700000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:21:22 2023","interfaces":[{"interface":"back","average":{"1min":7.5069999999999997,"5min":2.423,"15min":1.5},"min":{"1min":0.39800000000000002,"5min":0.29199999999999998,"15min":0.252},"max":{"1min":130.66999999999999,"5min":130.66999999999999,"15min":130.66999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":7.5469999999999997,"5min":2.4710000000000001,"15min":1.544},"min":{"1min":0.45500000000000002,"5min":0.35699999999999998,"15min":0.30099999999999999},"max":{"1min":130.702,"5min":130.702,"15min":130.702},"last":0.63800000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253164,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2379876,"kb_used_data":1595484,"kb_used_omap":9,"kb_used_meta":784374,"kb_avail":91365276,"statfs":{"total":95995035648,"available":93558042624,"internally_reserved":0,"allocated":1633775616,"data_stored":1632189363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":803199977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:20:38 2023","interfaces":[{"interface":"back","average":{"1min":6.7750000000000004,"5min":2.125,"15min":2.472},"min":{"1min":0.20599999999999999,"5min":0.188,"15min":0.183},"max":{"1min":108.87,"5min":108.87,"15min":108.87},"last":0.45900000000000002},{"interface":"front","average":{"1min":7.8109999999999999,"5min":2.702,"15min":2.9129999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":107.82899999999999,"5min":107.82899999999999,"15min":107.82899999999999},"last":0.64800000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:21:05 2023","interfaces":[{"interface":"back","average":{"1min":2.181,"5min":1.6240000000000001,"15min":2.5230000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":6.1319999999999997,"5min":14.223000000000001,"15min":23.105},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.496,"5min":1.024,"15min":2.109},"min":{"1min":0.26400000000000001,"5min":0.26000000000000001,"15min":0.22500000000000001},"max":{"1min":5.1159999999999997,"5min":6.6470000000000002,"15min":28.024999999999999},"last":0.61899999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:20:50 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.046,"15min":1.133},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1580000000000004,"5min":6.673,"15min":6.673},"last":0.57699999999999996},{"interface":"front","average":{"1min":2.024,"5min":1.256,"15min":1.5609999999999999},"min":{"1min":0.29599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.2859999999999996,"5min":7.2160000000000002,"15min":7.2160000000000002},"last":0.51600000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.1379999999999999,"15min":1.3799999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":5.2389999999999999,"5min":6.7300000000000004,"15min":21.620000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.5960000000000001,"15min":1.8520000000000001},"min":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.308},"max":{"1min":5.1109999999999998,"5min":10.497999999999999,"15min":12.904},"last":6.1459999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:20:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.5029999999999999,"15min":2.476},"min":{"1min":0.51000000000000001,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":5.0620000000000003,"5min":13.147,"15min":92.177999999999997},"last":4.1879999999999997},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.524,"15min":2.5750000000000002},"min":{"1min":0.35099999999999998,"5min":0.32000000000000001,"15min":0.29099999999999998},"max":{"1min":4.8360000000000003,"5min":13.170999999999999,"15min":91.869},"last":1.4750000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.2070000000000001,"15min":1.7869999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.2060000000000004,"5min":6.8540000000000001,"15min":21.670999999999999},"last":3.9420000000000002},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.4870000000000001,"15min":2.0169999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35099999999999998},"max":{"1min":3.8380000000000001,"5min":9.1660000000000004,"15min":14.471},"last":4.149}]},{"osd":7,"last update":"Mon Oct 9 10:20:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.486,"15min":2.1099999999999999},"min":{"1min":0.38200000000000001,"5min":0.35899999999999999,"15min":0.32300000000000001},"max":{"1min":4.8959999999999999,"5min":7.8010000000000002,"15min":16.331},"last":6.0730000000000004},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.391,"15min":1.8939999999999999},"min":{"1min":0.42799999999999999,"5min":0.38500000000000001,"15min":0.371},"max":{"1min":5.2649999999999997,"5min":10.627000000000001,"15min":21.593},"last":0.67000000000000004}]}]},{"osd":0,"up_from":79,"seq":339302416689,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3788040,"kb_used_data":3130880,"kb_used_omap":28,"kb_used_meta":657123,"kb_avail":89957112,"statfs":{"total":95995035648,"available":92116082688,"internally_reserved":0,"allocated":3206021120,"data_stored":3204398889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":672894800},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.3979999999999999,"15min":2.081},"min":{"1min":0.23100000000000001,"5min":0.19800000000000001,"15min":0.16600000000000001},"max":{"1min":1.8700000000000001,"5min":54.518000000000001,"15min":54.518000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.4610000000000001,"15min":3.093},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":2.6579999999999999,"5min":56.950000000000003,"15min":56.950000000000003},"last":0.46400000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:21:15 2023","interfaces":[{"interface":"back","average":{"1min":4.8840000000000003,"5min":2.0659999999999998,"15min":2.2320000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":47.215000000000003,"5min":53.988999999999997,"15min":53.988999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":5.2249999999999996,"5min":2.1419999999999999,"15min":2.2610000000000001},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":47.000999999999998,"5min":54.119,"15min":54.119},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:20:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.669,"15min":1.3400000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":16.879999999999999,"5min":56.984999999999999,"15min":56.984999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.385,"5min":1.6379999999999999,"15min":1.264},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.2469999999999999,"5min":56.475999999999999,"15min":56.475999999999999},"last":1.931}]},{"osd":4,"last update":"Mon Oct 9 10:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.157,"15min":2.2330000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":16.997,"5min":57.719000000000001,"15min":57.719000000000001},"last":0.379},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.9259999999999999,"15min":2.1659999999999999},"min":{"1min":0.35199999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.3689999999999998,"5min":56.378,"15min":56.378},"last":5.7759999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.9630000000000001,"15min":3.3039999999999998},"min":{"1min":0.32700000000000001,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.4770000000000003,"5min":56.445999999999998,"15min":229.589},"last":5.726},{"interface":"front","average":{"1min":2.198,"5min":2.0499999999999998,"15min":2.758},"min":{"1min":0.43099999999999999,"5min":0.31900000000000001,"15min":0.27000000000000002},"max":{"1min":8.7530000000000001,"5min":54.411000000000001,"15min":228.88900000000001},"last":0.66700000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2.548,"5min":2.2679999999999998,"15min":2.2170000000000001},"min":{"1min":0.47099999999999997,"5min":0.34000000000000002,"15min":0.33100000000000002},"max":{"1min":8.7810000000000006,"5min":54.482999999999997,"15min":54.482999999999997},"last":0.82099999999999995},{"interface":"front","average":{"1min":2.6179999999999999,"5min":2.3679999999999999,"15min":2.6930000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.954999999999998,"5min":57.753,"15min":57.753},"last":0.73699999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":2.1190000000000002,"15min":2.6070000000000002},"min":{"1min":0.38600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.4020000000000001,"5min":56.411999999999999,"15min":56.411999999999999},"last":7.9880000000000004},{"interface":"front","average":{"1min":2.4180000000000001,"5min":2.2370000000000001,"15min":2.1970000000000001},"min":{"1min":0.438,"5min":0.438,"15min":0.36499999999999999},"max":{"1min":8.8010000000000002,"5min":54.451999999999998,"15min":54.451999999999998},"last":0.63400000000000001}]}]},{"osd":2,"up_from":217,"seq":932007903424,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2968352,"kb_used_data":2388120,"kb_used_omap":83,"kb_used_meta":580140,"kb_avail":90776800,"statfs":{"total":95995035648,"available":92955443200,"internally_reserved":0,"allocated":2445434880,"data_stored":2443971402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":594063855},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8009999999999999,"5min":1.454,"15min":1.8959999999999999},"min":{"1min":0.219,"5min":0.19,"15min":0.17899999999999999},"max":{"1min":5.1779999999999999,"5min":33.289000000000001,"15min":33.289000000000001},"last":1.2170000000000001},{"interface":"front","average":{"1min":2.2490000000000001,"5min":1.609,"15min":2.6120000000000001},"min":{"1min":0.24199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":7.5949999999999998,"5min":33.347999999999999,"15min":33.347999999999999},"last":8.0299999999999994}]},{"osd":1,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.3560000000000001,"15min":2.2210000000000001},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.6909999999999998,"5min":34.893999999999998,"15min":34.893999999999998},"last":1.2450000000000001},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.4370000000000001,"15min":2.3490000000000002},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":5.492,"5min":34.962000000000003,"15min":34.962000000000003},"last":0.438}]},{"osd":3,"last update":"Mon Oct 9 10:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.577,"5min":1.319,"15min":1.0209999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.54,"5min":35.043999999999997,"15min":35.043999999999997},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.6040000000000001,"15min":1.21},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.0339999999999998,"5min":50.406999999999996,"15min":50.406999999999996},"last":0.56200000000000006}]},{"osd":4,"last update":"Mon Oct 9 10:21:04 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.87,"15min":1.3640000000000001},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.26600000000000001},"max":{"1min":3.6619999999999999,"5min":4.9279999999999999,"15min":18.873999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.534,"5min":1.3520000000000001,"15min":1.5389999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0810000000000004,"5min":24.352,"15min":24.352},"last":2.0009999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":1.3859999999999999,"15min":1.9650000000000001},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.27700000000000002},"max":{"1min":3.7490000000000001,"5min":24.23,"15min":57.298000000000002},"last":6.5750000000000002},{"interface":"front","average":{"1min":1.175,"5min":1.226,"15min":1.8500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.315},"max":{"1min":3.3199999999999998,"5min":6.5449999999999999,"15min":57.828000000000003},"last":2.4580000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3610000000000002,"5min":1.282,"15min":1.8029999999999999},"min":{"1min":0.40100000000000002,"5min":0.36599999999999999,"15min":0.33100000000000002},"max":{"1min":7.8600000000000003,"5min":7.8600000000000003,"15min":18.943999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.929,"5min":1.554,"15min":1.931},"min":{"1min":0.46100000000000002,"5min":0.41799999999999998,"15min":0.33300000000000002},"max":{"1min":9.8399999999999999,"5min":24.379000000000001,"15min":24.379000000000001},"last":6.6159999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:21:09 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.875,"15min":2.0600000000000001},"min":{"1min":0.435,"5min":0.41099999999999998,"15min":0.308},"max":{"1min":13.821,"5min":35.008000000000003,"15min":35.008000000000003},"last":8.1210000000000004},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.4630000000000001,"15min":2.1150000000000002},"min":{"1min":0.44900000000000001,"5min":0.39300000000000002,"15min":0.35899999999999999},"max":{"1min":3.6909999999999998,"5min":24.323,"15min":24.323},"last":6.6399999999999997}]}]},{"osd":3,"up_from":352,"seq":1511828488264,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1470076,"kb_used_data":1244468,"kb_used_omap":15,"kb_used_meta":225584,"kb_avail":92275076,"statfs":{"total":95995035648,"available":94489677824,"internally_reserved":0,"allocated":1274335232,"data_stored":1272434039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":230998470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.84399999999999997,"15min":1.0309999999999999},"min":{"1min":0.224,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":3.3140000000000001,"5min":7.4050000000000002,"15min":7.4050000000000002},"last":0.23300000000000001},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.2789999999999999,"15min":1.0449999999999999},"min":{"1min":0.23899999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.3070000000000004,"5min":8.3070000000000004,"15min":8.3070000000000004},"last":0.379}]},{"osd":1,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":8.6440000000000001,"5min":2.6869999999999998,"15min":1.6359999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":105.401,"5min":105.401,"15min":105.401},"last":0.35999999999999999},{"interface":"front","average":{"1min":9.5510000000000002,"5min":2.6709999999999998,"15min":1.3440000000000001},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":104.914,"5min":104.914,"15min":104.914},"last":1.5089999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":1.359,"15min":1.911},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.0919999999999996,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":1.3200000000000001},{"interface":"front","average":{"1min":0.90200000000000002,"5min":1.107,"15min":1.1830000000000001},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1379999999999999,"5min":16.018000000000001,"15min":16.018000000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2349999999999999,"5min":1.4059999999999999,"15min":0.90000000000000002},"min":{"1min":0.29799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":21.831,"5min":21.831,"15min":21.831},"last":0.92500000000000004},{"interface":"front","average":{"1min":3.3700000000000001,"5min":1.7150000000000001,"15min":1.423},"min":{"1min":0.39400000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":22.097999999999999,"5min":22.097999999999999,"15min":22.097999999999999},"last":2.0899999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.097,"15min":0.82899999999999996},"min":{"1min":0.373,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.3109999999999999,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":2.8879999999999999},{"interface":"front","average":{"1min":2.4220000000000002,"5min":1.5109999999999999,"15min":0.97399999999999998},"min":{"1min":0.433,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.2859999999999996,"5min":8.2859999999999996,"15min":8.2859999999999996},"last":5.2320000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.3700000000000001,"15min":1.3420000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.218,"5min":7.9219999999999997,"15min":7.9219999999999997},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.502,"5min":1.387,"15min":1.0589999999999999},"min":{"1min":0.45300000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.5789999999999997,"5min":10.698,"15min":10.698},"last":4.6340000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7690000000000001,"5min":1.7150000000000001,"15min":1.0720000000000001},"min":{"1min":0.39900000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":8.2680000000000007,"5min":10.459,"15min":10.459},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.635,"5min":1.6180000000000001,"15min":1.4370000000000001},"min":{"1min":0.41299999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":4.5300000000000002,"5min":8.827,"15min":8.827},"last":4.9260000000000002}]}]},{"osd":4,"up_from":266,"seq":1142461300893,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4268160,"kb_used_data":3937144,"kb_used_omap":35,"kb_used_meta":330908,"kb_avail":89476992,"statfs":{"total":95995035648,"available":91624439808,"internally_reserved":0,"allocated":4031635456,"data_stored":4030908876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":338850030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":9.2140000000000004,"5min":3.3290000000000002,"15min":1.8440000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":97.721999999999994,"5min":97.721999999999994,"15min":97.721999999999994},"last":3.754},{"interface":"front","average":{"1min":10.239000000000001,"5min":3.8610000000000002,"15min":2.3599999999999999},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":97.915999999999997,"5min":97.915999999999997,"15min":97.915999999999997},"last":10.526}]},{"osd":1,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":2.2160000000000002,"15min":1.944},"min":{"1min":0.34399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.281000000000001,"5min":71.457999999999998,"15min":71.457999999999998},"last":10.427},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.8440000000000001,"15min":1.776},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":6.7599999999999998,"5min":71.658000000000001,"15min":71.658000000000001},"last":3.5950000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.254,"15min":1.462},"min":{"1min":0.33800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.7279999999999998,"5min":6.7279999999999998,"15min":9.2469999999999999},"last":4.3339999999999996},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.2370000000000001,"15min":1.48},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":5.4950000000000001,"5min":5.4950000000000001,"15min":11.778},"last":4.4450000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.476,"15min":1.6100000000000001},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.3070000000000004,"5min":9.6170000000000009,"15min":9.6170000000000009},"last":12.074},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.246,"15min":1.125},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.6929999999999996,"5min":6.6929999999999996,"15min":6.6929999999999996},"last":3.6339999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":1.3939999999999999,"15min":1.472},"min":{"1min":0.25700000000000001,"5min":0.252,"15min":0.23799999999999999},"max":{"1min":9.8510000000000009,"5min":11.913,"15min":99.585999999999999},"last":3.5379999999999998},{"interface":"front","average":{"1min":1.478,"5min":1.05,"15min":1.1990000000000001},"min":{"1min":0.27700000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.632,"5min":10.632,"15min":99.384},"last":2.5249999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.151,"15min":0.94899999999999995},"min":{"1min":0.32200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.662000000000001,"5min":10.662000000000001,"15min":10.662000000000001},"last":2.488},{"interface":"front","average":{"1min":2.165,"5min":1.284,"15min":1.0580000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27000000000000002},"max":{"1min":7.1340000000000003,"5min":7.96,"15min":12.255000000000001},"last":10.465}]},{"osd":7,"last update":"Mon Oct 9 10:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.2669999999999999,"15min":1.115},"min":{"1min":0.315,"5min":0.307,"15min":0.307},"max":{"1min":7.1779999999999999,"5min":7.9850000000000003,"15min":12.237},"last":10.500999999999999},{"interface":"front","average":{"1min":1.9830000000000001,"5min":1.1950000000000001,"15min":1.125},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":9.9220000000000006,"5min":9.9220000000000006,"15min":14.964},"last":3.5760000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823078,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2879596,"kb_used_data":1970084,"kb_used_omap":0,"kb_used_meta":909504,"kb_avail":90865556,"statfs":{"total":95995035648,"available":93046329344,"internally_reserved":0,"allocated":2017366016,"data_stored":2016655104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931332096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,0,0,1,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:20:48 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":1.1120000000000001,"15min":1.536},"min":{"1min":0.24199999999999999,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":3.7050000000000001,"5min":8.0180000000000007,"15min":16.079000000000001},"last":2.694},{"interface":"front","average":{"1min":1.359,"5min":1.133,"15min":1.5489999999999999},"min":{"1min":0.247,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":9.657,"5min":9.657,"15min":15.907},"last":2.8860000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:21:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":2.008,"15min":1.962},"min":{"1min":0.28999999999999998,"5min":0.26200000000000001,"15min":0.23699999999999999},"max":{"1min":9.8059999999999992,"5min":29.370000000000001,"15min":29.370000000000001},"last":2.718},{"interface":"front","average":{"1min":2.339,"5min":1.9910000000000001,"15min":1.893},"min":{"1min":0.315,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.981,"5min":29.065000000000001,"15min":29.065000000000001},"last":1.9990000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:21:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.5,"15min":1.835},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27200000000000002},"max":{"1min":5.9249999999999998,"5min":12.553000000000001,"15min":15.973000000000001},"last":2.75},{"interface":"front","average":{"1min":3.044,"5min":1.655,"15min":1.6060000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":12.026,"5min":12.026,"15min":15.929},"last":2.8410000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":1.675,"15min":1.5409999999999999},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.029,"5min":10.029,"15min":10.029},"last":10.561999999999999},{"interface":"front","average":{"1min":1.306,"5min":1.627,"15min":1.522},"min":{"1min":0.39400000000000002,"5min":0.315,"15min":0.315},"max":{"1min":9.7439999999999998,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":4.9379999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:20:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.3420000000000001,"15min":1.0680000000000001},"min":{"1min":0.308,"5min":0.26400000000000001,"15min":0.24299999999999999},"max":{"1min":5.5190000000000001,"5min":8.468,"15min":9.9819999999999993},"last":2.7850000000000001},{"interface":"front","average":{"1min":1.877,"5min":1.498,"15min":0.97099999999999997},"min":{"1min":0.371,"5min":0.29299999999999998,"15min":0.23599999999999999},"max":{"1min":9.7970000000000006,"5min":9.7970000000000006,"15min":9.7970000000000006},"last":0.58599999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:20:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.3720000000000001,"15min":1.2589999999999999},"min":{"1min":0.36699999999999999,"5min":0.318,"15min":0.25},"max":{"1min":5.6740000000000004,"5min":12.772,"15min":13.483000000000001},"last":10.492000000000001},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.4850000000000001,"15min":1.2829999999999999},"min":{"1min":0.38800000000000001,"5min":0.28399999999999997,"15min":0.22600000000000001},"max":{"1min":6.399,"5min":9.4160000000000004,"15min":18.600999999999999},"last":3.0569999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:21:22 2023","interfaces":[{"interface":"back","average":{"1min":2.7959999999999998,"5min":1.712,"15min":1.343},"min":{"1min":0.40000000000000002,"5min":0.32700000000000001,"15min":0.25600000000000001},"max":{"1min":9.8789999999999996,"5min":9.8789999999999996,"15min":16.030999999999999},"last":14.006},{"interface":"front","average":{"1min":1.589,"5min":1.5109999999999999,"15min":0.97399999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.26800000000000002},"max":{"1min":11.956,"5min":11.956,"15min":11.956},"last":0.441}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3176595456,"data_stored":3176595456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1617297408,"data_stored":1617297408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2432892928,"data_stored":2432892928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1267535872,"data_stored":1267531800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4009955328,"data_stored":4009951256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2013200384,"data_stored":2013200384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2810777600,"data_stored":2810773528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2000617472,"data_stored":2000617472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:21:38.997 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:21:38.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-10-09T10:21:39.005 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:39.343 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:39.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:39.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:39.457 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:39.527 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:21:39.527 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:21:39.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:39.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:39 smithi012 ceph-mon[147926]: pgmap v1270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 102/294642 objects degraded (0.035%); 5254/294642 objects misplaced (1.783%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:21:39.582 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:39 smithi012 ceph-mon[152263]: pgmap v1270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 102/294642 objects degraded (0.035%); 5254/294642 objects misplaced (1.783%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:21:39.655 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:39 smithi110 ceph-mon[136032]: pgmap v1270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 102/294642 objects degraded (0.035%); 5254/294642 objects misplaced (1.783%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:21:39.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:39.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:39.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:39.767 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:39.838 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:39.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:21:39.885 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:21:39.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:39.939 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:39.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:40.041 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:40.138 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:40.176 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:40.251 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:40.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:40.362 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:40.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:40 smithi012 ceph-mon[147926]: from='client.56686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:21:40.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:40 smithi012 ceph-mon[152263]: from='client.56686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:21:40.437 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:40.511 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:40.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:40.620 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:40.667 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:40.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:40 smithi110 ceph-mon[136032]: from='client.56686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:21:41.372 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:41 smithi012 ceph-mon[147926]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 102/292521 objects degraded (0.035%); 5254/292521 objects misplaced (1.796%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:21:41.477 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:41.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:41.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:41.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:41.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:41 smithi012 ceph-mon[152263]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 102/292521 objects degraded (0.035%); 5254/292521 objects misplaced (1.796%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:21:41.671 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:41 smithi110 ceph-mon[136032]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 102/292521 objects degraded (0.035%); 5254/292521 objects misplaced (1.796%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:21:41.807 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:41.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:41.921 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:41.979 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:42.064 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:42.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:42.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:42.120 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:42.152 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:42.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:42.232 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:42.290 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:42.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:42.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:42.387 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3938387522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:42.387 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:42.396 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:42.400 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:21:42.470 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:42.511 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3938387522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:42.511 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:42.511 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3938387522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:42.511 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:42.563 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:42.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:42.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:42.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:42.752 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:42.868 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:42.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:42.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:42.980 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:43.058 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:43.429 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 10 pg_num to 26 2023-10-09T10:21:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 225/284607 objects degraded (0.079%); 5020/284607 objects misplaced (1.764%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:21:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 225/284607 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: failed to encode map e454 with expected crc 2023-10-09T10:21:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: failed to encode map e454 with expected crc 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4219253068' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 225/284607 objects degraded (0.079%); 5020/284607 objects misplaced (1.764%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 225/284607 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: failed to encode map e454 with expected crc 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T10:21:43.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: failed to encode map e454 with expected crc 2023-10-09T10:21:43.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4219253068' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:21:43.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:21:43.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 225/284607 objects degraded (0.079%); 5020/284607 objects misplaced (1.764%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T10:21:43.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 225/284607 objects degraded (0.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:43.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:43.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: failed to encode map e454 with expected crc 2023-10-09T10:21:43.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:43.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T10:21:43.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: failed to encode map e454 with expected crc 2023-10-09T10:21:43.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4219253068' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:21:43.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:21:43.986 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:43.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:44.099 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:44.164 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:44.255 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:44.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:44.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:44.426 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.427 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:21:44.427 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T10:21:44.427 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.427 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.427 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:21:44.427 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:21:44.427 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:44.427 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:44.452 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:21:44.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T10:21:44.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:21:44.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:21:44.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:44.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:44.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:21:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T10:21:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: failed to encode map e455 with expected crc 2023-10-09T10:21:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:21:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:21:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:44.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:45.096 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:45.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:45.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:45.231 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:45.269 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:45.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:45.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:45.362 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:21:45.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:45.470 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:45.584 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:45.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:45.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:45.639 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[152263]: pgmap v1275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 225/283335 objects degraded (0.079%); 5020/283335 objects misplaced (1.772%); 888 KiB/s, 13 objects/s recovering 2023-10-09T10:21:45.639 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[152263]: failed to encode map e456 with expected crc 2023-10-09T10:21:45.639 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:21:45.639 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[152263]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T10:21:45.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[152263]: failed to encode map e456 with expected crc 2023-10-09T10:21:45.640 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[147926]: pgmap v1275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 225/283335 objects degraded (0.079%); 5020/283335 objects misplaced (1.772%); 888 KiB/s, 13 objects/s recovering 2023-10-09T10:21:45.640 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[147926]: failed to encode map e456 with expected crc 2023-10-09T10:21:45.640 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:21:45.641 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[147926]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T10:21:45.641 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:45 smithi012 ceph-mon[147926]: failed to encode map e456 with expected crc 2023-10-09T10:21:45.653 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:21:45.653 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:21:45.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:45 smithi110 ceph-mon[136032]: pgmap v1275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 225/283335 objects degraded (0.079%); 5020/283335 objects misplaced (1.772%); 888 KiB/s, 13 objects/s recovering 2023-10-09T10:21:45.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:45 smithi110 ceph-mon[136032]: failed to encode map e456 with expected crc 2023-10-09T10:21:45.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:45 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:21:45.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:45 smithi110 ceph-mon[136032]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T10:21:45.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:45 smithi110 ceph-mon[136032]: failed to encode map e456 with expected crc 2023-10-09T10:21:45.713 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:45.767 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:45.896 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:45.933 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:45.973 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:45.986 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:46.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:46.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:46.015 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:46.045 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:46.081 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:46.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:46.110 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:46.144 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:46.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:21:46.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:21:46.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:21:46.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:21:46.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:21:46.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:21:46.151 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:21:46.151 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:21:46.173 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:46.206 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:21:46.504 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:46.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:46.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:46.568 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:46.617 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:46.665 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:46 smithi110 ceph-mon[136032]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:46 smithi110 ceph-mon[136032]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T10:21:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:46 smithi110 ceph-mon[136032]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:46 smithi110 ceph-mon[136032]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:46 smithi110 ceph-mon[136032]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:46 smithi110 ceph-mon[136032]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:46 smithi110 ceph-mon[136032]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:46 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2042277569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:46.719 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:46.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:46.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:46.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[147926]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[147926]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T10:21:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[147926]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[147926]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[147926]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[147926]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[147926]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2042277569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[152263]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[152263]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T10:21:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[152263]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[152263]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[152263]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[152263]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[152263]: failed to encode map e457 with expected crc 2023-10-09T10:21:46.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:46 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2042277569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:46.894 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:46.973 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:46.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:46.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:47.084 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:47.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:47.210 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:47.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:47.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:47.378 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:47.480 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:47.490 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:47.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:47.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:47.503 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:47 smithi012 ceph-mon[147926]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 10 unknown, 82 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/223401 objects degraded (0.143%); 4794/223401 objects misplaced (2.146%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T10:21:47.503 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:47 smithi012 ceph-mon[152263]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 10 unknown, 82 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/223401 objects degraded (0.143%); 4794/223401 objects misplaced (2.146%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T10:21:47.604 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:47.654 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:47.724 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:47.728 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:47 smithi110 ceph-mon[136032]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 10 unknown, 82 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/223401 objects degraded (0.143%); 4794/223401 objects misplaced (2.146%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T10:21:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:47.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:47.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:47.882 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:47.958 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:47.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:48.068 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:48.114 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:48.156 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:21:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:21:47] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.33.4" 2023-10-09T10:21:48.176 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:48.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:48.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:48.286 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:48.344 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:48.414 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:48.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:48.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:48.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:48.538 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:21:48.584 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:48.874 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 319/223401 objects degraded (0.143%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:48.875 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:48.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:48.891 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 319/223401 objects degraded (0.143%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:48.891 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 319/223401 objects degraded (0.143%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:48.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:49.041 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:49.100 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:49.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:49.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:49.161 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:21:49.161 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:21:49.161 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T10:21:49.161 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:21:49.162 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:21:49.162 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T10:21:49.195 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:21:49.195 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:21:49.195 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:21:49.195 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:21:49.196 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:21:49.211 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:49.263 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:49.324 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:49.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:49.427 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:49.476 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:49.499 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:21:49.662 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:49.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:49.685 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:49.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:49 smithi110 ceph-mon[136032]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 8 unknown, 84 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/222021 objects degraded (0.144%); 4794/222021 objects misplaced (2.159%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:21:49.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1299219953' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:49.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:49 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:49.745 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:21:49.745 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:49 smithi012 ceph-mon[152263]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 8 unknown, 84 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/222021 objects degraded (0.144%); 4794/222021 objects misplaced (2.159%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:21:49.746 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1299219953' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:49.746 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:49 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:49.746 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:49 smithi012 ceph-mon[147926]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 8 unknown, 84 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/222021 objects degraded (0.144%); 4794/222021 objects misplaced (2.159%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:21:49.746 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1299219953' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:49.746 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:49 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:21:49.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:49.837 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:49.903 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:49.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:49.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:50.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:50.101 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:50.278 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:50.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:50.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:50.385 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:50.453 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:50.549 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:50.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:50.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:50.659 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:50.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[152263]: 10.2 scrub starts 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:50 smithi012 ceph-mon[147926]: 10.2 scrub starts 2023-10-09T10:21:50.709 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:21:50.709 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:21:50.709 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T10:21:50.709 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.709 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.709 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.709 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.709 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.710 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: failed to encode map e458 with expected crc 2023-10-09T10:21:50.710 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:50 smithi110 ceph-mon[136032]: 10.2 scrub starts 2023-10-09T10:21:50.714 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:50.780 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:50.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:50.857 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:50.889 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:50.990 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:51.173 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:51.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:51.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:51.189 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:51.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:51.333 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:51.347 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:51.369 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:51.416 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:51.454 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:51.461 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:51.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:51.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:51.488 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:51.534 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:51.571 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:51.572 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:51.612 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:51.622 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:21:51.622 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:21:51.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:21:51.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:21:51.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:21:51.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:21:51.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:21:51.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:21:51.632 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:51.700 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:51.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:51.714 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:51 smithi012 ceph-mon[147926]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 6 unknown, 86 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/219777 objects degraded (0.145%); 4794/219777 objects misplaced (2.181%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:21:51.715 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:51 smithi012 ceph-mon[147926]: 10.0 scrub starts 2023-10-09T10:21:51.715 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3668324253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:51.716 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:51 smithi012 ceph-mon[152263]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 6 unknown, 86 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/219777 objects degraded (0.145%); 4794/219777 objects misplaced (2.181%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:21:51.716 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:51 smithi012 ceph-mon[152263]: 10.0 scrub starts 2023-10-09T10:21:51.716 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3668324253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:51.808 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:51.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:51 smithi110 ceph-mon[136032]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 peering, 6 unknown, 86 active+clean; 4.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 319/219777 objects degraded (0.145%); 4794/219777 objects misplaced (2.181%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T10:21:51.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:51 smithi110 ceph-mon[136032]: 10.0 scrub starts 2023-10-09T10:21:51.859 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3668324253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:51.867 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:51.920 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:51.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:52.033 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:52.089 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:52.163 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:52.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:52.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:52.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:21:52.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:52.377 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:52.645 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:52.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:52.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:52.838 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:53.073 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:53.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:53.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:53.128 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:21:53.128 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:21:53.128 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:21:53.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:21:53.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:53.287 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:21:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:53.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:21:53.393 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:21:53.394 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:21:53.395 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:53.481 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:53 smithi012 ceph-mon[147926]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 443/261594 objects degraded (0.169%); 4534/261594 objects misplaced (1.733%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:21:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:53 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:21:53.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:53 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 443/261594 objects degraded (0.169%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:53 smithi012 ceph-mon[152263]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 443/261594 objects degraded (0.169%); 4534/261594 objects misplaced (1.733%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:21:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:53 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:21:53.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:53 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 443/261594 objects degraded (0.169%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:53.777 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:53.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:53.883 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:53.928 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:53 smithi110 ceph-mon[136032]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 443/261594 objects degraded (0.169%); 4534/261594 objects misplaced (1.733%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T10:21:53.928 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:53 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:21:53.928 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:53 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 443/261594 objects degraded (0.169%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:53.934 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:54.009 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:54.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:54.079 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:21:54.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:54.173 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:54.175 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:54.247 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:54.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:54.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:54.324 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:21:54.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:54.448 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:54.527 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:54.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:54.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:54.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:54 smithi012 ceph-mon[152263]: 10.5 scrub starts 2023-10-09T10:21:54.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:54 smithi012 ceph-mon[152263]: 10.7 scrub starts 2023-10-09T10:21:54.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:54 smithi012 ceph-mon[147926]: 10.5 scrub starts 2023-10-09T10:21:54.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:54 smithi012 ceph-mon[147926]: 10.7 scrub starts 2023-10-09T10:21:54.717 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:54.787 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:54.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:54.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:54.899 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:54.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:54 smithi110 ceph-mon[136032]: 10.5 scrub starts 2023-10-09T10:21:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:54 smithi110 ceph-mon[136032]: 10.7 scrub starts 2023-10-09T10:21:54.965 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:54.981 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:55.453 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:55.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:55.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:55.567 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:55.595 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:55 smithi110 ceph-mon[136032]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 158 MiB/s wr, 3.57k op/s; 443/259779 objects degraded (0.171%); 4534/259779 objects misplaced (1.745%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:21:55.596 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:55 smithi110 ceph-mon[136032]: 10.12 scrub starts 2023-10-09T10:21:55.628 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:55.648 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:55.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:55 smithi012 ceph-mon[147926]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 158 MiB/s wr, 3.57k op/s; 443/259779 objects degraded (0.171%); 4534/259779 objects misplaced (1.745%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:21:55.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:55 smithi012 ceph-mon[147926]: 10.12 scrub starts 2023-10-09T10:21:55.737 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:55 smithi012 ceph-mon[152263]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 158 MiB/s wr, 3.57k op/s; 443/259779 objects degraded (0.171%); 4534/259779 objects misplaced (1.745%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T10:21:55.737 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:55 smithi012 ceph-mon[152263]: 10.12 scrub starts 2023-10-09T10:21:55.781 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:55.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:55.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:55.957 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:56.076 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:56.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:56.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:21:56.238 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:56.358 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:21:56.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:56.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:21:56.469 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:56.533 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:56.544 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:21:56.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/593469697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:56.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:56 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:56.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/593469697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:56.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:56 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:56.704 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:56.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:56.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:56.815 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:56.895 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/593469697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:56.896 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:56 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:21:56.901 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:57.005 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:21:57.005 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:21:57.033 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:57.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:57.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:57.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:21:57.192 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:57.345 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:21:57.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:57.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:21:57.434 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:21:57.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:57.468 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:21:57.509 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:21:57.513 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:57.544 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:21:57.583 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:21:57.609 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:57.615 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:21:57.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:57.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:57.617 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:21:57.646 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:21:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[147926]: pgmap v1284: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 130 MiB/s wr, 3.39k op/s; 642/246687 objects degraded (0.260%); 4215/246687 objects misplaced (1.709%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:21:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[147926]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T10:21:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[147926]: failed to encode map e459 with expected crc 2023-10-09T10:21:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[147926]: failed to encode map e459 with expected crc 2023-10-09T10:21:57.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[147926]: 10.10 deep-scrub starts 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/329813087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[152263]: pgmap v1284: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 130 MiB/s wr, 3.39k op/s; 642/246687 objects degraded (0.260%); 4215/246687 objects misplaced (1.709%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[152263]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[152263]: failed to encode map e459 with expected crc 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[152263]: failed to encode map e459 with expected crc 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[152263]: 10.10 deep-scrub starts 2023-10-09T10:21:57.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/329813087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:57.684 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:21:57.684 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:21:57.684 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:21:57.684 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:21:57.684 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:21:57.685 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:21:57.685 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:21:57.685 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:21:57.685 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:21:57.717 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:57.827 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:57 smithi110 ceph-mon[136032]: pgmap v1284: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 130 MiB/s wr, 3.39k op/s; 642/246687 objects degraded (0.260%); 4215/246687 objects misplaced (1.709%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:21:57.827 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:57 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:21:57.828 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:57 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:21:57.828 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:57 smithi110 ceph-mon[136032]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T10:21:57.828 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:57 smithi110 ceph-mon[136032]: failed to encode map e459 with expected crc 2023-10-09T10:21:57.828 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:57 smithi110 ceph-mon[136032]: failed to encode map e459 with expected crc 2023-10-09T10:21:57.828 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:57 smithi110 ceph-mon[136032]: 10.10 deep-scrub starts 2023-10-09T10:21:57.828 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/329813087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:21:57.843 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:57.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:57.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:57.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:58.058 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:58.148 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:21:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:21:57] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.33.4" 2023-10-09T10:21:58.156 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:58.226 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:58.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:58.331 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:58.420 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:58.451 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:21:58.496 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:58.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:58.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:58.607 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:21:58.665 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 642/246687 objects degraded (0.260%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:58.666 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:58.781 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:21:58.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:58.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:21:58.892 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:21:58.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 642/246687 objects degraded (0.260%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:58.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 642/246687 objects degraded (0.260%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:21:58.979 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:59.039 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:21:59.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:59.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:21:59.076 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:21:59.077 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:21:59.077 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T10:21:59.077 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:21:59.078 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:21:59.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T10:21:59.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:21:59.198 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:59.253 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:21:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:59.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:21:59.378 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:21:59.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:21:59.432 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:59.546 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:59 smithi012 ceph-mon[147926]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 159 MiB/s wr, 4.06k op/s; 642/244845 objects degraded (0.262%); 4215/244845 objects misplaced (1.721%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T10:21:59.546 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:59.547 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:21:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:59.599 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:21:59.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:59.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:21:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:59 smithi110 ceph-mon[136032]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 159 MiB/s wr, 4.06k op/s; 642/244845 objects degraded (0.262%); 4215/244845 objects misplaced (1.721%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T10:21:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:21:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:59.706 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:21:59.775 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:21:59.863 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:21:59.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:21:59.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:21:59.897 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:21:59.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:59 smithi012 ceph-mon[152263]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 159 MiB/s wr, 4.06k op/s; 642/244845 objects degraded (0.262%); 4215/244845 objects misplaced (1.721%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T10:21:59.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:21:59.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:21:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:21:59.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:00.070 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:00.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:22:00.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:00.235 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:00.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:00.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:00.361 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:22:00.361 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:22:00.361 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:22:00.362 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:22:00.362 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:22:00.407 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:00.446 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:00.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:00.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:00.721 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:00.808 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:00.904 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:22:00.978 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:00.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:00.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:01.083 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:01.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:00 smithi110 ceph-mon[136032]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 130 MiB/s wr, 3.42k op/s; 642/242301 objects degraded (0.265%); 4215/242301 objects misplaced (1.740%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:22:01.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:01.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:01.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:00 smithi012 ceph-mon[152263]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 130 MiB/s wr, 3.42k op/s; 642/242301 objects degraded (0.265%); 4215/242301 objects misplaced (1.740%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:22:01.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:01.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:01.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:00 smithi012 ceph-mon[147926]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 130 MiB/s wr, 3.42k op/s; 642/242301 objects degraded (0.265%); 4215/242301 objects misplaced (1.740%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:22:01.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:01.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:01.188 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:01.276 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:01.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:01.382 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:01.440 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:01.601 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:22:01.601 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "" 2023-10-09T10:22:01.601 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:22:01.667 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:01.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:01.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:01.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:01.844 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:22:01.847 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:02.000 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:02.013 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:02.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:02.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:02.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:01 smithi012 ceph-mon[147926]: 10.2 scrub starts 2023-10-09T10:22:02.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:02.024 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:01 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:02.025 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:01 smithi012 ceph-mon[152263]: 10.2 scrub starts 2023-10-09T10:22:02.025 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:02.025 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:01 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:02.125 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:02.204 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:01 smithi110 ceph-mon[136032]: 10.2 scrub starts 2023-10-09T10:22:02.205 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:02.205 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:01 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:02.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:02.370 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:02.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:02.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:02.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:02.552 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:02.614 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:02.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:02.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:02.721 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:02.775 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:02.836 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:02.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:02.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:02.947 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:03.003 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:03.128 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:03.134 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:02 smithi110 ceph-mon[136032]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 803/232866 objects degraded (0.345%); 3932/232866 objects misplaced (1.689%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:22:03.134 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:02 smithi110 ceph-mon[136032]: 10.0 scrub starts 2023-10-09T10:22:03.134 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:22:03.134 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:22:03.134 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:03.134 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:02 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3518891301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:03.134 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:02 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:03.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:03.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[147926]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 803/232866 objects degraded (0.345%); 3932/232866 objects misplaced (1.689%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:22:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[147926]: 10.0 scrub starts 2023-10-09T10:22:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:22:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:22:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3518891301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[152263]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 803/232866 objects degraded (0.345%); 3932/232866 objects misplaced (1.689%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T10:22:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[152263]: 10.0 scrub starts 2023-10-09T10:22:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:22:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:22:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3518891301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:03.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:02 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:03.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:03.291 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:03.357 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:03.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:03.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:03.462 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:03.468 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:22:03.468 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:22:03.468 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:22:03.468 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:22:03.468 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114937 9.c 9:3de29534:::benchmark_data_smithi012_313365_object2462:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 409.24481477400002, 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0079670510000001, 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114937 2023-10-09T10:22:03.469 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215215+0000", 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:22:03.470 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215214+0000", 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9450000000000005e-06 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215224+0000", 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4849999999999992e-06 2023-10-09T10:22:03.471 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215232+0000", 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3200000000000002e-06 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215239+0000", 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8306000000000002e-05 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.472 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215278+0000", 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3389999999999998e-06 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215284+0000", 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98804788700000001 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.203332+0000", 2023-10-09T10:22:03.473 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00200256 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.205335+0000", 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015448999999999999 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.205489+0000", 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013383071999999999 2023-10-09T10:22:03.474 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218872+0000", 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041514890000000004 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223024+0000", 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7812999999999997e-05 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.475 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223081+0000", 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000100445 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223182+0000", 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0079670510000001 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.476 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114938 9.c 9:317dc30b:::benchmark_data_smithi012_313365_object2463:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 409.24478145799998, 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0080620979999999, 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114938 2023-10-09T10:22:03.477 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.478 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215248+0000", 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.979e-06 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215250+0000", 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5199999999999997e-06 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.479 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215258+0000", 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1250000000000001e-06 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215261+0000", 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8232000000000001e-05 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215320+0000", 2023-10-09T10:22:03.480 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1710000000000002e-06 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215328+0000", 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99764452400000003 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.212972+0000", 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010863 2023-10-09T10:22:03.481 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213081+0000", 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012440199999999999 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213205+0000", 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056800169999999999 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.482 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218885+0000", 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043305829999999998 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223216+0000", 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1077000000000001e-05 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223247+0000", 2023-10-09T10:22:03.483 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3231000000000005e-05 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223310+0000", 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0080620979999999 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.484 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114946 9.c 9:3c5b83c8:::benchmark_data_smithi012_313365_object2471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 409.24474825800002, 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0081399559999999, 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114946 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.485 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.486 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215281+0000", 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6449999999999999e-06 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215283+0000", 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7349999999999998e-06 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.487 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215286+0000", 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7470000000000001e-06 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215289+0000", 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2694000000000001e-05 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215342+0000", 2023-10-09T10:22:03.488 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2049999999999998e-06 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.215345+0000", 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.000627545 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215973+0000", 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011626899999999999 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.489 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216089+0000", 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132232 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 7", 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216221+0000", 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027664209999999998 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.490 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.218988+0000", 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043527119999999999 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223340+0000", 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9572000000000001e-05 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223370+0000", 2023-10-09T10:22:03.491 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1471000000000002e-05 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.223421+0000", 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0081399559999999 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128021 9.c 9:3ed78aba:::benchmark_data_smithi012_313365_object15546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:22:03.492 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 355.75680650800001, 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999, 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128021 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.493 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703222+0000", 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7700000000000004e-06 2023-10-09T10:22:03.494 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703229+0000", 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1178e-05 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703240+0000", 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0509999999999996e-06 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.495 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703245+0000", 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9240000000000002e-05 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703304+0000", 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.429e-05 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703318+0000", 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.498177753 2023-10-09T10:22:03.496 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201496+0000", 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2643e-05 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201519+0000", 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1325606320000001 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.497 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334079+0000", 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016077600000000001 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334240+0000", 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016287699999999999 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334403+0000", 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050087500000000002 2023-10-09T10:22:03.498 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339412+0000", 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1050000000000007e-05 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339473+0000", 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013735 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.499 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339610+0000", 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041248470000000001 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343735+0000", 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9188999999999999e-05 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343784+0000", 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111032 2023-10-09T10:22:03.500 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343895+0000", 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128030 9.c 9:39ef457f:::benchmark_data_smithi012_313365_object15555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:22:03.501 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 355.616795432, 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999, 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128030 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.502 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843233+0000", 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.503 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843241+0000", 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3162e-05 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843254+0000", 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5490000000000001e-06 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.504 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843261+0000", 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2747999999999997e-05 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843314+0000", 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7970000000000006e-06 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843323+0000", 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35848033200000001 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.505 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201803+0000", 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6119999999999999e-06 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201812+0000", 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1353121740000001 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337124+0000", 2023-10-09T10:22:03.506 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8503000000000003e-05 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337212+0000", 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0490999999999998e-05 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337303+0000", 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002260742 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.507 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339564+0000", 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1705999999999994e-05 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339635+0000", 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002741145 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.342376+0000", 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001560173 2023-10-09T10:22:03.508 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343937+0000", 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6062000000000001e-05 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343973+0000", 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4776000000000002e-05 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.509 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.344068+0000", 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195623 9.c 9:350660d8:::benchmark_data_smithi012_313365_object83148:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.20276117500001, 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.178670812, 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.510 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195623 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.511 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257267+0000", 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8130000000000001e-06 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257273+0000", 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0166999999999999e-05 2023-10-09T10:22:03.512 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257283+0000", 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2350000000000002e-06 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257290+0000", 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001555255 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.513 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.258845+0000", 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2446000000000001e-05 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.258858+0000", 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99572670600000002 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254584+0000", 2023-10-09T10:22:03.514 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2269999999999999e-06 2023-10-09T10:22:03.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.515 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.515 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254594+0000", 2023-10-09T10:22:03.515 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.157542818 2023-10-09T10:22:03.515 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.515 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.517 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412136+0000", 2023-10-09T10:22:03.517 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010587299999999999 2023-10-09T10:22:03.517 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.517 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.517 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412242+0000", 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000452194 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412695+0000", 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010428244999999999 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423123+0000", 2023-10-09T10:22:03.518 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4022000000000001e-05 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423157+0000", 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012428642 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435585+0000", 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000185372 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.519 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435771+0000", 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5880000000000001e-05 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435817+0000", 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012248200000000001 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435939+0000", 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.178670812 2023-10-09T10:22:03.520 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195633 9.1c 9:384521d8:::benchmark_data_smithi012_313365_object83158:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.18270992999999, 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1456984170000002, 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.521 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195633 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:22:03.522 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277318+0000", 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4900000000000003e-06 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277326+0000", 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2858e-05 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.523 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277339+0000", 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1309999999999997e-06 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277344+0000", 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3996000000000002e-05 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277408+0000", 2023-10-09T10:22:03.524 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0821e-05 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277419+0000", 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97663378599999995 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254052+0000", 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1975e-05 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.525 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254064+0000", 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1520595250000001 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406124+0000", 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013994800000000001 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406264+0000", 2023-10-09T10:22:03.526 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045309899999999999 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406717+0000", 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051173240000000004 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411834+0000", 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3628999999999999e-05 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.527 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411878+0000", 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3909999999999998e-06 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411886+0000", 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010941673000000001 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.422828+0000", 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9455e-05 2023-10-09T10:22:03.528 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.422877+0000", 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014060400000000001 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423018+0000", 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1456984170000002 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.529 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195634 9.c 9:36e83a2e:::benchmark_data_smithi012_313365_object83159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.18245273299999, 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1586804549999998, 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195634 2023-10-09T10:22:03.530 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.531 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277576+0000", 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.331e-06 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277582+0000", 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2768e-05 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277595+0000", 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:22:03.532 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277601+0000", 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4212000000000003e-05 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277645+0000", 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1494999999999999e-05 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.533 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277657+0000", 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97695835200000003 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254615+0000", 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.798e-06 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254622+0000", 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1605278669999999 2023-10-09T10:22:03.534 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415150+0000", 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011404399999999999 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415264+0000", 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039553599999999999 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:22:03.535 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415659+0000", 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075350449999999998 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423194+0000", 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01213207 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435326+0000", 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062050799999999998 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.536 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435947+0000", 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013730799999999999 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436084+0000", 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2330000000000002e-05 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.537 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436137+0000", 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120761 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436257+0000", 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1586804549999998 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.538 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195652 9.1c 9:3e175140:::benchmark_data_smithi012_313365_object83177:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.15193248700001, 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1274454700000001, 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195652 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.539 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308095+0000", 2023-10-09T10:22:03.540 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.49e-06 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308104+0000", 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7926000000000001e-05 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308122+0000", 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6449999999999999e-06 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.541 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308128+0000", 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7748000000000003e-05 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308196+0000", 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3386e-05 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308210+0000", 2023-10-09T10:22:03.542 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.945878781 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254088+0000", 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5209999999999997e-06 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254096+0000", 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.15516445 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.543 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409260+0000", 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000137047 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409397+0000", 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039575500000000002 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409793+0000", 2023-10-09T10:22:03.544 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021158150000000001 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411909+0000", 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7689e-05 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411927+0000", 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011128549999999999 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.545 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423055+0000", 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012286897999999999 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435342+0000", 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6456000000000003e-05 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.546 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435399+0000", 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014397600000000001 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435543+0000", 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1274454700000001 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195653 9.c 9:3443b6fc:::benchmark_data_smithi012_313365_object83178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:22:03.547 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.15145994299999, 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1279840910000001, 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195653 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.548 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308569+0000", 2023-10-09T10:22:03.549 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1650000000000003e-06 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308575+0000", 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432000000000001e-05 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308587+0000", 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.648e-06 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.550 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308593+0000", 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3375000000000003e-05 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308646+0000", 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3016e-05 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.551 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308659+0000", 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94597990200000004 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254639+0000", 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.489e-06 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254646+0000", 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.163105716 2023-10-09T10:22:03.552 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417751+0000", 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109895 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417861+0000", 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033677300000000001 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.553 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.418198+0000", 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017758396999999999 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435956+0000", 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6774000000000002e-05 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.554 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436013+0000", 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5324000000000003e-05 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436049+0000", 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032668699999999998 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436375+0000", 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2172999999999997e-05 2023-10-09T10:22:03.555 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436427+0000", 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012633699999999999 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436554+0000", 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1279840910000001 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.556 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195656 9.c 9:310794b5:::benchmark_data_smithi012_313365_object83181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.136939982, 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1137697360000001, 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195656 2023-10-09T10:22:03.557 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.558 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323088+0000", 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0759e-05 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323099+0000", 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7932999999999999e-05 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323116+0000", 2023-10-09T10:22:03.559 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4930000000000003e-06 2023-10-09T10:22:03.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.560 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323123+0000", 2023-10-09T10:22:03.560 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0368000000000003e-05 2023-10-09T10:22:03.560 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.560 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.560 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323173+0000", 2023-10-09T10:22:03.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3370000000000006e-06 2023-10-09T10:22:03.561 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.561 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.561 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.561 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323183+0000", 2023-10-09T10:22:03.561 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93181647499999998 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254999+0000", 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0474e-05 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.255010+0000", 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.165299136 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.562 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420309+0000", 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117821 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420427+0000", 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000380281 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:22:03.563 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420807+0000", 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015156678999999999 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435964+0000", 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033305500000000002 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436297+0000", 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0358999999999998e-05 2023-10-09T10:22:03.564 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436337+0000", 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033388899999999997 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436671+0000", 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3112999999999998e-05 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.565 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436724+0000", 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000135403 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436859+0000", 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1137697360000001 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.566 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195663 9.1c 9:3f09486e:::benchmark_data_smithi012_313365_object83188:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 210.118860574, 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0945934909999999, 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195663 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.567 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.568 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341167+0000", 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5290000000000001e-05 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341182+0000", 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1254e-05 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341204+0000", 2023-10-09T10:22:03.569 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7949999999999996e-06 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341212+0000", 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6192000000000006e-05 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341279+0000", 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3284e-05 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.570 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341292+0000", 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91282471300000001 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254117+0000", 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.571 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254124+0000", 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.157826198 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411950+0000", 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012947999999999999 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412079+0000", 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000355423 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.572 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412435+0000", 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002552646 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.414987+0000", 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027370139999999999 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417724+0000", 2023-10-09T10:22:03.573 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053624290000000002 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423087+0000", 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012506992 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435594+0000", 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7438000000000002e-05 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.574 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435641+0000", 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012119 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435762+0000", 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0945934909999999 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.575 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195673 9.10 9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e400)", 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.20238507400001, 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6028094020000001, 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195673 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.576 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.577 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257643+0000", 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0550000000000002e-06 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257650+0000", 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8279999999999998e-06 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257660+0000", 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7660000000000001e-06 2023-10-09T10:22:03.578 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257666+0000", 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032584879999999999 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.260925+0000", 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0726000000000001e-05 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.579 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.260946+0000", 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9083686909999997 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.169314+0000", 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002054237 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.171369+0000", 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6018258909999998 2023-10-09T10:22:03.580 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.773194+0000", 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011103770000000001 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.774305+0000", 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055754660999999997 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.581 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.830060+0000", 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001693766 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.831753+0000", 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034851140000000001 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.835238+0000", 2023-10-09T10:22:03.582 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019324406999999998 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.854563+0000", 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000167465 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.854730+0000", 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058527999999999998 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.583 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.855316+0000", 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00087701200000000004 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.856193+0000", 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042613290000000003 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.860454+0000", 2023-10-09T10:22:03.584 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6028094020000001 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195674 9.c 9:313b05bb:::benchmark_data_smithi012_313365_object83199:head [delete] snapc 0=[] ondisk+write+known_if_redirected e400)", 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:34.315212+0000", 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 209.144817725, 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1218578619999999, 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.585 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195674 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315212+0000", 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.586 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315212+0000", 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315209+0000", 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2469999999999999e-05 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315222+0000", 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2121e-05 2023-10-09T10:22:03.587 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315244+0000", 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2990000000000002e-06 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315253+0000", 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2500000000000001e-05 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.588 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315316+0000", 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5967000000000001e-05 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.315332+0000", 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1078856859999999 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423217+0000", 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011951 2023-10-09T10:22:03.589 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423337+0000", 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035369399999999999 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423691+0000", 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012278888 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.590 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435970+0000", 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062949600000000001 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436599+0000", 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4643e-05 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436634+0000", 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000268072 2023-10-09T10:22:03.591 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436902+0000", 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8890000000000001e-05 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436951+0000", 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000119073 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.592 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.437070+0000", 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1218578619999999 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195693 9.10 9:0f588e8d:::benchmark_data_smithi012_313365_object83218:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 207.98647438699999, 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3933209699999995, 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.593 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195693 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.594 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473553+0000", 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0190000000000001e-05 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473563+0000", 2023-10-09T10:22:03.595 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2921999999999999e-05 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473576+0000", 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.994e-06 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473584+0000", 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.304343544 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.596 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.777928+0000", 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022209249999999999 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.780149+0000", 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013458505000000001 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.597 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.793607+0000", 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001306698 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.794914+0000", 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051671473000000002 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.846585+0000", 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014171050000000001 2023-10-09T10:22:03.598 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.848002+0000", 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003102222 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.851105+0000", 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0099542229999999999 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.599 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861059+0000", 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017274 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861231+0000", 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00060873299999999997 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861840+0000", 2023-10-09T10:22:03.600 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000843592 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.862684+0000", 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041923439999999998 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.866876+0000", 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3933209699999995 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.601 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195716 9.10 9:0de169be:::benchmark_data_smithi012_313365_object83241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 207.95030536900001, 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3902966980000002, 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.602 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195716 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:03.603 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509723+0000", 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0609999999999998e-06 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509730+0000", 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3033e-05 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.604 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509743+0000", 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.613e-06 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509749+0000", 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2706342480000004 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.780384+0000", 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023119730000000002 2023-10-09T10:22:03.605 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.782696+0000", 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020697949 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.803393+0000", 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001386583 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.606 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.804780+0000", 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.078157095999999995 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.882937+0000", 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001764306 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.884701+0000", 2023-10-09T10:22:03.607 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036488340000000001 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.888350+0000", 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003904832 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.892255+0000", 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00076447400000000001 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.608 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.893020+0000", 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022865630000000001 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.895306+0000", 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00084655199999999998 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.896153+0000", 2023-10-09T10:22:03.609 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038682040000000001 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.900021+0000", 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3902966980000002 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195733 9.10 9:0c074ac9:::benchmark_data_smithi012_313365_object83258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:22:03.610 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 207.88444232200001, 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3621552440000002, 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195733 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.611 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575586+0000", 2023-10-09T10:22:03.612 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5009999999999998e-06 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575593+0000", 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0510000000000002e-06 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575602+0000", 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9590000000000003e-06 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.613 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575607+0000", 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2073396159999996 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.782947+0000", 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00182449 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.784772+0000", 2023-10-09T10:22:03.614 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030197965 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.814970+0000", 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001106353 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.816076+0000", 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.101857857 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.615 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.917934+0000", 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001720475 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.919654+0000", 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003479429 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:22:03.616 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.923134+0000", 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041782599999999996 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.927312+0000", 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070182800000000004 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.928014+0000", 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043811029999999999 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.617 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.932395+0000", 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00094054900000000005 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.933335+0000", 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044071309999999999 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.618 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.937742+0000", 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3621552440000002 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(9.10 403/400 [PushOp(9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head, version: 334'5146, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head@334'5146, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 205.28773809699999, 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6005742729999999, 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.619 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.620 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9161999999999997e-05 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172390+0000", 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8474999999999998e-05 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.621 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172408+0000", 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0516000000000001e-05 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172419+0000", 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5926570120000001 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.765076+0000", 2023-10-09T10:22:03.622 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017349800000000001 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.765249+0000", 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076164220000000003 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.772866+0000", 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6005742729999999 2023-10-09T10:22:03.623 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195749 9.1c 9:3a233838:::benchmark_data_smithi012_313365_object83274:head [delete] snapc 0=[] ondisk+write+known_if_redirected e405)", 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:39.409149+0000", 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 204.05088019999999, 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94321279099999999, 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.624 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195749 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409149+0000", 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.625 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409149+0000", 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409147+0000", 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5019999999999998e-06 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409157+0000", 2023-10-09T10:22:03.626 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6129000000000001e-05 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409173+0000", 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5889999999999997e-06 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409181+0000", 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9796000000000001e-05 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.627 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409240+0000", 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9539999999999999e-06 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.409250+0000", 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.92036669000000004 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.628 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.329617+0000", 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011469299999999999 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.329732+0000", 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042454800000000002 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.330156+0000", 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0068902920000000001 2023-10-09T10:22:03.629 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.337047+0000", 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0054030570000000002 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.342450+0000", 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098152039999999992 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.630 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.352265+0000", 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3492999999999998e-05 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.352318+0000", 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3871000000000001e-05 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.352362+0000", 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94321279099999999 2023-10-09T10:22:03.631 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195754 9.1c 9:3939aea6:::benchmark_data_smithi012_313365_object83279:head [delete] snapc 0=[] ondisk+write+known_if_redirected e405)", 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:39.448462+0000", 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 204.01156727, 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91816715999999998, 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:22:03.632 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195754 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448462+0000", 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448462+0000", 2023-10-09T10:22:03.633 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448461+0000", 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9729999999999997e-06 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448469+0000", 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5580999999999999e-05 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.634 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448484+0000", 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.187e-06 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.448492+0000", 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012920550000000001 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.635 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.449784+0000", 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0560000000000005e-06 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.449792+0000", 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88260967599999995 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.332401+0000", 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010365999999999999 2023-10-09T10:22:03.636 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.332505+0000", 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027790600000000002 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.332783+0000", 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098019160000000008 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:22:03.637 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.342585+0000", 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0098189569999999997 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.352404+0000", 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014127015999999999 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366531+0000", 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4029000000000001e-05 2023-10-09T10:22:03.638 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366585+0000", 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4632000000000002e-05 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:40.366629+0000", 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91816715999999998 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.639 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.640 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:22:03.640 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:22:03.640 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:22:03.640 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:03.692 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:22:03.745 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:22:03.785 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:22:03.817 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:22:03.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:22:03.882 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:22:03.925 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:22:03.966 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:22:04.012 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:22:04.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:22:04.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:22:04.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:22:04.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:22:04.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:22:04.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:22:04.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:22:04.021 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:22:04.055 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[147926]: 10.12 scrub starts 2023-10-09T10:22:04.055 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 803/232866 objects degraded (0.345%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:04.056 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:04.056 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:22:04.056 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[147926]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T10:22:04.056 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2938296224' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:04.056 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[152263]: 10.12 scrub starts 2023-10-09T10:22:04.056 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 803/232866 objects degraded (0.345%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:04.057 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:04.057 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:22:04.057 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[152263]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T10:22:04.057 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2938296224' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:04.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:03 smithi110 ceph-mon[136032]: 10.12 scrub starts 2023-10-09T10:22:04.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 803/232866 objects degraded (0.345%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:04.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:03 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:04.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:03 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:22:04.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:03 smithi110 ceph-mon[136032]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T10:22:04.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2938296224' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:05.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:04 smithi012 ceph-mon[147926]: 10.7 scrub starts 2023-10-09T10:22:05.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:04 smithi012 ceph-mon[147926]: failed to encode map e460 with expected crc 2023-10-09T10:22:05.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:04 smithi012 ceph-mon[147926]: pgmap v1290: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 803/231087 objects degraded (0.347%); 3932/231087 objects misplaced (1.702%); 976 KiB/s, 15 objects/s recovering 2023-10-09T10:22:05.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:04 smithi012 ceph-mon[152263]: 10.7 scrub starts 2023-10-09T10:22:05.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:04 smithi012 ceph-mon[152263]: failed to encode map e460 with expected crc 2023-10-09T10:22:05.431 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:04 smithi012 ceph-mon[152263]: pgmap v1290: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 803/231087 objects degraded (0.347%); 3932/231087 objects misplaced (1.702%); 976 KiB/s, 15 objects/s recovering 2023-10-09T10:22:05.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:04 smithi110 ceph-mon[136032]: 10.7 scrub starts 2023-10-09T10:22:05.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:04 smithi110 ceph-mon[136032]: failed to encode map e460 with expected crc 2023-10-09T10:22:05.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:04 smithi110 ceph-mon[136032]: pgmap v1290: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 803/231087 objects degraded (0.347%); 3932/231087 objects misplaced (1.702%); 976 KiB/s, 15 objects/s recovering 2023-10-09T10:22:05.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:05.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:05.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:06.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:22:06.052 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:06.111 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:06.201 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:06.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:06.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:06.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:22:06.306 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:06.372 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:06.421 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:06.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:06.475 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:22:06.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:06.570 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:06.641 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:06.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:06.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:06.748 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:06.753 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:06.809 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:06.865 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:06.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:06.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:06.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:07.043 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:07.047 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 6, 7, 0, 2, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:22:07.047 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:22:07.047 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 3, 6, 7, 0, 2, 5, 4] 2023-10-09T10:22:07.047 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T10:22:07.118 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:07.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:07.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:07.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:06 smithi110 ceph-mon[136032]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 974/218721 objects degraded (0.445%); 3633/218721 objects misplaced (1.661%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:22:07.224 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:07.275 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:07.332 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:06 smithi012 ceph-mon[147926]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 974/218721 objects degraded (0.445%); 3633/218721 objects misplaced (1.661%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:22:07.332 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:06 smithi012 ceph-mon[152263]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 974/218721 objects degraded (0.445%); 3633/218721 objects misplaced (1.661%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T10:22:07.340 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:07.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:07.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:07.355 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:07.451 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:07.507 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:07.583 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:07.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:07.690 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:07.730 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:07.744 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:07.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:07.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:07.912 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:07.922 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:22:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:22:07] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.33.4" 2023-10-09T10:22:07.981 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:08.053 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:08.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:08.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:08.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:08.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:08.316 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:08.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:08.420 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:08.439 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:08.494 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:08.639 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:22:08.640 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:22:08.955 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:08.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:08.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:09.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:09.147 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:09.261 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:08 smithi012 ceph-mon[152263]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 974/216897 objects degraded (0.449%); 3633/216897 objects misplaced (1.675%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:22:09.261 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 974/216897 objects degraded (0.449%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:09.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:08 smithi012 ceph-mon[147926]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 974/216897 objects degraded (0.449%); 3633/216897 objects misplaced (1.675%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:22:09.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 974/216897 objects degraded (0.449%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:09.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:08 smithi110 ceph-mon[136032]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 974/216897 objects degraded (0.449%); 3633/216897 objects misplaced (1.675%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:22:09.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 974/216897 objects degraded (0.449%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:09.512 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:09.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:09.620 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:09.685 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:09.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:22:09.734 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:09.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:09.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:09.840 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:09.887 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:09.942 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:09.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:10.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:10.136 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:10.197 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:10.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:10.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:10.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:22:10.304 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:22:10.309 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:10.352 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:10.357 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:22:10.384 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:22:10.413 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:10.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:10.416 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:22:10.454 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:22:10.492 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:22:10.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:10.528 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:22:10.540 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:22:10.540 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:22:10.540 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:22:10.540 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:22:10.540 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:22:10.540 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:22:10.540 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:22:10.540 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:22:10.580 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:10.652 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:10.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:10.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:10.831 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:11.016 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:22:11.117 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:11.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:10 smithi012 ceph-mon[147926]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 974/213978 objects degraded (0.455%); 3633/213978 objects misplaced (1.698%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:22:11.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2753144781' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:11.119 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2190314323' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:11.119 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:10 smithi012 ceph-mon[152263]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 974/213978 objects degraded (0.455%); 3633/213978 objects misplaced (1.698%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:22:11.119 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2753144781' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:11.119 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2190314323' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:11.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:11.228 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:11.285 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:10 smithi110 ceph-mon[136032]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 974/213978 objects degraded (0.455%); 3633/213978 objects misplaced (1.698%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:22:11.285 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2753144781' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:11.285 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2190314323' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:11.297 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:11.361 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:11.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:11.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:11.466 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:11.519 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:11.788 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:11.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:11.826 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:11.894 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:11.935 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:12.004 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:12.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:12.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:12.177 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:11 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:11 smithi110 ceph-mon[136032]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2190314323' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:11 smithi110 ceph-mon[136032]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T10:22:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:11 smithi110 ceph-mon[136032]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:11 smithi110 ceph-mon[136032]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:11 smithi110 ceph-mon[136032]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.252 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:12.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:12.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:12.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[147926]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2190314323' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:12.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[147926]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T10:22:12.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[147926]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[147926]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[147926]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.280 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:12 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:22:12.112+0000 7f85a99d3700 -1 mon.a@0(leader).osd e461 definitely_dead 0 2023-10-09T10:22:12.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:12.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[152263]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2190314323' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:12.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[152263]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T10:22:12.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[152263]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[152263]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:11 smithi012 ceph-mon[152263]: failed to encode map e461 with expected crc 2023-10-09T10:22:12.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:12.428 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:12.633 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:12.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:12.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:12.744 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:12.808 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:12.893 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:12.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:12.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:12.998 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:13.043 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.1. 2023-10-09T10:22:13.061 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:13.106 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:13.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:13.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:13 smithi012 ceph-mon[147926]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 1127/203817 objects degraded (0.553%); 3339/203817 objects misplaced (1.638%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T10:22:13.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2808454673' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:22:13.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:22:13.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:13 smithi012 ceph-mon[147926]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:22:13.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:13 smithi012 ceph-mon[152263]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 1127/203817 objects degraded (0.553%); 3339/203817 objects misplaced (1.638%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T10:22:13.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2808454673' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:22:13.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:22:13.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:13 smithi012 ceph-mon[152263]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:22:13.209 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:13.237 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:13.248 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:22:13.249 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:22:13.249 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:22:13.249 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:22:13.249 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:22:13.271 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:13.273 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:13 smithi110 ceph-mon[136032]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 1127/203817 objects degraded (0.553%); 3339/203817 objects misplaced (1.638%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T10:22:13.273 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2808454673' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:22:13.273 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:22:13.273 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:13 smithi110 ceph-mon[136032]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:22:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:13.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:13.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:13.426 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:13.466 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:13.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:13.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:13.580 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:13.608 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:13.646 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:13.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:13.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:13.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:22:13.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:13.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:22:13.779 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:13.905 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:13.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:13.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:13.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:22:14.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:14.047 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:14.086 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:14.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:14.190 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:14.216 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:14.250 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:14.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:14.324 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:14.354 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:14.394 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:22:14.394 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: osdmap e462: 8 total, 8 up, 7 in 2023-10-09T10:22:14.394 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.395 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.395 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.395 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.395 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.395 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.396 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.397 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.397 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 1127/203817 objects degraded (0.553%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:14.397 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.397 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: osdmap e463: 8 total, 8 up, 7 in 2023-10-09T10:22:14.397 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.397 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:14.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:14.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:22:14.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: osdmap e462: 8 total, 8 up, 7 in 2023-10-09T10:22:14.399 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.399 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.399 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.399 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.399 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.399 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 1127/203817 objects degraded (0.553%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:14.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: osdmap e463: 8 total, 8 up, 7 in 2023-10-09T10:22:14.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.402 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.402 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:14.402 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:14.402 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:22:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: osdmap e462: 8 total, 8 up, 7 in 2023-10-09T10:22:14.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e462 with expected crc 2023-10-09T10:22:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 1127/203817 objects degraded (0.553%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: osdmap e463: 8 total, 8 up, 7 in 2023-10-09T10:22:14.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: failed to encode map e463 with expected crc 2023-10-09T10:22:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:14.448 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:14.470 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:14.617 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:14.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:14.738 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:14.791 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:15.146 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:15.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:15.211 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:15.251 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:15.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 1127/201807 objects degraded (0.558%); 3339/201807 objects misplaced (1.655%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:22:15.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: osdmap e464: 8 total, 8 up, 7 in 2023-10-09T10:22:15.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 461'10030 2023-10-09T10:22:15.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 461'8996 2023-10-09T10:22:15.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: 10.2 starting backfill to osd.6 from (0'0,0'0] MAX to 461'9004 2023-10-09T10:22:15.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: 10.4 starting backfill to osd.2 from (0'0,0'0] MAX to 461'9109 2023-10-09T10:22:15.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 461'8995 2023-10-09T10:22:15.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 461'9036 2023-10-09T10:22:15.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[147926]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 461'9047 2023-10-09T10:22:15.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 1127/201807 objects degraded (0.558%); 3339/201807 objects misplaced (1.655%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:22:15.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: osdmap e464: 8 total, 8 up, 7 in 2023-10-09T10:22:15.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 461'10030 2023-10-09T10:22:15.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 461'8996 2023-10-09T10:22:15.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: 10.2 starting backfill to osd.6 from (0'0,0'0] MAX to 461'9004 2023-10-09T10:22:15.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: 10.4 starting backfill to osd.2 from (0'0,0'0] MAX to 461'9109 2023-10-09T10:22:15.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 461'8995 2023-10-09T10:22:15.331 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 461'9036 2023-10-09T10:22:15.332 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:15 smithi012 ceph-mon[152263]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 461'9047 2023-10-09T10:22:15.345 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:15.449 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.450 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: failed to encode map e463 with expected crc 2023-10-09T10:22:15.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 1127/201807 objects degraded (0.558%); 3339/201807 objects misplaced (1.655%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:22:15.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: osdmap e464: 8 total, 8 up, 7 in 2023-10-09T10:22:15.451 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: failed to encode map e464 with expected crc 2023-10-09T10:22:15.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 461'10030 2023-10-09T10:22:15.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 461'8996 2023-10-09T10:22:15.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: 10.2 starting backfill to osd.6 from (0'0,0'0] MAX to 461'9004 2023-10-09T10:22:15.452 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: 10.4 starting backfill to osd.2 from (0'0,0'0] MAX to 461'9109 2023-10-09T10:22:15.453 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 461'8995 2023-10-09T10:22:15.453 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 461'9036 2023-10-09T10:22:15.453 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:15 smithi110 ceph-mon[136032]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 461'9047 2023-10-09T10:22:15.977 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:15.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:15.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:16.083 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:16.161 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:16.224 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:16.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:16.335 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:16.406 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:16 smithi110 ceph-mon[136032]: failed to encode map e464 with expected crc 2023-10-09T10:22:16.406 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:16 smithi110 ceph-mon[136032]: failed to encode map e464 with expected crc 2023-10-09T10:22:16.411 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:16.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:16 smithi012 ceph-mon[147926]: failed to encode map e464 with expected crc 2023-10-09T10:22:16.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:16 smithi012 ceph-mon[147926]: failed to encode map e464 with expected crc 2023-10-09T10:22:16.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:16 smithi012 ceph-mon[152263]: failed to encode map e464 with expected crc 2023-10-09T10:22:16.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:16 smithi012 ceph-mon[152263]: failed to encode map e464 with expected crc 2023-10-09T10:22:16.477 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:16.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:16.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:16.653 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:16.715 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:16.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:16.820 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:16.880 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:17.057 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:17.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:17.166 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:17.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:17 smithi012 ceph-mon[147926]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1224/193497 objects degraded (0.633%); 15016/193497 objects misplaced (7.760%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T10:22:17.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:17 smithi012 ceph-mon[152263]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1224/193497 objects degraded (0.633%); 15016/193497 objects misplaced (7.760%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T10:22:17.218 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:17.269 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:17.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:17.350 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:22:17.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:17.401 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:22:17.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:17 smithi110 ceph-mon[136032]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1224/193497 objects degraded (0.633%); 15016/193497 objects misplaced (7.760%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T10:22:17.445 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:17.447 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:22:17.499 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:22:17.516 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:17.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:17.538 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:22:17.571 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:22:17.616 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:22:17.624 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:17.649 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:22:17.658 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:22:17.658 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:22:17.666 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:22:17.666 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:22:17.666 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:22:17.666 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:22:17.666 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:22:17.666 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:22:17.666 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:22:17.667 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:22:17.681 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:17.766 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:17.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:17.880 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:17.935 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:17.989 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:17.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:18.085 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:22:18.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:18.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4123331259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:18.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3072521113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:18.180 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:22:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:22:17] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.33.4" 2023-10-09T10:22:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4123331259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:18.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3072521113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:18.189 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:18.267 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:18.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:18.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:18.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4123331259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:18.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3072521113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:18.459 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:18.497 INFO:tasks.thrashosds.thrasher:in_osds: [3, 6, 7, 0, 2, 5, 4] out_osds: [1] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:22:18.497 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:22:18.497 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [3, 6, 7, 0, 2, 5, 4] 2023-10-09T10:22:18.498 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T10:22:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: pgmap v1301: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 1224/191151 objects degraded (0.640%); 18627/191151 objects misplaced (9.745%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T10:22:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3072521113' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:22:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: osdmap e465: 8 total, 8 up, 7 in 2023-10-09T10:22:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 1224/191151 objects degraded (0.640%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: pgmap v1301: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 1224/191151 objects degraded (0.640%); 18627/191151 objects misplaced (9.745%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3072521113' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:22:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: osdmap e465: 8 total, 8 up, 7 in 2023-10-09T10:22:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:19 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 1224/191151 objects degraded (0.640%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:19.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: pgmap v1301: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 1224/191151 objects degraded (0.640%); 18627/191151 objects misplaced (9.745%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T10:22:19.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:19.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3072521113' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:22:19.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: osdmap e465: 8 total, 8 up, 7 in 2023-10-09T10:22:19.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: failed to encode map e465 with expected crc 2023-10-09T10:22:19.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:19 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 1224/191151 objects degraded (0.640%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:20.379 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:20 smithi012 ceph-mon[152263]: 10.7 scrub starts 2023-10-09T10:22:20.380 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:20 smithi012 ceph-mon[147926]: 10.7 scrub starts 2023-10-09T10:22:20.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:20 smithi110 ceph-mon[136032]: 10.7 scrub starts 2023-10-09T10:22:20.626 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:20.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:20.735 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:20.816 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:20.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:20.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:20.986 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:21.061 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:21.141 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:21.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:21.253 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:21.307 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:21 smithi110 ceph-mon[136032]: pgmap v1303: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1224/191151 objects degraded (0.640%); 18627/191151 objects misplaced (9.745%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T10:22:21.313 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:21.379 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:21.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:21.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:21.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:21 smithi012 ceph-mon[147926]: pgmap v1303: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1224/191151 objects degraded (0.640%); 18627/191151 objects misplaced (9.745%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T10:22:21.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:21 smithi012 ceph-mon[152263]: pgmap v1303: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1224/191151 objects degraded (0.640%); 18627/191151 objects misplaced (9.745%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T10:22:21.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:21.541 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:23.224 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:23.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:23.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:23.267 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:23 smithi012 ceph-mon[147926]: pgmap v1304: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 926 op/s; 1251/179550 objects degraded (0.697%); 17509/179550 objects misplaced (9.752%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T10:22:23.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:23 smithi012 ceph-mon[152263]: pgmap v1304: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 926 op/s; 1251/179550 objects degraded (0.697%); 17509/179550 objects misplaced (9.752%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T10:22:23.269 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:22:23.331 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:23.389 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:23 smithi110 ceph-mon[136032]: pgmap v1304: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 926 op/s; 1251/179550 objects degraded (0.697%); 17509/179550 objects misplaced (9.752%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T10:22:23.394 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:23.457 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:23.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:23.495 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:23.564 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:23.645 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:23.718 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:23.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:23.733 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:22:23.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:23.901 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:24.010 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:24.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:24.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:24.124 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:24.192 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:24.261 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:24.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:24.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:24.296 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:24 smithi012 ceph-mon[147926]: 10.7 scrub starts 2023-10-09T10:22:24.297 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:24 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 1251/179550 objects degraded (0.697%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:24.297 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:24 smithi012 ceph-mon[152263]: 10.7 scrub starts 2023-10-09T10:22:24.297 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:24 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 1251/179550 objects degraded (0.697%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:24.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:24.428 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:24 smithi110 ceph-mon[136032]: 10.7 scrub starts 2023-10-09T10:22:24.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:24 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 1251/179550 objects degraded (0.697%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:24.619 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:24.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:24.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:24.681 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:24.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:24.802 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:24.839 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:24.872 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:24.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:24.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:24.981 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:25.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:22:25.028 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:25.110 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:25.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:25.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:25.217 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:25.275 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:25.340 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:25.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:25.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:25 smithi110 ceph-mon[136032]: pgmap v1305: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 1251/176871 objects degraded (0.707%); 16772/176871 objects misplaced (9.483%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T10:22:25.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:25 smithi110 ceph-mon[136032]: 10.7 scrub starts 2023-10-09T10:22:25.453 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:25.506 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:25.558 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:25.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:25.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:25.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:25 smithi012 ceph-mon[152263]: pgmap v1305: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 1251/176871 objects degraded (0.707%); 16772/176871 objects misplaced (9.483%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T10:22:25.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:25 smithi012 ceph-mon[152263]: 10.7 scrub starts 2023-10-09T10:22:25.655 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:25 smithi012 ceph-mon[147926]: pgmap v1305: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 1251/176871 objects degraded (0.707%); 16772/176871 objects misplaced (9.483%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T10:22:25.655 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:25 smithi012 ceph-mon[147926]: 10.7 scrub starts 2023-10-09T10:22:25.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:25.721 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:25.907 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:25.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:26.017 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:26.081 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:26.154 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:26.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:26.268 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:26.349 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:26.387 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:26.411 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.6. 2023-10-09T10:22:26.424 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:26.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:26.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:26.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:22:26.183+0000 7f85a99d3700 -1 mon.a@0(leader).osd e465 definitely_dead 0 2023-10-09T10:22:26.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-mon[147926]: 10.7 scrub starts 2023-10-09T10:22:26.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3067805548' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T10:22:26.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T10:22:26.436 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-mon[147926]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T10:22:26.437 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-mon[152263]: 10.7 scrub starts 2023-10-09T10:22:26.437 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3067805548' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T10:22:26.437 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T10:22:26.437 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:26 smithi012 ceph-mon[152263]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T10:22:26.530 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:26.566 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:26 smithi110 ceph-mon[136032]: 10.7 scrub starts 2023-10-09T10:22:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3067805548' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T10:22:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T10:22:26.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:26 smithi110 ceph-mon[136032]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T10:22:26.889 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:22:26.889 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:22:27.385 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:22:27.571 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:27.574 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[147926]: pgmap v1306: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 1362/165519 objects degraded (0.823%); 15618/165519 objects misplaced (9.436%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T10:22:27.574 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[147926]: failed to encode map e466 with expected crc 2023-10-09T10:22:27.574 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T10:22:27.574 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[147926]: osdmap e466: 8 total, 8 up, 6 in 2023-10-09T10:22:27.574 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[147926]: failed to encode map e466 with expected crc 2023-10-09T10:22:27.574 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[147926]: 10.5 scrub starts 2023-10-09T10:22:27.574 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4201143613' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:27.574 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:27.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[152263]: pgmap v1306: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 1362/165519 objects degraded (0.823%); 15618/165519 objects misplaced (9.436%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T10:22:27.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[152263]: failed to encode map e466 with expected crc 2023-10-09T10:22:27.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T10:22:27.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[152263]: osdmap e466: 8 total, 8 up, 6 in 2023-10-09T10:22:27.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[152263]: failed to encode map e466 with expected crc 2023-10-09T10:22:27.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[152263]: 10.5 scrub starts 2023-10-09T10:22:27.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4201143613' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:27.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:27.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:27.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:27.677 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:27.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:27 smithi110 ceph-mon[136032]: pgmap v1306: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 1362/165519 objects degraded (0.823%); 15618/165519 objects misplaced (9.436%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T10:22:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:27 smithi110 ceph-mon[136032]: failed to encode map e466 with expected crc 2023-10-09T10:22:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T10:22:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:27 smithi110 ceph-mon[136032]: osdmap e466: 8 total, 8 up, 6 in 2023-10-09T10:22:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:27 smithi110 ceph-mon[136032]: failed to encode map e466 with expected crc 2023-10-09T10:22:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:27 smithi110 ceph-mon[136032]: 10.5 scrub starts 2023-10-09T10:22:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4201143613' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:27.711 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:27.922 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:22:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:22:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:22:27] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.33.4" 2023-10-09T10:22:28.233 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:28.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:28.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:28.339 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:28.373 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:28.471 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:22:28.471 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: osdmap e467: 8 total, 8 up, 6 in 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: 10.9 scrub starts 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: 10.b continuing backfill to osd.4 from (433'3775,465'10742] 10:d4ed475f:::benchmark_data_smithi012_366312_object75802:head to 465'10742 2023-10-09T10:22:28.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 465'10742 2023-10-09T10:22:28.473 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.473 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: 10.10 starting backfill to osd.5 from (0'0,0'0] MAX to 465'9320 2023-10-09T10:22:28.473 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: 10.12 starting backfill to osd.7 from (0'0,0'0] MAX to 465'9390 2023-10-09T10:22:28.473 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: 10.2 starting backfill to osd.7 from (0'0,0'0] MAX to 465'9344 2023-10-09T10:22:28.473 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: 10.10 continuing backfill to osd.7 from (428'1400,465'9320] MIN to 465'9320 2023-10-09T10:22:28.473 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 1362/165519 objects degraded (0.823%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:28.473 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:22:28.473 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: osdmap e467: 8 total, 8 up, 6 in 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: 10.9 scrub starts 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.474 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: 10.b continuing backfill to osd.4 from (433'3775,465'10742] 10:d4ed475f:::benchmark_data_smithi012_366312_object75802:head to 465'10742 2023-10-09T10:22:28.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 465'10742 2023-10-09T10:22:28.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: 10.10 starting backfill to osd.5 from (0'0,0'0] MAX to 465'9320 2023-10-09T10:22:28.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: 10.12 starting backfill to osd.7 from (0'0,0'0] MAX to 465'9390 2023-10-09T10:22:28.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: 10.2 starting backfill to osd.7 from (0'0,0'0] MAX to 465'9344 2023-10-09T10:22:28.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: 10.10 continuing backfill to osd.7 from (428'1400,465'9320] MIN to 465'9320 2023-10-09T10:22:28.475 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 1362/165519 objects degraded (0.823%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:28.576 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:28.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:28.682 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:28.700 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:22:28.700 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.700 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.700 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.700 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:28.701 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: osdmap e467: 8 total, 8 up, 6 in 2023-10-09T10:22:28.701 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.701 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: 10.9 scrub starts 2023-10-09T10:22:28.701 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.701 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.701 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.701 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.702 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: 10.b continuing backfill to osd.4 from (433'3775,465'10742] 10:d4ed475f:::benchmark_data_smithi012_366312_object75802:head to 465'10742 2023-10-09T10:22:28.702 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 465'10742 2023-10-09T10:22:28.702 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: failed to encode map e467 with expected crc 2023-10-09T10:22:28.702 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: 10.10 starting backfill to osd.5 from (0'0,0'0] MAX to 465'9320 2023-10-09T10:22:28.702 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: 10.12 starting backfill to osd.7 from (0'0,0'0] MAX to 465'9390 2023-10-09T10:22:28.702 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: 10.2 starting backfill to osd.7 from (0'0,0'0] MAX to 465'9344 2023-10-09T10:22:28.702 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: 10.10 continuing backfill to osd.7 from (428'1400,465'9320] MIN to 465'9320 2023-10-09T10:22:28.703 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 1362/165519 objects degraded (0.823%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:22:28.705 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:22:28.729 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:22:28.752 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:22:28.777 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:22:28.779 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:28.805 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:22:28.834 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:28.837 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:28.841 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:22:28.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:28.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:28.866 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:22:28.895 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:22:28.896 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:22:28.896 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:22:28.896 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:22:28.896 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:22:28.896 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:22:28.896 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:22:28.896 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:22:28.896 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:22:28.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:28.988 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:29.095 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:29.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:29.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:29.202 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:29.305 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:29.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:29.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:29.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:29.492 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:29.560 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: pgmap v1309: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 1362/163497 objects degraded (0.833%); 18532/163497 objects misplaced (11.335%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:22:29.560 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.560 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.560 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 465'10584 2023-10-09T10:22:29.560 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: osdmap e468: 8 total, 8 up, 6 in 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 465'10476 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: 10.0 starting backfill to osd.5 from (0'0,0'0] MAX to 465'9343 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: 10.a starting backfill to osd.7 from (0'0,0'0] MAX to 465'10489 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: 10.0 continuing backfill to osd.7 from (428'1400,465'9343] MIN to 465'9343 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/44089993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:29.561 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:29.567 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:29.661 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:29.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: pgmap v1309: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 1362/163497 objects degraded (0.833%); 18532/163497 objects misplaced (11.335%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:22:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 465'10584 2023-10-09T10:22:29.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: osdmap e468: 8 total, 8 up, 6 in 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 465'10476 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: 10.0 starting backfill to osd.5 from (0'0,0'0] MAX to 465'9343 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: 10.a starting backfill to osd.7 from (0'0,0'0] MAX to 465'10489 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: 10.0 continuing backfill to osd.7 from (428'1400,465'9343] MIN to 465'9343 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/44089993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: pgmap v1309: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 90 active+clean; 3.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 1362/163497 objects degraded (0.833%); 18532/163497 objects misplaced (11.335%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:22:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 465'10584 2023-10-09T10:22:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: osdmap e468: 8 total, 8 up, 6 in 2023-10-09T10:22:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 465'10476 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: 10.0 starting backfill to osd.5 from (0'0,0'0] MAX to 465'9343 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: failed to encode map e468 with expected crc 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: 10.a starting backfill to osd.7 from (0'0,0'0] MAX to 465'10489 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: 10.0 continuing backfill to osd.7 from (428'1400,465'9343] MIN to 465'9343 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/44089993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:29.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:29.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:29.810 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:29.867 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:29.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:29.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:29.974 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:30.022 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:22:30.070 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:30.140 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:30.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:30.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:30.247 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:30.312 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:30.394 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:30.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:30.424 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:30 smithi110 ceph-mon[136032]: osdmap e469: 8 total, 8 up, 6 in 2023-10-09T10:22:30.424 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:30 smithi110 ceph-mon[136032]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.425 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:30 smithi110 ceph-mon[136032]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.425 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:30 smithi110 ceph-mon[136032]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.425 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:30 smithi110 ceph-mon[136032]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.501 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:30.524 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:22:30.525 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:22:30.525 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:22:30.525 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:22:30.525 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:22:30.552 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:30.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[152263]: osdmap e469: 8 total, 8 up, 6 in 2023-10-09T10:22:30.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[152263]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.664 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[152263]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.664 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[152263]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.664 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[152263]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[147926]: osdmap e469: 8 total, 8 up, 6 in 2023-10-09T10:22:30.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[147926]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[147926]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[147926]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.664 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:30 smithi012 ceph-mon[147926]: failed to encode map e469 with expected crc 2023-10-09T10:22:30.920 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:30.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:30.977 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:31.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:31.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:22:31.107 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:31.168 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:31.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:31.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:31.273 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:31.324 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:31.388 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:31.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:31.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:31.545 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[152263]: pgmap v1312: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 89 active+clean; 3.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 1362/163131 objects degraded (0.835%); 21898/163131 objects misplaced (13.424%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:22:31.546 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[152263]: osdmap e470: 8 total, 8 up, 6 in 2023-10-09T10:22:31.546 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[152263]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.546 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[152263]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.546 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[152263]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.546 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[152263]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.546 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[147926]: pgmap v1312: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 89 active+clean; 3.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 1362/163131 objects degraded (0.835%); 21898/163131 objects misplaced (13.424%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:22:31.546 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[147926]: osdmap e470: 8 total, 8 up, 6 in 2023-10-09T10:22:31.547 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[147926]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.547 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[147926]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.547 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[147926]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.547 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:31 smithi012 ceph-mon[147926]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.549 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:31.629 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:22:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:31 smithi110 ceph-mon[136032]: pgmap v1312: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 89 active+clean; 3.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 1362/163131 objects degraded (0.835%); 21898/163131 objects misplaced (13.424%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:22:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:31 smithi110 ceph-mon[136032]: osdmap e470: 8 total, 8 up, 6 in 2023-10-09T10:22:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:31 smithi110 ceph-mon[136032]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:31 smithi110 ceph-mon[136032]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:31 smithi110 ceph-mon[136032]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:31 smithi110 ceph-mon[136032]: failed to encode map e470 with expected crc 2023-10-09T10:22:31.800 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:31.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:31.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:31.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:31.995 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:32.069 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:32.073 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:32.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:32.176 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:32.183 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 0, 2, 5, 4] out_osds: [1, 6] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:22:32.183 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:22:32.183 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:22:32.184 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.711714 2023-10-09T10:22:32.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0.7117141092246744 2023-10-09T10:22:32.237 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:32.323 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:32.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:32.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:32.428 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:32.500 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:32.598 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:32.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:32.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:32.779 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:32.898 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:32.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:33.007 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:33.101 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:33.167 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:33.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:33.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:33.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:33.370 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:33.434 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:33 smithi110 ceph-mon[136032]: pgmap v1314: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 85 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 3393/158502 objects degraded (2.141%); 32424/158502 objects misplaced (20.457%); 25 MiB/s, 404 objects/s recovering 2023-10-09T10:22:33.434 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3393/158502 objects degraded (2.141%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:33.679 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:33.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:33 smithi012 ceph-mon[147926]: pgmap v1314: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 85 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 3393/158502 objects degraded (2.141%); 32424/158502 objects misplaced (20.457%); 25 MiB/s, 404 objects/s recovering 2023-10-09T10:22:33.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3393/158502 objects degraded (2.141%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:33.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:33 smithi012 ceph-mon[152263]: pgmap v1314: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 85 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 3393/158502 objects degraded (2.141%); 32424/158502 objects misplaced (20.457%); 25 MiB/s, 404 objects/s recovering 2023-10-09T10:22:33.683 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3393/158502 objects degraded (2.141%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:33.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:33.694 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:33.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:33.846 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:34.446 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:22:34.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:34 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1463239516' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:34.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:34 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:34.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:34 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1463239516' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:34.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:34 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:34 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1463239516' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:34 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:22:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[147926]: pgmap v1315: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 85 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 364 op/s; 3393/156927 objects degraded (2.162%); 31725/156927 objects misplaced (20.216%); 19 MiB/s, 303 objects/s recovering 2023-10-09T10:22:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:22:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[147926]: osdmap e471: 8 total, 8 up, 6 in 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[147926]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[147926]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[147926]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[147926]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[152263]: pgmap v1315: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 85 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 364 op/s; 3393/156927 objects degraded (2.162%); 31725/156927 objects misplaced (20.216%); 19 MiB/s, 303 objects/s recovering 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[152263]: osdmap e471: 8 total, 8 up, 6 in 2023-10-09T10:22:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[152263]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[152263]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[152263]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:35 smithi012 ceph-mon[152263]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:35 smithi110 ceph-mon[136032]: pgmap v1315: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 85 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 364 op/s; 3393/156927 objects degraded (2.162%); 31725/156927 objects misplaced (20.216%); 19 MiB/s, 303 objects/s recovering 2023-10-09T10:22:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:35 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:22:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:35 smithi110 ceph-mon[136032]: osdmap e471: 8 total, 8 up, 6 in 2023-10-09T10:22:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:35 smithi110 ceph-mon[136032]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:35 smithi110 ceph-mon[136032]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:35 smithi110 ceph-mon[136032]: failed to encode map e471 with expected crc 2023-10-09T10:22:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:35 smithi110 ceph-mon[136032]: failed to encode map e471 with expected crc 2023-10-09T10:22:36.086 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:36.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:36.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:36.195 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:36.242 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:36.277 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114948 9.18 9:1ed3790b:::benchmark_data_smithi012_313365_object2473:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 442.06648031399999, 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9028626509999995, 2023-10-09T10:22:36.295 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114948 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.296 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223502+0000", 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223501+0000", 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223504+0000", 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.932e-06 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223510+0000", 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6890000000000001e-06 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223515+0000", 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7820000000000001e-06 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223563+0000", 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8705000000000002e-05 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223573+0000", 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3149999999999998e-06 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.185664+0000", 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96209155300000004 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.185676+0000", 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2165e-05 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.092908+0000", 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9072312829999998 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093028+0000", 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120679 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093121+0000", 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2170999999999996e-05 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.105712+0000", 2023-10-09T10:22:36.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012591150000000001 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124350+0000", 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018638268 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124362+0000", 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2362e-05 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.124426+0000", 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3637000000000001e-05 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.126364+0000", 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019383549999999999 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.303 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114976 9.8 9:131c68e0:::benchmark_data_smithi012_313365_object2501:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 440.984101157, 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.83748516, 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114976 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305881+0000", 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.305 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305879+0000", 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305887+0000", 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7619999999999995e-06 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305900+0000", 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3369e-05 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.305912+0000", 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2313000000000001e-05 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.309645+0000", 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003732815 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:36.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.309659+0000", 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3890000000000001e-05 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.063120+0000", 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7534609549999995 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.073887+0000", 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010766901000000001 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.106337+0000", 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032450537000000002 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.110335+0000", 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003997212 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.118546+0000", 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0082116299999999993 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.127135+0000", 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085889409999999992 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.128467+0000", 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001332176 2023-10-09T10:22:36.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.130828+0000", 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00236046 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.133144+0000", 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023158419999999998 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.143366+0000", 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010222337 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114990 9.8 9:10d5a285:::benchmark_data_smithi012_313365_object2515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 440.02750225599999, 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0309713049999996, 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.312 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114990 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262480+0000", 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262478+0000", 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.262492+0000", 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4115e-05 2023-10-09T10:22:36.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.267636+0000", 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051444480000000002 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.267653+0000", 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6095000000000001e-05 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.144210+0000", 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8765578109999996 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155494+0000", 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011283985999999999 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.201082+0000", 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.045587640999999998 2023-10-09T10:22:36.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.205016+0000", 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039338280000000003 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.214090+0000", 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090739989999999993 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.243654+0000", 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029564373000000001 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.265457+0000", 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021802591 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.267777+0000", 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023206949999999998 2023-10-09T10:22:36.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.271653+0000", 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038760219999999998 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.293451+0000", 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021797512000000002 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115028 9.18 9:1ec11435:::benchmark_data_smithi012_313365_object2553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.88569993300001, 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7778839790000003, 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115028 2023-10-09T10:22:36.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404282+0000", 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404280+0000", 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404289+0000", 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5809999999999997e-06 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404316+0000", 2023-10-09T10:22:36.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6720000000000002e-05 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.404326+0000", 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0453e-05 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155549+0000", 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7512224199999995 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155657+0000", 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108597 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.155763+0000", 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106253 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.167754+0000", 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011990404 2023-10-09T10:22:36.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179961+0000", 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012206995999999999 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179968+0000", 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9029999999999998e-06 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.180006+0000", 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8136000000000002e-05 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.182166+0000", 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021601250000000002 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115046 9.8 9:10ac3fd1:::benchmark_data_smithi012_313365_object2571:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.83145116399999, 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8549557910000001, 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115046 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458531+0000", 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458530+0000", 2023-10-09T10:22:36.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458533+0000", 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.79e-06 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458539+0000", 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4199999999999998e-06 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.458543+0000", 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8060000000000002e-06 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.168371+0000", 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.709827089 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.179894+0000", 2023-10-09T10:22:36.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011523197000000001 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.244426+0000", 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064532795000000004 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.249276+0000", 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004849969 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.257311+0000", 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0080343040000000008 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.295944+0000", 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.038633611999999998 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.297677+0000", 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001732803 2023-10-09T10:22:36.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.298328+0000", 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00065045599999999999 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.301789+0000", 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034609559999999998 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313487+0000", 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011697982000000001 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115063 9.8 9:12e13310:::benchmark_data_smithi012_313365_object2588:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.76840812500001, 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9951984300000003, 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.334 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115063 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521574+0000", 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521572+0000", 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521582+0000", 2023-10-09T10:22:36.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3870000000000001e-06 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521600+0000", 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8029e-05 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.521609+0000", 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7890000000000006e-06 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.314437+0000", 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7928272840000004 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330107+0000", 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015670738 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.392854+0000", 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062746280000000001 2023-10-09T10:22:36.338 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.396715+0000", 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038610379999999998 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.405209+0000", 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084946369999999993 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.468006+0000", 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062796789000000006 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.492245+0000", 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024238563000000001 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.495737+0000", 2023-10-09T10:22:36.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034923879999999999 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.499136+0000", 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0033991400000000001 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.516772+0000", 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017636013999999998 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.341 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115064 9.18 9:1d8cd058:::benchmark_data_smithi012_313365_object2589:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.76461103000003, 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8195911320000002, 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.342 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115064 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525371+0000", 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525369+0000", 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525377+0000", 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4749999999999993e-06 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525393+0000", 2023-10-09T10:22:36.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6319000000000001e-05 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.525405+0000", 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1484000000000001e-05 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313537+0000", 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7881318369999999 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313674+0000", 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137074 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.313794+0000", 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120281 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330174+0000", 2023-10-09T10:22:36.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016380212000000002 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330198+0000", 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3269e-05 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342558+0000", 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012360884000000001 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342624+0000", 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5277000000000005e-05 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.344962+0000", 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023383079999999999 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.348 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 347/342 [PushOp(9:0c166afc:::benchmark_data_smithi012_313365_object2594:head, version: 320'166, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0c166afc:::benchmark_data_smithi012_313365_object2594:head@320'166, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.74314824599998, 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4620482260000003, 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.349 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546834+0000", 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546828+0000", 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999948 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546890+0000", 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1514999999999996e-05 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546920+0000", 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0239e-05 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546947+0000", 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7192e-05 2023-10-09T10:22:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.007036+0000", 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4600890619999998 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.007054+0000", 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.785e-05 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.008882+0000", 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001827841 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115082 9.18 9:1d0726ed:::benchmark_data_smithi012_313365_object2607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.66041628099998, 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8115987789999997, 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.353 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115082 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.354 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629566+0000", 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629565+0000", 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629569+0000", 2023-10-09T10:22:36.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.377e-06 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629577+0000", 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0830000000000004e-06 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.629584+0000", 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315693+0000", 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6861087850000001 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315831+0000", 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013798599999999999 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.315971+0000", 2023-10-09T10:22:36.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000139887 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330217+0000", 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014246325000000001 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330237+0000", 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9962e-05 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.438747+0000", 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10850990200000001 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.438804+0000", 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6765000000000003e-05 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.441164+0000", 2023-10-09T10:22:36.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023603669999999999 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115093 9.8 9:161687e0:::benchmark_data_smithi012_313365_object2618:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.64221873100001, 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8841582529999998, 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.360 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115093 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647763+0000", 2023-10-09T10:22:36.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647762+0000", 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647769+0000", 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2479999999999999e-06 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647781+0000", 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2565000000000001e-05 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.647791+0000", 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3360000000000001e-06 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.330722+0000", 2023-10-09T10:22:36.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6829310240000002 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.342528+0000", 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011806542999999999 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.439337+0000", 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.096808401000000002 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.364 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.445587+0000", 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062502110000000003 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.458972+0000", 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013385052 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.493763+0000", 2023-10-09T10:22:36.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.034790897000000001 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.517445+0000", 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023681722999999998 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.519054+0000", 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016091409999999999 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.521336+0000", 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022823409999999998 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.531921+0000", 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010585331 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.367 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115107 9.8 9:160e4e76:::benchmark_data_smithi012_313365_object2632:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.614453298, 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9200908160000001, 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115107 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675529+0000", 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675527+0000", 2023-10-09T10:22:36.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675535+0000", 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.695e-06 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675546+0000", 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0796e-05 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.675553+0000", 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0149999999999998e-06 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.343412+0000", 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6678597110000002 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.360622+0000", 2023-10-09T10:22:36.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017209067000000002 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532751+0000", 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.172129479 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.539246+0000", 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0064948999999999996 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.549540+0000", 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010293656999999999 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.577693+0000", 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028153504999999999 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578635+0000", 2023-10-09T10:22:36.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094198399999999999 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.581004+0000", 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023692980000000002 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.584057+0000", 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003052768 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.374 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595619+0000", 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011562329999999999 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 347/342 [PushOp(9:0b40069f:::benchmark_data_smithi012_313365_object2631:head, version: 320'167, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0b40069f:::benchmark_data_smithi012_313365_object2631:head@320'167, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.60451141800002, 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3934248150000004, 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.375 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685471+0000", 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685469+0000", 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685554+0000", 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5025999999999995e-05 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685577+0000", 2023-10-09T10:22:36.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.319e-05 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.685596+0000", 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8238000000000001e-05 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.077623+0000", 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3920269019999996 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.077631+0000", 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0590000000000003e-06 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.078895+0000", 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001264668 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115110 9.18 9:189e6899:::benchmark_data_smithi012_313365_object2635:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.60176219599998, 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.870415833, 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115110 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688220+0000", 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688219+0000", 2023-10-09T10:22:36.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688222+0000", 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2199999999999999e-06 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688226+0000", 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9859999999999996e-06 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.688229+0000", 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2629999999999999e-06 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.531968+0000", 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8437393259999997 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532080+0000", 2023-10-09T10:22:36.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111876 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.532180+0000", 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9580999999999994e-05 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.536128+0000", 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039479809999999997 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.536166+0000", 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8197e-05 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556671+0000", 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020505332000000001 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556708+0000", 2023-10-09T10:22:36.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6332999999999999e-05 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.558636+0000", 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019280180000000001 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115117 9.18 9:1bb15d0c:::benchmark_data_smithi012_313365_object2642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.386 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.58518217099999, 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8692873749999999, 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115117 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704800+0000", 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704799+0000", 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704805+0000", 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0100000000000001e-06 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704817+0000", 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2054e-05 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.704823+0000", 2023-10-09T10:22:36.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9900000000000002e-06 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.533734+0000", 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.828911078 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.533880+0000", 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014652400000000001 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.534039+0000", 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015923399999999999 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.556665+0000", 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022625698 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.558646+0000", 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019810460000000002 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571804+0000", 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013158174 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571859+0000", 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5074000000000002e-05 2023-10-09T10:22:36.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.574087+0000", 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002227672 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115136 9.8 9:102eeaef:::benchmark_data_smithi012_313365_object2661:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.50893705099998, 2023-10-09T10:22:36.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9345335690000001, 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115136 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:22:36.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781045+0000", 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781043+0000", 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781051+0000", 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2010000000000002e-06 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781062+0000", 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1688000000000001e-05 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.781071+0000", 2023-10-09T10:22:36.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3089999999999996e-06 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.557513+0000", 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7764423310000002 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.571744+0000", 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014230893 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.636487+0000", 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064743060000000005 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.642690+0000", 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0062035450000000004 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.660819+0000", 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018128339 2023-10-09T10:22:36.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.700703+0000", 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.039884233999999998 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.702344+0000", 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016405899999999999 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.702817+0000", 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000473578 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.705696+0000", 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028788009999999998 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.715578+0000", 2023-10-09T10:22:36.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0098824919999999997 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115142 9.18 9:1de397b1:::benchmark_data_smithi012_313365_object2667:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 439.47943800399997, 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7874058789999996, 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:36.401 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115142 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810544+0000", 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810542+0000", 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810549+0000", 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7070000000000003e-06 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810562+0000", 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2543e-05 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.810570+0000", 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6279999999999994e-06 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578683+0000", 2023-10-09T10:22:36.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7681127830000003 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.578869+0000", 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018629700000000001 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.579038+0000", 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000168355 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.582715+0000", 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0036773040000000002 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.582723+0000", 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0139999999999998e-06 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595701+0000", 2023-10-09T10:22:36.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012977700999999999 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.595774+0000", 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2996000000000004e-05 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.597950+0000", 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021762019999999999 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.407 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPushReply(9.8 349/346 [PushReplyOp(9:12cf827d:::benchmark_data_smithi012_313365_object76143:head)])", 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:33.511271+0000", 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 422.77871139799998, 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.487463483, 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511271+0000", 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511271+0000", 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511269+0000", 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511278+0000", 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3359999999999995e-06 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511298+0000", 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0188999999999999e-05 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:33.511320+0000", 2023-10-09T10:22:36.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2050999999999999e-05 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:35.976548+0000", 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4652284409999998 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:35.998734+0000", 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022185547 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.411 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 354/342 [PushOp(9:0d3d12ba:::benchmark_data_smithi012_313365_object10048:head, version: 321'604, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0d3d12ba:::benchmark_data_smithi012_313365_object10048:head@321'604, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.142237+0000", 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 405.14774541100002, 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0200850130000001, 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142237+0000", 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142237+0000", 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142236+0000", 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142266+0000", 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0227e-05 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142281+0000", 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4569999999999999e-05 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.142298+0000", 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7076999999999999e-05 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.160817+0000", 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0185196150000002 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.160830+0000", 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2294e-05 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.162322+0000", 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014918100000000001 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 354/342 [PushOp(9:092dbf94:::benchmark_data_smithi012_313365_object10086:head, version: 321'606, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:092dbf94:::benchmark_data_smithi012_313365_object10086:head@321'606, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.525640+0000", 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 404.76434170700003, 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1090250359999998, 2023-10-09T10:22:36.416 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525640+0000", 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525640+0000", 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525639+0000", 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525682+0000", 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2657999999999997e-05 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525701+0000", 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8756000000000001e-05 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.525717+0000", 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6877000000000001e-05 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.633112+0000", 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1073941170000001 2023-10-09T10:22:36.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.633126+0000", 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4323e-05 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.634665+0000", 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001539428 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.10 354/342 [PushOp(9:0a3555ea:::benchmark_data_smithi012_313365_object10214:head, version: 321'613, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0a3555ea:::benchmark_data_smithi012_313365_object10214:head@321'613, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.639337+0000", 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 404.65064540100002, 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9968960120000001, 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639337+0000", 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639337+0000", 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639336+0000", 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639443+0000", 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000107241 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639486+0000", 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3460000000000001e-05 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.639510+0000", 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4189000000000002e-05 2023-10-09T10:22:36.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.634718+0000", 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9952079889999998 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.634731+0000", 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2605e-05 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:36.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.636233+0000", 2023-10-09T10:22:36.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0015015009999999999 2023-10-09T10:22:36.425 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.425 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.425 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.425 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:36.425 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:36.425 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:22:36.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:22:37.299 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:37.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:37.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:37.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:37.456 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:37.564 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:37.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:37.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:37.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:37.748 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:37 smithi110 ceph-mon[136032]: pgmap v1317: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 3406/148236 objects degraded (2.298%); 29966/148236 objects misplaced (20.215%); 21 MiB/s, 337 objects/s recovering 2023-10-09T10:22:37.749 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:37.813 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:37.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:37.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:37 smithi012 ceph-mon[147926]: pgmap v1317: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 3406/148236 objects degraded (2.298%); 29966/148236 objects misplaced (20.215%); 21 MiB/s, 337 objects/s recovering 2023-10-09T10:22:37.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:37 smithi012 ceph-mon[152263]: pgmap v1317: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 3406/148236 objects degraded (2.298%); 29966/148236 objects misplaced (20.215%); 21 MiB/s, 337 objects/s recovering 2023-10-09T10:22:37.918 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:37.983 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:38.021 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:38.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:38.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:38.128 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:38.151 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:22:38.178 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:22:38.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:22:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:22:37] "GET /metrics HTTP/1.1" 200 36666 "" "Prometheus/2.33.4" 2023-10-09T10:22:38.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:38.206 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:22:38.234 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:22:38.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:22:38.243 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:38.284 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:38.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:38.298 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:22:38.338 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:22:38.369 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:22:38.391 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:38.416 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:22:38.421 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:22:38.421 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:22:38.421 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:22:38.421 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:22:38.421 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:22:38.421 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:22:38.421 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:22:38.422 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:22:38.448 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:38.510 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:38.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:38.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:38.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:38.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4252803133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:38.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3406/148236 objects degraded (2.298%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:38.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4252803133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:38.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3406/148236 objects degraded (2.298%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:38.682 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:38.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4252803133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3406/148236 objects degraded (2.298%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:39.155 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:39.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:39.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:39.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:39.290 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:39.328 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:39.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:39.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:39.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:39.475 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:39.498 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.4 primary-affinity to 0.711714 (8466422) 2023-10-09T10:22:39.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:39 smithi012 ceph-mon[147926]: pgmap v1318: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 3406/146271 objects degraded (2.329%); 29629/146271 objects misplaced (20.256%); 16 MiB/s, 265 objects/s recovering 2023-10-09T10:22:39.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3203261849' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]: dispatch 2023-10-09T10:22:39.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:39 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]: dispatch 2023-10-09T10:22:39.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:39 smithi012 ceph-mon[152263]: pgmap v1318: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 3406/146271 objects degraded (2.329%); 29629/146271 objects misplaced (20.256%); 16 MiB/s, 265 objects/s recovering 2023-10-09T10:22:39.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3203261849' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]: dispatch 2023-10-09T10:22:39.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:39 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]: dispatch 2023-10-09T10:22:39.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:39 smithi110 ceph-mon[136032]: pgmap v1318: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 3406/146271 objects degraded (2.329%); 29629/146271 objects misplaced (20.256%); 16 MiB/s, 265 objects/s recovering 2023-10-09T10:22:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3203261849' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]: dispatch 2023-10-09T10:22:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:39 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]: dispatch 2023-10-09T10:22:39.992 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:39.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:40.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:40.127 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:40.284 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:40.315 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:40.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:40.419 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:40.446 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:40.479 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:40.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:40.571 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]': finished 2023-10-09T10:22:40.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: osdmap e472: 8 total, 8 up, 6 in 2023-10-09T10:22:40.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.572 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.573 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.573 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.573 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.573 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[147926]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.574 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]': finished 2023-10-09T10:22:40.574 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: osdmap e472: 8 total, 8 up, 6 in 2023-10-09T10:22:40.574 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.575 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:40 smithi012 ceph-mon[152263]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:40.642 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:40.813 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:40.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:40.920 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7117141092246744}]': finished 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: osdmap e472: 8 total, 8 up, 6 in 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:40 smithi110 ceph-mon[136032]: failed to encode map e472 with expected crc 2023-10-09T10:22:40.994 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:41.063 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:41.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:41.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:41.175 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:41.281 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:41.380 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:41.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:41.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:41.485 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:41.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: pgmap v1320: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 537 op/s; 3406/145593 objects degraded (2.339%); 29405/145593 objects misplaced (20.197%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:22:41.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: osdmap e473: 8 total, 8 up, 6 in 2023-10-09T10:22:41.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: 10.0 continuing backfill to osd.5 from (428'1400,471'9572] MIN to 471'9572 2023-10-09T10:22:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: 10.0 continuing backfill to osd.7 from (428'1400,471'9572] MIN to 471'9572 2023-10-09T10:22:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: 10.10 continuing backfill to osd.5 from (428'1400,471'9557] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 471'9557 2023-10-09T10:22:41.544 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:41 smithi110 ceph-mon[136032]: 10.10 continuing backfill to osd.7 from (428'1400,471'9557] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 471'9557 2023-10-09T10:22:41.548 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:41.609 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: pgmap v1320: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 537 op/s; 3406/145593 objects degraded (2.339%); 29405/145593 objects misplaced (20.197%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: osdmap e473: 8 total, 8 up, 6 in 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: 10.0 continuing backfill to osd.5 from (428'1400,471'9572] MIN to 471'9572 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: 10.0 continuing backfill to osd.7 from (428'1400,471'9572] MIN to 471'9572 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: 10.10 continuing backfill to osd.5 from (428'1400,471'9557] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 471'9557 2023-10-09T10:22:41.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[147926]: 10.10 continuing backfill to osd.7 from (428'1400,471'9557] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 471'9557 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: pgmap v1320: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 537 op/s; 3406/145593 objects degraded (2.339%); 29405/145593 objects misplaced (20.197%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: osdmap e473: 8 total, 8 up, 6 in 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: failed to encode map e473 with expected crc 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: 10.0 continuing backfill to osd.5 from (428'1400,471'9572] MIN to 471'9572 2023-10-09T10:22:41.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: 10.0 continuing backfill to osd.7 from (428'1400,471'9572] MIN to 471'9572 2023-10-09T10:22:41.612 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: 10.10 continuing backfill to osd.5 from (428'1400,471'9557] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 471'9557 2023-10-09T10:22:41.612 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:41 smithi012 ceph-mon[152263]: 10.10 continuing backfill to osd.7 from (428'1400,471'9557] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 471'9557 2023-10-09T10:22:41.612 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:41.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:41.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:41.648 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:22:41.648 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:22:41.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:41.812 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:41.889 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:41.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:41.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:22:41.999 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:42.084 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:42.147 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:22:42.219 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:42.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:42.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:42.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:42.390 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:42.460 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:42.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:42.519 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2368727430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:42.519 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:42.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2368727430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:42.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:42.544 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:22:42.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:42.652 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:42.726 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:42.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:42.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:42.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:42.878 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:42.951 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2368727430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:42.951 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:43.026 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:43.083 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:43.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:43.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:43.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:43.249 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:43.285 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:43.310 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:43.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:43.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:43.414 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:43.483 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:43.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: pgmap v1322: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 3434/139443 objects degraded (2.463%); 22232/139443 objects misplaced (15.943%); 5.4 MiB/s, 1 keys/s, 85 objects/s recovering 2023-10-09T10:22:43.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:22:43.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:43.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:43.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: osdmap e474: 8 total, 8 up, 6 in 2023-10-09T10:22:43.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.523 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3434/139443 objects degraded (2.463%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:43.523 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: pgmap v1322: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 3434/139443 objects degraded (2.463%); 22232/139443 objects misplaced (15.943%); 5.4 MiB/s, 1 keys/s, 85 objects/s recovering 2023-10-09T10:22:43.523 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:22:43.523 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:43.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:43.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: osdmap e474: 8 total, 8 up, 6 in 2023-10-09T10:22:43.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.524 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3434/139443 objects degraded (2.463%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:43.565 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:43.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:43.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:43.739 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: pgmap v1322: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 3434/139443 objects degraded (2.463%); 22232/139443 objects misplaced (15.943%); 5.4 MiB/s, 1 keys/s, 85 objects/s recovering 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: osdmap e474: 8 total, 8 up, 6 in 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: failed to encode map e474 with expected crc 2023-10-09T10:22:43.740 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3434/139443 objects degraded (2.463%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:43.741 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:43.804 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:43.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:43.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:43.914 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:43.972 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:44.103 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:44.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:44.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:44.210 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:44.314 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:44.384 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:44.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:44.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:44.550 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:44.716 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:44.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:44.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:44.779 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:22:44.821 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:44.827 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:22:44.860 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:22:44.901 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:22:44.906 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:44.950 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:22:44.965 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:44.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:44.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:44.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:44 smithi012 ceph-mon[147926]: pgmap v1324: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 3434/136146 objects degraded (2.522%); 21961/136146 objects misplaced (16.130%); 3.2 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-09T10:22:44.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:44.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:44.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2169374596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:44.997 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:22:45.016 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 0, 2, 5, 4] out_osds: [1, 6] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:22:45.017 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:22:45.017 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T10:22:45.017 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:22:45.030 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:22:45.063 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:22:45.070 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:45.071 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:22:45.071 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:22:45.071 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:22:45.071 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:22:45.071 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:22:45.071 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:22:45.072 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:22:45.072 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:22:45.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:45.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:22:45.219 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:45.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:44 smithi110 ceph-mon[136032]: pgmap v1324: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 3434/136146 objects degraded (2.522%); 21961/136146 objects misplaced (16.130%); 3.2 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-09T10:22:45.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:45.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:45.224 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2169374596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:45.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:45.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:45.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:44 smithi012 ceph-mon[152263]: pgmap v1324: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 79 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 3434/136146 objects degraded (2.522%); 21961/136146 objects misplaced (16.130%); 3.2 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-09T10:22:45.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:45.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:45.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2169374596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:45.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:45.400 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:45.452 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:45.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:45.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:45.480 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:22:45.480 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:22:45.480 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:22:45.481 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:22:45.481 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:22:45.562 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:45.646 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:45.701 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:45.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:45.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:45.868 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:46.091 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:46.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:46.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:46.119 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:22:46.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:46 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T10:22:46.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:46 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T10:22:46.202 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:46.273 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:46.292 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:46.334 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:46.343 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:46.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:46 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T10:22:46.458 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:46.525 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:46.581 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:46.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:46.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:46.782 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:46.864 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:46.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:46.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:46.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:47.041 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:47 smithi012 ceph-mon[147926]: pgmap v1325: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 3437/129354 objects degraded (2.657%); 26063/129354 objects misplaced (20.149%); 5.9 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T10:22:47.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:47 smithi012 ceph-mon[152263]: pgmap v1325: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 3437/129354 objects degraded (2.657%); 26063/129354 objects misplaced (20.149%); 5.9 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T10:22:47.042 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:47 smithi110 ceph-mon[136032]: pgmap v1325: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 3437/129354 objects degraded (2.657%); 26063/129354 objects misplaced (20.149%); 5.9 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T10:22:47.066 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:47.279 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:47.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:47.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:47.390 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:47.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:47.561 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:47.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:47.613 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:22:47.671 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:47.751 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:47.914 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:22:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:22:47] "GET /metrics HTTP/1.1" 200 36672 "" "Prometheus/2.33.4" 2023-10-09T10:22:49.124 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:49.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:49.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:49.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:49 smithi012 ceph-mon[147926]: pgmap v1326: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 827 op/s; 3437/125739 objects degraded (2.733%); 25659/125739 objects misplaced (20.407%); 4.6 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-09T10:22:49.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:49 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3437/125739 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:49 smithi012 ceph-mon[152263]: pgmap v1326: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 827 op/s; 3437/125739 objects degraded (2.733%); 25659/125739 objects misplaced (20.407%); 4.6 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-09T10:22:49.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:49 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3437/125739 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:49.230 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:49.249 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:49.292 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:49.380 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:49.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:49.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:49.395 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:49 smithi110 ceph-mon[136032]: pgmap v1326: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 827 op/s; 3437/125739 objects degraded (2.733%); 25659/125739 objects misplaced (20.407%); 4.6 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-09T10:22:49.395 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:49 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3437/125739 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:49.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:49.510 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:49.568 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:49.797 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:49.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:49.903 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:50.003 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:50.108 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:50.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:50.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:50.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:50.282 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:50.361 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:50.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:50.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:50.468 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:50.557 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:50.636 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:50.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:50.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:50.660 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:22:50.665 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:22:50.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:50.774 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:50.844 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:50.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:50.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:51.033 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:51.106 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:22:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:51 smithi012 ceph-mon[147926]: pgmap v1327: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 3437/124998 objects degraded (2.750%); 25412/124998 objects misplaced (20.330%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T10:22:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3320283871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:51.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:51 smithi012 ceph-mon[152263]: pgmap v1327: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 3437/124998 objects degraded (2.750%); 25412/124998 objects misplaced (20.330%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T10:22:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3320283871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:51.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:51 smithi110 ceph-mon[136032]: pgmap v1327: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 3437/124998 objects degraded (2.750%); 25412/124998 objects misplaced (20.330%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T10:22:51.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3320283871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:51.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:22:51.603 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:51.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:51.645 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1327,"stamp":"2023-10-09T10:22:49.713268+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2729580472,"num_objects":41666,"num_object_clones":0,"num_object_copies":124998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3437,"num_objects_misplaced":25412,"num_objects_unfound":0,"num_objects_dirty":41666,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":181272,"num_write_kb":7127310,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6117,"num_bytes_recovered":394681496,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":210001,"ondisk_log_size":210001,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10314400,"kb_used_data":6579312,"kb_used_omap":163,"kb_used_meta":3734748,"kb_avail":552156512,"statfs":{"total":575970213888,"available":565408268288,"internally_reserved":0,"allocated":6737215488,"data_stored":6729661104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":3824382963},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-315555840,"num_objects":-4815,"num_object_clones":0,"num_object_copies":-14445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":3180,"num_objects_unfound":0,"num_objects_dirty":-4815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4815,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":18087936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.218446"},"pg_stats":[{"pgid":"10.13","version":"474'9975","reported_seq":10042,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:44.819059+0000","last_change":"2023-10-09T10:21:45.524555+0000","last_active":"2023-10-09T10:22:44.819059+0000","last_peered":"2023-10-09T10:22:44.819059+0000","last_clean":"2023-10-09T10:22:44.819059+0000","last_became_active":"2023-10-09T10:21:45.500658+0000","last_became_peered":"2023-10-09T10:21:45.500658+0000","last_unstale":"2023-10-09T10:22:44.819059+0000","last_undegraded":"2023-10-09T10:22:44.819059+0000","last_fullsized":"2023-10-09T10:22:44.819059+0000","mapping_epoch":456,"log_start":"428'1263","ondisk_log_start":"428'1263","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8712,"log_dups_size":0,"ondisk_log_size":8712,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85786624,"num_objects":1309,"num_object_clones":0,"num_object_copies":3927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5784,"num_write_kb":226976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":67,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505163+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:22:40.505163+0000","last_peered":"2023-10-09T10:22:40.505163+0000","last_clean":"2023-10-09T10:22:40.505163+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:22:40.505163+0000","last_undegraded":"2023-10-09T10:22:40.505163+0000","last_fullsized":"2023-10-09T10:22:40.505163+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":419,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505548+0000","last_change":"2023-10-09T10:22:27.502142+0000","last_active":"2023-10-09T10:22:40.505548+0000","last_peered":"2023-10-09T10:22:40.505548+0000","last_clean":"2023-10-09T10:22:40.505548+0000","last_became_active":"2023-10-09T10:22:27.501338+0000","last_became_peered":"2023-10-09T10:22:27.501338+0000","last_unstale":"2023-10-09T10:22:40.505548+0000","last_undegraded":"2023-10-09T10:22:40.505548+0000","last_fullsized":"2023-10-09T10:22:40.505548+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.10","version":"474'9666","reported_seq":9786,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:44.959787+0000","last_change":"2023-10-09T10:22:40.563991+0000","last_active":"2023-10-09T10:22:44.959787+0000","last_peered":"2023-10-09T10:22:44.959787+0000","last_clean":"2023-10-09T10:22:12.925413+0000","last_became_active":"2023-10-09T10:22:40.537317+0000","last_became_peered":"2023-10-09T10:22:40.537317+0000","last_unstale":"2023-10-09T10:22:44.959787+0000","last_undegraded":"2023-10-09T10:22:44.959787+0000","last_fullsized":"2023-10-09T10:22:44.959787+0000","mapping_epoch":472,"log_start":"428'1400","ondisk_log_start":"428'1400","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8266,"log_dups_size":0,"ondisk_log_size":8266,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87195648,"num_objects":1331,"num_object_clones":0,"num_object_copies":3993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2654,"num_objects_unfound":0,"num_objects_dirty":1331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5584,"num_write_kb":221248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[4,6,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":1331}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":487,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.504303+0000","last_change":"2023-10-09T10:22:27.432604+0000","last_active":"2023-10-09T10:22:40.504303+0000","last_peered":"2023-10-09T10:22:40.504303+0000","last_clean":"2023-10-09T10:22:40.504303+0000","last_became_active":"2023-10-09T10:22:27.429232+0000","last_became_peered":"2023-10-09T10:22:27.429232+0000","last_unstale":"2023-10-09T10:22:40.504303+0000","last_undegraded":"2023-10-09T10:22:40.504303+0000","last_fullsized":"2023-10-09T10:22:40.504303+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":379,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499524+0000","last_change":"2023-10-09T10:22:13.362862+0000","last_active":"2023-10-09T10:22:40.499524+0000","last_peered":"2023-10-09T10:22:40.499524+0000","last_clean":"2023-10-09T10:22:40.499524+0000","last_became_active":"2023-10-09T10:22:13.362506+0000","last_became_peered":"2023-10-09T10:22:13.362506+0000","last_unstale":"2023-10-09T10:22:40.499524+0000","last_undegraded":"2023-10-09T10:22:40.499524+0000","last_fullsized":"2023-10-09T10:22:40.499524+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:55.200012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"474'9719","reported_seq":9791,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:47.433094+0000","last_change":"2023-10-09T10:22:40.524572+0000","last_active":"2023-10-09T10:22:47.433094+0000","last_peered":"2023-10-09T10:22:47.433094+0000","last_clean":"2023-10-09T10:22:47.433094+0000","last_became_active":"2023-10-09T10:22:40.509173+0000","last_became_peered":"2023-10-09T10:22:40.509173+0000","last_unstale":"2023-10-09T10:22:47.433094+0000","last_undegraded":"2023-10-09T10:22:47.433094+0000","last_fullsized":"2023-10-09T10:22:47.433094+0000","mapping_epoch":472,"log_start":"428'1500","ondisk_log_start":"428'1500","created":456,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8219,"log_dups_size":0,"ondisk_log_size":8219,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81756160,"num_objects":1248,"num_object_clones":0,"num_object_copies":3744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5633,"num_write_kb":220160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":390,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.530380+0000","last_change":"2023-10-09T10:22:13.355331+0000","last_active":"2023-10-09T10:22:40.530380+0000","last_peered":"2023-10-09T10:22:40.530380+0000","last_clean":"2023-10-09T10:22:40.530380+0000","last_became_active":"2023-10-09T10:22:13.354983+0000","last_became_peered":"2023-10-09T10:22:13.354983+0000","last_unstale":"2023-10-09T10:22:40.530380+0000","last_undegraded":"2023-10-09T10:22:40.530380+0000","last_fullsized":"2023-10-09T10:22:40.530380+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:31.067024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":31,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499626+0000","last_change":"2023-10-09T10:22:27.399114+0000","last_active":"2023-10-09T10:22:40.499626+0000","last_peered":"2023-10-09T10:22:40.499626+0000","last_clean":"2023-10-09T10:22:40.499626+0000","last_became_active":"2023-10-09T10:22:27.398806+0000","last_became_peered":"2023-10-09T10:22:27.398806+0000","last_unstale":"2023-10-09T10:22:40.499626+0000","last_undegraded":"2023-10-09T10:22:40.499626+0000","last_fullsized":"2023-10-09T10:22:40.499626+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:06.160841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":413,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499900+0000","last_change":"2023-10-09T10:22:27.392579+0000","last_active":"2023-10-09T10:22:40.499900+0000","last_peered":"2023-10-09T10:22:40.499900+0000","last_clean":"2023-10-09T10:22:40.499900+0000","last_became_active":"2023-10-09T10:22:27.392040+0000","last_became_peered":"2023-10-09T10:22:27.392040+0000","last_unstale":"2023-10-09T10:22:40.499900+0000","last_undegraded":"2023-10-09T10:22:40.499900+0000","last_fullsized":"2023-10-09T10:22:40.499900+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:20.831225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":27,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.547281+0000","last_change":"2023-10-09T10:22:40.543504+0000","last_active":"2023-10-09T10:22:40.547281+0000","last_peered":"2023-10-09T10:22:40.547281+0000","last_clean":"2023-10-09T10:22:40.547281+0000","last_became_active":"2023-10-09T10:22:40.542810+0000","last_became_peered":"2023-10-09T10:22:40.542810+0000","last_unstale":"2023-10-09T10:22:40.547281+0000","last_undegraded":"2023-10-09T10:22:40.547281+0000","last_fullsized":"2023-10-09T10:22:40.547281+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":131,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.502612+0000","last_change":"2023-10-09T10:22:13.355925+0000","last_active":"2023-10-09T10:22:40.502612+0000","last_peered":"2023-10-09T10:22:40.502612+0000","last_clean":"2023-10-09T10:22:40.502612+0000","last_became_active":"2023-10-09T10:22:13.355129+0000","last_became_peered":"2023-10-09T10:22:13.355129+0000","last_unstale":"2023-10-09T10:22:40.502612+0000","last_undegraded":"2023-10-09T10:22:40.502612+0000","last_fullsized":"2023-10-09T10:22:40.502612+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":394,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.530835+0000","last_change":"2023-10-09T10:22:27.394556+0000","last_active":"2023-10-09T10:22:40.530835+0000","last_peered":"2023-10-09T10:22:40.530835+0000","last_clean":"2023-10-09T10:22:40.530835+0000","last_became_active":"2023-10-09T10:22:27.394086+0000","last_became_peered":"2023-10-09T10:22:27.394086+0000","last_unstale":"2023-10-09T10:22:40.530835+0000","last_undegraded":"2023-10-09T10:22:40.530835+0000","last_fullsized":"2023-10-09T10:22:40.530835+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:30:25.017559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":64,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505824+0000","last_change":"2023-10-09T10:19:43.299477+0000","last_active":"2023-10-09T10:22:40.505824+0000","last_peered":"2023-10-09T10:22:40.505824+0000","last_clean":"2023-10-09T10:22:40.505824+0000","last_became_active":"2023-10-09T10:19:43.274448+0000","last_became_peered":"2023-10-09T10:19:43.274448+0000","last_unstale":"2023-10-09T10:22:40.505824+0000","last_undegraded":"2023-10-09T10:22:40.505824+0000","last_fullsized":"2023-10-09T10:22:40.505824+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":30,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.500444+0000","last_change":"2023-10-09T10:22:27.399642+0000","last_active":"2023-10-09T10:22:40.500444+0000","last_peered":"2023-10-09T10:22:40.500444+0000","last_clean":"2023-10-09T10:22:40.500444+0000","last_became_active":"2023-10-09T10:22:27.399069+0000","last_became_peered":"2023-10-09T10:22:27.399069+0000","last_unstale":"2023-10-09T10:22:40.500444+0000","last_undegraded":"2023-10-09T10:22:40.500444+0000","last_fullsized":"2023-10-09T10:22:40.500444+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":87,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.506085+0000","last_change":"2023-10-09T10:22:27.523164+0000","last_active":"2023-10-09T10:22:40.506085+0000","last_peered":"2023-10-09T10:22:40.506085+0000","last_clean":"2023-10-09T10:22:40.506085+0000","last_became_active":"2023-10-09T10:22:27.465781+0000","last_became_peered":"2023-10-09T10:22:27.465781+0000","last_unstale":"2023-10-09T10:22:40.506085+0000","last_undegraded":"2023-10-09T10:22:40.506085+0000","last_fullsized":"2023-10-09T10:22:40.506085+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":76,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.504597+0000","last_change":"2023-10-09T10:22:27.502829+0000","last_active":"2023-10-09T10:22:40.504597+0000","last_peered":"2023-10-09T10:22:40.504597+0000","last_clean":"2023-10-09T10:22:40.504597+0000","last_became_active":"2023-10-09T10:22:27.502361+0000","last_became_peered":"2023-10-09T10:22:27.502361+0000","last_unstale":"2023-10-09T10:22:40.504597+0000","last_undegraded":"2023-10-09T10:22:40.504597+0000","last_fullsized":"2023-10-09T10:22:40.504597+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":497,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.512954+0000","last_change":"2023-10-09T10:22:40.512327+0000","last_active":"2023-10-09T10:22:40.512954+0000","last_peered":"2023-10-09T10:22:40.512954+0000","last_clean":"2023-10-09T10:22:40.512954+0000","last_became_active":"2023-10-09T10:22:40.511936+0000","last_became_peered":"2023-10-09T10:22:40.511936+0000","last_unstale":"2023-10-09T10:22:40.512954+0000","last_undegraded":"2023-10-09T10:22:40.512954+0000","last_fullsized":"2023-10-09T10:22:40.512954+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":383,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.531202+0000","last_change":"2023-10-09T10:22:13.355370+0000","last_active":"2023-10-09T10:22:40.531202+0000","last_peered":"2023-10-09T10:22:40.531202+0000","last_clean":"2023-10-09T10:22:40.531202+0000","last_became_active":"2023-10-09T10:22:13.355003+0000","last_became_peered":"2023-10-09T10:22:13.355003+0000","last_unstale":"2023-10-09T10:22:40.531202+0000","last_undegraded":"2023-10-09T10:22:40.531202+0000","last_fullsized":"2023-10-09T10:22:40.531202+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:08.408252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":377,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499133+0000","last_change":"2023-10-09T10:22:13.360714+0000","last_active":"2023-10-09T10:22:40.499133+0000","last_peered":"2023-10-09T10:22:40.499133+0000","last_clean":"2023-10-09T10:22:40.499133+0000","last_became_active":"2023-10-09T10:22:13.357250+0000","last_became_peered":"2023-10-09T10:22:13.357250+0000","last_unstale":"2023-10-09T10:22:40.499133+0000","last_undegraded":"2023-10-09T10:22:40.499133+0000","last_fullsized":"2023-10-09T10:22:40.499133+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:28.076425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501935+0000","last_change":"2023-10-09T10:16:18.255609+0000","last_active":"2023-10-09T10:22:40.501935+0000","last_peered":"2023-10-09T10:22:40.501935+0000","last_clean":"2023-10-09T10:22:40.501935+0000","last_became_active":"2023-10-09T10:16:18.244618+0000","last_became_peered":"2023-10-09T10:16:18.244618+0000","last_unstale":"2023-10-09T10:22:40.501935+0000","last_undegraded":"2023-10-09T10:22:40.501935+0000","last_fullsized":"2023-10-09T10:22:40.501935+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.18","version":"474'9786","reported_seq":9855,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:47.497553+0000","last_change":"2023-10-09T10:21:45.527702+0000","last_active":"2023-10-09T10:22:47.497553+0000","last_peered":"2023-10-09T10:22:47.497553+0000","last_clean":"2023-10-09T10:22:47.497553+0000","last_became_active":"2023-10-09T10:21:45.513973+0000","last_became_peered":"2023-10-09T10:21:45.513973+0000","last_unstale":"2023-10-09T10:22:47.497553+0000","last_undegraded":"2023-10-09T10:22:47.497553+0000","last_fullsized":"2023-10-09T10:22:47.497553+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":9786,"log_dups_size":0,"ondisk_log_size":9786,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83230720,"num_objects":1270,"num_object_clones":0,"num_object_copies":3810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5697,"num_write_kb":222944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":67,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.504473+0000","last_change":"2023-10-09T10:22:27.465342+0000","last_active":"2023-10-09T10:22:40.504473+0000","last_peered":"2023-10-09T10:22:40.504473+0000","last_clean":"2023-10-09T10:22:40.504473+0000","last_became_active":"2023-10-09T10:22:27.464647+0000","last_became_peered":"2023-10-09T10:22:27.464647+0000","last_unstale":"2023-10-09T10:22:40.504473+0000","last_undegraded":"2023-10-09T10:22:40.504473+0000","last_fullsized":"2023-10-09T10:22:40.504473+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":46,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.531354+0000","last_change":"2023-10-09T10:20:37.462345+0000","last_active":"2023-10-09T10:22:40.531354+0000","last_peered":"2023-10-09T10:22:40.531354+0000","last_clean":"2023-10-09T10:22:40.531354+0000","last_became_active":"2023-10-09T10:20:37.462057+0000","last_became_peered":"2023-10-09T10:20:37.462057+0000","last_unstale":"2023-10-09T10:22:40.531354+0000","last_undegraded":"2023-10-09T10:22:40.531354+0000","last_fullsized":"2023-10-09T10:22:40.531354+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:35.024832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.19","version":"474'9777","reported_seq":9844,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:44.750531+0000","last_change":"2023-10-09T10:21:45.513359+0000","last_active":"2023-10-09T10:22:44.750531+0000","last_peered":"2023-10-09T10:22:44.750531+0000","last_clean":"2023-10-09T10:22:44.750531+0000","last_became_active":"2023-10-09T10:21:45.501223+0000","last_became_peered":"2023-10-09T10:21:45.501223+0000","last_unstale":"2023-10-09T10:22:44.750531+0000","last_undegraded":"2023-10-09T10:22:44.750531+0000","last_fullsized":"2023-10-09T10:22:44.750531+0000","mapping_epoch":456,"log_start":"428'854","ondisk_log_start":"428'854","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8923,"log_dups_size":0,"ondisk_log_size":8923,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87785472,"num_objects":1340,"num_object_clones":0,"num_object_copies":4020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5645,"num_write_kb":223488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":408,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.502461+0000","last_change":"2023-10-09T10:22:27.407286+0000","last_active":"2023-10-09T10:22:40.502461+0000","last_peered":"2023-10-09T10:22:40.502461+0000","last_clean":"2023-10-09T10:22:40.502461+0000","last_became_active":"2023-10-09T10:22:27.395638+0000","last_became_peered":"2023-10-09T10:22:27.395638+0000","last_unstale":"2023-10-09T10:22:40.502461+0000","last_undegraded":"2023-10-09T10:22:40.502461+0000","last_fullsized":"2023-10-09T10:22:40.502461+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.502168+0000","last_change":"2023-10-09T10:22:27.395329+0000","last_active":"2023-10-09T10:22:40.502168+0000","last_peered":"2023-10-09T10:22:40.502168+0000","last_clean":"2023-10-09T10:22:40.502168+0000","last_became_active":"2023-10-09T10:22:27.394723+0000","last_became_peered":"2023-10-09T10:22:27.394723+0000","last_unstale":"2023-10-09T10:22:40.502168+0000","last_undegraded":"2023-10-09T10:22:40.502168+0000","last_fullsized":"2023-10-09T10:22:40.502168+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.6","version":"474'9722","reported_seq":9790,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:44.780107+0000","last_change":"2023-10-09T10:21:45.455204+0000","last_active":"2023-10-09T10:22:44.780107+0000","last_peered":"2023-10-09T10:22:44.780107+0000","last_clean":"2023-10-09T10:22:44.780107+0000","last_became_active":"2023-10-09T10:21:45.446086+0000","last_became_peered":"2023-10-09T10:21:45.446086+0000","last_unstale":"2023-10-09T10:22:44.780107+0000","last_undegraded":"2023-10-09T10:22:44.780107+0000","last_fullsized":"2023-10-09T10:22:44.780107+0000","mapping_epoch":424,"log_start":"428'756","ondisk_log_start":"428'756","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8966,"log_dups_size":0,"ondisk_log_size":8966,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90570752,"num_objects":1382,"num_object_clones":0,"num_object_copies":4146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5597,"num_write_kb":223328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":72,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499766+0000","last_change":"2023-10-09T10:22:13.396641+0000","last_active":"2023-10-09T10:22:40.499766+0000","last_peered":"2023-10-09T10:22:40.499766+0000","last_clean":"2023-10-09T10:22:40.499766+0000","last_became_active":"2023-10-09T10:22:13.356686+0000","last_became_peered":"2023-10-09T10:22:13.356686+0000","last_unstale":"2023-10-09T10:22:40.499766+0000","last_undegraded":"2023-10-09T10:22:40.499766+0000","last_fullsized":"2023-10-09T10:22:40.499766+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:31.123346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.502367+0000","last_change":"2023-10-09T10:22:27.559733+0000","last_active":"2023-10-09T10:22:40.502367+0000","last_peered":"2023-10-09T10:22:40.502367+0000","last_clean":"2023-10-09T10:22:40.502367+0000","last_became_active":"2023-10-09T10:22:27.558941+0000","last_became_peered":"2023-10-09T10:22:27.558941+0000","last_unstale":"2023-10-09T10:22:40.502367+0000","last_undegraded":"2023-10-09T10:22:40.502367+0000","last_fullsized":"2023-10-09T10:22:40.502367+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"474'9509","reported_seq":10141,"reported_epoch":474,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:22:44.782286+0000","last_change":"2023-10-09T10:22:28.717556+0000","last_active":"2023-10-09T10:22:44.782286+0000","last_peered":"2023-10-09T10:22:44.782286+0000","last_clean":"2023-10-09T10:22:26.409885+0000","last_became_active":"2023-10-09T10:22:28.578083+0000","last_became_peered":"2023-10-09T10:22:28.578083+0000","last_unstale":"2023-10-09T10:22:44.782286+0000","last_undegraded":"2023-10-09T10:22:28.461271+0000","last_fullsized":"2023-10-09T10:22:28.404656+0000","mapping_epoch":467,"log_start":"428'692","ondisk_log_start":"428'692","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8817,"log_dups_size":0,"ondisk_log_size":8817,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84934656,"num_objects":1296,"num_object_clones":0,"num_object_copies":3888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":45,"num_objects_misplaced":1279,"num_objects_unfound":0,"num_objects_dirty":1296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5496,"num_write_kb":217344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-28},{"shards":"4,5,6","objects":1279}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":75,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.508068+0000","last_change":"2023-10-09T10:22:13.526659+0000","last_active":"2023-10-09T10:22:40.508068+0000","last_peered":"2023-10-09T10:22:40.508068+0000","last_clean":"2023-10-09T10:22:40.508068+0000","last_became_active":"2023-10-09T10:22:13.366867+0000","last_became_peered":"2023-10-09T10:22:13.366867+0000","last_unstale":"2023-10-09T10:22:40.508068+0000","last_undegraded":"2023-10-09T10:22:40.508068+0000","last_fullsized":"2023-10-09T10:22:40.508068+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":73,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:42.528124+0000","last_change":"2023-10-09T10:22:40.509408+0000","last_active":"2023-10-09T10:22:42.528124+0000","last_peered":"2023-10-09T10:22:42.528124+0000","last_clean":"2023-10-09T10:22:42.528124+0000","last_became_active":"2023-10-09T10:22:40.509174+0000","last_became_peered":"2023-10-09T10:22:40.509174+0000","last_unstale":"2023-10-09T10:22:42.528124+0000","last_undegraded":"2023-10-09T10:22:42.528124+0000","last_fullsized":"2023-10-09T10:22:42.528124+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:12.890481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.4","version":"474'9823","reported_seq":12417,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:45.121638+0000","last_change":"2023-10-09T10:22:28.480698+0000","last_active":"2023-10-09T10:22:45.121638+0000","last_peered":"2023-10-09T10:22:45.121638+0000","last_clean":"2023-10-09T10:22:12.998671+0000","last_became_active":"2023-10-09T10:22:14.566556+0000","last_became_peered":"2023-10-09T10:22:14.566556+0000","last_unstale":"2023-10-09T10:22:45.121638+0000","last_undegraded":"2023-10-09T10:22:45.121638+0000","last_fullsized":"2023-10-09T10:22:45.121638+0000","mapping_epoch":463,"log_start":"428'1671","ondisk_log_start":"428'1671","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8152,"log_dups_size":0,"ondisk_log_size":8152,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82051072,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":255,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5687,"num_write_kb":222048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1257,"num_bytes_recovered":82247680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1252}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":139,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501697+0000","last_change":"2023-10-09T10:22:27.440685+0000","last_active":"2023-10-09T10:22:40.501697+0000","last_peered":"2023-10-09T10:22:40.501697+0000","last_clean":"2023-10-09T10:22:40.501697+0000","last_became_active":"2023-10-09T10:22:27.394703+0000","last_became_peered":"2023-10-09T10:22:27.394703+0000","last_unstale":"2023-10-09T10:22:40.501697+0000","last_undegraded":"2023-10-09T10:22:40.501697+0000","last_fullsized":"2023-10-09T10:22:40.501697+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":30,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.500915+0000","last_change":"2023-10-09T10:22:27.399497+0000","last_active":"2023-10-09T10:22:40.500915+0000","last_peered":"2023-10-09T10:22:40.500915+0000","last_clean":"2023-10-09T10:22:40.500915+0000","last_became_active":"2023-10-09T10:22:27.398744+0000","last_became_peered":"2023-10-09T10:22:27.398744+0000","last_unstale":"2023-10-09T10:22:40.500915+0000","last_undegraded":"2023-10-09T10:22:40.500915+0000","last_fullsized":"2023-10-09T10:22:40.500915+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"474'9890","reported_seq":9961,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:47.461764+0000","last_change":"2023-10-09T10:22:27.381738+0000","last_active":"2023-10-09T10:22:47.461764+0000","last_peered":"2023-10-09T10:22:47.461764+0000","last_clean":"2023-10-09T10:22:47.461764+0000","last_became_active":"2023-10-09T10:21:45.495805+0000","last_became_peered":"2023-10-09T10:21:45.495805+0000","last_unstale":"2023-10-09T10:22:47.461764+0000","last_undegraded":"2023-10-09T10:22:47.461764+0000","last_fullsized":"2023-10-09T10:22:47.461764+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":9890,"log_dups_size":0,"ondisk_log_size":9890,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86081536,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5735,"num_write_kb":225568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":140,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501524+0000","last_change":"2023-10-09T10:22:27.394423+0000","last_active":"2023-10-09T10:22:40.501524+0000","last_peered":"2023-10-09T10:22:40.501524+0000","last_clean":"2023-10-09T10:22:40.501524+0000","last_became_active":"2023-10-09T10:22:27.394040+0000","last_became_peered":"2023-10-09T10:22:27.394040+0000","last_unstale":"2023-10-09T10:22:40.501524+0000","last_undegraded":"2023-10-09T10:22:40.501524+0000","last_fullsized":"2023-10-09T10:22:40.501524+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":373,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:42.528247+0000","last_change":"2023-10-09T10:22:13.366463+0000","last_active":"2023-10-09T10:22:42.528247+0000","last_peered":"2023-10-09T10:22:42.528247+0000","last_clean":"2023-10-09T10:22:42.528247+0000","last_became_active":"2023-10-09T10:22:13.366125+0000","last_became_peered":"2023-10-09T10:22:13.366125+0000","last_unstale":"2023-10-09T10:22:42.528247+0000","last_undegraded":"2023-10-09T10:22:42.528247+0000","last_fullsized":"2023-10-09T10:22:42.528247+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:41.113925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"474'9695","reported_seq":9782,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:45.090298+0000","last_change":"2023-10-09T10:22:27.587164+0000","last_active":"2023-10-09T10:22:45.090298+0000","last_peered":"2023-10-09T10:22:45.090298+0000","last_clean":"2023-10-09T10:22:12.977049+0000","last_became_active":"2023-10-09T10:22:27.558911+0000","last_became_peered":"2023-10-09T10:22:27.558911+0000","last_unstale":"2023-10-09T10:22:45.090298+0000","last_undegraded":"2023-10-09T10:22:45.090298+0000","last_fullsized":"2023-10-09T10:22:45.090298+0000","mapping_epoch":466,"log_start":"428'1500","ondisk_log_start":"428'1500","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8195,"log_dups_size":0,"ondisk_log_size":8195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88899584,"num_objects":1356,"num_object_clones":0,"num_object_copies":4068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1356,"num_objects_unfound":0,"num_objects_dirty":1356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5601,"num_write_kb":222656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1356}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":56,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499093+0000","last_change":"2023-10-09T10:22:13.371260+0000","last_active":"2023-10-09T10:22:40.499093+0000","last_peered":"2023-10-09T10:22:40.499093+0000","last_clean":"2023-10-09T10:22:40.499093+0000","last_became_active":"2023-10-09T10:22:13.356722+0000","last_became_peered":"2023-10-09T10:22:13.356722+0000","last_unstale":"2023-10-09T10:22:40.499093+0000","last_undegraded":"2023-10-09T10:22:40.499093+0000","last_fullsized":"2023-10-09T10:22:40.499093+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:02:10.174833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":67,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505994+0000","last_change":"2023-10-09T10:22:27.429350+0000","last_active":"2023-10-09T10:22:40.505994+0000","last_peered":"2023-10-09T10:22:40.505994+0000","last_clean":"2023-10-09T10:22:40.505994+0000","last_became_active":"2023-10-09T10:22:27.429075+0000","last_became_peered":"2023-10-09T10:22:27.429075+0000","last_unstale":"2023-10-09T10:22:40.505994+0000","last_undegraded":"2023-10-09T10:22:40.505994+0000","last_fullsized":"2023-10-09T10:22:40.505994+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"474'9963","reported_seq":10031,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:44.788848+0000","last_change":"2023-10-09T10:21:45.483914+0000","last_active":"2023-10-09T10:22:44.788848+0000","last_peered":"2023-10-09T10:22:44.788848+0000","last_clean":"2023-10-09T10:22:44.788848+0000","last_became_active":"2023-10-09T10:21:45.474650+0000","last_became_peered":"2023-10-09T10:21:45.474650+0000","last_unstale":"2023-10-09T10:22:44.788848+0000","last_undegraded":"2023-10-09T10:22:44.788848+0000","last_fullsized":"2023-10-09T10:22:44.788848+0000","mapping_epoch":424,"log_start":"428'1145","ondisk_log_start":"428'1145","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8818,"log_dups_size":0,"ondisk_log_size":8818,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86573056,"num_objects":1321,"num_object_clones":0,"num_object_copies":3963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5772,"num_write_kb":226976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505010+0000","last_change":"2023-10-09T10:22:27.444402+0000","last_active":"2023-10-09T10:22:40.505010+0000","last_peered":"2023-10-09T10:22:40.505010+0000","last_clean":"2023-10-09T10:22:40.505010+0000","last_became_active":"2023-10-09T10:22:27.443887+0000","last_became_peered":"2023-10-09T10:22:27.443887+0000","last_unstale":"2023-10-09T10:22:40.505010+0000","last_undegraded":"2023-10-09T10:22:40.505010+0000","last_fullsized":"2023-10-09T10:22:40.505010+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":73,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.530169+0000","last_change":"2023-10-09T10:22:40.529911+0000","last_active":"2023-10-09T10:22:40.530169+0000","last_peered":"2023-10-09T10:22:40.530169+0000","last_clean":"2023-10-09T10:22:40.530169+0000","last_became_active":"2023-10-09T10:22:40.529496+0000","last_became_peered":"2023-10-09T10:22:40.529496+0000","last_unstale":"2023-10-09T10:22:40.530169+0000","last_undegraded":"2023-10-09T10:22:40.530169+0000","last_fullsized":"2023-10-09T10:22:40.530169+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.0","version":"474'9657","reported_seq":9760,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:44.988724+0000","last_change":"2023-10-09T10:22:40.536955+0000","last_active":"2023-10-09T10:22:44.988724+0000","last_peered":"2023-10-09T10:22:44.988724+0000","last_clean":"2023-10-09T10:22:12.951978+0000","last_became_active":"2023-10-09T10:22:40.510422+0000","last_became_peered":"2023-10-09T10:22:40.510422+0000","last_unstale":"2023-10-09T10:22:44.988724+0000","last_undegraded":"2023-10-09T10:22:44.988724+0000","last_fullsized":"2023-10-09T10:22:44.988724+0000","mapping_epoch":472,"log_start":"428'1400","ondisk_log_start":"428'1400","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8257,"log_dups_size":0,"ondisk_log_size":8257,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87785472,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2678,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5574,"num_write_kb":221248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":1339}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":58,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.504596+0000","last_change":"2023-10-09T10:19:43.288603+0000","last_active":"2023-10-09T10:22:40.504596+0000","last_peered":"2023-10-09T10:22:40.504596+0000","last_clean":"2023-10-09T10:22:40.504596+0000","last_became_active":"2023-10-09T10:19:43.287994+0000","last_became_peered":"2023-10-09T10:19:43.287994+0000","last_unstale":"2023-10-09T10:22:40.504596+0000","last_undegraded":"2023-10-09T10:22:40.504596+0000","last_fullsized":"2023-10-09T10:22:40.504596+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":85,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.508032+0000","last_change":"2023-10-09T10:22:27.429388+0000","last_active":"2023-10-09T10:22:40.508032+0000","last_peered":"2023-10-09T10:22:40.508032+0000","last_clean":"2023-10-09T10:22:40.508032+0000","last_became_active":"2023-10-09T10:22:27.429067+0000","last_became_peered":"2023-10-09T10:22:27.429067+0000","last_unstale":"2023-10-09T10:22:40.508032+0000","last_undegraded":"2023-10-09T10:22:40.508032+0000","last_fullsized":"2023-10-09T10:22:40.508032+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"474'9733","reported_seq":9808,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:47.430285+0000","last_change":"2023-10-09T10:22:40.540614+0000","last_active":"2023-10-09T10:22:47.430285+0000","last_peered":"2023-10-09T10:22:47.430285+0000","last_clean":"2023-10-09T10:22:47.430285+0000","last_became_active":"2023-10-09T10:22:40.524435+0000","last_became_peered":"2023-10-09T10:22:40.524435+0000","last_unstale":"2023-10-09T10:22:47.430285+0000","last_undegraded":"2023-10-09T10:22:47.430285+0000","last_fullsized":"2023-10-09T10:22:47.430285+0000","mapping_epoch":472,"log_start":"428'1500","ondisk_log_start":"428'1500","created":424,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8233,"log_dups_size":0,"ondisk_log_size":8233,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80838656,"num_objects":1233,"num_object_clones":0,"num_object_copies":3699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5646,"num_write_kb":220160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":399,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.547303+0000","last_change":"2023-10-09T10:22:40.543714+0000","last_active":"2023-10-09T10:22:40.547303+0000","last_peered":"2023-10-09T10:22:40.547303+0000","last_clean":"2023-10-09T10:22:40.547303+0000","last_became_active":"2023-10-09T10:22:40.543222+0000","last_became_peered":"2023-10-09T10:22:40.543222+0000","last_unstale":"2023-10-09T10:22:40.547303+0000","last_undegraded":"2023-10-09T10:22:40.547303+0000","last_fullsized":"2023-10-09T10:22:40.547303+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":479,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505286+0000","last_change":"2023-10-09T10:22:13.367996+0000","last_active":"2023-10-09T10:22:40.505286+0000","last_peered":"2023-10-09T10:22:40.505286+0000","last_clean":"2023-10-09T10:22:40.505286+0000","last_became_active":"2023-10-09T10:22:13.367583+0000","last_became_peered":"2023-10-09T10:22:13.367583+0000","last_unstale":"2023-10-09T10:22:40.505286+0000","last_undegraded":"2023-10-09T10:22:40.505286+0000","last_fullsized":"2023-10-09T10:22:40.505286+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"474'11260","reported_seq":11324,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:47.424183+0000","last_change":"2023-10-09T10:22:40.536685+0000","last_active":"2023-10-09T10:22:47.424183+0000","last_peered":"2023-10-09T10:22:47.424183+0000","last_clean":"2023-10-09T10:22:47.424183+0000","last_became_active":"2023-10-09T10:22:40.511610+0000","last_became_peered":"2023-10-09T10:22:40.511610+0000","last_unstale":"2023-10-09T10:22:47.424183+0000","last_undegraded":"2023-10-09T10:22:47.424183+0000","last_fullsized":"2023-10-09T10:22:47.424183+0000","mapping_epoch":472,"log_start":"441'5950","ondisk_log_start":"441'5950","created":424,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":5310,"log_dups_size":0,"ondisk_log_size":5310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:31.313534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168034304,"num_objects":2564,"num_object_clones":0,"num_object_copies":7692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11260,"num_write_kb":442368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":175,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499142+0000","last_change":"2023-10-09T10:22:13.356443+0000","last_active":"2023-10-09T10:22:40.499142+0000","last_peered":"2023-10-09T10:22:40.499142+0000","last_clean":"2023-10-09T10:22:40.499142+0000","last_became_active":"2023-10-09T10:22:13.356073+0000","last_became_peered":"2023-10-09T10:22:13.356073+0000","last_unstale":"2023-10-09T10:22:40.499142+0000","last_undegraded":"2023-10-09T10:22:40.499142+0000","last_fullsized":"2023-10-09T10:22:40.499142+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:27.200439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501863+0000","last_change":"2023-10-09T10:22:27.397595+0000","last_active":"2023-10-09T10:22:40.501863+0000","last_peered":"2023-10-09T10:22:40.501863+0000","last_clean":"2023-10-09T10:22:40.501863+0000","last_became_active":"2023-10-09T10:22:27.393982+0000","last_became_peered":"2023-10-09T10:22:27.393982+0000","last_unstale":"2023-10-09T10:22:40.501863+0000","last_undegraded":"2023-10-09T10:22:40.501863+0000","last_fullsized":"2023-10-09T10:22:40.501863+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":438,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.500990+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:22:40.500990+0000","last_peered":"2023-10-09T10:22:40.500990+0000","last_clean":"2023-10-09T10:22:40.500990+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:22:40.500990+0000","last_undegraded":"2023-10-09T10:22:40.500990+0000","last_fullsized":"2023-10-09T10:22:40.500990+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"474'11184","reported_seq":11253,"reported_epoch":474,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:22:44.736555+0000","last_change":"2023-10-09T10:22:28.683842+0000","last_active":"2023-10-09T10:22:44.736555+0000","last_peered":"2023-10-09T10:22:44.736555+0000","last_clean":"2023-10-09T10:22:26.368789+0000","last_became_active":"2023-10-09T10:22:28.606052+0000","last_became_peered":"2023-10-09T10:22:28.606052+0000","last_unstale":"2023-10-09T10:22:44.736555+0000","last_undegraded":"2023-10-09T10:22:44.736555+0000","last_fullsized":"2023-10-09T10:22:44.736555+0000","mapping_epoch":467,"log_start":"441'5917","ondisk_log_start":"441'5917","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":5267,"log_dups_size":0,"ondisk_log_size":5267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169213952,"num_objects":2582,"num_object_clones":0,"num_object_copies":7746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2582,"num_objects_unfound":0,"num_objects_dirty":2582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11184,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5,6","objects":2582}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":140,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.500503+0000","last_change":"2023-10-09T10:22:27.399349+0000","last_active":"2023-10-09T10:22:40.500503+0000","last_peered":"2023-10-09T10:22:40.500503+0000","last_clean":"2023-10-09T10:22:40.500503+0000","last_became_active":"2023-10-09T10:22:27.398276+0000","last_became_peered":"2023-10-09T10:22:27.398276+0000","last_unstale":"2023-10-09T10:22:40.500503+0000","last_undegraded":"2023-10-09T10:22:40.500503+0000","last_fullsized":"2023-10-09T10:22:40.500503+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.504516+0000","last_change":"2023-10-09T10:22:27.502663+0000","last_active":"2023-10-09T10:22:40.504516+0000","last_peered":"2023-10-09T10:22:40.504516+0000","last_clean":"2023-10-09T10:22:40.504516+0000","last_became_active":"2023-10-09T10:22:27.501953+0000","last_became_peered":"2023-10-09T10:22:27.501953+0000","last_unstale":"2023-10-09T10:22:40.504516+0000","last_undegraded":"2023-10-09T10:22:40.504516+0000","last_fullsized":"2023-10-09T10:22:40.504516+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":17,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501185+0000","last_change":"2023-10-09T10:22:27.395593+0000","last_active":"2023-10-09T10:22:40.501185+0000","last_peered":"2023-10-09T10:22:40.501185+0000","last_clean":"2023-10-09T10:22:40.501185+0000","last_became_active":"2023-10-09T10:22:27.394741+0000","last_became_peered":"2023-10-09T10:22:27.394741+0000","last_unstale":"2023-10-09T10:22:40.501185+0000","last_undegraded":"2023-10-09T10:22:40.501185+0000","last_fullsized":"2023-10-09T10:22:40.501185+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"474'9847","reported_seq":9917,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:44.794735+0000","last_change":"2023-10-09T10:22:27.377887+0000","last_active":"2023-10-09T10:22:44.794735+0000","last_peered":"2023-10-09T10:22:44.794735+0000","last_clean":"2023-10-09T10:22:44.794735+0000","last_became_active":"2023-10-09T10:21:45.487312+0000","last_became_peered":"2023-10-09T10:21:45.487312+0000","last_unstale":"2023-10-09T10:22:44.794735+0000","last_undegraded":"2023-10-09T10:22:44.794735+0000","last_fullsized":"2023-10-09T10:22:44.794735+0000","mapping_epoch":424,"log_start":"428'1157","ondisk_log_start":"428'1157","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8690,"log_dups_size":0,"ondisk_log_size":8690,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83197952,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5714,"num_write_kb":223488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":404,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499602+0000","last_change":"2023-10-09T10:22:27.441855+0000","last_active":"2023-10-09T10:22:40.499602+0000","last_peered":"2023-10-09T10:22:40.499602+0000","last_clean":"2023-10-09T10:22:40.499602+0000","last_became_active":"2023-10-09T10:22:27.441539+0000","last_became_peered":"2023-10-09T10:22:27.441539+0000","last_unstale":"2023-10-09T10:22:40.499602+0000","last_undegraded":"2023-10-09T10:22:40.499602+0000","last_fullsized":"2023-10-09T10:22:40.499602+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:46.202562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":31,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.498764+0000","last_change":"2023-10-09T10:22:27.392415+0000","last_active":"2023-10-09T10:22:40.498764+0000","last_peered":"2023-10-09T10:22:40.498764+0000","last_clean":"2023-10-09T10:22:40.498764+0000","last_became_active":"2023-10-09T10:22:27.392069+0000","last_became_peered":"2023-10-09T10:22:27.392069+0000","last_unstale":"2023-10-09T10:22:40.498764+0000","last_undegraded":"2023-10-09T10:22:40.498764+0000","last_fullsized":"2023-10-09T10:22:40.498764+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:51:06.172359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":460,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.503059+0000","last_change":"2023-10-09T10:22:13.367244+0000","last_active":"2023-10-09T10:22:40.503059+0000","last_peered":"2023-10-09T10:22:40.503059+0000","last_clean":"2023-10-09T10:22:40.503059+0000","last_became_active":"2023-10-09T10:22:13.366925+0000","last_became_peered":"2023-10-09T10:22:13.366925+0000","last_unstale":"2023-10-09T10:22:40.503059+0000","last_undegraded":"2023-10-09T10:22:40.503059+0000","last_fullsized":"2023-10-09T10:22:40.503059+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"474'11560","reported_seq":14752,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:47.463690+0000","last_change":"2023-10-09T10:22:28.548566+0000","last_active":"2023-10-09T10:22:47.463690+0000","last_peered":"2023-10-09T10:22:47.463690+0000","last_clean":"2023-10-09T10:22:12.975597+0000","last_became_active":"2023-10-09T10:22:27.537029+0000","last_became_peered":"2023-10-09T10:22:27.537029+0000","last_unstale":"2023-10-09T10:22:47.463690+0000","last_undegraded":"2023-10-09T10:22:47.463690+0000","last_fullsized":"2023-10-09T10:22:47.463690+0000","mapping_epoch":466,"log_start":"434'4375","ondisk_log_start":"434'4375","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":7185,"log_dups_size":0,"ondisk_log_size":7185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:10.307806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170131456,"num_objects":2596,"num_object_clones":0,"num_object_copies":7788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4077,"num_objects_unfound":0,"num_objects_dirty":2596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11560,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1554,"num_bytes_recovered":101842944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2596}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":119,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501536+0000","last_change":"2023-10-09T10:16:18.477748+0000","last_active":"2023-10-09T10:22:40.501536+0000","last_peered":"2023-10-09T10:22:40.501536+0000","last_clean":"2023-10-09T10:22:40.501536+0000","last_became_active":"2023-10-09T10:16:18.477398+0000","last_became_peered":"2023-10-09T10:16:18.477398+0000","last_unstale":"2023-10-09T10:22:40.501536+0000","last_undegraded":"2023-10-09T10:22:40.501536+0000","last_fullsized":"2023-10-09T10:22:40.501536+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":680,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.548149+0000","last_change":"2023-10-09T10:22:40.541772+0000","last_active":"2023-10-09T10:22:40.548149+0000","last_peered":"2023-10-09T10:22:40.548149+0000","last_clean":"2023-10-09T10:22:40.548149+0000","last_became_active":"2023-10-09T10:22:40.541385+0000","last_became_peered":"2023-10-09T10:22:40.541385+0000","last_unstale":"2023-10-09T10:22:40.548149+0000","last_undegraded":"2023-10-09T10:22:40.548149+0000","last_fullsized":"2023-10-09T10:22:40.548149+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":149,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.500405+0000","last_change":"2023-10-09T10:22:27.505137+0000","last_active":"2023-10-09T10:22:40.500405+0000","last_peered":"2023-10-09T10:22:40.500405+0000","last_clean":"2023-10-09T10:22:40.500405+0000","last_became_active":"2023-10-09T10:22:27.504833+0000","last_became_peered":"2023-10-09T10:22:27.504833+0000","last_unstale":"2023-10-09T10:22:40.500405+0000","last_undegraded":"2023-10-09T10:22:40.500405+0000","last_fullsized":"2023-10-09T10:22:40.500405+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":67,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505736+0000","last_change":"2023-10-09T10:22:27.457041+0000","last_active":"2023-10-09T10:22:40.505736+0000","last_peered":"2023-10-09T10:22:40.505736+0000","last_clean":"2023-10-09T10:22:40.505736+0000","last_became_active":"2023-10-09T10:22:27.456442+0000","last_became_peered":"2023-10-09T10:22:27.456442+0000","last_unstale":"2023-10-09T10:22:40.505736+0000","last_undegraded":"2023-10-09T10:22:40.505736+0000","last_fullsized":"2023-10-09T10:22:40.505736+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"474'9751","reported_seq":9821,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:47.464648+0000","last_change":"2023-10-09T10:21:45.478095+0000","last_active":"2023-10-09T10:22:47.464648+0000","last_peered":"2023-10-09T10:22:47.464648+0000","last_clean":"2023-10-09T10:22:47.464648+0000","last_became_active":"2023-10-09T10:21:45.470115+0000","last_became_peered":"2023-10-09T10:21:45.470115+0000","last_unstale":"2023-10-09T10:22:47.464648+0000","last_undegraded":"2023-10-09T10:22:47.464648+0000","last_fullsized":"2023-10-09T10:22:47.464648+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":424,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":9751,"log_dups_size":0,"ondisk_log_size":9751,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85524480,"num_objects":1305,"num_object_clones":0,"num_object_copies":3915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5662,"num_write_kb":222944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":148,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.498892+0000","last_change":"2023-10-09T10:22:13.360439+0000","last_active":"2023-10-09T10:22:40.498892+0000","last_peered":"2023-10-09T10:22:40.498892+0000","last_clean":"2023-10-09T10:22:40.498892+0000","last_became_active":"2023-10-09T10:22:13.360046+0000","last_became_peered":"2023-10-09T10:22:13.360046+0000","last_unstale":"2023-10-09T10:22:40.498892+0000","last_undegraded":"2023-10-09T10:22:40.498892+0000","last_fullsized":"2023-10-09T10:22:40.498892+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.142326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":407,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499694+0000","last_change":"2023-10-09T10:19:43.292390+0000","last_active":"2023-10-09T10:22:40.499694+0000","last_peered":"2023-10-09T10:22:40.499694+0000","last_clean":"2023-10-09T10:22:40.499694+0000","last_became_active":"2023-10-09T10:19:43.290733+0000","last_became_peered":"2023-10-09T10:19:43.290733+0000","last_unstale":"2023-10-09T10:22:40.499694+0000","last_undegraded":"2023-10-09T10:22:40.499694+0000","last_fullsized":"2023-10-09T10:22:40.499694+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:53.407598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":21,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.502283+0000","last_change":"2023-10-09T10:22:13.356105+0000","last_active":"2023-10-09T10:22:40.502283+0000","last_peered":"2023-10-09T10:22:40.502283+0000","last_clean":"2023-10-09T10:22:40.502283+0000","last_became_active":"2023-10-09T10:22:13.355605+0000","last_became_peered":"2023-10-09T10:22:13.355605+0000","last_unstale":"2023-10-09T10:22:40.502283+0000","last_undegraded":"2023-10-09T10:22:40.502283+0000","last_fullsized":"2023-10-09T10:22:40.502283+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"474'11281","reported_seq":17135,"reported_epoch":474,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:22:45.150137+0000","last_change":"2023-10-09T10:22:28.916651+0000","last_active":"2023-10-09T10:22:45.150137+0000","last_peered":"2023-10-09T10:22:45.150137+0000","last_clean":"2023-10-09T10:20:57.308413+0000","last_became_active":"2023-10-09T10:22:28.712042+0000","last_became_peered":"2023-10-09T10:22:28.712042+0000","last_unstale":"2023-10-09T10:22:45.150137+0000","last_undegraded":"2023-10-09T10:22:45.150137+0000","last_fullsized":"2023-10-09T10:22:28.400675+0000","mapping_epoch":467,"log_start":"446'6272","ondisk_log_start":"446'6272","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":5009,"log_dups_size":0,"ondisk_log_size":5009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181010456,"num_objects":2763,"num_object_clones":0,"num_object_copies":8289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2761,"num_objects_unfound":0,"num_objects_dirty":2763,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11281,"num_write_kb":449345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2884,"num_bytes_recovered":189005824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"3,4","objects":2},{"shards":"3,4,6","objects":955},{"shards":"4,6,7","objects":1806}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":76,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505904+0000","last_change":"2023-10-09T10:22:27.460155+0000","last_active":"2023-10-09T10:22:40.505904+0000","last_peered":"2023-10-09T10:22:40.505904+0000","last_clean":"2023-10-09T10:22:40.505904+0000","last_became_active":"2023-10-09T10:22:27.459820+0000","last_became_peered":"2023-10-09T10:22:27.459820+0000","last_unstale":"2023-10-09T10:22:40.505904+0000","last_undegraded":"2023-10-09T10:22:40.505904+0000","last_fullsized":"2023-10-09T10:22:40.505904+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":437,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501279+0000","last_change":"2023-10-09T10:22:27.429171+0000","last_active":"2023-10-09T10:22:40.501279+0000","last_peered":"2023-10-09T10:22:40.501279+0000","last_clean":"2023-10-09T10:22:40.501279+0000","last_became_active":"2023-10-09T10:22:27.428849+0000","last_became_peered":"2023-10-09T10:22:27.428849+0000","last_unstale":"2023-10-09T10:22:40.501279+0000","last_undegraded":"2023-10-09T10:22:40.501279+0000","last_fullsized":"2023-10-09T10:22:40.501279+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":366,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499683+0000","last_change":"2023-10-09T10:16:18.477779+0000","last_active":"2023-10-09T10:22:40.499683+0000","last_peered":"2023-10-09T10:22:40.499683+0000","last_clean":"2023-10-09T10:22:40.499683+0000","last_became_active":"2023-10-09T10:16:18.477390+0000","last_became_peered":"2023-10-09T10:16:18.477390+0000","last_unstale":"2023-10-09T10:22:40.499683+0000","last_undegraded":"2023-10-09T10:22:40.499683+0000","last_fullsized":"2023-10-09T10:22:40.499683+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:06.576807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"474'11100","reported_seq":11165,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:45.113754+0000","last_change":"2023-10-09T10:22:28.600250+0000","last_active":"2023-10-09T10:22:45.113754+0000","last_peered":"2023-10-09T10:22:45.113754+0000","last_clean":"2023-10-09T10:22:26.382138+0000","last_became_active":"2023-10-09T10:22:28.549005+0000","last_became_peered":"2023-10-09T10:22:28.549005+0000","last_unstale":"2023-10-09T10:22:45.113754+0000","last_undegraded":"2023-10-09T10:22:45.113754+0000","last_fullsized":"2023-10-09T10:22:45.113754+0000","mapping_epoch":467,"log_start":"446'6158","ondisk_log_start":"446'6158","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":4942,"log_dups_size":0,"ondisk_log_size":4942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174981120,"num_objects":2670,"num_object_clones":0,"num_object_copies":8010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2670,"num_objects_unfound":0,"num_objects_dirty":2670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11100,"num_write_kb":440640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,6],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2670}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":259,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499860+0000","last_change":"2023-10-09T10:18:45.347229+0000","last_active":"2023-10-09T10:22:40.499860+0000","last_peered":"2023-10-09T10:22:40.499860+0000","last_clean":"2023-10-09T10:22:40.499860+0000","last_became_active":"2023-10-09T10:18:45.346901+0000","last_became_peered":"2023-10-09T10:18:45.346901+0000","last_unstale":"2023-10-09T10:22:40.499860+0000","last_undegraded":"2023-10-09T10:22:40.499860+0000","last_fullsized":"2023-10-09T10:22:40.499860+0000","mapping_epoch":409,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:53.628052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":446,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.507972+0000","last_change":"2023-10-09T10:22:27.442726+0000","last_active":"2023-10-09T10:22:40.507972+0000","last_peered":"2023-10-09T10:22:40.507972+0000","last_clean":"2023-10-09T10:22:40.507972+0000","last_became_active":"2023-10-09T10:22:27.442396+0000","last_became_peered":"2023-10-09T10:22:27.442396+0000","last_unstale":"2023-10-09T10:22:40.507972+0000","last_undegraded":"2023-10-09T10:22:40.507972+0000","last_fullsized":"2023-10-09T10:22:40.507972+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":330,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.502557+0000","last_change":"2023-10-09T10:22:27.559876+0000","last_active":"2023-10-09T10:22:40.502557+0000","last_peered":"2023-10-09T10:22:40.502557+0000","last_clean":"2023-10-09T10:22:40.502557+0000","last_became_active":"2023-10-09T10:22:27.559306+0000","last_became_peered":"2023-10-09T10:22:27.559306+0000","last_unstale":"2023-10-09T10:22:40.502557+0000","last_undegraded":"2023-10-09T10:22:40.502557+0000","last_fullsized":"2023-10-09T10:22:40.502557+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.f","version":"474'11487","reported_seq":11583,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:49.292900+0000","last_change":"2023-10-09T10:22:28.714873+0000","last_active":"2023-10-09T10:22:49.292900+0000","last_peered":"2023-10-09T10:22:49.292900+0000","last_clean":"2023-10-09T10:22:26.347721+0000","last_became_active":"2023-10-09T10:22:28.523280+0000","last_became_peered":"2023-10-09T10:22:28.523280+0000","last_unstale":"2023-10-09T10:22:49.292900+0000","last_undegraded":"2023-10-09T10:22:49.292900+0000","last_fullsized":"2023-10-09T10:22:49.292900+0000","mapping_epoch":467,"log_start":"438'4855","ondisk_log_start":"438'4855","created":424,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":6632,"log_dups_size":0,"ondisk_log_size":6632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:48:27.031952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163643392,"num_objects":2497,"num_object_clones":0,"num_object_copies":7491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2489,"num_objects_unfound":0,"num_objects_dirty":2497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11487,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[2,3,6],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":2497}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":155,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505092+0000","last_change":"2023-10-09T10:22:27.395071+0000","last_active":"2023-10-09T10:22:40.505092+0000","last_peered":"2023-10-09T10:22:40.505092+0000","last_clean":"2023-10-09T10:22:40.505092+0000","last_became_active":"2023-10-09T10:22:27.394757+0000","last_became_peered":"2023-10-09T10:22:27.394757+0000","last_unstale":"2023-10-09T10:22:40.505092+0000","last_undegraded":"2023-10-09T10:22:40.505092+0000","last_fullsized":"2023-10-09T10:22:40.505092+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":135,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.508125+0000","last_change":"2023-10-09T10:22:13.440553+0000","last_active":"2023-10-09T10:22:40.508125+0000","last_peered":"2023-10-09T10:22:40.508125+0000","last_clean":"2023-10-09T10:22:40.508125+0000","last_became_active":"2023-10-09T10:22:13.366508+0000","last_became_peered":"2023-10-09T10:22:13.366508+0000","last_unstale":"2023-10-09T10:22:40.508125+0000","last_undegraded":"2023-10-09T10:22:40.508125+0000","last_fullsized":"2023-10-09T10:22:40.508125+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":145,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.513829+0000","last_change":"2023-10-09T10:22:40.513566+0000","last_active":"2023-10-09T10:22:40.513829+0000","last_peered":"2023-10-09T10:22:40.513829+0000","last_clean":"2023-10-09T10:22:40.513829+0000","last_became_active":"2023-10-09T10:22:40.513228+0000","last_became_peered":"2023-10-09T10:22:40.513228+0000","last_unstale":"2023-10-09T10:22:40.513829+0000","last_undegraded":"2023-10-09T10:22:40.513829+0000","last_fullsized":"2023-10-09T10:22:40.513829+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.12","version":"474'9755","reported_seq":9841,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:45.078603+0000","last_change":"2023-10-09T10:22:27.587182+0000","last_active":"2023-10-09T10:22:45.078603+0000","last_peered":"2023-10-09T10:22:45.078603+0000","last_clean":"2023-10-09T10:22:12.926811+0000","last_became_active":"2023-10-09T10:22:27.566952+0000","last_became_peered":"2023-10-09T10:22:27.566952+0000","last_unstale":"2023-10-09T10:22:45.078603+0000","last_undegraded":"2023-10-09T10:22:45.078603+0000","last_fullsized":"2023-10-09T10:22:45.078603+0000","mapping_epoch":466,"log_start":"428'1608","ondisk_log_start":"428'1608","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8147,"log_dups_size":0,"ondisk_log_size":8147,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84967424,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1297,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5662,"num_write_kb":222656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1297}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":73,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.512981+0000","last_change":"2023-10-09T10:22:40.512157+0000","last_active":"2023-10-09T10:22:40.512981+0000","last_peered":"2023-10-09T10:22:40.512981+0000","last_clean":"2023-10-09T10:22:40.512981+0000","last_became_active":"2023-10-09T10:22:40.511797+0000","last_became_peered":"2023-10-09T10:22:40.511797+0000","last_unstale":"2023-10-09T10:22:40.512981+0000","last_undegraded":"2023-10-09T10:22:40.512981+0000","last_fullsized":"2023-10-09T10:22:40.512981+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":440,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505909+0000","last_change":"2023-10-09T10:22:13.358654+0000","last_active":"2023-10-09T10:22:40.505909+0000","last_peered":"2023-10-09T10:22:40.505909+0000","last_clean":"2023-10-09T10:22:40.505909+0000","last_became_active":"2023-10-09T10:22:13.358366+0000","last_became_peered":"2023-10-09T10:22:13.358366+0000","last_unstale":"2023-10-09T10:22:40.505909+0000","last_undegraded":"2023-10-09T10:22:40.505909+0000","last_fullsized":"2023-10-09T10:22:40.505909+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"474'9944","reported_seq":10012,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:47.411194+0000","last_change":"2023-10-09T10:21:45.520248+0000","last_active":"2023-10-09T10:22:47.411194+0000","last_peered":"2023-10-09T10:22:47.411194+0000","last_clean":"2023-10-09T10:22:47.411194+0000","last_became_active":"2023-10-09T10:21:45.509240+0000","last_became_peered":"2023-10-09T10:21:45.509240+0000","last_unstale":"2023-10-09T10:22:47.411194+0000","last_undegraded":"2023-10-09T10:22:47.411194+0000","last_fullsized":"2023-10-09T10:22:47.411194+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":9944,"log_dups_size":0,"ondisk_log_size":9944,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82542592,"num_objects":1260,"num_object_clones":0,"num_object_copies":3780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5790,"num_write_kb":225568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":178,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501431+0000","last_change":"2023-10-09T10:22:27.541328+0000","last_active":"2023-10-09T10:22:40.501431+0000","last_peered":"2023-10-09T10:22:40.501431+0000","last_clean":"2023-10-09T10:22:40.501431+0000","last_became_active":"2023-10-09T10:22:27.434795+0000","last_became_peered":"2023-10-09T10:22:27.434795+0000","last_unstale":"2023-10-09T10:22:40.501431+0000","last_undegraded":"2023-10-09T10:22:40.501431+0000","last_fullsized":"2023-10-09T10:22:40.501431+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":67,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.504674+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:22:40.504674+0000","last_peered":"2023-10-09T10:22:40.504674+0000","last_clean":"2023-10-09T10:22:40.504674+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:22:40.504674+0000","last_undegraded":"2023-10-09T10:22:40.504674+0000","last_fullsized":"2023-10-09T10:22:40.504674+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"474'9752","reported_seq":9856,"reported_epoch":474,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:22:45.040838+0000","last_change":"2023-10-09T10:22:28.543065+0000","last_active":"2023-10-09T10:22:45.040838+0000","last_peered":"2023-10-09T10:22:45.040838+0000","last_clean":"2023-10-09T10:22:12.964931+0000","last_became_active":"2023-10-09T10:22:14.566546+0000","last_became_peered":"2023-10-09T10:22:14.566546+0000","last_unstale":"2023-10-09T10:22:45.040838+0000","last_undegraded":"2023-10-09T10:22:45.040838+0000","last_fullsized":"2023-10-09T10:22:45.040838+0000","mapping_epoch":463,"log_start":"428'1600","ondisk_log_start":"428'1600","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8152,"log_dups_size":0,"ondisk_log_size":8152,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86704128,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1314,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5616,"num_write_kb":222048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1323}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":161,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.501785+0000","last_change":"2023-10-09T10:22:27.534153+0000","last_active":"2023-10-09T10:22:40.501785+0000","last_peered":"2023-10-09T10:22:40.501785+0000","last_clean":"2023-10-09T10:22:40.501785+0000","last_became_active":"2023-10-09T10:22:27.430411+0000","last_became_peered":"2023-10-09T10:22:27.430411+0000","last_unstale":"2023-10-09T10:22:40.501785+0000","last_undegraded":"2023-10-09T10:22:40.501785+0000","last_fullsized":"2023-10-09T10:22:40.501785+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":295,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499040+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:22:40.499040+0000","last_peered":"2023-10-09T10:22:40.499040+0000","last_clean":"2023-10-09T10:22:40.499040+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:22:40.499040+0000","last_undegraded":"2023-10-09T10:22:40.499040+0000","last_fullsized":"2023-10-09T10:22:40.499040+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.17","version":"474'9532","reported_seq":9607,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:22:44.751151+0000","last_change":"2023-10-09T10:22:28.697587+0000","last_active":"2023-10-09T10:22:44.751151+0000","last_peered":"2023-10-09T10:22:44.751151+0000","last_clean":"2023-10-09T10:22:26.367638+0000","last_became_active":"2023-10-09T10:22:28.664116+0000","last_became_peered":"2023-10-09T10:22:28.664116+0000","last_unstale":"2023-10-09T10:22:44.751151+0000","last_undegraded":"2023-10-09T10:22:28.465312+0000","last_fullsized":"2023-10-09T10:22:28.404066+0000","mapping_epoch":467,"log_start":"428'640","ondisk_log_start":"428'640","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8892,"log_dups_size":0,"ondisk_log_size":8892,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83427328,"num_objects":1273,"num_object_clones":0,"num_object_copies":3819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5519,"num_write_kb":217344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-393},{"shards":"4,5,6","objects":1288}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":18,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499435+0000","last_change":"2023-10-09T10:22:27.398373+0000","last_active":"2023-10-09T10:22:40.499435+0000","last_peered":"2023-10-09T10:22:40.499435+0000","last_clean":"2023-10-09T10:22:40.499435+0000","last_became_active":"2023-10-09T10:22:27.398010+0000","last_became_peered":"2023-10-09T10:22:27.398010+0000","last_unstale":"2023-10-09T10:22:40.499435+0000","last_undegraded":"2023-10-09T10:22:40.499435+0000","last_fullsized":"2023-10-09T10:22:40.499435+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:20.017053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":516,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.499503+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:22:40.499503+0000","last_peered":"2023-10-09T10:22:40.499503+0000","last_clean":"2023-10-09T10:22:40.499503+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:22:40.499503+0000","last_undegraded":"2023-10-09T10:22:40.499503+0000","last_fullsized":"2023-10-09T10:22:40.499503+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.16","version":"474'9851","reported_seq":9918,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-10-09T10:22:44.686523+0000","last_change":"2023-10-09T10:21:45.512742+0000","last_active":"2023-10-09T10:22:44.686523+0000","last_peered":"2023-10-09T10:22:44.686523+0000","last_clean":"2023-10-09T10:22:44.686523+0000","last_became_active":"2023-10-09T10:21:45.498940+0000","last_became_peered":"2023-10-09T10:21:45.498940+0000","last_unstale":"2023-10-09T10:22:44.686523+0000","last_undegraded":"2023-10-09T10:22:44.686523+0000","last_fullsized":"2023-10-09T10:22:44.686523+0000","mapping_epoch":456,"log_start":"428'1199","ondisk_log_start":"428'1199","created":456,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:19:42.247108+0000","last_clean_scrub_stamp":"2023-10-09T10:19:42.247108+0000","objects_scrubbed":0,"log_size":8652,"log_dups_size":0,"ondisk_log_size":8652,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82116608,"num_objects":1253,"num_object_clones":0,"num_object_copies":3759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5726,"num_write_kb":223328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":463,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.502905+0000","last_change":"2023-10-09T10:22:27.629185+0000","last_active":"2023-10-09T10:22:40.502905+0000","last_peered":"2023-10-09T10:22:40.502905+0000","last_clean":"2023-10-09T10:22:40.502905+0000","last_became_active":"2023-10-09T10:22:27.503302+0000","last_became_peered":"2023-10-09T10:22:27.503302+0000","last_unstale":"2023-10-09T10:22:40.502905+0000","last_undegraded":"2023-10-09T10:22:40.502905+0000","last_fullsized":"2023-10-09T10:22:40.502905+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":76,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-10-09T10:22:40.505122+0000","last_change":"2023-10-09T10:22:27.441829+0000","last_active":"2023-10-09T10:22:40.505122+0000","last_peered":"2023-10-09T10:22:40.505122+0000","last_clean":"2023-10-09T10:22:40.505122+0000","last_became_active":"2023-10-09T10:22:27.441479+0000","last_became_peered":"2023-10-09T10:22:27.441479+0000","last_unstale":"2023-10-09T10:22:40.505122+0000","last_undegraded":"2023-10-09T10:22:40.505122+0000","last_fullsized":"2023-10-09T10:22:40.505122+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":2728984600,"num_objects":41642,"num_object_clones":0,"num_object_copies":124926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3437,"num_objects_misplaced":25412,"num_objects_unfound":0,"num_objects_dirty":41642,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181012,"num_write_kb":7124865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6000,"num_bytes_recovered":392888320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8619569152,"data_stored":8619556936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":209807,"ondisk_log_size":209807,"up":78,"acting":75,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":111,"num_bytes_recovered":22072,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430637,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1973648,"kb_used_data":1057032,"kb_used_omap":0,"kb_used_meta":916544,"kb_avail":91771504,"statfs":{"total":95995035648,"available":93974020096,"internally_reserved":0,"allocated":1082400768,"data_stored":1081632762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":938541056},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.3080000000000001,"15min":1.667},"min":{"1min":0.23300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":6.5759999999999996,"5min":7.7160000000000002,"15min":11.164},"last":2.6619999999999999},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.3720000000000001,"15min":1.6930000000000001},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":5.5759999999999996,"5min":10.029,"15min":15.974},"last":0.61499999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:22:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.7389999999999999,"15min":2.2559999999999998},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.23799999999999999},"max":{"1min":5.3220000000000001,"5min":129.53700000000001,"15min":129.53700000000001},"last":2.6899999999999999},{"interface":"front","average":{"1min":2.0190000000000001,"5min":2.9350000000000001,"15min":2.4300000000000002},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":5.6399999999999997,"5min":128.85599999999999,"15min":128.85599999999999},"last":1.083}]},{"osd":2,"last update":"Mon Oct 9 10:22:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.9710000000000001,"15min":2.0089999999999999},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":6.3920000000000003,"5min":42.725999999999999,"15min":42.725999999999999},"last":3.411},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.9990000000000001,"15min":2.0630000000000002},"min":{"1min":0.33600000000000002,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":9.4209999999999994,"5min":42.752000000000002,"15min":42.752000000000002},"last":0.65200000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:21:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":1.7070000000000001,"15min":1.6080000000000001},"min":{"1min":0.34999999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.0640000000000001,"5min":12.436,"15min":12.436},"last":0.67700000000000005},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.7050000000000001,"15min":1.639},"min":{"1min":0.32900000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.0510000000000002,"5min":12.161,"15min":12.161},"last":6.1150000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:22:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.429,"15min":1.3089999999999999},"min":{"1min":0.26400000000000001,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":12.805,"5min":12.805,"15min":12.805},"last":2.6230000000000002},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.8129999999999999,"15min":1.3049999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25700000000000001},"max":{"1min":13.631,"5min":13.631,"15min":13.631},"last":0.71999999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:22:17 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":1.7709999999999999,"15min":1.5620000000000001},"min":{"1min":0.36699999999999999,"5min":0.29499999999999998,"15min":0.219},"max":{"1min":10.952,"5min":21.712,"15min":25.277999999999999},"last":2.7480000000000002},{"interface":"front","average":{"1min":3.738,"5min":2.2610000000000001,"15min":1.623},"min":{"1min":0.40000000000000002,"5min":0.28299999999999997,"15min":0.22600000000000001},"max":{"1min":14.983000000000001,"5min":22.824999999999999,"15min":25.754000000000001},"last":0.69799999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:22:17 2023","interfaces":[{"interface":"back","average":{"1min":3.1920000000000002,"5min":1.764,"15min":1.3280000000000001},"min":{"1min":0.40799999999999997,"5min":0.33900000000000002,"15min":0.26100000000000001},"max":{"1min":9.4819999999999993,"5min":9.4819999999999993,"15min":9.4819999999999993},"last":3.0779999999999998},{"interface":"front","average":{"1min":1.956,"5min":1.4530000000000001,"15min":1.2070000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.27200000000000002},"max":{"1min":7.7670000000000003,"5min":7.7670000000000003,"15min":10.625},"last":6.1520000000000001}]}]},{"osd":6,"up_from":0,"seq":180388626865,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":360777253179,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416703,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2040976,"kb_used_data":1325512,"kb_used_omap":28,"kb_used_meta":715427,"kb_avail":91704176,"statfs":{"total":95995035648,"available":93905076224,"internally_reserved":0,"allocated":1357324288,"data_stored":1355610393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":732598096},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":1.492,"15min":1.99},"min":{"1min":0.22600000000000001,"5min":0.20699999999999999,"15min":0.184},"max":{"1min":8.7040000000000006,"5min":54.518000000000001,"15min":54.518000000000001},"last":4.2460000000000004},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.76,"15min":2.7509999999999999},"min":{"1min":0.246,"5min":0.192,"15min":0.192},"max":{"1min":5.8040000000000003,"5min":56.950000000000003,"15min":56.950000000000003},"last":0.56599999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":2.032,"15min":2.1349999999999998},"min":{"1min":0.29099999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.4049999999999998,"5min":53.988999999999997,"15min":53.988999999999997},"last":4.2869999999999999},{"interface":"front","average":{"1min":1.262,"5min":2.1829999999999998,"15min":2.1120000000000001},"min":{"1min":0.34200000000000003,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":3.5110000000000001,"5min":54.119,"15min":54.119},"last":4.5709999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:21:53 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.726,"15min":1.359},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":8.6050000000000004,"5min":56.984999999999999,"15min":56.984999999999999},"last":3.718},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.665,"15min":1.2729999999999999},"min":{"1min":0.36899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.7830000000000004,"5min":56.475999999999999,"15min":56.475999999999999},"last":0.53700000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:22:07 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":2.1680000000000001,"15min":2.2349999999999999},"min":{"1min":0.30499999999999999,"5min":0.251,"15min":0.251},"max":{"1min":6.819,"5min":57.719000000000001,"15min":57.719000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.946,"5min":2.0539999999999998,"15min":2.1669999999999998},"min":{"1min":0.39600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":10.819000000000001,"5min":56.378,"15min":56.378},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:22:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":2.1669999999999998,"15min":3.133},"min":{"1min":0.39700000000000002,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":8.5760000000000005,"5min":56.445999999999998,"15min":229.589},"last":3.4670000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":2.0350000000000001,"15min":2.6920000000000002},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.27000000000000002},"max":{"1min":3.3700000000000001,"5min":54.411000000000001,"15min":228.88900000000001},"last":4.5039999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:22:07 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":2.2810000000000001,"15min":2.1589999999999998},"min":{"1min":0.53700000000000003,"5min":0.34899999999999998,"15min":0.33100000000000002},"max":{"1min":4.6790000000000003,"5min":54.482999999999997,"15min":54.482999999999997},"last":4.5410000000000004},{"interface":"front","average":{"1min":2.194,"5min":2.4940000000000002,"15min":2.6480000000000001},"min":{"1min":0.52500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.2249999999999996,"5min":57.753,"15min":57.753},"last":0.90100000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:22:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.3380000000000001,"15min":2.4279999999999999},"min":{"1min":0.432,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.9880000000000004,"5min":56.411999999999999,"15min":56.411999999999999},"last":2.5049999999999999},{"interface":"front","average":{"1min":1.7649999999999999,"5min":2.2770000000000001,"15min":2.1459999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.36499999999999999},"max":{"1min":6.0220000000000002,"5min":54.451999999999998,"15min":54.451999999999998},"last":11.175000000000001}]}]},{"osd":2,"up_from":217,"seq":932007903439,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1638384,"kb_used_data":1017128,"kb_used_omap":83,"kb_used_meta":621164,"kb_avail":92106768,"statfs":{"total":95995035648,"available":94317330432,"internally_reserved":0,"allocated":1041539072,"data_stored":1039979900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":636072431},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:22:11 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":1.4399999999999999,"15min":1.8300000000000001},"min":{"1min":0.21199999999999999,"5min":0.19,"15min":0.17899999999999999},"max":{"1min":2.5219999999999998,"5min":33.289000000000001,"15min":33.289000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.597,"5min":1.6599999999999999,"15min":2.5219999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.0299999999999994,"5min":33.347999999999999,"15min":33.347999999999999},"last":0.28899999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:22:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.3740000000000001,"15min":2.1259999999999999},"min":{"1min":0.24099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.4080000000000004,"5min":34.893999999999998,"15min":34.893999999999998},"last":0.318},{"interface":"front","average":{"1min":1.337,"5min":1.381,"15min":2.2589999999999999},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":6.3529999999999998,"5min":34.962000000000003,"15min":34.962000000000003},"last":0.45500000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:22:47 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.121,"15min":1.109},"min":{"1min":0.27700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":6.4980000000000002,"5min":7.54,"15min":35.043999999999997},"last":0.375},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.91500000000000004,"15min":1.198},"min":{"1min":0.25900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":6.617,"5min":6.617,"15min":50.406999999999996},"last":0.39300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:22:05 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":0.91000000000000003,"15min":1.4219999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26600000000000001},"max":{"1min":6.4349999999999996,"5min":6.4349999999999996,"15min":18.873999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.929,"5min":1.5209999999999999,"15min":1.613},"min":{"1min":0.371,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.0069999999999997,"5min":24.352,"15min":24.352},"last":0.42299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":1.5529999999999999,"15min":2.012},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.27700000000000002},"max":{"1min":9.4580000000000002,"5min":24.23,"15min":57.298000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":2,"5min":1.4590000000000001,"15min":1.8660000000000001},"min":{"1min":0.46200000000000002,"5min":0.32600000000000001,"15min":0.315},"max":{"1min":6.2350000000000003,"5min":6.5449999999999999,"15min":57.828000000000003},"last":0.52400000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2919999999999998,"5min":1.427,"15min":1.893},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":9.6039999999999992,"5min":9.6039999999999992,"15min":18.943999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.081,"5min":1.7390000000000001,"15min":1.954},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.33300000000000002},"max":{"1min":7.2309999999999999,"5min":24.379000000000001,"15min":24.379000000000001},"last":0.505}]},{"osd":7,"last update":"Mon Oct 9 10:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":1.9610000000000001,"15min":2.1320000000000001},"min":{"1min":0.42699999999999999,"5min":0.41699999999999998,"15min":0.308},"max":{"1min":8.1210000000000004,"5min":35.008000000000003,"15min":35.008000000000003},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.819,"5min":1.5649999999999999,"15min":2.1320000000000001},"min":{"1min":0.438,"5min":0.40999999999999998,"15min":0.35899999999999999},"max":{"1min":6.6399999999999997,"5min":24.323,"15min":24.323},"last":0.56000000000000005}]}]},{"osd":3,"up_from":352,"seq":1511828488278,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":835212,"kb_used_data":577604,"kb_used_omap":15,"kb_used_meta":257584,"kb_avail":92909940,"statfs":{"total":95995035648,"available":95139778560,"internally_reserved":0,"allocated":591466496,"data_stored":589465995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":263766470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:57 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":1.111,"15min":1.1200000000000001},"min":{"1min":0.23100000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.9909999999999997,"5min":8.9909999999999997,"15min":8.9909999999999997},"last":3.4319999999999999},{"interface":"front","average":{"1min":2.3159999999999998,"5min":1.556,"15min":1.137},"min":{"1min":0.252,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.931,"5min":8.3070000000000004,"15min":8.3070000000000004},"last":5.9450000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:21:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2749999999999999,"5min":2.9199999999999999,"15min":1.714},"min":{"1min":0.28000000000000003,"5min":0.251,"15min":0.251},"max":{"1min":6.7830000000000004,"5min":105.401,"15min":105.401},"last":0.498},{"interface":"front","average":{"1min":2.665,"5min":3.0680000000000001,"15min":1.476},"min":{"1min":0.39600000000000002,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":9.9499999999999993,"5min":104.914,"15min":104.914},"last":6.0119999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:21:57 2023","interfaces":[{"interface":"back","average":{"1min":3.3900000000000001,"5min":1.599,"15min":1.9910000000000001},"min":{"1min":0.32200000000000001,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":8.1630000000000003,"5min":8.1630000000000003,"15min":8.7970000000000006},"last":4.0629999999999997},{"interface":"front","average":{"1min":1.974,"5min":1.2569999999999999,"15min":1.2330000000000001},"min":{"1min":0.35499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.6760000000000002,"5min":16.018000000000001,"15min":16.018000000000001},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:21:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0600000000000001,"5min":1.8879999999999999,"15min":1.0600000000000001},"min":{"1min":0.35099999999999998,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":20.326000000000001,"5min":21.831,"15min":21.831},"last":5.8970000000000002},{"interface":"front","average":{"1min":3.4409999999999998,"5min":2.1480000000000001,"15min":1.5680000000000001},"min":{"1min":0.46700000000000003,"5min":0.32100000000000001,"15min":0.30199999999999999},"max":{"1min":14.07,"5min":22.097999999999999,"15min":22.097999999999999},"last":5.5449999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:21:57 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":1.4930000000000001,"15min":0.96099999999999997},"min":{"1min":0.44700000000000001,"5min":0.371,"15min":0.32100000000000001},"max":{"1min":6.9390000000000001,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":3.4790000000000001},{"interface":"front","average":{"1min":3.1699999999999999,"5min":2.004,"15min":1.1379999999999999},"min":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":13.394,"5min":13.394,"15min":13.394},"last":5.9809999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:21:57 2023","interfaces":[{"interface":"back","average":{"1min":4.2130000000000001,"5min":1.9470000000000001,"15min":1.534},"min":{"1min":0.36699999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":18.213000000000001,"5min":18.213000000000001,"15min":18.213000000000001},"last":1.7270000000000001},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.7849999999999999,"15min":1.1919999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":9.6099999999999994,"5min":10.698,"15min":10.698},"last":0.53400000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:21:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2749999999999999,"5min":2.0190000000000001,"15min":1.173},"min":{"1min":0.45800000000000002,"5min":0.39500000000000002,"15min":0.38400000000000001},"max":{"1min":7.0110000000000001,"5min":10.459,"15min":10.459},"last":6.0460000000000003},{"interface":"front","average":{"1min":2.895,"5min":1.9279999999999999,"15min":1.54},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":8.1980000000000004,"5min":8.827,"15min":8.827},"last":4.0279999999999996}]}]},{"osd":4,"up_from":266,"seq":1142461300907,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2074892,"kb_used_data":1756228,"kb_used_omap":35,"kb_used_meta":318620,"kb_avail":91670260,"statfs":{"total":95995035648,"available":93870346240,"internally_reserved":0,"allocated":1798377472,"data_stored":1797613494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":326267118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":3.6419999999999999,"15min":1.9670000000000001},"min":{"1min":0.23300000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":11.923999999999999,"5min":97.721999999999994,"15min":97.721999999999994},"last":0.751},{"interface":"front","average":{"1min":4.3390000000000004,"5min":4.3540000000000001,"15min":2.5859999999999999},"min":{"1min":0.48999999999999999,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":11.257,"5min":97.915999999999997,"15min":97.915999999999997},"last":5.391}]},{"osd":1,"last update":"Mon Oct 9 10:22:01 2023","interfaces":[{"interface":"back","average":{"1min":3.472,"5min":2.702,"15min":2.121},"min":{"1min":0.434,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.32,"5min":71.457999999999998,"15min":71.457999999999998},"last":5.4450000000000003},{"interface":"front","average":{"1min":3.238,"5min":2.2799999999999998,"15min":1.907},"min":{"1min":0.32700000000000001,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":7.6020000000000003,"5min":71.658000000000001,"15min":71.658000000000001},"last":7.9210000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.8180000000000001,"5min":1.554,"15min":1.583},"min":{"1min":0.33700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.6669999999999998,"5min":7.6669999999999998,"15min":9.2469999999999999},"last":7.9530000000000003},{"interface":"front","average":{"1min":2.6899999999999999,"5min":1.552,"15min":1.599},"min":{"1min":0.317,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":11.962999999999999,"5min":11.962999999999999,"15min":11.962999999999999},"last":0.64900000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:21:48 2023","interfaces":[{"interface":"back","average":{"1min":4.1429999999999998,"5min":1.9690000000000001,"15min":1.7749999999999999},"min":{"1min":0.35199999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.074,"5min":12.074,"15min":12.074},"last":5.3639999999999999},{"interface":"front","average":{"1min":3.1640000000000001,"5min":1.6659999999999999,"15min":1.2649999999999999},"min":{"1min":0.40500000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":9.1690000000000005,"5min":9.1690000000000005,"15min":9.1690000000000005},"last":7.883}]},{"osd":5,"last update":"Mon Oct 9 10:22:01 2023","interfaces":[{"interface":"back","average":{"1min":3.8860000000000001,"5min":1.9830000000000001,"15min":1.6830000000000001},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.23799999999999999},"max":{"1min":14.162000000000001,"5min":14.162000000000001,"15min":99.585999999999999},"last":8.0289999999999999},{"interface":"front","average":{"1min":3.2189999999999999,"5min":1.5740000000000001,"15min":1.371},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.994,"5min":11.994,"15min":99.384},"last":0.72999999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:22:01 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":1.6919999999999999,"15min":1.1319999999999999},"min":{"1min":0.373,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":12.019,"5min":12.019,"15min":12.019},"last":0.62},{"interface":"front","average":{"1min":4.5030000000000001,"5min":2.0089999999999999,"15min":1.319},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":13.016,"5min":13.016,"15min":13.016},"last":5.4219999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:22:01 2023","interfaces":[{"interface":"back","average":{"1min":4.7610000000000001,"5min":2.0609999999999999,"15min":1.391},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":17.369,"5min":17.369,"15min":17.369},"last":7.9349999999999996},{"interface":"front","average":{"1min":2.9969999999999999,"5min":1.623,"15min":1.2689999999999999},"min":{"1min":0.38600000000000001,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":7.7949999999999999,"5min":9.9220000000000006,"15min":14.964},"last":7.9980000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823092,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1751288,"kb_used_data":845808,"kb_used_omap":0,"kb_used_meta":905408,"kb_avail":91993864,"statfs":{"total":95995035648,"available":94201716736,"internally_reserved":0,"allocated":866107392,"data_stored":865358560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927137792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:21:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":1.587,"15min":1.613},"min":{"1min":0.23899999999999999,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":22.469999999999999,"5min":22.469999999999999,"15min":22.469999999999999},"last":0.25900000000000001},{"interface":"front","average":{"1min":4.516,"5min":1.893,"15min":1.6950000000000001},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.22900000000000001},"max":{"1min":22.437000000000001,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.315}]},{"osd":1,"last update":"Mon Oct 9 10:22:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":2.3180000000000001,"15min":2.0550000000000002},"min":{"1min":0.33400000000000002,"5min":0.26700000000000002,"15min":0.23699999999999999},"max":{"1min":8.343,"5min":29.370000000000001,"15min":29.370000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.016,"5min":2.0019999999999998,"15min":1.8899999999999999},"min":{"1min":0.251,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":13.143000000000001,"5min":29.065000000000001,"15min":29.065000000000001},"last":1.044}]},{"osd":2,"last update":"Mon Oct 9 10:22:16 2023","interfaces":[{"interface":"back","average":{"1min":3.1139999999999999,"5min":1.7,"15min":1.8600000000000001},"min":{"1min":0.41299999999999998,"5min":0.28999999999999998,"15min":0.27200000000000002},"max":{"1min":7.5830000000000002,"5min":8.0760000000000005,"15min":15.973000000000001},"last":13.965999999999999},{"interface":"front","average":{"1min":2.6349999999999998,"5min":1.883,"15min":1.613},"min":{"1min":0.317,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":13.095000000000001,"5min":13.095000000000001,"15min":15.929},"last":2.6299999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:21:48 2023","interfaces":[{"interface":"back","average":{"1min":2.927,"5min":1.966,"15min":1.6379999999999999},"min":{"1min":0.35299999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.561999999999999,"5min":10.561999999999999,"15min":10.561999999999999},"last":4.1070000000000002},{"interface":"front","average":{"1min":3.464,"5min":2.0259999999999998,"15min":1.655},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.315},"max":{"1min":13.624000000000001,"5min":13.624000000000001,"15min":13.624000000000001},"last":0.35099999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:21:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.649,"15min":1.1830000000000001},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.24299999999999999},"max":{"1min":6.5890000000000004,"5min":8.468,"15min":9.9819999999999993},"last":4.1859999999999999},{"interface":"front","average":{"1min":2.7759999999999998,"5min":1.8899999999999999,"15min":1.1060000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":11.956,"5min":11.956,"15min":11.956},"last":4.1710000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:21:48 2023","interfaces":[{"interface":"back","average":{"1min":3.468,"5min":1.923,"15min":1.391},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.25},"max":{"1min":10.492000000000001,"5min":12.772,"15min":13.483000000000001},"last":4.1449999999999996},{"interface":"front","average":{"1min":2.8050000000000002,"5min":1.8080000000000001,"15min":1.409},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.22600000000000001},"max":{"1min":9.8499999999999996,"5min":9.8499999999999996,"15min":18.600999999999999},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:22:25 2023","interfaces":[{"interface":"back","average":{"1min":3.4689999999999999,"5min":2.0569999999999999,"15min":1.478},"min":{"1min":0.38500000000000001,"5min":0.32700000000000001,"15min":0.25600000000000001},"max":{"1min":14.006,"5min":14.006,"15min":16.030999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.79,"5min":1.643,"15min":1.0409999999999999},"min":{"1min":0.34899999999999998,"5min":0.32100000000000001,"15min":0.26800000000000002},"max":{"1min":11.994,"5min":11.994,"15min":11.994},"last":0.44}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1334312960,"data_stored":1334312960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":645660672,"data_stored":645660672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1030750208,"data_stored":1030750208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":581177344,"data_stored":581173272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1785204736,"data_stored":1785200664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":853671936,"data_stored":853671936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1318195200,"data_stored":1318191128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1070596096,"data_stored":1070596096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:22:51.655 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:22:51.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 36 2023-10-09T10:22:51.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:51.820 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[152263]: from='client.46724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:22:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[152263]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:22:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[152263]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T10:22:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[152263]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[152263]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[152263]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[147926]: from='client.46724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[147926]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[147926]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[147926]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[147926]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:52 smithi012 ceph-mon[147926]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.191 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:52.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:52.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:52.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:52.366 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:52.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:52 smithi110 ceph-mon[136032]: from='client.46724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:22:52.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:52 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:22:52.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:52 smithi110 ceph-mon[136032]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:52 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:22:52.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:52 smithi110 ceph-mon[136032]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T10:22:52.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:52 smithi110 ceph-mon[136032]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:52 smithi110 ceph-mon[136032]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:52 smithi110 ceph-mon[136032]: failed to encode map e475 with expected crc 2023-10-09T10:22:52.475 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:52.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:52.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:52.581 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:22:52.656 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:52.660 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114935 9.f 9:f4378248:::benchmark_data_smithi012_313365_object2460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 458.49407319199997, 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2871666070000001, 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.661 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114935 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.662 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162480+0000", 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162479+0000", 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.663 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162485+0000", 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3759999999999996e-06 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162497+0000", 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.664 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162503+0000", 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9770000000000003e-06 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174344+0000", 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01184062 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174510+0000", 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000166026 2023-10-09T10:22:52.665 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.174655+0000", 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000144563 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.181694+0000", 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007039694 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.666 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.214204+0000", 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.03250981 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.222717+0000", 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085130050000000006 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.667 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.222730+0000", 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3229e-05 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196525+0000", 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97379490400000002 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196531+0000", 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.879e-06 2023-10-09T10:22:52.668 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.428687+0000", 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.232155509 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.428882+0000", 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019557999999999999 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.669 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.449518+0000", 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020635767999999999 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.449647+0000", 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000129014 2023-10-09T10:22:52.670 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114936 9.a 9:58a6faa8:::benchmark_data_smithi012_313365_object2461:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 458.49404248000002, 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.092944401, 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.671 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114936 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.672 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162511+0000", 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162513+0000", 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1610000000000001e-06 2023-10-09T10:22:52.673 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162518+0000", 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7310000000000001e-06 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.162522+0000", 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9920000000000001e-06 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.674 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164493+0000", 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019717620000000002 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164686+0000", 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00019211 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:22:52.675 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.164874+0000", 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018879699999999999 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.168656+0000", 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037811670000000002 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.169482+0000", 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082636199999999997 2023-10-09T10:22:52.676 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.212142+0000", 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042660245999999999 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.212158+0000", 2023-10-09T10:22:52.677 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5977999999999998e-05 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.195199+0000", 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98304075999999996 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.195213+0000", 2023-10-09T10:22:52.678 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4025e-05 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236218+0000", 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0410054230000001 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:22:52.679 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.236372+0000", 2023-10-09T10:22:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015330899999999999 2023-10-09T10:22:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.680 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255344+0000", 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018972025 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.255456+0000", 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111928 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.681 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114940 9.f 9:f2b30a6a:::benchmark_data_smithi012_313365_object2465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 458.48293762999998, 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2912956169999998, 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114940 2023-10-09T10:22:52.682 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173616+0000", 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.683 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173614+0000", 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173632+0000", 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8349999999999999e-05 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173651+0000", 2023-10-09T10:22:52.684 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8372e-05 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.173660+0000", 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0429999999999996e-06 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.177952+0000", 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042923120000000004 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.685 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.178136+0000", 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018356399999999999 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.178304+0000", 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016803400000000001 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.686 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.181873+0000", 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035693090000000001 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.214244+0000", 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032370566000000003 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223012+0000", 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087685690000000004 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.687 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223019+0000", 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196547+0000", 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97352829299999999 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.688 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196551+0000", 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9949999999999999e-06 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.434365+0000", 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2378136799999999 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.434545+0000", 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000180044 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.689 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.464846+0000", 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030301600000000001 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.464912+0000", 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5380000000000001e-05 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.690 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:114950 9.f 9:f115a5aa:::benchmark_data_smithi012_313365_object2475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 458.43292633900001, 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2705686389999999, 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 114950 2023-10-09T10:22:52.691 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223627+0000", 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.692 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223625+0000", 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223632+0000", 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8639999999999998e-06 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223644+0000", 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1824000000000001e-05 2023-10-09T10:22:52.693 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223651+0000", 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5499999999999997e-06 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223694+0000", 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2765999999999997e-05 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.694 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:14.223699+0000", 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2569999999999998e-06 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196570+0000", 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.97287036000000005 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:15.196576+0000", 2023-10-09T10:22:52.695 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5710000000000003e-06 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.439509+0000", 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.242932758 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.440273+0000", 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00076375 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.696 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474247+0000", 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033973640999999999 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474363+0000", 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116765 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.474458+0000", 2023-10-09T10:22:52.697 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4829999999999998e-05 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.479012+0000", 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045542890000000004 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.481102+0000", 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020900659999999998 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.698 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.489625+0000", 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085221180000000004 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.489680+0000", 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5405999999999998e-05 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.699 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.494196+0000", 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045159930000000003 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115069 9.10 9:0c166afc:::benchmark_data_smithi012_313365_object2594:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:22:52.700 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 456.110906516, 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5496961840000001, 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115069 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:22:52.701 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545647+0000", 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545646+0000", 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.702 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545653+0000", 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5399999999999998e-06 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545670+0000", 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7144000000000001e-05 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.703 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545679+0000", 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5830000000000007e-06 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.545730+0000", 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1313000000000002e-05 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.704 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.546723+0000", 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00099291799999999997 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079001+0000", 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5322773119999997 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.705 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079100+0000", 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9010000000000002e-05 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.079372+0000", 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000272782 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.080850+0000", 2023-10-09T10:22:52.706 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014771509999999999 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.082687+0000", 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001837336 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.082730+0000", 2023-10-09T10:22:52.707 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3612999999999997e-05 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093025+0000", 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010295011 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.093094+0000", 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8725000000000004e-05 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.708 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095343+0000", 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022491519999999999 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:115106 9.10 9:0b40069f:::benchmark_data_smithi012_313365_object2631:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 455.97272181099999, 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4133900090000004, 2023-10-09T10:22:52.709 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115106 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.710 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683832+0000", 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683831+0000", 2023-10-09T10:22:52.711 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683836+0000", 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0440000000000003e-06 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.712 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683846+0000", 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9250000000000007e-06 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683854+0000", 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7959999999999996e-06 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.683942+0000", 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7602000000000006e-05 2023-10-09T10:22:52.713 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:16.684891+0000", 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094994899999999995 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089254+0000", 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4043629649999998 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.714 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089396+0000", 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000141886 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.089758+0000", 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00036181000000000001 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.091841+0000", 2023-10-09T10:22:52.715 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020825930000000002 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095413+0000", 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035718400000000002 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095477+0000", 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4685999999999997e-05 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.716 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095503+0000", 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5575000000000002e-05 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.095538+0000", 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5116000000000001e-05 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:21.097222+0000", 2023-10-09T10:22:52.717 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001683941 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122337 9.10 9:0bd6554e:::benchmark_data_smithi012_313365_object9862:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 421.75957437599999, 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6698403750000002, 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.718 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122337 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.719 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896979+0000", 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896978+0000", 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896984+0000", 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7479999999999999e-06 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.720 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.896996+0000", 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.199e-05 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897001+0000", 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8189999999999997e-06 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897056+0000", 2023-10-09T10:22:52.721 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4818999999999998e-05 2023-10-09T10:22:52.722 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.722 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.722 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:50.897145+0000", 2023-10-09T10:22:52.722 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8943000000000005e-05 2023-10-09T10:22:52.722 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.722 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.722 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.517542+0000", 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6203965519999999 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.517672+0000", 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013004499999999999 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.518031+0000", 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035930199999999998 2023-10-09T10:22:52.723 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.536908+0000", 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.018877416000000001 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.536918+0000", 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9369999999999999e-06 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.724 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.537422+0000", 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00050323499999999999 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564538+0000", 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.027116735999999999 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.564585+0000", 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6592999999999999e-05 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.566820+0000", 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022347140000000001 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122523 9.10 9:0d3d12ba:::benchmark_data_smithi012_313365_object10048:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.726 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 421.21422508400002, 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7874849670000001, 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122523 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.727 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442329+0000", 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442327+0000", 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.728 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442334+0000", 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.221e-06 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442347+0000", 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2809000000000001e-05 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.729 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442355+0000", 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.0760000000000003e-06 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442400+0000", 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4570000000000002e-05 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.442467+0000", 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7063999999999994e-05 2023-10-09T10:22:52.730 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194401+0000", 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7519345159999999 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194518+0000", 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000116872 2023-10-09T10:22:52.731 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.732 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.732 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.732 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.194929+0000", 2023-10-09T10:22:52.732 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041051599999999998 2023-10-09T10:22:52.732 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.732 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.732 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.733 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.211495+0000", 2023-10-09T10:22:52.733 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016566498999999998 2023-10-09T10:22:52.733 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.733 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.733 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.733 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.211544+0000", 2023-10-09T10:22:52.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9190999999999997e-05 2023-10-09T10:22:52.734 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.734 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.734 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.734 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.214081+0000", 2023-10-09T10:22:52.734 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025366910000000002 2023-10-09T10:22:52.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.735 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.735 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.227445+0000", 2023-10-09T10:22:52.735 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013363808 2023-10-09T10:22:52.735 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.735 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.736 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.736 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.227497+0000", 2023-10-09T10:22:52.736 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1860000000000002e-05 2023-10-09T10:22:52.736 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.736 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.737 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.737 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.229814+0000", 2023-10-09T10:22:52.737 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023168160000000002 2023-10-09T10:22:52.737 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.737 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.737 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.738 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.738 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.738 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122561 9.10 9:092dbf94:::benchmark_data_smithi012_313365_object10086:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.738 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:22:52.738 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 421.11511601299998, 2023-10-09T10:22:52.738 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1569061710000001, 2023-10-09T10:22:52.739 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.739 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.739 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.739 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.739 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.739 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122561 2023-10-09T10:22:52.739 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.740 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.740 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.740 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.740 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:22:52.740 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.740 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.741 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.741 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541438+0000", 2023-10-09T10:22:52.741 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.741 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.741 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.742 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541437+0000", 2023-10-09T10:22:52.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:22:52.742 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.742 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.742 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.742 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541441+0000", 2023-10-09T10:22:52.742 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1919999999999998e-06 2023-10-09T10:22:52.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.743 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.743 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.743 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541449+0000", 2023-10-09T10:22:52.743 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.746e-06 2023-10-09T10:22:52.743 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.744 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.744 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541454+0000", 2023-10-09T10:22:52.744 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4639999999999997e-06 2023-10-09T10:22:52.744 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.744 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.745 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541494+0000", 2023-10-09T10:22:52.745 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9505999999999997e-05 2023-10-09T10:22:52.745 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.745 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.745 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.541543+0000", 2023-10-09T10:22:52.746 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9872e-05 2023-10-09T10:22:52.746 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.746 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.746 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.746 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.660487+0000", 2023-10-09T10:22:52.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1189437330000001 2023-10-09T10:22:52.747 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.747 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.747 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.747 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.660622+0000", 2023-10-09T10:22:52.747 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135024 2023-10-09T10:22:52.748 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.748 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.748 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.748 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.661025+0000", 2023-10-09T10:22:52.748 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040268299999999998 2023-10-09T10:22:52.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.749 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.749 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.749 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.668459+0000", 2023-10-09T10:22:52.749 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074342699999999998 2023-10-09T10:22:52.749 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.750 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.750 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.668490+0000", 2023-10-09T10:22:52.750 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1279999999999999e-05 2023-10-09T10:22:52.750 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.750 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.751 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683619+0000", 2023-10-09T10:22:52.751 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015128694999999999 2023-10-09T10:22:52.751 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.751 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.751 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.696066+0000", 2023-10-09T10:22:52.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012446472 2023-10-09T10:22:52.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.752 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.752 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.752 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.696109+0000", 2023-10-09T10:22:52.752 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3887999999999997e-05 2023-10-09T10:22:52.752 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.753 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.753 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.753 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698344+0000", 2023-10-09T10:22:52.753 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022343480000000001 2023-10-09T10:22:52.753 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.753 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.753 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.753 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.754 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.754 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122689 9.10 9:0a3555ea:::benchmark_data_smithi012_313365_object10214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.754 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:22:52.754 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 420.701698502, 2023-10-09T10:22:52.754 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7453082640000002, 2023-10-09T10:22:52.754 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.754 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.754 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122689 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.755 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954855+0000", 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954853+0000", 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.756 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954862+0000", 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2959999999999999e-06 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954874+0000", 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2072000000000001e-05 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954882+0000", 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1030000000000002e-06 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.757 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.954938+0000", 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6165999999999997e-05 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:51.955019+0000", 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1001000000000006e-05 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.758 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.666823+0000", 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7118038929999999 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.666922+0000", 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8803999999999998e-05 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.667247+0000", 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00032501299999999997 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.759 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683704+0000", 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016457423999999998 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683804+0000", 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100046 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.683814+0000", 2023-10-09T10:22:52.760 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0083e-05 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698412+0000", 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014598266 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.698458+0000", 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5503999999999997e-05 2023-10-09T10:22:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.700163+0000", 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001705441 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:122759 9.10 9:0b678b9b:::benchmark_data_smithi012_313365_object10284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:22:52.763 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 420.53204666200003, 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.605785577, 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 122759 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.764 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124507+0000", 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124506+0000", 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.765 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124513+0000", 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124528+0000", 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4893e-05 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.766 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124536+0000", 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1319999999999994e-06 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124601+0000", 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5038999999999999e-05 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:52.124689+0000", 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7894999999999998e-05 2023-10-09T10:22:52.767 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704065+0000", 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5793759000000001 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704190+0000", 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012584799999999999 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.768 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.704594+0000", 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040309699999999999 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.727961+0000", 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023367024 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728012+0000", 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1174000000000002e-05 2023-10-09T10:22:52.769 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728024+0000", 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1966e-05 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728058+0000", 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3911000000000001e-05 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.770 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.728098+0000", 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0286e-05 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:53.730293+0000", 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021946410000000002 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.771 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124410 9.10 9:0a0e0dea:::benchmark_data_smithi012_313365_object11935:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 415.95359967799999, 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5937443060000001, 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124410 2023-10-09T10:22:52.772 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702954+0000", 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.773 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702952+0000", 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702960+0000", 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6920000000000002e-06 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.774 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702975+0000", 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4969e-05 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.702983+0000", 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7510000000000008e-06 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.707988+0000", 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050051440000000004 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.775 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:56.708039+0000", 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0998999999999997e-05 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.196596+0000", 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.488557455 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.776 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.196709+0000", 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000113075 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.197066+0000", 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00035631299999999997 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.198754+0000", 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016883169999999999 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.777 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.216476+0000", 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017721654 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.216545+0000", 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8888999999999997e-05 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.778 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.294372+0000", 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.077827269000000004 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.294431+0000", 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9001e-05 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.296698+0000", 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00226742 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.779 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124495 9.10 9:0df9feaa:::benchmark_data_smithi012_313365_object12020:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 415.65411014300003, 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6452444429999999, 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.780 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124495 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002444+0000", 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.781 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002442+0000", 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002451+0000", 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6459999999999999e-06 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.782 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002465+0000", 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4531e-05 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002478+0000", 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3091999999999999e-05 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002543+0000", 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5003000000000001e-05 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.002642+0000", 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8486000000000005e-05 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604276+0000", 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.601633879 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604435+0000", 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000159295 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.604865+0000", 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00043062800000000001 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613177+0000", 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0083113730000000004 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613227+0000", 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9805000000000002e-05 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.613271+0000", 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4691999999999998e-05 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645599+0000", 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032327910000000001 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645659+0000", 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9821000000000002e-05 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647688+0000", 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020288989999999998 2023-10-09T10:22:52.787 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124499 9.10 9:0914df8f:::benchmark_data_smithi012_313365_object12024:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 415.63767636099999, 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6316283970000001, 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.788 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124499 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018877+0000", 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018876+0000", 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018883+0000", 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2869999999999999e-06 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018893+0000", 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0190999999999999e-05 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018900+0000", 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2099999999999996e-06 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.018964+0000", 2023-10-09T10:22:52.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.355e-05 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.019054+0000", 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0099999999999995e-05 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611351+0000", 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5922972340000001 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611460+0000", 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108608 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.611836+0000", 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037601000000000003 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.621012+0000", 2023-10-09T10:22:52.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.009175753 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645527+0000", 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.024515678999999999 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.645566+0000", 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8294000000000003e-05 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647904+0000", 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023386240000000001 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647967+0000", 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2964000000000002e-05 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.650506+0000", 2023-10-09T10:22:52.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025385540000000002 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124527 9.10 9:0cb84cd4:::benchmark_data_smithi012_313365_object12052:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 415.55014281799998, 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5734285209999999, 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.796 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124527 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106411+0000", 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106409+0000", 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106417+0000", 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2060000000000001e-06 2023-10-09T10:22:52.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106431+0000", 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4693e-05 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106440+0000", 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4649999999999994e-06 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106537+0000", 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7021000000000005e-05 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.106637+0000", 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010021799999999999 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.617803+0000", 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5111660599999999 2023-10-09T10:22:52.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.617924+0000", 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012099600000000001 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.618302+0000", 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037780300000000001 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643413+0000", 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025111018999999998 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647753+0000", 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043399199999999997 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647820+0000", 2023-10-09T10:22:52.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7353000000000004e-05 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.677556+0000", 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029735927999999998 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.677618+0000", 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2069000000000005e-05 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.679839+0000", 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022211359999999999 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:124541 9.10 9:0fc113d5:::benchmark_data_smithi012_313365_object12066:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 415.45218028199997, 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.506778945, 2023-10-09T10:22:52.804 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 124541 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204373+0000", 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204372+0000", 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.806 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204381+0000", 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9970000000000004e-06 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204402+0000", 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0761999999999999e-05 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204412+0000", 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0368e-05 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204473+0000", 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1490999999999998e-05 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:57.204558+0000", 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4437999999999994e-05 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643445+0000", 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4388867670000001 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643565+0000", 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000120537 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.643947+0000", 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00038192800000000002 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.647844+0000", 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0038965330000000002 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.652834+0000", 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049904190000000003 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.652889+0000", 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4759000000000002e-05 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.708684+0000", 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.05579493 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.708750+0000", 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6202000000000001e-05 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:58.711152+0000", 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024023360000000001 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128019 9.a 9:56d43b6a:::benchmark_data_smithi012_313365_object15544:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 405.003264301, 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.955188565, 2023-10-09T10:22:52.812 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128019 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653288+0000", 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653296+0000", 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.79e-06 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653308+0000", 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2376e-05 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653323+0000", 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4844000000000001e-05 2023-10-09T10:22:52.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653379+0000", 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6054999999999998e-05 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653595+0000", 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021573 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.653860+0000", 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026534600000000002 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.696431+0000", 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042570967000000001 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.696443+0000", 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1739000000000001e-05 2023-10-09T10:22:52.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.323819+0000", 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.62737651100000003 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.323833+0000", 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3869000000000001e-05 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.494433+0000", 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1706003599999999 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for missing object", 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.494639+0000", 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00020562600000000001 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.608361+0000", 2023-10-09T10:22:52.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.113721818 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.608478+0000", 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011719500000000001 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128025 9.f 9:f78d4616:::benchmark_data_smithi012_313365_object15550:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:22:52.820 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 404.84105718000001, 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.792284043, 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128025 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815496+0000", 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815495+0000", 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815501+0000", 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815513+0000", 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1770000000000001e-05 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815520+0000", 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1010000000000003e-06 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815583+0000", 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2779000000000003e-05 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:07.815595+0000", 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2153999999999999e-05 2023-10-09T10:22:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.227751+0000", 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.41215553100000002 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:08.227764+0000", 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3131e-05 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.825 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.482847+0000", 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.255083368 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.483568+0000", 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00072077100000000004 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.582946+0000", 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.099378280999999999 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.826 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.583074+0000", 2023-10-09T10:22:52.827 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012746599999999999 2023-10-09T10:22:52.827 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.827 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.827 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T10:22:52.827 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.583609+0000", 2023-10-09T10:22:52.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00053539899999999999 2023-10-09T10:22:52.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.829 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.829 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.595736+0000", 2023-10-09T10:22:52.829 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012126994 2023-10-09T10:22:52.829 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.829 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.599249+0000", 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035130600000000001 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.602352+0000", 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031027540000000001 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.603699+0000", 2023-10-09T10:22:52.830 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001347284 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.603770+0000", 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1014999999999997e-05 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.607781+0000", 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.004010319 2023-10-09T10:22:52.831 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:195679 9.3 9:c12d9ed3:::benchmark_data_smithi012_313365_object83204:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 257.220335304, 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9465798859999999, 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.832 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195679 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:22:52.833 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436217+0000", 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436223+0000", 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9850000000000003e-06 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.834 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436235+0000", 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1229e-05 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436243+0000", 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9789999999999993e-06 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436300+0000", 2023-10-09T10:22:52.835 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7265999999999997e-05 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436951+0000", 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00065154099999999995 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345069+0000", 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.90811741 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.836 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345197+0000", 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128496 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345622+0000", 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042431400000000002 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.347037+0000", 2023-10-09T10:22:52.837 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014153449999999999 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.348160+0000", 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001123097 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382157+0000", 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033996849000000003 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.838 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382214+0000", 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7012000000000002e-05 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382798+0000", 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058430099999999996 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.839 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:195683 9.13 9:c8f1253c:::benchmark_data_smithi012_313365_object83208:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 257.21511326299998, 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9973604489999999, 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195683 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.840 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:22:52.841 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441439+0000", 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441447+0000", 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4549999999999995e-06 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441463+0000", 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5675000000000002e-05 2023-10-09T10:22:52.842 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441471+0000", 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3100000000000001e-06 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441521+0000", 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9709999999999997e-05 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:22:52.843 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.442142+0000", 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062057699999999998 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383458+0000", 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.941316144 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383530+0000", 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2574e-05 2023-10-09T10:22:52.844 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383875+0000", 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00034479399999999999 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.386609+0000", 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027342930000000001 2023-10-09T10:22:52.845 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.386879+0000", 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026999800000000003 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.437748+0000", 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.050869079999999997 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.846 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.437820+0000", 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1427999999999994e-05 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.438801+0000", 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00098100300000000004 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:22:52.847 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:22:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:52.848 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:52.849 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:22:52.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:22:52.883 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:22:52.917 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:22:52.942 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:22:52.950 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:52.970 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:22:53.003 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:22:53.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:53.037 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:22:53.072 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:22:53.109 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:22:53.113 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:22:53.113 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:22:53.113 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:22:53.114 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:22:53.114 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:22:53.114 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:22:53.114 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:22:53.114 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:22:53.132 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:53.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:53.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:53 smithi012 ceph-mon[147926]: pgmap v1329: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 3464/118551 objects degraded (2.922%); 23898/118551 objects misplaced (20.158%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:22:53.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2257675941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:53.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:53 smithi012 ceph-mon[152263]: pgmap v1329: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 3464/118551 objects degraded (2.922%); 23898/118551 objects misplaced (20.158%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:22:53.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2257675941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:53.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:53.272 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:22:53.312 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:53 smithi110 ceph-mon[136032]: pgmap v1329: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.4 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 3464/118551 objects degraded (2.922%); 23898/118551 objects misplaced (20.158%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:22:53.313 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2257675941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:53.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:53.428 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:53.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:53.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:53.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:53.578 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:53.679 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:53.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:53.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:53.830 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:54.093 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:54.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:54.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:54.201 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:54.253 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:22:54.272 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:54.320 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:22:54.336 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:54.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:54.389 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:54 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3464/118551 objects degraded (2.922%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:54.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:54 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3464/118551 objects degraded (2.922%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:54.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:54 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3464/118551 objects degraded (2.922%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:54.444 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:54.524 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:54.649 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:54.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:54.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:54.761 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:54.827 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:54.887 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:54.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:54.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:54.993 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:55.103 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:55.114 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:55.384 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:55 smithi110 ceph-mon[136032]: pgmap v1330: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.3 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 3464/115053 objects degraded (3.011%); 23513/115053 objects misplaced (20.437%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T10:22:55.385 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1038419779' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:22:55.392 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 10 pg_num to 36 2023-10-09T10:22:55.429 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:55.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:55.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:55.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:55.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:55 smithi012 ceph-mon[147926]: pgmap v1330: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.3 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 3464/115053 objects degraded (3.011%); 23513/115053 objects misplaced (20.437%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T10:22:55.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1038419779' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:22:55.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:55 smithi012 ceph-mon[152263]: pgmap v1330: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.3 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 3464/115053 objects degraded (3.011%); 23513/115053 objects misplaced (20.437%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T10:22:55.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1038419779' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T10:22:55.594 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:55.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:22:55.844 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:55.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:55.954 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:56.026 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:56.160 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:56.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:56.267 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:56.323 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:56.376 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:56.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:56.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:56.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:56.574 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:22:56.617 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1038419779' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-10-09T10:22:56.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T10:22:56.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.618 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.619 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:22:56.619 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:22:56.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1038419779' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-10-09T10:22:56.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T10:22:56.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:22:56.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:56 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:22:56.626 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:56.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1038419779' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "36"}]': finished 2023-10-09T10:22:56.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T10:22:56.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: failed to encode map e476 with expected crc 2023-10-09T10:22:56.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:22:56.698 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:56 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T10:22:56.890 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:22:56.891 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:22:57.010 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:57.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:57.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:57.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:57.142 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:57.267 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:57.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:57.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:22:57.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:57.385 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[147926]: pgmap v1332: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 3471/107751 objects degraded (3.221%); 21684/107751 objects misplaced (20.124%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:22:57.385 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T10:22:57.385 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[147926]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T10:22:57.386 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[147926]: failed to encode map e477 with expected crc 2023-10-09T10:22:57.386 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[147926]: failed to encode map e477 with expected crc 2023-10-09T10:22:57.386 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[152263]: pgmap v1332: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 3471/107751 objects degraded (3.221%); 21684/107751 objects misplaced (20.124%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:22:57.386 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T10:22:57.386 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[152263]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T10:22:57.387 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[152263]: failed to encode map e477 with expected crc 2023-10-09T10:22:57.387 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-mon[152263]: failed to encode map e477 with expected crc 2023-10-09T10:22:57.427 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:57.469 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:57.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:57.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:57.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:57.605 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:57 smithi110 ceph-mon[136032]: pgmap v1332: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 3471/107751 objects degraded (3.221%); 21684/107751 objects misplaced (20.124%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:22:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:57 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T10:22:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:57 smithi110 ceph-mon[136032]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T10:22:57.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:57 smithi110 ceph-mon[136032]: failed to encode map e477 with expected crc 2023-10-09T10:22:57.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:57 smithi110 ceph-mon[136032]: failed to encode map e477 with expected crc 2023-10-09T10:22:57.696 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:57.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:57.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:22:57.829 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:57.958 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:22:57.961 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:22:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:22:57] "GET /metrics HTTP/1.1" 200 36672 "" "Prometheus/2.33.4" 2023-10-09T10:22:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:57.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:22:58.032 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:22:58.062 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:58.095 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:58.159 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:58.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:58.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:58.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:58.295 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:58.331 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:58.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:58.366 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:22:58.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:22:58.441 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:22:58.466 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:22:58.469 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:58.487 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:22:58.507 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:22:58.509 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:22:58.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:58.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:22:58.534 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:22:58.556 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:22:58.576 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:22:58.596 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:22:58.597 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:22:58.597 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:22:58.597 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:22:58.597 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:22:58.597 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:22:58.597 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:22:58.598 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:22:58.598 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:22:58.616 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:22:58.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T10:22:58.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: 10.b continuing backfill to osd.4 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:22:58.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: 10.b continuing backfill to osd.7 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:22:58.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: 10.f continuing backfill to osd.4 from (438'4855,476'11800] MIN to 476'11800 2023-10-09T10:22:58.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: 10.0 continuing backfill to osd.5 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: 10.0 continuing backfill to osd.7 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: 10.2 continuing backfill to osd.7 from (429'1728,476'9946] MIN to 476'9946 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: 10.e starting backfill to osd.5 from (448'6658,476'11590] MIN to 476'11592 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: 10.a continuing backfill to osd.7 from (447'6417,476'11656] MIN to 476'11656 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1817819433' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3471/107751 objects degraded (3.221%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1817819433' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[152263]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T10:22:58.623 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: 10.b continuing backfill to osd.4 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: 10.b continuing backfill to osd.7 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: 10.f continuing backfill to osd.4 from (438'4855,476'11800] MIN to 476'11800 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: 10.0 continuing backfill to osd.5 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: 10.0 continuing backfill to osd.7 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: 10.2 continuing backfill to osd.7 from (429'1728,476'9946] MIN to 476'9946 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: 10.e starting backfill to osd.5 from (448'6658,476'11590] MIN to 476'11592 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: 10.a continuing backfill to osd.7 from (447'6417,476'11656] MIN to 476'11656 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1817819433' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3471/107751 objects degraded (3.221%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1817819433' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:58.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:58 smithi012 ceph-mon[147926]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T10:22:58.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T10:22:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: 10.b continuing backfill to osd.4 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: 10.b continuing backfill to osd.7 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: 10.f continuing backfill to osd.4 from (438'4855,476'11800] MIN to 476'11800 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: failed to encode map e478 with expected crc 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: 10.0 continuing backfill to osd.5 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: 10.0 continuing backfill to osd.7 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: 10.2 continuing backfill to osd.7 from (429'1728,476'9946] MIN to 476'9946 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: 10.e starting backfill to osd.5 from (448'6658,476'11590] MIN to 476'11592 2023-10-09T10:22:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: 10.a continuing backfill to osd.7 from (447'6417,476'11656] MIN to 476'11656 2023-10-09T10:22:58.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1817819433' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:22:58.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3471/107751 objects degraded (3.221%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:22:58.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:22:58.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1817819433' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:22:58.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:58 smithi110 ceph-mon[136032]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T10:22:58.796 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:22:58.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:58.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:22:58.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:22:58.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:58.972 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:22:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:58.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:22:59.076 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:59.106 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:59.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:59.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:59.288 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:22:59.288 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:22:59.288 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:22:59.288 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:22:59.288 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:22:59.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:22:59.326 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:59.360 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:22:59.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:59.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:22:59.464 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:22:59.530 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:59.650 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:22:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[147926]: pgmap v1335: 109 pgs: 8 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 87 active+clean; 2.1 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 3471/105186 objects degraded (3.300%); 19702/105186 objects misplaced (18.731%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:22:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[147926]: failed to encode map e479 with expected crc 2023-10-09T10:22:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[147926]: failed to encode map e479 with expected crc 2023-10-09T10:22:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/719940918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:59.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:59.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[147926]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T10:22:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[152263]: pgmap v1335: 109 pgs: 8 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 87 active+clean; 2.1 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 3471/105186 objects degraded (3.300%); 19702/105186 objects misplaced (18.731%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:22:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[152263]: failed to encode map e479 with expected crc 2023-10-09T10:22:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[152263]: failed to encode map e479 with expected crc 2023-10-09T10:22:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/719940918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:59.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:22:59 smithi012 ceph-mon[152263]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T10:22:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:59.680 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:59 smithi110 ceph-mon[136032]: pgmap v1335: 109 pgs: 8 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 87 active+clean; 2.1 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 3471/105186 objects degraded (3.300%); 19702/105186 objects misplaced (18.731%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:22:59.680 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:59 smithi110 ceph-mon[136032]: failed to encode map e479 with expected crc 2023-10-09T10:22:59.680 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:59 smithi110 ceph-mon[136032]: failed to encode map e479 with expected crc 2023-10-09T10:22:59.680 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/719940918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:22:59.680 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:22:59.680 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:22:59.680 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:22:59 smithi110 ceph-mon[136032]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T10:22:59.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:22:59.692 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:22:59.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:22:59.834 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:22:59.878 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:22:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:22:59.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:22:59.985 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:00.077 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:00.175 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:00.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:00.282 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:00.338 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:00.386 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:00.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:00.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:00.491 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:00.529 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:00.583 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:00.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: 10.1f continuing backfill to osd.4 from (438'4855,476'11800] MIN to 476'11800 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: 10.1b continuing backfill to osd.4 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: 10.1b continuing backfill to osd.7 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: 10.20 continuing backfill to osd.5 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: 10.20 continuing backfill to osd.7 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: 10.22 continuing backfill to osd.7 from (429'1728,476'9946] MIN to 476'9946 2023-10-09T10:23:00.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: 10.1e starting backfill to osd.5 from (448'6658,476'11590] MIN to 476'11592 2023-10-09T10:23:00.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[147926]: 10.1a continuing backfill to osd.7 from (447'6417,476'11656] MIN to 476'11656 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: 10.1f continuing backfill to osd.4 from (438'4855,476'11800] MIN to 476'11800 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: 10.1b continuing backfill to osd.4 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: 10.1b continuing backfill to osd.7 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: 10.20 continuing backfill to osd.5 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:23:00.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: 10.20 continuing backfill to osd.7 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:23:00.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: 10.22 continuing backfill to osd.7 from (429'1728,476'9946] MIN to 476'9946 2023-10-09T10:23:00.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: 10.1e starting backfill to osd.5 from (448'6658,476'11590] MIN to 476'11592 2023-10-09T10:23:00.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:00 smithi012 ceph-mon[152263]: 10.1a continuing backfill to osd.7 from (447'6417,476'11656] MIN to 476'11656 2023-10-09T10:23:00.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: 10.1f continuing backfill to osd.4 from (438'4855,476'11800] MIN to 476'11800 2023-10-09T10:23:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: 10.1b continuing backfill to osd.4 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:23:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: 10.1b continuing backfill to osd.7 from (437'4775,476'11976] MIN to 476'11976 2023-10-09T10:23:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: 10.20 continuing backfill to osd.5 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:23:00.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: 10.20 continuing backfill to osd.7 from (429'1763,476'9898] MIN to 476'9898 2023-10-09T10:23:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: failed to encode map e480 with expected crc 2023-10-09T10:23:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: 10.22 continuing backfill to osd.7 from (429'1728,476'9946] MIN to 476'9946 2023-10-09T10:23:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: 10.1e starting backfill to osd.5 from (448'6658,476'11590] MIN to 476'11592 2023-10-09T10:23:00.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:00 smithi110 ceph-mon[136032]: 10.1a continuing backfill to osd.7 from (447'6417,476'11656] MIN to 476'11656 2023-10-09T10:23:00.751 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:00.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:00.869 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:00.911 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 0, 2, 5, 4] out_osds: [1, 6] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:23:00.911 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:23:00.912 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T10:23:00.912 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:23:00.912 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:23:00.912 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T10:23:00.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:00.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:01.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:01.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:01.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:23:01.046 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:23:01.102 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:01.226 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:01.315 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:01.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:01.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:01.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:01.496 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:01 smithi012 ceph-mon[152263]: pgmap v1338: 109 pgs: 1 remapped+peering, 6 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 87 active+clean; 2.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 27 MiB/s wr, 961 op/s; 3471/106575 objects degraded (3.257%); 19623/106575 objects misplaced (18.412%); 0 B/s, 0 objects/s recovering 2023-10-09T10:23:01.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:01 smithi012 ceph-mon[147926]: pgmap v1338: 109 pgs: 1 remapped+peering, 6 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 87 active+clean; 2.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 27 MiB/s wr, 961 op/s; 3471/106575 objects degraded (3.257%); 19623/106575 objects misplaced (18.412%); 0 B/s, 0 objects/s recovering 2023-10-09T10:23:01.503 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:01.565 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:01.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:01.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:01.678 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:01.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:01 smithi110 ceph-mon[136032]: pgmap v1338: 109 pgs: 1 remapped+peering, 6 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 87 active+clean; 2.2 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 27 MiB/s wr, 961 op/s; 3471/106575 objects degraded (3.257%); 19623/106575 objects misplaced (18.412%); 0 B/s, 0 objects/s recovering 2023-10-09T10:23:01.747 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:01.751 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:01.879 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:01.980 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:02.763 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:02.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:02.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:02.911 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:03.153 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:03.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:03.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:03.260 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:03.343 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:03.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:03 smithi110 ceph-mon[136032]: pgmap v1339: 109 pgs: 1 remapped+peering, 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+remapped+backfill_wait, 88 active+clean; 2.1 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 358 op/s; 7065/102024 objects degraded (6.925%); 15592/102024 objects misplaced (15.283%); 15 MiB/s, 247 objects/s recovering 2023-10-09T10:23:03.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:03 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:23:03.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 7065/102024 objects degraded (6.925%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:03.719 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:03 smithi012 ceph-mon[147926]: pgmap v1339: 109 pgs: 1 remapped+peering, 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+remapped+backfill_wait, 88 active+clean; 2.1 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 358 op/s; 7065/102024 objects degraded (6.925%); 15592/102024 objects misplaced (15.283%); 15 MiB/s, 247 objects/s recovering 2023-10-09T10:23:03.719 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:03 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:23:03.719 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 7065/102024 objects degraded (6.925%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:03.720 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:03 smithi012 ceph-mon[152263]: pgmap v1339: 109 pgs: 1 remapped+peering, 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+remapped+backfill_wait, 88 active+clean; 2.1 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 358 op/s; 7065/102024 objects degraded (6.925%); 15592/102024 objects misplaced (15.283%); 15 MiB/s, 247 objects/s recovering 2023-10-09T10:23:03.720 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:03 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:23:03.720 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 7065/102024 objects degraded (6.925%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:03.974 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:03.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:04.037 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:04.074 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:04.082 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:04.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:04.147 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:04.163 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:04.211 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:04.216 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:04.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:04.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:04.255 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:04.305 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:04.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:04.356 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:04.375 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:23:04.375 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:23:04.375 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:23:04.375 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:23:04.375 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:23:04.375 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:23:04.375 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:23:04.376 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:23:04.411 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:04.473 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:04.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:04.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:04.530 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:23:04.530 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "" 2023-10-09T10:23:04.530 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:23:04.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:04.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3957605308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:04.670 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3957605308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:04.671 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:04.708 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3957605308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:04.945 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:04.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:05.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:05.136 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:05.159 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:23:05.159 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:23:05.219 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:05.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:05.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:05.384 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:05.464 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:05.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:05.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:05.469 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:05 smithi012 ceph-mon[147926]: pgmap v1340: 109 pgs: 1 remapped+peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 88 active+clean; 2.0 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 449 op/s; 7065/99222 objects degraded (7.120%); 16936/99222 objects misplaced (17.069%); 13 MiB/s, 208 objects/s recovering 2023-10-09T10:23:05.469 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1388898724' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:05.470 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:05 smithi012 ceph-mon[152263]: pgmap v1340: 109 pgs: 1 remapped+peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 88 active+clean; 2.0 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 449 op/s; 7065/99222 objects degraded (7.120%); 16936/99222 objects misplaced (17.069%); 13 MiB/s, 208 objects/s recovering 2023-10-09T10:23:05.470 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1388898724' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:05.485 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:23:05.535 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:23:05.569 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:05.617 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:05.906 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:05.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:05.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:05 smithi110 ceph-mon[136032]: pgmap v1340: 109 pgs: 1 remapped+peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 88 active+clean; 2.0 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 449 op/s; 7065/99222 objects degraded (7.120%); 16936/99222 objects misplaced (17.069%); 13 MiB/s, 208 objects/s recovering 2023-10-09T10:23:05.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1388898724' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:06.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:06.049 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:06.107 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:06.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:06.214 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:06.266 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:06.333 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:06.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:06.440 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:06.507 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:06.528 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:06 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:06 smithi012 ceph-mon[147926]: failed to encode map e481 with expected crc 2023-10-09T10:23:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1388898724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:23:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:06 smithi012 ceph-mon[147926]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T10:23:06.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:06 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:06.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:06 smithi012 ceph-mon[152263]: failed to encode map e481 with expected crc 2023-10-09T10:23:06.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1388898724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:23:06.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:06 smithi012 ceph-mon[152263]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T10:23:06.692 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:06.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:06.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:06.837 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:06 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:06.837 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:06 smithi110 ceph-mon[136032]: failed to encode map e481 with expected crc 2023-10-09T10:23:06.838 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1388898724' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:23:06.838 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:06 smithi110 ceph-mon[136032]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T10:23:06.854 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:06.906 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:06.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:07.010 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:07.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:23:07.092 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:07.175 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:07.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:07.284 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:07.361 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:07.484 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:07.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:07.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:07.592 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:07.676 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:07.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:07 smithi012 ceph-mon[152263]: pgmap v1342: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.8 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 7229/89382 objects degraded (8.088%); 15706/89382 objects misplaced (17.572%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:23:07.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:07 smithi012 ceph-mon[147926]: pgmap v1342: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.8 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 7229/89382 objects degraded (8.088%); 15706/89382 objects misplaced (17.572%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:23:07.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:07 smithi110 ceph-mon[136032]: pgmap v1342: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.8 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 7229/89382 objects degraded (8.088%); 15706/89382 objects misplaced (17.572%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:23:07.761 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:07.767 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:07.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:07.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:07.873 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:07.940 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:07.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:23:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:23:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:23:07] "GET /metrics HTTP/1.1" 200 36656 "" "Prometheus/2.33.4" 2023-10-09T10:23:08.184 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:08.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:08.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:08.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:08.348 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:08.471 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:23:08.471 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:23:08.597 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:08.600 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 7229/89382 objects degraded (8.088%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:08.600 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 7229/89382 objects degraded (8.088%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:08.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:08.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:08.702 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:08.779 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 7229/89382 objects degraded (8.088%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:08.786 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:08.908 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:08.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:08.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:08.932 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:23:09.019 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:09.085 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:09.154 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:09.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:09.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:09.285 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:09.371 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:09.381 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:09.399 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:09.427 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:09.455 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:09.460 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:09.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:09.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:09.498 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:09.534 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:09.562 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:09.580 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:09.610 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:09.613 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:09.615 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:23:09.615 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:23:09.615 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:23:09.615 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:23:09.615 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:23:09.616 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:23:09.616 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:23:09.616 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:23:09.685 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:09.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:09 smithi110 ceph-mon[136032]: pgmap v1343: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 7229/85299 objects degraded (8.475%); 15154/85299 objects misplaced (17.766%); 12 MiB/s, 199 objects/s recovering 2023-10-09T10:23:09.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2351365725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:09.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:09.793 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:09.866 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:09.915 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:09.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:09 smithi012 ceph-mon[147926]: pgmap v1343: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 7229/85299 objects degraded (8.475%); 15154/85299 objects misplaced (17.766%); 12 MiB/s, 199 objects/s recovering 2023-10-09T10:23:09.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2351365725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:09.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:09 smithi012 ceph-mon[152263]: pgmap v1343: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 7229/85299 objects degraded (8.475%); 15154/85299 objects misplaced (17.766%); 12 MiB/s, 199 objects/s recovering 2023-10-09T10:23:09.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2351365725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:09.933 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:09.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:10.045 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:10.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:10.255 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 0, 2, 5, 4] out_osds: [1, 6] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:23:10.255 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:23:10.255 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T10:23:10.255 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:23:10.255 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:23:10.256 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T10:23:10.331 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:10.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:10.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:10.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:10.569 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:10.814 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:10.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:10.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:10.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:10.963 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:11.084 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:11.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:11.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:11.193 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:11.321 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:11.580 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:11.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:11.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:11.587 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:11 smithi110 ceph-mon[136032]: pgmap v1344: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.7 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 397 op/s; 7229/84282 objects degraded (8.577%); 14747/84282 objects misplaced (17.497%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:23:11.587 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:11 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:11.588 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:11 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:11.669 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:11.688 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:11.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:11 smithi012 ceph-mon[147926]: pgmap v1344: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.7 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 397 op/s; 7229/84282 objects degraded (8.577%); 14747/84282 objects misplaced (17.497%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:23:11.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:11 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:11.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:11 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:11.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:11 smithi012 ceph-mon[152263]: pgmap v1344: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.7 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 397 op/s; 7229/84282 objects degraded (8.577%); 14747/84282 objects misplaced (17.497%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:23:11.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:11 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:11.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:11 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:11.737 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:11.793 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:11.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:11.901 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:11.981 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:12.011 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:23:12.011 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:23:12.011 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:23:12.011 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:23:12.012 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:23:12.058 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:12.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:12.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:12.172 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:12.220 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:12.269 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:12.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:12.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:12.414 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:12.470 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:12.471 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4084269682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:12.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:12 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:12.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:12.472 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:12.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:12.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:23:12.490 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:23:12.575 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:12.612 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4084269682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:12.612 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:12 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:12.612 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:12.612 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:12.693 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:12.695 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:23:12.779 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:12.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:12.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:12.894 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4084269682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:12 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:13.014 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:13.137 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:13.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:13.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:13.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:13.335 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:13.390 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:13.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:13.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:13.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:13.571 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:13.588 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:13.656 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:13.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: pgmap v1345: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 7422/76383 objects degraded (9.717%); 13100/76383 objects misplaced (17.150%); 3.4 MiB/s, 66 objects/s recovering 2023-10-09T10:23:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T10:23:13.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 7422/76383 objects degraded (9.717%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:13.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:23:13.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:23:13.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:13.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: pgmap v1345: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 7422/76383 objects degraded (9.717%); 13100/76383 objects misplaced (17.150%); 3.4 MiB/s, 66 objects/s recovering 2023-10-09T10:23:13.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 7422/76383 objects degraded (9.717%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:23:13.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:13 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:13.683 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: pgmap v1345: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 7422/76383 objects degraded (9.717%); 13100/76383 objects misplaced (17.150%); 3.4 MiB/s, 66 objects/s recovering 2023-10-09T10:23:13.683 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: failed to encode map e482 with expected crc 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 7422/76383 objects degraded (9.717%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:23:13.684 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:13 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:13.766 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:23:13.767 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:23:13.767 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:23:13.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:13.842 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:13.848 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:13.898 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:13.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:13.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:14.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:14.048 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:14.323 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:14.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:14.429 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:14.515 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:14.834 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:14.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:14.910 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:14.937 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:14.945 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:14.970 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:15.012 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:15.021 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:15.052 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:15.081 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:15.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:15.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:15.101 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:15.128 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:15.156 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:15.175 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:23:15.175 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:23:15.175 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:23:15.175 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:23:15.176 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:23:15.176 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:23:15.176 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:23:15.176 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:23:15.190 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:15.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:14 smithi110 ceph-mon[136032]: pgmap v1347: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.5 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 7422/72384 objects degraded (10.254%); 12546/72384 objects misplaced (17.333%); 1.5 MiB/s, 39 objects/s recovering 2023-10-09T10:23:15.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:15.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:15.197 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2379546447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:15.288 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:15.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:23:15.318 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:14 smithi012 ceph-mon[147926]: pgmap v1347: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.5 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 7422/72384 objects degraded (10.254%); 12546/72384 objects misplaced (17.333%); 1.5 MiB/s, 39 objects/s recovering 2023-10-09T10:23:15.318 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:15.318 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:15.318 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2379546447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:15.318 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:14 smithi012 ceph-mon[152263]: pgmap v1347: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.5 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 7422/72384 objects degraded (10.254%); 12546/72384 objects misplaced (17.333%); 1.5 MiB/s, 39 objects/s recovering 2023-10-09T10:23:15.318 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:15.318 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:15.319 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2379546447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:15.660 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:15.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:15.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:15.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:15.830 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:15.890 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:15.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:15.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:15.996 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:16.040 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:16.101 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:16.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:16.116 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:16.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:16.263 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:16.378 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:16.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:16.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:16.544 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:16.798 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:16.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:16.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:16.966 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:17.048 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:17.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:17.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:17.151 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:16 smithi012 ceph-mon[147926]: pgmap v1348: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.3 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 7620/63462 objects degraded (12.007%); 10660/63462 objects misplaced (16.797%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T10:23:17.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:17.161 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:16 smithi012 ceph-mon[152263]: pgmap v1348: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.3 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 7620/63462 objects degraded (12.007%); 10660/63462 objects misplaced (16.797%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T10:23:17.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:16 smithi110 ceph-mon[136032]: pgmap v1348: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.3 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 7620/63462 objects degraded (12.007%); 10660/63462 objects misplaced (16.797%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T10:23:17.222 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:17.282 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:17.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:17.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:17.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:17.469 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:17.535 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:17.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:17.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:17.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:17.674 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:23:17.728 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:17.820 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:17.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:17.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:17.982 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:17.989 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:23:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:23:17] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.33.4" 2023-10-09T10:23:17.989 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:17 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2708516794' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:17.990 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:17 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2708516794' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:17.993 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:17 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2708516794' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:18.035 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:23:18.053 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:18.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:18.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:18.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:18.212 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:18.274 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:23:18.277 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:23:18.277 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:18.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:18.326 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:18.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:18.436 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:18.705 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:23:18.993 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:18 smithi110 ceph-mon[136032]: pgmap v1349: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.2 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 7620/59058 objects degraded (12.903%); 10072/59058 objects misplaced (17.054%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T10:23:18.993 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:18 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:18.993 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2708516794' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:23:18.993 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:18 smithi110 ceph-mon[136032]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T10:23:18.993 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:18 smithi110 ceph-mon[136032]: failed to encode map e483 with expected crc 2023-10-09T10:23:18.993 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:18 smithi110 ceph-mon[136032]: failed to encode map e483 with expected crc 2023-10-09T10:23:18.994 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 7620/59058 objects degraded (12.903%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[147926]: pgmap v1349: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.2 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 7620/59058 objects degraded (12.903%); 10072/59058 objects misplaced (17.054%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T10:23:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2708516794' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:23:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[147926]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T10:23:19.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[147926]: failed to encode map e483 with expected crc 2023-10-09T10:23:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[147926]: failed to encode map e483 with expected crc 2023-10-09T10:23:19.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 7620/59058 objects degraded (12.903%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[152263]: pgmap v1349: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.2 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 7620/59058 objects degraded (12.903%); 10072/59058 objects misplaced (17.054%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T10:23:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2708516794' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:23:19.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[152263]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T10:23:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[152263]: failed to encode map e483 with expected crc 2023-10-09T10:23:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[152263]: failed to encode map e483 with expected crc 2023-10-09T10:23:19.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 7620/59058 objects degraded (12.903%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:19.465 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:19.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:19.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:19.495 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:19.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:19.624 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:20.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:20 smithi012 ceph-mon[147926]: failed to encode map e483 with expected crc 2023-10-09T10:23:20.254 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:20 smithi012 ceph-mon[152263]: failed to encode map e483 with expected crc 2023-10-09T10:23:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:20 smithi110 ceph-mon[136032]: failed to encode map e483 with expected crc 2023-10-09T10:23:20.686 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:20.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:20.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:20.695 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 0, 2, 5, 4] out_osds: [1, 6] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:23:20.695 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:23:20.695 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [3, 7, 0, 2, 5, 4] 2023-10-09T10:23:20.695 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T10:23:20.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:20.870 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:21.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:21 smithi012 ceph-mon[147926]: pgmap v1351: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.2 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 7620/57897 objects degraded (13.161%); 9608/57897 objects misplaced (16.595%); 688 KiB/s, 40 objects/s recovering 2023-10-09T10:23:21.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:21 smithi012 ceph-mon[152263]: pgmap v1351: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.2 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 7620/57897 objects degraded (13.161%); 9608/57897 objects misplaced (16.595%); 688 KiB/s, 40 objects/s recovering 2023-10-09T10:23:21.110 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:21.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:21.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:21.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:21.283 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:21 smithi110 ceph-mon[136032]: pgmap v1351: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.2 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 7620/57897 objects degraded (13.161%); 9608/57897 objects misplaced (16.595%); 688 KiB/s, 40 objects/s recovering 2023-10-09T10:23:21.294 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:21.529 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:21.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:21.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:21.644 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:21.673 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:21.725 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:21.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:21.766 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:21.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:21.824 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:21.847 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:21.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:21.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:21.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:21.894 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:21.935 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:21.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:21.966 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:21.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:23:21.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:23:21.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:23:21.984 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:23:21.985 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:23:21.985 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:23:21.985 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:23:21.985 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:23:22.027 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:22.169 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2374509566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:22.338 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2374509566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:22.338 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2374509566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:22.476 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:22.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:22.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:22.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:22.643 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:22.717 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:23:22.724 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:22.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:22.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:22.879 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:22.965 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:22.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:22.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:23.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:23.160 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:23.176 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:23:23.176 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:23:23.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:209876 10.8 e483/456 10:17708bc2:::benchmark_data_smithi012_366312_object98547:head v 483'10626, mlcod=483'10624)", 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:23.170747+0000", 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00074224999999999996, 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00078570500000000004, 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.170747+0000", 2023-10-09T10:23:23.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.170747+0000", 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.170746+0000", 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5489999999999995e-06 2023-10-09T10:23:23.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:23.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.170755+0000", 2023-10-09T10:23:23.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1008e-05 2023-10-09T10:23:23.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.170766+0000", 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9589999999999998e-06 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.170773+0000", 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8415999999999998e-05 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.170831+0000", 2023-10-09T10:23:23.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8876e-05 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.170850+0000", 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103829 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:209878 10.b e483/477)", 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:23.171340+0000", 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00014881000000000001, 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025940700000000002, 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:23.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.171340+0000", 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.171340+0000", 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:23.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.171339+0000", 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4490000000000002e-06 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.171347+0000", 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9090000000000006e-06 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.171356+0000", 2023-10-09T10:23:23.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.263e-06 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:23.171362+0000", 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1869999999999999e-05 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:23.185 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:23.186 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:23:23.186 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 2 2023-10-09T10:23:23.186 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:23:23.269 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:23.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:23.301 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:23 smithi110 ceph-mon[136032]: pgmap v1352: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.0 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 971 op/s; 7816/49455 objects degraded (15.804%); 8005/49455 objects misplaced (16.186%); 1.2 MiB/s, 73 objects/s recovering 2023-10-09T10:23:23.381 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:23.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:23 smithi012 ceph-mon[152263]: pgmap v1352: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.0 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 971 op/s; 7816/49455 objects degraded (15.804%); 8005/49455 objects misplaced (16.186%); 1.2 MiB/s, 73 objects/s recovering 2023-10-09T10:23:23.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:23 smithi012 ceph-mon[147926]: pgmap v1352: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 1.0 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 971 op/s; 7816/49455 objects degraded (15.804%); 8005/49455 objects misplaced (16.186%); 1.2 MiB/s, 73 objects/s recovering 2023-10-09T10:23:23.445 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:23.524 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:23.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:23.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:23.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:23.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:23:23.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:23.729 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:23.783 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:23.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:23.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:23.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:23.945 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:23 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:23:23.691+0000 7f85a99d3700 -1 mon.a@0(leader).osd e483 definitely_dead 0 2023-10-09T10:23:23.946 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:24.009 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:24.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:24.114 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:24.186 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:24.249 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:24.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:24.297 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:24 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 7816/49455 objects degraded (15.804%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:24.297 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1084250789' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:23:24.297 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:23:24.297 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:24 smithi012 ceph-mon[147926]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:23:24.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:24 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 7816/49455 objects degraded (15.804%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:24.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1084250789' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:23:24.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:23:24.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:24 smithi012 ceph-mon[152263]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:23:24.360 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:24.423 INFO:teuthology.orchestra.run.smithi012.stderr:marked out osd.7. 2023-10-09T10:23:24.423 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:24 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 7816/49455 objects degraded (15.804%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:24.423 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1084250789' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:23:24.423 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:23:24.424 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:24 smithi110 ceph-mon[136032]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:23:24.431 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:24.491 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:24.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:24.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:24.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:23:24.588 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:24.595 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:24.626 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:24.967 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:24.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:25.072 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:25.100 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:25.191 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:25.423 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:23:25.429 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:25.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:25.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:25.575 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: pgmap v1353: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 938 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 910 op/s; 7816/45072 objects degraded (17.341%); 7413/45072 objects misplaced (16.447%); 1.1 MiB/s, 68 objects/s recovering 2023-10-09T10:23:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:23:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: osdmap e484: 8 total, 8 up, 5 in 2023-10-09T10:23:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1327218224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: pgmap v1353: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 938 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 910 op/s; 7816/45072 objects degraded (17.341%); 7413/45072 objects misplaced (16.447%); 1.1 MiB/s, 68 objects/s recovering 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: osdmap e484: 8 total, 8 up, 5 in 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1327218224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:25.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:25.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: pgmap v1353: 109 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 88 active+clean; 938 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 910 op/s; 7816/45072 objects degraded (17.341%); 7413/45072 objects misplaced (16.447%); 1.1 MiB/s, 68 objects/s recovering 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: osdmap e484: 8 total, 8 up, 5 in 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: failed to encode map e484 with expected crc 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1327218224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:25.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:25.815 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:25.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:25.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:25.948 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:26.189 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:26.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:26.294 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:26.322 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:26.482 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:23:26.482 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.482 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.482 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: osdmap e485: 8 total, 8 up, 5 in 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.d starting backfill to osd.2 from (0'0,0'0] MAX to 483'12551 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.0 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10272 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10723 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.20 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10250 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.10 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10619 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.2 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10296 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.1b starting backfill to osd.2 from (0'0,0'0] MAX to 483'12701 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.1a starting backfill to osd.4 from (0'0,0'0] MAX to 483'12361 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.20 continuing backfill to osd.5 from (429'1763,483'10250] MIN to 483'10250 2023-10-09T10:23:26.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.b starting backfill to osd.2 from (0'0,0'0] MAX to 483'12692 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.1b continuing backfill to osd.4 from (437'4775,483'12701] MIN to 483'12701 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.a starting backfill to osd.4 from (0'0,0'0] MAX to 483'12321 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.b continuing backfill to osd.4 from (437'4775,483'12692] MIN to 483'12692 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.22 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10314 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.0 continuing backfill to osd.5 from (429'1763,483'10272] MIN to 483'10272 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[147926]: 10.10 continuing backfill to osd.5 from (437'4516,483'10619] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 483'10619 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.484 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: osdmap e485: 8 total, 8 up, 5 in 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.d starting backfill to osd.2 from (0'0,0'0] MAX to 483'12551 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.0 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10272 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10723 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.20 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10250 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.10 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10619 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.2 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10296 2023-10-09T10:23:26.485 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.1b starting backfill to osd.2 from (0'0,0'0] MAX to 483'12701 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.1a starting backfill to osd.4 from (0'0,0'0] MAX to 483'12361 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.20 continuing backfill to osd.5 from (429'1763,483'10250] MIN to 483'10250 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.b starting backfill to osd.2 from (0'0,0'0] MAX to 483'12692 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.1b continuing backfill to osd.4 from (437'4775,483'12701] MIN to 483'12701 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.a starting backfill to osd.4 from (0'0,0'0] MAX to 483'12321 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.b continuing backfill to osd.4 from (437'4775,483'12692] MIN to 483'12692 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.22 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10314 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.0 continuing backfill to osd.5 from (429'1763,483'10272] MIN to 483'10272 2023-10-09T10:23:26.486 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:26 smithi012 ceph-mon[152263]: 10.10 continuing backfill to osd.5 from (437'4516,483'10619] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 483'10619 2023-10-09T10:23:26.608 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:26.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:26.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:26.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:23:26.617 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.618 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.618 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:26.618 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: osdmap e485: 8 total, 8 up, 5 in 2023-10-09T10:23:26.618 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.d starting backfill to osd.2 from (0'0,0'0] MAX to 483'12551 2023-10-09T10:23:26.619 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.0 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10272 2023-10-09T10:23:26.619 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10723 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: failed to encode map e485 with expected crc 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.20 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10250 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.10 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10619 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.2 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10296 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.1b starting backfill to osd.2 from (0'0,0'0] MAX to 483'12701 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.1a starting backfill to osd.4 from (0'0,0'0] MAX to 483'12361 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.20 continuing backfill to osd.5 from (429'1763,483'10250] MIN to 483'10250 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.b starting backfill to osd.2 from (0'0,0'0] MAX to 483'12692 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.1b continuing backfill to osd.4 from (437'4775,483'12701] MIN to 483'12701 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.a starting backfill to osd.4 from (0'0,0'0] MAX to 483'12321 2023-10-09T10:23:26.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.b continuing backfill to osd.4 from (437'4775,483'12692] MIN to 483'12692 2023-10-09T10:23:26.621 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.22 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10314 2023-10-09T10:23:26.621 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.0 continuing backfill to osd.5 from (429'1763,483'10272] MIN to 483'10272 2023-10-09T10:23:26.621 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:26 smithi110 ceph-mon[136032]: 10.10 continuing backfill to osd.5 from (437'4516,483'10619] 10:0804b7b5:::benchmark_data_smithi012_366312_object107564:head to 483'10619 2023-10-09T10:23:26.717 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:26.743 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:26.780 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:26.788 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:26.811 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:26.843 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:26.872 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:26.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:26.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:26.878 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:26.908 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:26.938 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:26.970 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:26.975 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:23:26.975 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:23:26.975 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:23:26.975 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:23:26.975 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:23:26.975 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:23:26.976 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:23:26.976 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:23:26.979 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:27.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:27.171 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:27.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:27.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:27.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:27.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: pgmap v1356: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 10 remapped+peering, 39 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 49 active+clean; 762 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 3693/36630 objects degraded (10.082%); 2572/36630 objects misplaced (7.022%); 5.6 MiB/s, 152 objects/s recovering 2023-10-09T10:23:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: osdmap e486: 8 total, 8 up, 5 in 2023-10-09T10:23:27.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.21 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10285 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.1c starting backfill to osd.2 from (0'0,0'0] MAX to 483'12359 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.18 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10701 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10662 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.c starting backfill to osd.2 from (0'0,0'0] MAX to 483'12368 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.1d starting backfill to osd.2 from (0'0,0'0] MAX to 483'12528 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.11 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10546 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 483'10780 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 483'10705 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: 10.1 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10287 2023-10-09T10:23:27.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1701922034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:27.445 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:27.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:27.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:27.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: pgmap v1356: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 10 remapped+peering, 39 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 49 active+clean; 762 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 3693/36630 objects degraded (10.082%); 2572/36630 objects misplaced (7.022%); 5.6 MiB/s, 152 objects/s recovering 2023-10-09T10:23:27.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: osdmap e486: 8 total, 8 up, 5 in 2023-10-09T10:23:27.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.592 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.21 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10285 2023-10-09T10:23:27.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.1c starting backfill to osd.2 from (0'0,0'0] MAX to 483'12359 2023-10-09T10:23:27.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.18 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10701 2023-10-09T10:23:27.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10662 2023-10-09T10:23:27.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.c starting backfill to osd.2 from (0'0,0'0] MAX to 483'12368 2023-10-09T10:23:27.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.1d starting backfill to osd.2 from (0'0,0'0] MAX to 483'12528 2023-10-09T10:23:27.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.11 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10546 2023-10-09T10:23:27.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 483'10780 2023-10-09T10:23:27.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 483'10705 2023-10-09T10:23:27.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: 10.1 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10287 2023-10-09T10:23:27.594 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1701922034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:27.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: pgmap v1356: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 10 remapped+peering, 39 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 49 active+clean; 762 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 3693/36630 objects degraded (10.082%); 2572/36630 objects misplaced (7.022%); 5.6 MiB/s, 152 objects/s recovering 2023-10-09T10:23:27.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: osdmap e486: 8 total, 8 up, 5 in 2023-10-09T10:23:27.594 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: failed to encode map e486 with expected crc 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.21 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10285 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.1c starting backfill to osd.2 from (0'0,0'0] MAX to 483'12359 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.18 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10701 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 483'10662 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.c starting backfill to osd.2 from (0'0,0'0] MAX to 483'12368 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.1d starting backfill to osd.2 from (0'0,0'0] MAX to 483'12528 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.11 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10546 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 483'10780 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 483'10705 2023-10-09T10:23:27.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: 10.1 starting backfill to osd.2 from (0'0,0'0] MAX to 483'10287 2023-10-09T10:23:27.596 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1701922034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:27.596 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:27.649 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:27.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:27.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:27.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:27.839 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:27.915 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:27.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:27.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:28.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:28.071 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:28.138 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:28.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:28.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:23:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:23:27] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.33.4" 2023-10-09T10:23:28.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:23:28.243 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:28.314 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:28.316 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:23:28.316 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:23:28.373 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:28.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:28.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:28.536 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3693/36630 objects degraded (10.082%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:28.536 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3693/36630 objects degraded (10.082%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:28.551 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:28.625 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:28.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:28.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:28.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3693/36630 objects degraded (10.082%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:23:28.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:28.798 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:28.853 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:28.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:28.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:28.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:23:28.958 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:29.006 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:29.036 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:29.138 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:29.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:29.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:29.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:29.293 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:29.347 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:23:29.435 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:29 smithi110 ceph-mon[136032]: pgmap v1358: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 10 remapped+peering, 39 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 43 active+clean; 715 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 838 op/s; 3693/34350 objects degraded (10.751%); 4607/34350 objects misplaced (13.412%); 6.2 MiB/s, 136 objects/s recovering 2023-10-09T10:23:29.435 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:29.435 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:29 smithi012 ceph-mon[147926]: pgmap v1358: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 10 remapped+peering, 39 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 43 active+clean; 715 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 838 op/s; 3693/34350 objects degraded (10.751%); 4607/34350 objects misplaced (13.412%); 6.2 MiB/s, 136 objects/s recovering 2023-10-09T10:23:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:29.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:29 smithi012 ceph-mon[152263]: pgmap v1358: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 10 remapped+peering, 39 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+remapped+backfill_wait, 43 active+clean; 715 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 838 op/s; 3693/34350 objects degraded (10.751%); 4607/34350 objects misplaced (13.412%); 6.2 MiB/s, 136 objects/s recovering 2023-10-09T10:23:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:29.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:29.787 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:29.987 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4] out_osds: [1, 6, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:23:29.987 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:23:29.987 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T10:23:29.987 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:23:29.987 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:23:29.988 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T10:23:30.975 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:30.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:30.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:31.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:31.128 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:31.269 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:31.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:31.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:31.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:31.430 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:31.510 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:23:31.510 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:23:31.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:31 smithi012 ceph-mon[147926]: pgmap v1359: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 9 remapped+peering, 39 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 42 active+clean; 703 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 3695/33777 objects degraded (10.939%); 4764/33777 objects misplaced (14.104%); 6.2 MiB/s, 137 objects/s recovering 2023-10-09T10:23:31.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:31 smithi012 ceph-mon[152263]: pgmap v1359: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 9 remapped+peering, 39 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 42 active+clean; 703 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 3695/33777 objects degraded (10.939%); 4764/33777 objects misplaced (14.104%); 6.2 MiB/s, 137 objects/s recovering 2023-10-09T10:23:31.715 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:31 smithi110 ceph-mon[136032]: pgmap v1359: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 9 remapped+peering, 39 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 42 active+clean; 703 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 3695/33777 objects degraded (10.939%); 4764/33777 objects misplaced (14.104%); 6.2 MiB/s, 137 objects/s recovering 2023-10-09T10:23:31.840 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:31.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:31.946 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:31.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:23:32.004 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:32.080 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:32.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:32.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:32.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:32.303 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:32.563 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:32.574 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:32.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:32.682 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:32.746 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:32.763 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:32.841 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:32.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:32.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:33.035 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:33.099 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:33.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:33.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:33.208 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:33.282 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:33.340 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:33.345 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:33.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:33.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:33.454 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:33.525 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:33.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:33 smithi012 ceph-mon[147926]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 583 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 5304/28008 objects degraded (18.937%); 6850/28008 objects misplaced (24.457%); 6.7 MiB/s, 171 objects/s recovering 2023-10-09T10:23:33.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5304/28008 objects degraded (18.937%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:33.645 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:33 smithi012 ceph-mon[152263]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 583 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 5304/28008 objects degraded (18.937%); 6850/28008 objects misplaced (24.457%); 6.7 MiB/s, 171 objects/s recovering 2023-10-09T10:23:33.645 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5304/28008 objects degraded (18.937%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:33.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:33 smithi110 ceph-mon[136032]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 583 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 5304/28008 objects degraded (18.937%); 6850/28008 objects misplaced (24.457%); 6.7 MiB/s, 171 objects/s recovering 2023-10-09T10:23:33.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5304/28008 objects degraded (18.937%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:33.969 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:33.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:34.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:34.158 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:34.735 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:34.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:34.847 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:34.933 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:35.010 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:35.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:35.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:35.117 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:35.192 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:35.383 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:35.401 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:35.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:35.423 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:35.466 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:35.480 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:23:35.480 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "" 2023-10-09T10:23:35.480 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:23:35.513 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:35.525 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:35.543 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:23:35.566 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:35.592 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:35.618 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:35.650 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:35.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:35.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:35.671 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:35.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:35 smithi012 ceph-mon[147926]: pgmap v1361: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 439 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 5222/21096 objects degraded (24.754%); 4994/21096 objects misplaced (23.673%); 2.0 MiB/s, 86 objects/s recovering 2023-10-09T10:23:35.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1186961544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:35.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:35.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2668267316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:35.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:35 smithi012 ceph-mon[152263]: pgmap v1361: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 439 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 5222/21096 objects degraded (24.754%); 4994/21096 objects misplaced (23.673%); 2.0 MiB/s, 86 objects/s recovering 2023-10-09T10:23:35.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1186961544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:35.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:35.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2668267316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:35.714 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:35.732 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:23:35.732 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:23:35.732 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:23:35.733 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:23:35.733 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:23:35.733 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:23:35.733 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:23:35.733 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:23:35.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:35.844 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:35.913 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:35.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:35.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:35 smithi110 ceph-mon[136032]: pgmap v1361: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 439 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 5222/21096 objects degraded (24.754%); 4994/21096 objects misplaced (23.673%); 2.0 MiB/s, 86 objects/s recovering 2023-10-09T10:23:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1186961544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:23:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2668267316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:35.956 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:23:35.956 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:23:35.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.956 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:219728 10.18 e487/485 10:1fa45e07:::benchmark_data_smithi012_366312_object108399:head v 487'10981, mlcod=487'10979)", 2023-10-09T10:23:35.956 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:23:35.948233+0000", 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0071802439999999997, 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072112039999999997, 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948233+0000", 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:23:35.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948233+0000", 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948232+0000", 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:23:35.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948242+0000", 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0251999999999999e-05 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948263+0000", 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1174000000000001e-05 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:23:35.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948285+0000", 2023-10-09T10:23:35.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1829999999999999e-05 2023-10-09T10:23:35.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.953419+0000", 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051336719999999997 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.953445+0000", 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6942999999999999e-05 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:23:35.961 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:219730 10.11 e487/485 10:8e329eb9:::benchmark_data_smithi012_366312_object108401:head v 487'10842, mlcod=487'10840)", 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:23:35.948303+0000", 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0071097529999999999, 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072099709999999999, 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:23:35.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948303+0000", 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948303+0000", 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948303+0000", 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:23:35.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948308+0000", 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1710000000000001e-06 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948322+0000", 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4080000000000001e-05 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:23:35.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948329+0000", 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1210000000000001e-06 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.953677+0000", 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053479549999999997 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.953704+0000", 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7285999999999999e-05 2023-10-09T10:23:35.965 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:219732 10.19 e487/485)", 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:23:35.948295+0000", 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0071181059999999999, 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072773990000000004, 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:23:35.966 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948295+0000", 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948295+0000", 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948294+0000", 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948302+0000", 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6090000000000003e-06 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:23:35.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948333+0000", 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1699000000000003e-05 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.948341+0000", 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9840000000000001e-06 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:219740 10.1e e487/479 10:7aa9a2ed:::benchmark_data_smithi012_366312_object108411:head v 487'12620, mlcod=487'12618)", 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:23:35.955016+0000", 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00039675700000000002, 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00060767000000000004, 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955016+0000", 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955016+0000", 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955016+0000", 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.971 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955021+0000", 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7180000000000001e-06 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955028+0000", 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:23:35.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955032+0000", 2023-10-09T10:23:35.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.014e-06 2023-10-09T10:23:35.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:23:35.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955075+0000", 2023-10-09T10:23:35.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3470000000000002e-05 2023-10-09T10:23:35.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:23:35.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955100+0000", 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5147000000000002e-05 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:219739 10.9 e487/485)", 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:23:35.955186+0000", 2023-10-09T10:23:35.975 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00022710100000000001, 2023-10-09T10:23:35.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00049568799999999999, 2023-10-09T10:23:35.976 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:23:35.976 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:23:35.976 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955186+0000", 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955186+0000", 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955185+0000", 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955190+0000", 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.169e-06 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955198+0000", 2023-10-09T10:23:35.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4499999999999998e-06 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:35.955203+0000", 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5439999999999998e-06 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:23:35.979 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:23:35.980 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 5 2023-10-09T10:23:35.980 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:23:36.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:36.091 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:36.415 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:36.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:36.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:36.587 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:36.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:36.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:36.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[147926]: osdmap e487: 8 total, 8 up, 5 in 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[147926]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[147926]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[147926]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[147926]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[147926]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:36.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:23:36.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[152263]: osdmap e487: 8 total, 8 up, 5 in 2023-10-09T10:23:36.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[152263]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[152263]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[152263]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[152263]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:36 smithi012 ceph-mon[152263]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.779 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:23:36.779 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:36.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:36 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:36.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:23:36.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:36 smithi110 ceph-mon[136032]: osdmap e487: 8 total, 8 up, 5 in 2023-10-09T10:23:36.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:36 smithi110 ceph-mon[136032]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.845 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:36 smithi110 ceph-mon[136032]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:36 smithi110 ceph-mon[136032]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:36 smithi110 ceph-mon[136032]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.846 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:36 smithi110 ceph-mon[136032]: failed to encode map e487 with expected crc 2023-10-09T10:23:36.852 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:36.931 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:36.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:36.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:37.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:37.109 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:37.186 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:37.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:37.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:37.375 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:37.432 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:37.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:37.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:37.495 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:37.539 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:37.598 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:37 smithi012 ceph-mon[147926]: pgmap v1363: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 321 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 756 op/s; 5269/15474 objects degraded (34.051%); 3801/15474 objects misplaced (24.564%); 2.4 MiB/s, 87 objects/s recovering 2023-10-09T10:23:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:37 smithi012 ceph-mon[152263]: pgmap v1363: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 321 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 756 op/s; 5269/15474 objects degraded (34.051%); 3801/15474 objects misplaced (24.564%); 2.4 MiB/s, 87 objects/s recovering 2023-10-09T10:23:37.683 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:37.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:37.797 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:37.869 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:37 smithi110 ceph-mon[136032]: pgmap v1363: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 321 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 756 op/s; 5269/15474 objects degraded (34.051%); 3801/15474 objects misplaced (24.564%); 2.4 MiB/s, 87 objects/s recovering 2023-10-09T10:23:37.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:37.939 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:37.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:38.040 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:23:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:23:37] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.33.4" 2023-10-09T10:23:38.051 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:38.121 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:38.321 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:38.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:38.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:38.427 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:38.506 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:38.572 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:38.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:38.669 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5269/15474 objects degraded (34.051%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:38.669 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5269/15474 objects degraded (34.051%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:38.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:38.751 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:38.868 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:38.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:38.911 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Removed 111326 objects 2023-10-09T10:23:38.912 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Clean up completed and total clean up time :140.876 2023-10-09T10:23:38.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5269/15474 objects degraded (34.051%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:38.957 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:23:38.957 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:23:38.957 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:23:38.957 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:23:38.957 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:23:38.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:23:38.979 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:38.989 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:23:39.005 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:39.039 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:39.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:39.144 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:39.170 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:39.209 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:39.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:39.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:39.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:39.343 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:39.379 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:39.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:39.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:23:39.485 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:39.519 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:39.562 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:39.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:39.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:39.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:39 smithi012 ceph-mon[147926]: pgmap v1364: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 226 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 5269/10908 objects degraded (48.304%); 2585/10908 objects misplaced (23.698%); 2.2 MiB/s, 81 objects/s recovering 2023-10-09T10:23:39.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:39 smithi012 ceph-mon[152263]: pgmap v1364: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 226 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 5269/10908 objects degraded (48.304%); 2585/10908 objects misplaced (23.698%); 2.2 MiB/s, 81 objects/s recovering 2023-10-09T10:23:39.694 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:39.696 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:39.728 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:39.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:39.737 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:39.799 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T10:23:39.800 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T10:23:39.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:39.875 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:39.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:39 smithi110 ceph-mon[136032]: pgmap v1364: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 226 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 5269/10908 objects degraded (48.304%); 2585/10908 objects misplaced (23.698%); 2.2 MiB/s, 81 objects/s recovering 2023-10-09T10:23:40.068 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:40.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:40.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:40.172 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:40.199 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:40.234 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:40.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:40.338 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:40.363 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:40.397 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:40.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:40.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:40.502 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:40.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:40.563 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:40.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:40.656 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:40.669 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:40.696 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:40.736 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:40.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:40.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:40.841 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:40.868 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:40.901 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:40.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:40.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:41.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:41.039 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:41.177 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:41.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:41.282 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:41.307 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:41.341 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:41.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:41.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:41.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:41.479 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:41.518 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:41.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:41.623 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:41.647 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:41.681 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:41.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:41.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:41.718 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4] out_osds: [1, 6, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:23:41.719 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:23:41.719 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T10:23:41.719 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:23:41.719 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:23:41.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T10:23:41.747 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[147926]: pgmap v1365: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 116 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 943 op/s; 5108/5613 objects degraded (91.003%); 1121/5613 objects misplaced (19.971%); 3.3 MiB/s, 140 objects/s recovering 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[147926]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[147926]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[147926]: osdmap e488: 8 total, 8 up, 5 in 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[147926]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[152263]: pgmap v1365: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 116 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 943 op/s; 5108/5613 objects degraded (91.003%); 1121/5613 objects misplaced (19.971%); 3.3 MiB/s, 140 objects/s recovering 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[152263]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[152263]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[152263]: osdmap e488: 8 total, 8 up, 5 in 2023-10-09T10:23:41.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:41 smithi012 ceph-mon[152263]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.785 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:41.815 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:41.891 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:41.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:41.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:41.896 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:41.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:41 smithi110 ceph-mon[136032]: pgmap v1365: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 116 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 943 op/s; 5108/5613 objects degraded (91.003%); 1121/5613 objects misplaced (19.971%); 3.3 MiB/s, 140 objects/s recovering 2023-10-09T10:23:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:41 smithi110 ceph-mon[136032]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:41 smithi110 ceph-mon[136032]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:41 smithi110 ceph-mon[136032]: osdmap e488: 8 total, 8 up, 5 in 2023-10-09T10:23:41.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:41 smithi110 ceph-mon[136032]: failed to encode map e488 with expected crc 2023-10-09T10:23:41.997 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:42.022 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:42.063 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:42.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:42.167 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:42.192 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:42.218 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:42.225 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:42.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:42.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:42.237 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:42.256 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:42.277 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:42.298 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:42.319 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:42.330 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:42.341 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:42.365 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:42.367 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:42.372 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:23:42.372 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:23:42.372 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:23:42.372 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:23:42.373 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:23:42.373 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:23:42.373 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:23:42.373 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:23:42.405 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:42.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:42.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:42.540 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:42.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[152263]: failed to encode map e489 with expected crc 2023-10-09T10:23:42.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[152263]: osdmap e489: 8 total, 8 up, 5 in 2023-10-09T10:23:42.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[152263]: 10.d continuing backfill to osd.2 from (450'7188,487'12945] MIN to 487'12945 2023-10-09T10:23:42.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1962940282' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:42.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:42.576 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2287487964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:42.587 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:23:42.704 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:23:42.705 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:23:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[147926]: failed to encode map e489 with expected crc 2023-10-09T10:23:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[147926]: osdmap e489: 8 total, 8 up, 5 in 2023-10-09T10:23:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[147926]: 10.d continuing backfill to osd.2 from (450'7188,487'12945] MIN to 487'12945 2023-10-09T10:23:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1962940282' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2287487964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:42 smithi110 ceph-mon[136032]: failed to encode map e489 with expected crc 2023-10-09T10:23:42.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:42 smithi110 ceph-mon[136032]: osdmap e489: 8 total, 8 up, 5 in 2023-10-09T10:23:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:42 smithi110 ceph-mon[136032]: 10.d continuing backfill to osd.2 from (450'7188,487'12945] MIN to 487'12945 2023-10-09T10:23:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1962940282' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:23:42.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2287487964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:42.980 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:42.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:42.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:43.085 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:43.110 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:43.143 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:43.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:43.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:43.247 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:43.272 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:43.311 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:43.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:43.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:43.416 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:43.442 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:43.474 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:43.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:43.579 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:43.609 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:43 smithi110 ceph-mon[136032]: pgmap v1368: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 55 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 718 op/s; 5149/2673 objects degraded (192.630%); 609/2673 objects misplaced (22.783%); 2.3 MiB/s, 110 objects/s recovering 2023-10-09T10:23:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:43 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:43 smithi110 ceph-mon[136032]: osdmap e490: 8 total, 8 up, 5 in 2023-10-09T10:23:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 5149/2673 objects degraded (192.630%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:43.755 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:43.758 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[147926]: pgmap v1368: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 55 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 718 op/s; 5149/2673 objects degraded (192.630%); 609/2673 objects misplaced (22.783%); 2.3 MiB/s, 110 objects/s recovering 2023-10-09T10:23:43.758 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:43.758 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:43.758 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[147926]: osdmap e490: 8 total, 8 up, 5 in 2023-10-09T10:23:43.758 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 5149/2673 objects degraded (192.630%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:43.759 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[152263]: pgmap v1368: 109 pgs: 4 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 55 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 718 op/s; 5149/2673 objects degraded (192.630%); 609/2673 objects misplaced (22.783%); 2.3 MiB/s, 110 objects/s recovering 2023-10-09T10:23:43.759 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:43.759 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:23:43.759 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[152263]: osdmap e490: 8 total, 8 up, 5 in 2023-10-09T10:23:43.759 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 5149/2673 objects degraded (192.630%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:23:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:43.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:43.794 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:43.823 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:23:43.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:43.895 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:44.133 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:44.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:44.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:44.243 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:44.269 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:44.305 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:44.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:44.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:44.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:44.430 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:44.439 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:44.685 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:44.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[147926]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.687 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[147926]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[147926]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[147926]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.688 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:44.689 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:44.689 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2017184451' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:23:44.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[152263]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[152263]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[152263]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[152263]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:44.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:44.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2017184451' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:23:44.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:44.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:44.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:44.822 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:44.908 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:44.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:44.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:44.936 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:23:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:44 smithi110 ceph-mon[136032]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:44 smithi110 ceph-mon[136032]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:44 smithi110 ceph-mon[136032]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:44 smithi110 ceph-mon[136032]: failed to encode map e490 with expected crc 2023-10-09T10:23:44.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:23:44.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:23:44.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2017184451' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:23:44.974 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_4' removed 2023-10-09T10:23:45.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:45.044 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:45.135 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:23:45.136 INFO:teuthology.orchestra.run.smithi012.stdout: "success": "" 2023-10-09T10:23:45.136 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:23:45.422 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:45.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:45.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:45.502 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:23:45.503 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:23:45.503 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T10:23:45.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T10:23:45.529 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:45.556 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:45 smithi012 ceph-mon[147926]: pgmap v1370: 109 pgs: 4 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped, 77 active+clean; 28 MiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 600 op/s; 5119/1380 objects degraded (370.942%); 257/1380 objects misplaced (18.623%); 2.7 MiB/s, 240 objects/s recovering 2023-10-09T10:23:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2017184451' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:23:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:45 smithi012 ceph-mon[147926]: osdmap e491: 8 total, 8 up, 5 in 2023-10-09T10:23:45.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:45 smithi012 ceph-mon[147926]: failed to encode map e491 with expected crc 2023-10-09T10:23:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:45 smithi012 ceph-mon[152263]: pgmap v1370: 109 pgs: 4 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped, 77 active+clean; 28 MiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 600 op/s; 5119/1380 objects degraded (370.942%); 257/1380 objects misplaced (18.623%); 2.7 MiB/s, 240 objects/s recovering 2023-10-09T10:23:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2017184451' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:23:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:45 smithi012 ceph-mon[152263]: osdmap e491: 8 total, 8 up, 5 in 2023-10-09T10:23:45.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:45 smithi012 ceph-mon[152263]: failed to encode map e491 with expected crc 2023-10-09T10:23:45.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:23:45.875 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:45 smithi110 ceph-mon[136032]: pgmap v1370: 109 pgs: 4 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 23 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 1 active+remapped, 77 active+clean; 28 MiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 600 op/s; 5119/1380 objects degraded (370.942%); 257/1380 objects misplaced (18.623%); 2.7 MiB/s, 240 objects/s recovering 2023-10-09T10:23:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2017184451' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:23:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:45 smithi110 ceph-mon[136032]: osdmap e491: 8 total, 8 up, 5 in 2023-10-09T10:23:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:45 smithi110 ceph-mon[136032]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.092 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:46.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:46.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:46.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:46.225 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:46.262 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:46.262 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:46.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:46.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:46.367 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:46.393 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:46.431 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:46.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:46.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:46.535 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:46.567 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:46.637 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:46.639 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:46.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:46.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:46.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:46 smithi110 ceph-mon[136032]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:46 smithi110 ceph-mon[136032]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:46 smithi110 ceph-mon[136032]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:46 smithi110 ceph-mon[136032]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:46 smithi110 ceph-mon[136032]: failed to encode map e492 with expected crc 2023-10-09T10:23:46.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:46 smithi110 ceph-mon[136032]: failed to encode map e492 with expected crc 2023-10-09T10:23:46.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:46 smithi110 ceph-mon[136032]: osdmap e492: 8 total, 8 up, 5 in 2023-10-09T10:23:46.741 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:46.772 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:46.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[147926]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[147926]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[147926]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[147926]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[147926]: failed to encode map e492 with expected crc 2023-10-09T10:23:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[147926]: failed to encode map e492 with expected crc 2023-10-09T10:23:46.876 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[147926]: osdmap e492: 8 total, 8 up, 5 in 2023-10-09T10:23:46.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[152263]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[152263]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[152263]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[152263]: failed to encode map e491 with expected crc 2023-10-09T10:23:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[152263]: failed to encode map e492 with expected crc 2023-10-09T10:23:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[152263]: failed to encode map e492 with expected crc 2023-10-09T10:23:46.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:46 smithi012 ceph-mon[152263]: osdmap e492: 8 total, 8 up, 5 in 2023-10-09T10:23:46.959 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:46.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:47.063 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:47.092 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:47.227 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:47.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:47.331 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:47.332 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:23:47.332 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:23:47.345 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:47.345 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:47.349 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:47.360 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:47.369 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:47.388 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:47.406 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:47.424 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:47.441 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:47.442 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:23:47.443 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:23:47.443 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:23:47.443 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:23:47.443 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:23:47.443 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:23:47.443 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:23:47.443 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:23:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-mon[147926]: pgmap v1372: 73 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects degraded (2.778%); 2/72 objects misplaced (2.778%) 2023-10-09T10:23:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-mon[147926]: failed to encode map e493 with expected crc 2023-10-09T10:23:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-mon[147926]: osdmap e493: 8 total, 8 up, 5 in 2023-10-09T10:23:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1345798867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:47.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-mon[152263]: pgmap v1372: 73 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects degraded (2.778%); 2/72 objects misplaced (2.778%) 2023-10-09T10:23:47.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-mon[152263]: failed to encode map e493 with expected crc 2023-10-09T10:23:47.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-mon[152263]: osdmap e493: 8 total, 8 up, 5 in 2023-10-09T10:23:47.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1345798867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:47.691 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:47 smithi110 ceph-mon[136032]: pgmap v1372: 73 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects degraded (2.778%); 2/72 objects misplaced (2.778%) 2023-10-09T10:23:47.691 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:47 smithi110 ceph-mon[136032]: failed to encode map e493 with expected crc 2023-10-09T10:23:47.691 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:47 smithi110 ceph-mon[136032]: osdmap e493: 8 total, 8 up, 5 in 2023-10-09T10:23:47.691 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1345798867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:47.758 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:23:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:23:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:23:47] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.33.4" 2023-10-09T10:23:48.298 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:48.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:48.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:48.354 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:48.404 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:48.430 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:48.464 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:48.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:48.568 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:48.597 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:48.631 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:48.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:48.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:48 smithi110 ceph-mon[136032]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:48 smithi110 ceph-mon[136032]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:48 smithi110 ceph-mon[136032]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:48 smithi110 ceph-mon[136032]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:23:48.735 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:48.767 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[152263]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.767 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[152263]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.767 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[152263]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.767 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[152263]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.768 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:23:48.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[147926]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[147926]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[147926]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[147926]: failed to encode map e493 with expected crc 2023-10-09T10:23:48.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:23:48.770 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:49.566 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:23:49.566 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:23:49.566 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:23:49.566 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:23:49.566 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:23:49.902 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:49 smithi110 ceph-mon[136032]: pgmap v1375: 73 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects degraded (2.778%); 2/72 objects misplaced (2.778%) 2023-10-09T10:23:49.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:49 smithi012 ceph-mon[147926]: pgmap v1375: 73 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects degraded (2.778%); 2/72 objects misplaced (2.778%) 2023-10-09T10:23:49.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:49 smithi012 ceph-mon[152263]: pgmap v1375: 73 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects degraded (2.778%); 2/72 objects misplaced (2.778%) 2023-10-09T10:23:49.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:23:50.220 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:50.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:50.325 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:50.353 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:50.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:50.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:50.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:50.491 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:50.517 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:50.549 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:50.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:50.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:50.568 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:23:50.600 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:50 smithi012 ceph-mon[152263]: pgmap v1376: 73 pgs: 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:50.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:50.675 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4] out_osds: [1, 6, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:23:50.675 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:23:50.688 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:50 smithi110 ceph-mon[136032]: pgmap v1376: 73 pgs: 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:50.880 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:50.882 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:50 smithi012 ceph-mon[147926]: pgmap v1376: 73 pgs: 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:50.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:50.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:50.984 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:51.025 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:51.061 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:51.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:51.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:51.165 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:51.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:51.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:51.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:51.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:51.442 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:51.527 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:51.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:51.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:51.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:51.665 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:51.727 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:23:51.727 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128021 9.c 9:3ed78aba:::benchmark_data_smithi012_313365_object15546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 464.02065277100002, 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999, 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.728 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128021 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703223+0000", 2023-10-09T10:23:51.729 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703222+0000", 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7700000000000004e-06 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703229+0000", 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1178e-05 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.730 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703240+0000", 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0509999999999996e-06 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703245+0000", 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9240000000000002e-05 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.731 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703304+0000", 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.429e-05 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.703318+0000", 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.498177753 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201496+0000", 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2643e-05 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.732 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201519+0000", 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1325606320000001 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334079+0000", 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016077600000000001 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.733 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334240+0000", 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016287699999999999 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.334403+0000", 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0050087500000000002 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339412+0000", 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1050000000000007e-05 2023-10-09T10:23:51.734 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339473+0000", 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013735 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339610+0000", 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041248470000000001 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.735 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343735+0000", 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9188999999999999e-05 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343784+0000", 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000111032 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343895+0000", 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6406721609999999 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.736 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128030 9.c 9:39ef457f:::benchmark_data_smithi012_313365_object15555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 463.88064169500001, 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999, 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.737 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128030 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843234+0000", 2023-10-09T10:23:51.738 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843233+0000", 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843241+0000", 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3162e-05 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.739 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843254+0000", 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5490000000000001e-06 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843261+0000", 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2747999999999997e-05 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843314+0000", 2023-10-09T10:23:51.740 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7970000000000006e-06 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.843323+0000", 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.35848033200000001 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201803+0000", 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6119999999999999e-06 2023-10-09T10:23:51.741 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.201812+0000", 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1353121740000001 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337124+0000", 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8503000000000003e-05 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.742 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337212+0000", 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0490999999999998e-05 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.337303+0000", 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002260742 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339564+0000", 2023-10-09T10:23:51.743 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1705999999999994e-05 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.339635+0000", 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002741145 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.342376+0000", 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001560173 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.744 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343937+0000", 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6062000000000001e-05 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.343973+0000", 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4776000000000002e-05 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:09.344068+0000", 2023-10-09T10:23:51.745 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5008333679999999 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195623 9.c 9:350660d8:::benchmark_data_smithi012_313365_object83148:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 318.46660743799998, 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.178670812, 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.746 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195623 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.747 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257268+0000", 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257267+0000", 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8130000000000001e-06 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257273+0000", 2023-10-09T10:23:51.748 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0166999999999999e-05 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257283+0000", 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2350000000000002e-06 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.257290+0000", 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001555255 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.749 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.258845+0000", 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2446000000000001e-05 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.258858+0000", 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99572670600000002 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254584+0000", 2023-10-09T10:23:51.750 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2269999999999999e-06 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254594+0000", 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.157542818 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412136+0000", 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010587299999999999 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.751 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412242+0000", 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000452194 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412695+0000", 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010428244999999999 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.752 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423123+0000", 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4022000000000001e-05 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423157+0000", 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012428642 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435585+0000", 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000185372 2023-10-09T10:23:51.753 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435771+0000", 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5880000000000001e-05 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435817+0000", 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012248200000000001 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.754 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435939+0000", 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.178670812 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195633 9.1c 9:384521d8:::benchmark_data_smithi012_313365_object83158:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 318.44655619299999, 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1456984170000002, 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.755 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195633 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.756 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277320+0000", 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277318+0000", 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4900000000000003e-06 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277326+0000", 2023-10-09T10:23:51.757 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2858e-05 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277339+0000", 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1309999999999997e-06 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277344+0000", 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3996000000000002e-05 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.758 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277408+0000", 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0821e-05 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277419+0000", 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97663378599999995 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.759 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254052+0000", 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1975e-05 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254064+0000", 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1520595250000001 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406124+0000", 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013994800000000001 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406264+0000", 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045309899999999999 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.406717+0000", 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051173240000000004 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411834+0000", 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3628999999999999e-05 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411878+0000", 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3909999999999998e-06 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411886+0000", 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010941673000000001 2023-10-09T10:23:51.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.422828+0000", 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9455e-05 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.422877+0000", 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014060400000000001 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423018+0000", 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1456984170000002 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195634 9.c 9:36e83a2e:::benchmark_data_smithi012_313365_object83159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 318.446298996, 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1586804549999998, 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.764 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195634 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277577+0000", 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277576+0000", 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.331e-06 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277582+0000", 2023-10-09T10:23:51.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2768e-05 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277595+0000", 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277601+0000", 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4212000000000003e-05 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277645+0000", 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1494999999999999e-05 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.277657+0000", 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97695835200000003 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254615+0000", 2023-10-09T10:23:51.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.798e-06 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254622+0000", 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1605278669999999 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415150+0000", 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011404399999999999 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415264+0000", 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039553599999999999 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.415659+0000", 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0075350449999999998 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.770 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423194+0000", 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.01213207 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435326+0000", 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062050799999999998 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435947+0000", 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013730799999999999 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436084+0000", 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2330000000000002e-05 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436137+0000", 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120761 2023-10-09T10:23:51.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436257+0000", 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1586804549999998 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195652 9.1c 9:3e175140:::benchmark_data_smithi012_313365_object83177:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:23:51.773 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 318.41577875000002, 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1274454700000001, 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195652 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308097+0000", 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308095+0000", 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.49e-06 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308104+0000", 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7926000000000001e-05 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308122+0000", 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6449999999999999e-06 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308128+0000", 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7748000000000003e-05 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308196+0000", 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3386e-05 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308210+0000", 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.945878781 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254088+0000", 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5209999999999997e-06 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254096+0000", 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.15516445 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409260+0000", 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000137047 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409397+0000", 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039575500000000002 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.409793+0000", 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021158150000000001 2023-10-09T10:23:51.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411909+0000", 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7689e-05 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411927+0000", 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011128549999999999 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.780 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423055+0000", 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012286897999999999 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435342+0000", 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6456000000000003e-05 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435399+0000", 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014397600000000001 2023-10-09T10:23:51.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435543+0000", 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1274454700000001 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195653 9.c 9:3443b6fc:::benchmark_data_smithi012_313365_object83178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:23:51.782 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 318.41530620600003, 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1279840910000001, 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195653 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308570+0000", 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308569+0000", 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1650000000000003e-06 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308575+0000", 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432000000000001e-05 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308587+0000", 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.648e-06 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308593+0000", 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3375000000000003e-05 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308646+0000", 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3016e-05 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.308659+0000", 2023-10-09T10:23:51.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.94597990200000004 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254639+0000", 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.489e-06 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254646+0000", 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.163105716 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417751+0000", 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000109895 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417861+0000", 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033677300000000001 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.418198+0000", 2023-10-09T10:23:51.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017758396999999999 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435956+0000", 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6774000000000002e-05 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436013+0000", 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5324000000000003e-05 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436049+0000", 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032668699999999998 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436375+0000", 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2172999999999997e-05 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436427+0000", 2023-10-09T10:23:51.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012633699999999999 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436554+0000", 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1279840910000001 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195656 9.c 9:310794b5:::benchmark_data_smithi012_313365_object83181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:23:51.791 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 318.40078624500001, 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1137697360000001, 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195656 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323090+0000", 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323088+0000", 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0759e-05 2023-10-09T10:23:51.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323099+0000", 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7932999999999999e-05 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323116+0000", 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4930000000000003e-06 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323123+0000", 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0368000000000003e-05 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323173+0000", 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3370000000000006e-06 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.323183+0000", 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93181647499999998 2023-10-09T10:23:51.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254999+0000", 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0474e-05 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.255010+0000", 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.165299136 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420309+0000", 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117821 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420427+0000", 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000380281 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.420807+0000", 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015156678999999999 2023-10-09T10:23:51.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435964+0000", 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033305500000000002 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436297+0000", 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0358999999999998e-05 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436337+0000", 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033388899999999997 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436671+0000", 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3112999999999998e-05 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436724+0000", 2023-10-09T10:23:51.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000135403 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.436859+0000", 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1137697360000001 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195663 9.1c 9:3f09486e:::benchmark_data_smithi012_313365_object83188:head [delete] snapc 0=[] ondisk+write+known_if_redirected e399)", 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:23:51.800 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 318.382706837, 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0945934909999999, 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195663 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.801 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341169+0000", 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341167+0000", 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5290000000000001e-05 2023-10-09T10:23:51.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341182+0000", 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1254e-05 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341204+0000", 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7949999999999996e-06 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341212+0000", 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6192000000000006e-05 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341279+0000", 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3284e-05 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:33.341292+0000", 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91282471300000001 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254117+0000", 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.254124+0000", 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.157826198 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.411950+0000", 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012947999999999999 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412079+0000", 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000355423 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.412435+0000", 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002552646 2023-10-09T10:23:51.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.414987+0000", 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027370139999999999 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.417724+0000", 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053624290000000002 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.423087+0000", 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012506992 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435594+0000", 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7438000000000002e-05 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435641+0000", 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012119 2023-10-09T10:23:51.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.435762+0000", 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0945934909999999 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195673 9.10 9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e400)", 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 317.46623133700001, 2023-10-09T10:23:51.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6028094020000001, 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195673 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257644+0000", 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257643+0000", 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0550000000000002e-06 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257650+0000", 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8279999999999998e-06 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257660+0000", 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7660000000000001e-06 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.257666+0000", 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032584879999999999 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.260925+0000", 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0726000000000001e-05 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:34.260946+0000", 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9083686909999997 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.169314+0000", 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002054237 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.171369+0000", 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6018258909999998 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.773194+0000", 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011103770000000001 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.774305+0000", 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.055754660999999997 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.830060+0000", 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001693766 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.831753+0000", 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0034851140000000001 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.835238+0000", 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.019324406999999998 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.854563+0000", 2023-10-09T10:23:51.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000167465 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.854730+0000", 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058527999999999998 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.855316+0000", 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00087701200000000004 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.856193+0000", 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042613290000000003 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.860454+0000", 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6028094020000001 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195693 9.10 9:0f588e8d:::benchmark_data_smithi012_313365_object83218:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 316.25032064999999, 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3933209699999995, 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195693 2023-10-09T10:23:51.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473555+0000", 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473553+0000", 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0190000000000001e-05 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473563+0000", 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2921999999999999e-05 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473576+0000", 2023-10-09T10:23:51.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.994e-06 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.473584+0000", 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.304343544 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.777928+0000", 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022209249999999999 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.780149+0000", 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013458505000000001 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.793607+0000", 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001306698 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.794914+0000", 2023-10-09T10:23:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.051671473000000002 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.846585+0000", 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014171050000000001 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.848002+0000", 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003102222 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.851105+0000", 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0099542229999999999 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861059+0000", 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017274 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861231+0000", 2023-10-09T10:23:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00060873299999999997 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.861840+0000", 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000843592 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.862684+0000", 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041923439999999998 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.866876+0000", 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3933209699999995 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195716 9.10 9:0de169be:::benchmark_data_smithi012_313365_object83241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 316.21415163199998, 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3902966980000002, 2023-10-09T10:23:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195716 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509724+0000", 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509723+0000", 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0609999999999998e-06 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509730+0000", 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3033e-05 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509743+0000", 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.613e-06 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.509749+0000", 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2706342480000004 2023-10-09T10:23:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.780384+0000", 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023119730000000002 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.782696+0000", 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020697949 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.803393+0000", 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001386583 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.804780+0000", 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.078157095999999995 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.882937+0000", 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001764306 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.884701+0000", 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0036488340000000001 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.888350+0000", 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003904832 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.892255+0000", 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00076447400000000001 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.893020+0000", 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022865630000000001 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.895306+0000", 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00084655199999999998 2023-10-09T10:23:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.896153+0000", 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038682040000000001 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.900021+0000", 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3902966980000002 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:195733 9.10 9:0c074ac9:::benchmark_data_smithi012_313365_object83258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 316.14828858499999, 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3621552440000002, 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:23:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 195733 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575587+0000", 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575586+0000", 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5009999999999998e-06 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575593+0000", 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0510000000000002e-06 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575602+0000", 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9590000000000003e-06 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:35.575607+0000", 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2073396159999996 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.782947+0000", 2023-10-09T10:23:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00182449 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.784772+0000", 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.030197965 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.814970+0000", 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001106353 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.816076+0000", 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.101857857 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.917934+0000", 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001720475 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.919654+0000", 2023-10-09T10:23:51.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003479429 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,7", 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.923134+0000", 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041782599999999996 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.927312+0000", 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070182800000000004 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.928014+0000", 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043811029999999999 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.932395+0000", 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00094054900000000005 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.933335+0000", 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044071309999999999 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.937742+0000", 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3621552440000002 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.844 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPush(9.10 403/400 [PushOp(9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head, version: 334'5146, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:0fbe37ae:::benchmark_data_smithi012_313365_object83198:head@334'5146, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 313.55158435999999, 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6005742729999999, 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172291+0000", 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9161999999999997e-05 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172390+0000", 2023-10-09T10:23:51.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8474999999999998e-05 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172408+0000", 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0516000000000001e-05 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:38.172419+0000", 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5926570120000001 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.765076+0000", 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017349800000000001 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.765249+0000", 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076164220000000003 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:18:39.772866+0000", 2023-10-09T10:23:51.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6005742729999999 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158234 10.2 10:41c2e2c6:::benchmark_data_smithi012_366312_object46905:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.041780+0000", 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.682095748999998, 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4942225009999999, 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.849 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158234 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041780+0000", 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041780+0000", 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041779+0000", 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9330000000000007e-06 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041788+0000", 2023-10-09T10:23:51.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6970000000000005e-06 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041798+0000", 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0059999999999998e-06 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041804+0000", 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027629389999999998 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.044567+0000", 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2949999999999996e-06 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.044574+0000", 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085963280000000003 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.053170+0000", 2023-10-09T10:23:51.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6649999999999992e-06 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.053179+0000", 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29553761699999997 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.348717+0000", 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5679999999999999e-05 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.348732+0000", 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.167581639 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.516314+0000", 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011239 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.516426+0000", 2023-10-09T10:23:51.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039882999999999997 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.516825+0000", 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021040099999999999 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.518929+0000", 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021660400000000002 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.521095+0000", 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001823414 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.522919+0000", 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012894950000000001 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.535814+0000", 2023-10-09T10:23:51.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7305000000000001e-05 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.535871+0000", 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013163699999999999 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.536003+0000", 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4942225009999999 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.858 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158238 10.4 10:21665022:::benchmark_data_smithi012_366312_object46909:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.046937+0000", 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.676939328000003, 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.571134107, 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:23:51.859 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158238 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046937+0000", 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046937+0000", 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:23:51.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046935+0000", 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1009999999999998e-06 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046944+0000", 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8352e-05 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046962+0000", 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1799999999999996e-06 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046970+0000", 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013846494000000001 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.060817+0000", 2023-10-09T10:23:51.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7120000000000006e-06 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.060826+0000", 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.291419873 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352245+0000", 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5809999999999999e-06 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352253+0000", 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.238754991 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.591008+0000", 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105715 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.591114+0000", 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042542200000000001 2023-10-09T10:23:51.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3", 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.591539+0000", 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006005074 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.597544+0000", 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044712470000000002 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.602015+0000", 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051608729999999998 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.607176+0000", 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010717366000000001 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.617894+0000", 2023-10-09T10:23:51.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6595000000000003e-05 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.617950+0000", 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120309 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618071+0000", 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.571134107 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158239 10.4 10:23993120:::benchmark_data_smithi012_366312_object46910:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.051553+0000", 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.672322899999998, 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.566721287, 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.870 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158239 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051553+0000", 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051553+0000", 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051552+0000", 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0249999999999997e-06 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051559+0000", 2023-10-09T10:23:51.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0814e-05 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051569+0000", 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8150000000000006e-06 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051577+0000", 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009272789 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.060850+0000", 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.311e-06 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.060857+0000", 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29142648199999999 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352284+0000", 2023-10-09T10:23:51.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0299999999999999e-06 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352293+0000", 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.241035501 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.593328+0000", 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010223 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.593431+0000", 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033623600000000001 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3", 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.593767+0000", 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059739930000000004 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.599741+0000", 2023-10-09T10:23:51.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023597510000000002 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.602101+0000", 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051075469999999996 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.607208+0000", 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0109065 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618115+0000", 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1894000000000002e-05 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618156+0000", 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011776400000000001 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618274+0000", 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.566721287 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158249 10.14 10:2e30cd7c:::benchmark_data_smithi012_366312_object46920:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.076034+0000", 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.647841666999994, 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.542011485, 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.879 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158249 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076034+0000", 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076034+0000", 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076033+0000", 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0129999999999996e-06 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076040+0000", 2023-10-09T10:23:51.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0220999999999999e-05 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076050+0000", 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.913e-06 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076056+0000", 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5566999999999998e-05 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076111+0000", 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0560000000000001e-05 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076122+0000", 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.276081191 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352203+0000", 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7429999999999997e-06 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352213+0000", 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.234992906 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.587206+0000", 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012481999999999999 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.587331+0000", 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036589800000000003 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3", 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.587697+0000", 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078394450000000004 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595536+0000", 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9175000000000001e-05 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595565+0000", 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020002290000000001 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.597565+0000", 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020290898000000002 2023-10-09T10:23:51.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.617856+0000", 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7027000000000001e-05 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.617913+0000", 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132366 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618046+0000", 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.542011485 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158251 10.4 10:231b664c:::benchmark_data_smithi012_366312_object46922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.082556+0000", 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 98.641320019000005, 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5359710120000001, 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.888 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158251 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082556+0000", 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082556+0000", 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082554+0000", 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2155e-05 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082566+0000", 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3308e-05 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082580+0000", 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3039999999999999e-06 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082587+0000", 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3467000000000007e-05 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082661+0000", 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5810000000000003e-06 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082670+0000", 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26964073300000002 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352311+0000", 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7440000000000007e-06 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352319+0000", 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432332580000001 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595552+0000", 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3499999999999996e-05 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595645+0000", 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028873199999999999 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3", 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595934+0000", 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060391780000000001 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.601973+0000", 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016949700000000001 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.602143+0000", 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051037610000000001 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.607246+0000", 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011098742999999999 2023-10-09T10:23:51.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618345+0000", 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9413999999999998e-05 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618395+0000", 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013218 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618527+0000", 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5359710120000001 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:210964 10.d 10:b494e22a:::benchmark_data_smithi012_366312_object99635:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:24.454180+0000", 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 27.269695905999999, 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8139022659999999, 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:23:51.897 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 210964 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454180+0000", 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454180+0000", 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454180+0000", 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.779e-06 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454182+0000", 2023-10-09T10:23:51.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6509999999999998e-06 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454188+0000", 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.118e-06 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454193+0000", 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002309167 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.456502+0000", 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5301000000000001e-05 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.456518+0000", 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.251861756 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.708379+0000", 2023-10-09T10:23:51.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112889 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.708492+0000", 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037031599999999998 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,7", 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.708863+0000", 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001691901 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.710554+0000", 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6291999999999994e-05 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.710631+0000", 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015809680000000001 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.712212+0000", 2023-10-09T10:23:51.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.55551082600000001 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.267723+0000", 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8658999999999994e-05 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.267791+0000", 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029103199999999999 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:23:51.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:23:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.268082+0000", 2023-10-09T10:23:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8139022659999999 2023-10-09T10:23:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:23:51.905 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:23:51.905 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:23:51.906 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:51.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:51 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T10:23:51.906 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:51 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T10:23:51.907 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:51 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T10:23:51.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:51.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:52.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:52.036 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:52.070 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:52.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:52.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:52.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:23:52.175 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:52.203 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:52.238 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:52.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:52.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:52.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:52.369 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:52.403 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:52.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:52.507 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:52.533 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:52.566 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:52.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:52.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:52.622 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_5' created 2023-10-09T10:23:52.670 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:52.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:52 smithi110 ceph-mon[136032]: pgmap v1377: 73 pgs: 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%); 85 KiB/s, 0 objects/s recovering 2023-10-09T10:23:52.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3256901365' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T10:23:52.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:52 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T10:23:52.696 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:52.728 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:52.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:52.832 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:52.858 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:52.890 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:52.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:52.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:52.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:23:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:52 smithi012 ceph-mon[147926]: pgmap v1377: 73 pgs: 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%); 85 KiB/s, 0 objects/s recovering 2023-10-09T10:23:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3256901365' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T10:23:52.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:52 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T10:23:52.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:52 smithi012 ceph-mon[152263]: pgmap v1377: 73 pgs: 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%); 85 KiB/s, 0 objects/s recovering 2023-10-09T10:23:52.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3256901365' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T10:23:52.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:52 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T10:23:52.995 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:53.021 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:53.053 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:53.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:53.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:53.106 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:53.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T10:23:53.156 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:53.216 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:53.358 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:53.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:53.464 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:53.493 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:53.580 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:53.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:53.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:53.623 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:23:53.687 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:53.714 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:53.748 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:53.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:53.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:53.879 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T10:23:53.879 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[152263]: osdmap e494: 8 total, 8 up, 5 in 2023-10-09T10:23:53.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[152263]: failed to encode map e494 with expected crc 2023-10-09T10:23:53.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[152263]: failed to encode map e494 with expected crc 2023-10-09T10:23:53.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3717947529' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:53.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:53.880 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T10:23:53.880 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[147926]: osdmap e494: 8 total, 8 up, 5 in 2023-10-09T10:23:53.880 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[147926]: failed to encode map e494 with expected crc 2023-10-09T10:23:53.880 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[147926]: failed to encode map e494 with expected crc 2023-10-09T10:23:53.881 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3717947529' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:53.881 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:53 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:53.889 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:53.896 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:53.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:53 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T10:23:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:53 smithi110 ceph-mon[136032]: osdmap e494: 8 total, 8 up, 5 in 2023-10-09T10:23:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:53 smithi110 ceph-mon[136032]: failed to encode map e494 with expected crc 2023-10-09T10:23:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:53 smithi110 ceph-mon[136032]: failed to encode map e494 with expected crc 2023-10-09T10:23:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3717947529' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:53 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:23:54.214 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:54.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:54.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:54.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:54.351 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:54.440 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:54.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:54.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:54.544 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:54.574 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:54.608 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:54.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:54.618 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:54.712 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:54.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:54.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[152263]: failed to encode map e495 with expected crc 2023-10-09T10:23:54.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:23:54.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[152263]: osdmap e495: 8 total, 8 up, 5 in 2023-10-09T10:23:54.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[152263]: failed to encode map e495 with expected crc 2023-10-09T10:23:54.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[152263]: pgmap v1380: 89 pgs: 4 creating+peering, 12 unknown, 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%); 85 KiB/s, 0 objects/s recovering 2023-10-09T10:23:54.739 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:54.740 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[147926]: failed to encode map e495 with expected crc 2023-10-09T10:23:54.740 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:23:54.740 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[147926]: osdmap e495: 8 total, 8 up, 5 in 2023-10-09T10:23:54.740 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[147926]: failed to encode map e495 with expected crc 2023-10-09T10:23:54.740 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:54 smithi012 ceph-mon[147926]: pgmap v1380: 89 pgs: 4 creating+peering, 12 unknown, 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%); 85 KiB/s, 0 objects/s recovering 2023-10-09T10:23:54.743 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:54.823 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:23:54.823 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:23:54.823 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:23:54.823 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:23:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:54 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:23:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:54 smithi110 ceph-mon[136032]: failed to encode map e495 with expected crc 2023-10-09T10:23:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:54 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:23:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:54 smithi110 ceph-mon[136032]: osdmap e495: 8 total, 8 up, 5 in 2023-10-09T10:23:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:54 smithi110 ceph-mon[136032]: failed to encode map e495 with expected crc 2023-10-09T10:23:54.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:54 smithi110 ceph-mon[136032]: pgmap v1380: 89 pgs: 4 creating+peering, 12 unknown, 4 active+clean+remapped, 69 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%); 85 KiB/s, 0 objects/s recovering 2023-10-09T10:23:54.978 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:54.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:55.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:55.112 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:55.441 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:23:55.447 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:55.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:55.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:55.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:23:55.582 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:55.619 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:23:55.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:55.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:23:55.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:55.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:55 smithi012 ceph-mon[152263]: failed to encode map e496 with expected crc 2023-10-09T10:23:55.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:55 smithi012 ceph-mon[152263]: osdmap e496: 8 total, 8 up, 5 in 2023-10-09T10:23:55.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:55 smithi012 ceph-mon[147926]: failed to encode map e496 with expected crc 2023-10-09T10:23:55.751 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:55 smithi012 ceph-mon[147926]: osdmap e496: 8 total, 8 up, 5 in 2023-10-09T10:23:55.754 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:55.900 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:55.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:55.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:55 smithi110 ceph-mon[136032]: failed to encode map e496 with expected crc 2023-10-09T10:23:55.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:55 smithi110 ceph-mon[136032]: osdmap e496: 8 total, 8 up, 5 in 2023-10-09T10:23:56.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:56.040 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:56.153 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:56.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:56.217 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:23:56.240 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:23:56.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:56.260 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:23:56.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:23:56.289 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:56.290 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:23:56.312 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:23:56.326 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:56.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:56.332 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:23:56.365 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:23:56.383 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:23:56.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:23:56.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:23:56.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:23:56.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:23:56.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:23:56.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:23:56.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:23:56.385 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:23:56.432 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:56.452 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:56.478 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:56.723 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:56.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:56.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:56.827 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:56.829 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:56 smithi012 ceph-mon[147926]: pgmap v1382: 89 pgs: 1 active+undersized, 4 creating+peering, 6 unknown, 4 active+clean+remapped, 74 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:56.829 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2749457853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:56.829 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:56 smithi012 ceph-mon[152263]: pgmap v1382: 89 pgs: 1 active+undersized, 4 creating+peering, 6 unknown, 4 active+clean+remapped, 74 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:56.829 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2749457853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:56.860 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:56.898 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:56.900 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:56 smithi110 ceph-mon[136032]: pgmap v1382: 89 pgs: 1 active+undersized, 4 creating+peering, 6 unknown, 4 active+clean+remapped, 74 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:56.901 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2749457853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:23:56.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:56.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:57.003 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:57.032 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:57.068 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:57.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:57.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:57.144 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:57.173 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:23:57.212 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:57.401 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:23:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:57.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:23:57.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:57.542 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:57.610 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:57.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:57.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:57.643 INFO:teuthology.orchestra.run.smithi012.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T10:23:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2362890757' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:23:57.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:57 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:23:57.665 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2362890757' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:23:57.665 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:57 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:23:57.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:23:57.749 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:57.892 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:23:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:57.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:23:57.927 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:23:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:23:57] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.33.4" 2023-10-09T10:23:57.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2362890757' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:23:57.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:57 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:23:57.998 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:58.030 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:58.066 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:58.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:58.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:58.171 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:23:58.201 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:58.234 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:23:58.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:58.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:23:58.260 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:23:58.260 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:23:58.338 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:58.370 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:58.838 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:58 smithi012 ceph-mon[147926]: failed to encode map e497 with expected crc 2023-10-09T10:23:58.838 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:23:58.838 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:58 smithi012 ceph-mon[147926]: osdmap e497: 8 total, 8 up, 5 in 2023-10-09T10:23:58.839 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:58 smithi012 ceph-mon[147926]: pgmap v1384: 89 pgs: 1 active+undersized, 4 creating+peering, 4 active+clean+remapped, 80 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:58.839 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:58 smithi012 ceph-mon[152263]: failed to encode map e497 with expected crc 2023-10-09T10:23:58.839 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:23:58.839 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:58 smithi012 ceph-mon[152263]: osdmap e497: 8 total, 8 up, 5 in 2023-10-09T10:23:58.839 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:58 smithi012 ceph-mon[152263]: pgmap v1384: 89 pgs: 1 active+undersized, 4 creating+peering, 4 active+clean+remapped, 80 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:58.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:23:58.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:58 smithi110 ceph-mon[136032]: failed to encode map e497 with expected crc 2023-10-09T10:23:58.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:23:58.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:58 smithi110 ceph-mon[136032]: osdmap e497: 8 total, 8 up, 5 in 2023-10-09T10:23:58.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:58 smithi110 ceph-mon[136032]: pgmap v1384: 89 pgs: 1 active+undersized, 4 creating+peering, 4 active+clean+remapped, 80 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:23:58.950 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:23:59.081 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:59.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:59.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:59.187 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:23:59.217 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:59.252 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:23:59.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:59.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:23:59.356 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:59.384 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:59.418 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:23:59.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:59.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:23:59.522 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:23:59.551 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:59.566 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:59.585 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:23:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:59.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:23:59.690 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:23:59.695 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:23:59.724 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:23:59.859 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:23:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:23:59.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:23:59.964 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:23:59.979 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:23:59.994 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:00.029 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:00.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:00.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:00.056 DEBUG:teuthology.orchestra.run.smithi012:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T10:24:00.056 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-10-09T10:24:00.057 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:24:00.059 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:24:00.060 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:24:00.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:00.093 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:00.093 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:59 smithi012 ceph-mon[147926]: failed to encode map e497 with expected crc 2023-10-09T10:24:00.093 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:23:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2618294145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:00.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:00.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:00.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:59 smithi012 ceph-mon[152263]: failed to encode map e497 with expected crc 2023-10-09T10:24:00.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:23:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2618294145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:00.122 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:hints = 1 2023-10-09T10:24:00.122 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:24:00.122 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object prefix: benchmark_data_smithi012_404402 2023-10-09T10:24:00.123 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:24:00.123 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:24:00.134 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:00.175 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:00.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:00.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:00.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:59 smithi110 ceph-mon[136032]: failed to encode map e497 with expected crc 2023-10-09T10:24:00.238 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:23:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2618294145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:00.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:00.377 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:00.421 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:00.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:00.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:00.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:00.581 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:01.042 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:01.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:01.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:01.122 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 1 16 3656 3640 227.468 227.5 0.00340534 0.00438429 2023-10-09T10:24:01.149 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:01.150 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: failed to encode map e497 with expected crc 2023-10-09T10:24:01.150 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: failed to encode map e497 with expected crc 2023-10-09T10:24:01.150 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: pgmap v1385: 89 pgs: 2 active+undersized, 4 active+clean+remapped, 83 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:24:01.150 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:24:01.150 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:24:01.150 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: failed to encode map e498 with expected crc 2023-10-09T10:24:01.150 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: failed to encode map e498 with expected crc 2023-10-09T10:24:01.150 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2618294145' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[147926]: osdmap e498: 8 total, 8 up, 5 in 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: failed to encode map e497 with expected crc 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: failed to encode map e497 with expected crc 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: pgmap v1385: 89 pgs: 2 active+undersized, 4 active+clean+remapped, 83 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: failed to encode map e498 with expected crc 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: failed to encode map e498 with expected crc 2023-10-09T10:24:01.151 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2618294145' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:01.152 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:00 smithi012 ceph-mon[152263]: osdmap e498: 8 total, 8 up, 5 in 2023-10-09T10:24:01.216 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:01.280 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:01.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:01.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: failed to encode map e497 with expected crc 2023-10-09T10:24:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: failed to encode map e497 with expected crc 2023-10-09T10:24:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: pgmap v1385: 89 pgs: 2 active+undersized, 4 active+clean+remapped, 83 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:24:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:24:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:24:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: failed to encode map e498 with expected crc 2023-10-09T10:24:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: failed to encode map e498 with expected crc 2023-10-09T10:24:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2618294145' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:01.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:00 smithi110 ceph-mon[136032]: osdmap e498: 8 total, 8 up, 5 in 2023-10-09T10:24:01.451 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:01.566 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:01.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:01.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:01.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:24:01.637 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:01.673 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:01.675 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:24:01.719 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:24:01.744 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:01.769 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:24:01.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:24:01.847 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:24:01.847 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:01.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:01.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:01.882 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:24:01.915 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:24:01.924 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:24:01.924 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:24:01.924 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:24:01.924 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:24:01.924 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:24:01.924 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:24:01.924 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:24:01.924 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:24:01.936 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:24:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:24:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:24:01.936 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121890 9.1 9.4309a8a1 (undecoded) ondisk+retry+write+known_if_redirected e352)", 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 500.63271238599998, 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99073293299999998, 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:24:01.937 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121890 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300151+0000", 2023-10-09T10:24:01.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300149+0000", 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300155+0000", 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2280000000000003e-06 2023-10-09T10:24:01.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300163+0000", 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8429999999999993e-06 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300170+0000", 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5989999999999998e-06 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300211+0000", 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1511000000000003e-05 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.300219+0000", 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6190000000000002e-06 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.290871+0000", 2023-10-09T10:24:01.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.99065166100000002 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.290884+0000", 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2998e-05 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121893 9.11 9.c051ab51 (undecoded) ondisk+write+known_if_redirected e352)", 2023-10-09T10:24:01.942 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 500.626803741, 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98772148100000001, 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121893 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306059+0000", 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306058+0000", 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:24:01.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306064+0000", 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1700000000000002e-06 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306074+0000", 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0094e-05 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306084+0000", 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6579999999999997e-06 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306174+0000", 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0655000000000005e-05 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:24:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.306187+0000", 2023-10-09T10:24:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2376e-05 2023-10-09T10:24:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.293771+0000", 2023-10-09T10:24:01.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98758401100000004 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.293781+0000", 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0017000000000001e-05 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.948 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:121898 9.c 9.187a12cc (undecoded) ondisk+write+known_if_redirected e352)", 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 500.59956554799999, 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94351869099999996, 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 121898 2023-10-09T10:24:01.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333298+0000", 2023-10-09T10:24:01.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333295+0000", 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333306+0000", 2023-10-09T10:24:01.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0361e-05 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333346+0000", 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0145999999999998e-05 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333357+0000", 2023-10-09T10:24:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1377e-05 2023-10-09T10:24:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333428+0000", 2023-10-09T10:24:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0835999999999997e-05 2023-10-09T10:24:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:41.333441+0000", 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2449000000000001e-05 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.276804+0000", 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.94336297999999996 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:15:42.276816+0000", 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2670999999999999e-05 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128066 9.1 9:8603ff6a:::benchmark_data_smithi012_313365_object15591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-10-09T10:24:01.955 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 472.25070554899997, 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55978837800000003, 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128066 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.956 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682155+0000", 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682168+0000", 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2682e-05 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682184+0000", 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6339000000000001e-05 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682194+0000", 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4420000000000004e-06 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682258+0000", 2023-10-09T10:24:01.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4463999999999998e-05 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682271+0000", 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3166000000000001e-05 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237630+0000", 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55535865299999998 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237768+0000", 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013818000000000001 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237891+0000", 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012287299999999999 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.240596+0000", 2023-10-09T10:24:01.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002704559 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241710+0000", 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00111455 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241856+0000", 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014592000000000001 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241905+0000", 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9042000000000002e-05 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241946+0000", 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0655999999999998e-05 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156571 9.11 9:88722a35:::benchmark_data_smithi012_313365_object44096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 413.53243930100001, 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.21718912700000001, 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156571 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400422+0000", 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400431+0000", 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2720000000000003e-06 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400450+0000", 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8981e-05 2023-10-09T10:24:01.966 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400462+0000", 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2333e-05 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.402769+0000", 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023067069999999999 2023-10-09T10:24:01.967 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.402909+0000", 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013978499999999999 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.403031+0000", 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012171300000000001 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:24:01.968 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.446104+0000", 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.043073587000000003 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575808+0000", 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12970353000000001 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617506+0000", 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041698357999999998 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.969 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617561+0000", 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5191000000000001e-05 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617613+0000", 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1649e-05 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.970 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156574 9.21 9:87a74460:::benchmark_data_smithi012_313365_object44099:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 413.52742664900001, 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0291358530000001, 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:24:01.971 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156574 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:24:01.972 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405435+0000", 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405441+0000", 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7290000000000003e-06 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.973 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405454+0000", 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2811e-05 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405465+0000", 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0448000000000001e-05 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.974 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575963+0000", 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17049810900000001 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575975+0000", 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.221e-05 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.430808+0000", 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85483296499999994 2023-10-09T10:24:01.975 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.430955+0000", 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014732499999999999 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.431079+0000", 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012408499999999999 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.976 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.432894+0000", 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001814593 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434249+0000", 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001355537 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434463+0000", 2023-10-09T10:24:01.977 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000213165 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434525+0000", 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2205000000000005e-05 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434572+0000", 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7500000000000003e-05 2023-10-09T10:24:01.978 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156578 9.11 9:8e383e1c:::benchmark_data_smithi012_313365_object44103:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 413.42124137899998, 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.19283726200000001, 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:01.979 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156578 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.980 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511620+0000", 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511628+0000", 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7139999999999999e-06 2023-10-09T10:24:01.981 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511644+0000", 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6018000000000001e-05 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511654+0000", 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9119999999999993e-06 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511718+0000", 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3587e-05 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511874+0000", 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000156126 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.512003+0000", 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012904400000000001 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575815+0000", 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063811965999999998 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.656235+0000", 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.080420144999999998 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704358+0000", 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.048122282000000002 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704415+0000", 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7090999999999999e-05 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704459+0000", 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4347000000000001e-05 2023-10-09T10:24:01.986 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27047 10.4 e428/424 10:2e4baf45:::benchmark_data_smithi012_366312_object27046:head v 428'1731, mlcod=428'1729)", 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.25261222500001, 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16898123100000001, 2023-10-09T10:24:01.987 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680249+0000", 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680286+0000", 2023-10-09T10:24:01.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6359000000000002e-05 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680298+0000", 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2072999999999999e-05 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680305+0000", 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2400000000000001e-06 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680357+0000", 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1610000000000002e-05 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.991 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680379+0000", 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1732000000000001e-05 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849191+0000", 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.168812924 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.992 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849232+0000", 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0663e-05 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27057 10.5 e428/424 10:ac997561:::benchmark_data_smithi012_366312_object27056:head v 428'1679, mlcod=428'1677)", 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.252027263, 2023-10-09T10:24:01.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.170571852, 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.994 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680835+0000", 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680874+0000", 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9499000000000002e-05 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680887+0000", 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.278e-05 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680895+0000", 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.644e-06 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680981+0000", 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6104000000000001e-05 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.681001+0000", 2023-10-09T10:24:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0417000000000001e-05 2023-10-09T10:24:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851371+0000", 2023-10-09T10:24:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.170369565 2023-10-09T10:24:01.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851408+0000", 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6757000000000001e-05 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27067 10.5 e428/424 10:a3c37c8c:::benchmark_data_smithi012_366312_object27066:head v 428'1680, mlcod=428'1677)", 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:24:01.999 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.24856578800001, 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16713378700000001, 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.000 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684296+0000", 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684335+0000", 2023-10-09T10:24:02.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9345999999999999e-05 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848078+0000", 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16374229300000001 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848092+0000", 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4222e-05 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849076+0000", 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00098447500000000006 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849097+0000", 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0258000000000002e-05 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851412+0000", 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002315517 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851431+0000", 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8884000000000001e-05 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27069 10.4 e428/424 10:2f6e2957:::benchmark_data_smithi012_366312_object27068:head v 428'1732, mlcod=428'1729)", 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.685774+0000", 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.24708951299999, 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16706146599999999, 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.685774+0000", 2023-10-09T10:24:02.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.685774+0000", 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.685773+0000", 2023-10-09T10:24:02.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.685820+0000", 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6873999999999999e-05 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848088+0000", 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.162268312 2023-10-09T10:24:02.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848099+0000", 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1205e-05 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848146+0000", 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6663999999999999e-05 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848168+0000", 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1919000000000001e-05 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.852794+0000", 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046263420000000003 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.852835+0000", 2023-10-09T10:24:02.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1016000000000002e-05 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27078 10.f e428/424 10:f852e76d:::benchmark_data_smithi012_366312_object27077:head v 428'1695, mlcod=428'1693)", 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.690195+0000", 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.24266850999999, 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.160981017, 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.010 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.690195+0000", 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.690195+0000", 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.690183+0000", 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999881 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.690236+0000", 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2896999999999999e-05 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848098+0000", 2023-10-09T10:24:02.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15786182700000001 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848105+0000", 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4800000000000004e-06 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848324+0000", 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000219018 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848345+0000", 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1382999999999998e-05 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851127+0000", 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027814850000000002 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851176+0000", 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8639e-05 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:100348 10.d e446/443 10:b23cf803:::benchmark_data_smithi012_366312_object100347:head v 446'6294, mlcod=428'83)", 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:03.790773+0000", 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 178.14208999900001, 2023-10-09T10:24:02.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066822567999999999, 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790773+0000", 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790773+0000", 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790773+0000", 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790814+0000", 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1415999999999998e-05 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790822+0000", 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6120000000000001e-06 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790827+0000", 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5489999999999997e-06 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.018 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.798748+0000", 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0079209510000000007 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.798781+0000", 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2410000000000003e-05 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.857546+0000", 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058765036 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.857596+0000", 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9935999999999998e-05 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.020 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(10.d 447/443 [PushOp(10:b1f42904:::benchmark_data_smithi012_366312_object2884:head, version: 428'173, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:b1f42904:::benchmark_data_smithi012_366312_object2884:head@428'173, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:08.006676+0000", 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 173.92618680199999, 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067742063000000005, 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006676+0000", 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006676+0000", 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006675+0000", 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006729+0000", 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4329000000000001e-05 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006750+0000", 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0428000000000002e-05 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006767+0000", 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6912999999999999e-05 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.071299+0000", 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064532701999999997 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.071318+0000", 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8717000000000001e-05 2023-10-09T10:24:02.024 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.074418+0000", 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031001650000000002 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135021 10.d e457/443 10:b65cfb4e:::benchmark_data_smithi012_366312_object23692:head v 457'8492, mlcod=428'433)", 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.049439+0000", 2023-10-09T10:24:02.025 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 132.883423854, 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.075575955, 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049439+0000", 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049439+0000", 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049438+0000", 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049449+0000", 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.163e-05 2023-10-09T10:24:02.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049466+0000", 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6501999999999999e-05 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049475+0000", 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9390000000000003e-06 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.057081+0000", 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076062170000000002 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.057107+0000", 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5797000000000001e-05 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.124962+0000", 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067854822999999995 2023-10-09T10:24:02.029 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.125015+0000", 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3579999999999999e-05 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135039 10.d e457/443 10:b7499b5c:::benchmark_data_smithi012_366312_object23710:head v 457'8493, mlcod=428'433)", 2023-10-09T10:24:02.030 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.060346+0000", 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 132.87251713000001, 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070359356999999997, 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060346+0000", 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.031 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060346+0000", 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060345+0000", 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060355+0000", 2023-10-09T10:24:02.032 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.062e-05 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060374+0000", 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8312e-05 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060384+0000", 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0268000000000001e-05 2023-10-09T10:24:02.033 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067438+0000", 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00705435 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067464+0000", 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5636e-05 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.034 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130662+0000", 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063198174999999995 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130705+0000", 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3129e-05 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.035 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135046 10.5 e457/456 10:a6a4524b:::benchmark_data_smithi012_366312_object23717:head v 457'8418, mlcod=457'8416)", 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.061370+0000", 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 132.87149293900001, 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063689329000000003, 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061370+0000", 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.036 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061370+0000", 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061369+0000", 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061378+0000", 2023-10-09T10:24:02.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7499999999999992e-06 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061396+0000", 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7628000000000001e-05 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061414+0000", 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8592000000000001e-05 2023-10-09T10:24:02.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.065364+0000", 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003949508 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.065384+0000", 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0625e-05 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.125023+0000", 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059638459999999997 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.125059+0000", 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6664e-05 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135042 10.d e457/443 10:b1be933a:::benchmark_data_smithi012_366312_object23713:head v 457'8494, mlcod=428'433)", 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.067349+0000", 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 132.86551436799999, 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063424356000000001, 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067349+0000", 2023-10-09T10:24:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067349+0000", 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067348+0000", 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067357+0000", 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8330000000000006e-06 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067372+0000", 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4978e-05 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067381+0000", 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7360000000000007e-06 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.078524+0000", 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011143125 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.078551+0000", 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6531999999999999e-05 2023-10-09T10:24:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130748+0000", 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052196996000000002 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130773+0000", 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5272000000000001e-05 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135049 10.5 e457/456 10:a35ad3f9:::benchmark_data_smithi012_366312_object23720:head v 457'8419, mlcod=457'8416)", 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.068476+0000", 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 132.864387118, 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062269128999999999, 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068476+0000", 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068476+0000", 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068475+0000", 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068484+0000", 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8040000000000007e-06 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068497+0000", 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.295e-05 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068504+0000", 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3409999999999997e-06 2023-10-09T10:24:02.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.076436+0000", 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0079316730000000002 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.076460+0000", 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4379000000000002e-05 2023-10-09T10:24:02.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130711+0000", 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054251313000000002 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130745+0000", 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3639000000000001e-05 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.050 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:136118 10.d e458/443 10:bce59347:::benchmark_data_smithi012_366312_object24789:head v 458'8570, mlcod=428'433)", 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:50.181296+0000", 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 131.75156740200001, 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063169832999999995, 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:02.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181296+0000", 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181296+0000", 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181295+0000", 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181301+0000", 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5779999999999998e-06 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181325+0000", 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4624999999999999e-05 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:02.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181332+0000", 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5420000000000002e-06 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.188034+0000", 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067019790000000003 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.188065+0000", 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0951999999999998e-05 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.244405+0000", 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.056340135999999999 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.244466+0000", 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0458000000000003e-05 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.055 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:02.056 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:02.056 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:24:02.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:02.057 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3461618125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:02.123 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 2 16 6266 6250 195.291 163.125 0.00277448 0.00510497 2023-10-09T10:24:02.127 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:02.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3461618125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:02.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3461618125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:02.683 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:02.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:02.737 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:24:02.789 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:02.832 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:02.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:24:02.886 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:02.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:02.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:02.991 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:03.027 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:03.071 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:03.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:03.123 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 3 16 9344 9328 194.313 192.375 0.00313012 0.00513583 2023-10-09T10:24:03.176 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:03.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:02 smithi012 ceph-mon[147926]: pgmap v1387: 89 pgs: 2 active+undersized, 4 active+clean+remapped, 83 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:24:03.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:02 smithi012 ceph-mon[152263]: pgmap v1387: 89 pgs: 2 active+undersized, 4 active+clean+remapped, 83 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:24:03.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:02 smithi110 ceph-mon[136032]: pgmap v1387: 89 pgs: 2 active+undersized, 4 active+clean+remapped, 83 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 2/72 objects misplaced (2.778%) 2023-10-09T10:24:03.241 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:03.305 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:03.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:03.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:03.410 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:03.455 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:03.515 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:03.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:03.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:03.622 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:03.659 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:03.705 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:03.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:03.771 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:03.810 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:03.837 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:03.886 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:03.967 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:03.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:03.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:04.059 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:24:04.072 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:24:04.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:04.123 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 4 16 11826 11810 184.497 155.125 0.00682631 0.00541055 2023-10-09T10:24:04.141 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:04.359 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:04.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:04.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:04.503 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:04.554 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:04.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:04.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:04.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:24:04.660 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1388,"stamp":"2023-10-09T10:24:03.735935+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":342366112,"num_objects":5239,"num_object_clones":0,"num_object_copies":15717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":567,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":5239,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":5475,"num_write_kb":336205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":1798058,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5409,"ondisk_log_size":5409,"up":261,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":262,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4152364,"kb_used_data":594820,"kb_used_omap":163,"kb_used_meta":3557340,"kb_avail":464573396,"statfs":{"total":479975178240,"available":475723157504,"internally_reserved":0,"allocated":609095680,"data_stored":601778694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":3642717171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":341770240,"num_objects":5215,"num_object_clones":0,"num_object_copies":15645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":567,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5215,"num_write_kb":333760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002320"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":91,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.310766+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:23:59.310766+0000","last_peered":"2023-10-09T10:23:59.310766+0000","last_clean":"2023-10-09T10:23:59.310766+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:23:59.310766+0000","last_undegraded":"2023-10-09T10:23:59.310766+0000","last_fullsized":"2023-10-09T10:23:59.310766+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":443,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311223+0000","last_change":"2023-10-09T10:22:27.502142+0000","last_active":"2023-10-09T10:23:59.311223+0000","last_peered":"2023-10-09T10:23:59.311223+0000","last_clean":"2023-10-09T10:23:59.311223+0000","last_became_active":"2023-10-09T10:22:27.501338+0000","last_became_peered":"2023-10-09T10:22:27.501338+0000","last_unstale":"2023-10-09T10:23:59.311223+0000","last_undegraded":"2023-10-09T10:23:59.311223+0000","last_fullsized":"2023-10-09T10:23:59.311223+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":520,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.308150+0000","last_change":"2023-10-09T10:23:25.445976+0000","last_active":"2023-10-09T10:23:59.308150+0000","last_peered":"2023-10-09T10:23:59.308150+0000","last_clean":"2023-10-09T10:23:59.308150+0000","last_became_active":"2023-10-09T10:23:25.424379+0000","last_became_peered":"2023-10-09T10:23:25.424379+0000","last_unstale":"2023-10-09T10:23:59.308150+0000","last_undegraded":"2023-10-09T10:23:59.308150+0000","last_fullsized":"2023-10-09T10:23:59.308150+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":412,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146693+0000","last_change":"2023-10-09T10:23:25.438653+0000","last_active":"2023-10-09T10:23:59.146693+0000","last_peered":"2023-10-09T10:23:59.146693+0000","last_clean":"2023-10-09T10:23:59.146693+0000","last_became_active":"2023-10-09T10:23:25.438243+0000","last_became_peered":"2023-10-09T10:23:25.438243+0000","last_unstale":"2023-10-09T10:23:59.146693+0000","last_undegraded":"2023-10-09T10:23:59.146693+0000","last_fullsized":"2023-10-09T10:23:59.146693+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:59.024327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":22,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630046+0000","last_change":"2023-10-09T10:23:25.411688+0000","last_active":"2023-10-09T10:23:54.630046+0000","last_peered":"2023-10-09T10:23:54.630046+0000","last_clean":"2023-10-09T10:23:54.630046+0000","last_became_active":"2023-10-09T10:23:25.411340+0000","last_became_peered":"2023-10-09T10:23:25.411340+0000","last_unstale":"2023-10-09T10:23:54.630046+0000","last_undegraded":"2023-10-09T10:23:54.630046+0000","last_fullsized":"2023-10-09T10:23:54.630046+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":64,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.147230+0000","last_change":"2023-10-09T10:23:25.433215+0000","last_active":"2023-10-09T10:23:59.147230+0000","last_peered":"2023-10-09T10:23:59.147230+0000","last_clean":"2023-10-09T10:23:59.147230+0000","last_became_active":"2023-10-09T10:23:25.432369+0000","last_became_peered":"2023-10-09T10:23:25.432369+0000","last_unstale":"2023-10-09T10:23:59.147230+0000","last_undegraded":"2023-10-09T10:23:59.147230+0000","last_fullsized":"2023-10-09T10:23:59.147230+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:00.938191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":437,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.147669+0000","last_change":"2023-10-09T10:22:27.392579+0000","last_active":"2023-10-09T10:23:59.147669+0000","last_peered":"2023-10-09T10:23:59.147669+0000","last_clean":"2023-10-09T10:23:59.147669+0000","last_became_active":"2023-10-09T10:22:27.392040+0000","last_became_peered":"2023-10-09T10:22:27.392040+0000","last_unstale":"2023-10-09T10:23:59.147669+0000","last_undegraded":"2023-10-09T10:23:59.147669+0000","last_fullsized":"2023-10-09T10:23:59.147669+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:20.831225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":50,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630175+0000","last_change":"2023-10-09T10:22:40.543504+0000","last_active":"2023-10-09T10:23:54.630175+0000","last_peered":"2023-10-09T10:23:54.630175+0000","last_clean":"2023-10-09T10:23:54.630175+0000","last_became_active":"2023-10-09T10:22:40.542810+0000","last_became_peered":"2023-10-09T10:22:40.542810+0000","last_unstale":"2023-10-09T10:23:54.630175+0000","last_undegraded":"2023-10-09T10:23:54.630175+0000","last_fullsized":"2023-10-09T10:23:54.630175+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":163,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.631182+0000","last_change":"2023-10-09T10:23:25.517220+0000","last_active":"2023-10-09T10:23:54.631182+0000","last_peered":"2023-10-09T10:23:54.631182+0000","last_clean":"2023-10-09T10:23:54.631182+0000","last_became_active":"2023-10-09T10:23:25.516909+0000","last_became_peered":"2023-10-09T10:23:25.516909+0000","last_unstale":"2023-10-09T10:23:54.631182+0000","last_undegraded":"2023-10-09T10:23:54.631182+0000","last_fullsized":"2023-10-09T10:23:54.631182+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146284+0000","last_change":"2023-10-09T10:23:25.414305+0000","last_active":"2023-10-09T10:23:59.146284+0000","last_peered":"2023-10-09T10:23:59.146284+0000","last_clean":"2023-10-09T10:23:59.146284+0000","last_became_active":"2023-10-09T10:23:25.413716+0000","last_became_peered":"2023-10-09T10:23:25.413716+0000","last_unstale":"2023-10-09T10:23:59.146284+0000","last_undegraded":"2023-10-09T10:23:59.146284+0000","last_fullsized":"2023-10-09T10:23:59.146284+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:52.696335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":103,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311460+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:23:59.311460+0000","last_peered":"2023-10-09T10:23:59.311460+0000","last_clean":"2023-10-09T10:23:59.311460+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:23:59.311460+0000","last_undegraded":"2023-10-09T10:23:59.311460+0000","last_fullsized":"2023-10-09T10:23:59.311460+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":62,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630420+0000","last_change":"2023-10-09T10:23:25.681695+0000","last_active":"2023-10-09T10:23:54.630420+0000","last_peered":"2023-10-09T10:23:54.630420+0000","last_clean":"2023-10-09T10:23:54.630420+0000","last_became_active":"2023-10-09T10:23:25.609955+0000","last_became_peered":"2023-10-09T10:23:25.609955+0000","last_unstale":"2023-10-09T10:23:54.630420+0000","last_undegraded":"2023-10-09T10:23:54.630420+0000","last_fullsized":"2023-10-09T10:23:54.630420+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":130,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311111+0000","last_change":"2023-10-09T10:23:26.105223+0000","last_active":"2023-10-09T10:23:59.311111+0000","last_peered":"2023-10-09T10:23:59.311111+0000","last_clean":"2023-10-09T10:23:59.311111+0000","last_became_active":"2023-10-09T10:23:25.529288+0000","last_became_peered":"2023-10-09T10:23:25.529288+0000","last_unstale":"2023-10-09T10:23:59.311111+0000","last_undegraded":"2023-10-09T10:23:59.311111+0000","last_fullsized":"2023-10-09T10:23:59.311111+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.310951+0000","last_change":"2023-10-09T10:22:27.502829+0000","last_active":"2023-10-09T10:23:59.310951+0000","last_peered":"2023-10-09T10:23:59.310951+0000","last_clean":"2023-10-09T10:23:59.310951+0000","last_became_active":"2023-10-09T10:22:27.502361+0000","last_became_peered":"2023-10-09T10:22:27.502361+0000","last_unstale":"2023-10-09T10:23:59.310951+0000","last_undegraded":"2023-10-09T10:23:59.310951+0000","last_fullsized":"2023-10-09T10:23:59.310951+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":30,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146707+0000","last_change":"2023-10-09T10:23:26.091433+0000","last_active":"2023-10-09T10:23:59.146707+0000","last_peered":"2023-10-09T10:23:59.146707+0000","last_clean":"2023-10-09T10:23:59.146707+0000","last_became_active":"2023-10-09T10:23:25.437180+0000","last_became_peered":"2023-10-09T10:23:25.437180+0000","last_unstale":"2023-10-09T10:23:59.146707+0000","last_undegraded":"2023-10-09T10:23:59.146707+0000","last_fullsized":"2023-10-09T10:23:59.146707+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:18.902210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":22,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630878+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:23:54.630878+0000","last_peered":"2023-10-09T10:23:54.630878+0000","last_clean":"2023-10-09T10:23:54.630878+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:23:54.630878+0000","last_undegraded":"2023-10-09T10:23:54.630878+0000","last_fullsized":"2023-10-09T10:23:54.630878+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":411,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146677+0000","last_change":"2023-10-09T10:23:25.421838+0000","last_active":"2023-10-09T10:23:59.146677+0000","last_peered":"2023-10-09T10:23:59.146677+0000","last_clean":"2023-10-09T10:23:59.146677+0000","last_became_active":"2023-10-09T10:23:25.421388+0000","last_became_peered":"2023-10-09T10:23:25.421388+0000","last_unstale":"2023-10-09T10:23:59.146677+0000","last_undegraded":"2023-10-09T10:23:59.146677+0000","last_fullsized":"2023-10-09T10:23:59.146677+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:56.171644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":154,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.631099+0000","last_change":"2023-10-09T10:23:25.449285+0000","last_active":"2023-10-09T10:23:54.631099+0000","last_peered":"2023-10-09T10:23:54.631099+0000","last_clean":"2023-10-09T10:23:54.631099+0000","last_became_active":"2023-10-09T10:23:25.448967+0000","last_became_peered":"2023-10-09T10:23:25.448967+0000","last_unstale":"2023-10-09T10:23:54.631099+0000","last_undegraded":"2023-10-09T10:23:54.631099+0000","last_fullsized":"2023-10-09T10:23:54.631099+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":100,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.308242+0000","last_change":"2023-10-09T10:23:25.447085+0000","last_active":"2023-10-09T10:23:59.308242+0000","last_peered":"2023-10-09T10:23:59.308242+0000","last_clean":"2023-10-09T10:23:59.308242+0000","last_became_active":"2023-10-09T10:23:25.442003+0000","last_became_peered":"2023-10-09T10:23:25.442003+0000","last_unstale":"2023-10-09T10:23:59.308242+0000","last_undegraded":"2023-10-09T10:23:59.308242+0000","last_fullsized":"2023-10-09T10:23:59.308242+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":22,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630687+0000","last_change":"2023-10-09T10:23:25.452755+0000","last_active":"2023-10-09T10:23:54.630687+0000","last_peered":"2023-10-09T10:23:54.630687+0000","last_clean":"2023-10-09T10:23:54.630687+0000","last_became_active":"2023-10-09T10:23:25.452438+0000","last_became_peered":"2023-10-09T10:23:25.452438+0000","last_unstale":"2023-10-09T10:23:54.630687+0000","last_undegraded":"2023-10-09T10:23:54.630687+0000","last_fullsized":"2023-10-09T10:23:54.630687+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":450,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630468+0000","last_change":"2023-10-09T10:23:26.100317+0000","last_active":"2023-10-09T10:23:54.630468+0000","last_peered":"2023-10-09T10:23:54.630468+0000","last_clean":"2023-10-09T10:23:54.630468+0000","last_became_active":"2023-10-09T10:23:25.558059+0000","last_became_peered":"2023-10-09T10:23:25.558059+0000","last_unstale":"2023-10-09T10:23:54.630468+0000","last_undegraded":"2023-10-09T10:23:54.630468+0000","last_fullsized":"2023-10-09T10:23:54.630468+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":49,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630720+0000","last_change":"2023-10-09T10:23:25.521718+0000","last_active":"2023-10-09T10:23:54.630720+0000","last_peered":"2023-10-09T10:23:54.630720+0000","last_clean":"2023-10-09T10:23:54.630720+0000","last_became_active":"2023-10-09T10:23:25.521075+0000","last_became_peered":"2023-10-09T10:23:25.521075+0000","last_unstale":"2023-10-09T10:23:54.630720+0000","last_undegraded":"2023-10-09T10:23:54.630720+0000","last_fullsized":"2023-10-09T10:23:54.630720+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":11,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630784+0000","last_change":"2023-10-09T10:23:53.628989+0000","last_active":"2023-10-09T10:23:54.630784+0000","last_peered":"2023-10-09T10:23:54.630784+0000","last_clean":"2023-10-09T10:23:54.630784+0000","last_became_active":"2023-10-09T10:23:53.628725+0000","last_became_peered":"2023-10-09T10:23:53.628725+0000","last_unstale":"2023-10-09T10:23:54.630784+0000","last_undegraded":"2023-10-09T10:23:54.630784+0000","last_fullsized":"2023-10-09T10:23:54.630784+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":109,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.147398+0000","last_change":"2023-10-09T10:23:25.461650+0000","last_active":"2023-10-09T10:23:59.147398+0000","last_peered":"2023-10-09T10:23:59.147398+0000","last_clean":"2023-10-09T10:23:59.147398+0000","last_became_active":"2023-10-09T10:23:25.432695+0000","last_became_peered":"2023-10-09T10:23:25.432695+0000","last_unstale":"2023-10-09T10:23:59.147398+0000","last_undegraded":"2023-10-09T10:23:59.147398+0000","last_fullsized":"2023-10-09T10:23:59.147398+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:15.291716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":163,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630288+0000","last_change":"2023-10-09T10:23:25.518495+0000","last_active":"2023-10-09T10:23:54.630288+0000","last_peered":"2023-10-09T10:23:54.630288+0000","last_clean":"2023-10-09T10:23:54.630288+0000","last_became_active":"2023-10-09T10:23:25.517781+0000","last_became_peered":"2023-10-09T10:23:25.517781+0000","last_unstale":"2023-10-09T10:23:54.630288+0000","last_undegraded":"2023-10-09T10:23:54.630288+0000","last_fullsized":"2023-10-09T10:23:54.630288+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"498'576","reported_seq":588,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:03.086152+0000","last_change":"2023-10-09T10:23:53.788722+0000","last_active":"2023-10-09T10:24:03.086152+0000","last_peered":"2023-10-09T10:24:03.086152+0000","last_clean":"2023-10-09T10:24:03.086152+0000","last_became_active":"2023-10-09T10:23:53.788364+0000","last_became_peered":"2023-10-09T10:23:53.788364+0000","last_unstale":"2023-10-09T10:24:03.086152+0000","last_undegraded":"2023-10-09T10:24:03.086152+0000","last_fullsized":"2023-10-09T10:24:03.086152+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":576,"log_dups_size":0,"ondisk_log_size":576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37748736,"num_objects":576,"num_object_clones":0,"num_object_copies":1728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":576,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":112,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.312154+0000","last_change":"2023-10-09T10:23:25.742454+0000","last_active":"2023-10-09T10:23:59.312154+0000","last_peered":"2023-10-09T10:23:59.312154+0000","last_clean":"2023-10-09T10:23:59.312154+0000","last_became_active":"2023-10-09T10:23:25.526430+0000","last_became_peered":"2023-10-09T10:23:25.526430+0000","last_unstale":"2023-10-09T10:23:59.312154+0000","last_undegraded":"2023-10-09T10:23:59.312154+0000","last_fullsized":"2023-10-09T10:23:59.312154+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":106,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146506+0000","last_change":"2023-10-09T10:23:25.446111+0000","last_active":"2023-10-09T10:23:59.146506+0000","last_peered":"2023-10-09T10:23:59.146506+0000","last_clean":"2023-10-09T10:23:59.146506+0000","last_became_active":"2023-10-09T10:23:25.445584+0000","last_became_peered":"2023-10-09T10:23:25.445584+0000","last_unstale":"2023-10-09T10:23:59.146506+0000","last_undegraded":"2023-10-09T10:23:59.146506+0000","last_fullsized":"2023-10-09T10:23:59.146506+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:40:04.357388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":11,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630930+0000","last_change":"2023-10-09T10:23:53.628972+0000","last_active":"2023-10-09T10:23:54.630930+0000","last_peered":"2023-10-09T10:23:54.630930+0000","last_clean":"2023-10-09T10:23:54.630930+0000","last_became_active":"2023-10-09T10:23:53.628676+0000","last_became_peered":"2023-10-09T10:23:53.628676+0000","last_unstale":"2023-10-09T10:23:54.630930+0000","last_undegraded":"2023-10-09T10:23:54.630930+0000","last_fullsized":"2023-10-09T10:23:54.630930+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":175,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630657+0000","last_change":"2023-10-09T10:23:26.107824+0000","last_active":"2023-10-09T10:23:54.630657+0000","last_peered":"2023-10-09T10:23:54.630657+0000","last_clean":"2023-10-09T10:23:54.630657+0000","last_became_active":"2023-10-09T10:23:25.617347+0000","last_became_peered":"2023-10-09T10:23:25.617347+0000","last_unstale":"2023-10-09T10:23:54.630657+0000","last_undegraded":"2023-10-09T10:23:54.630657+0000","last_fullsized":"2023-10-09T10:23:54.630657+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":62,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630693+0000","last_change":"2023-10-09T10:23:25.681497+0000","last_active":"2023-10-09T10:23:54.630693+0000","last_peered":"2023-10-09T10:23:54.630693+0000","last_clean":"2023-10-09T10:23:54.630693+0000","last_became_active":"2023-10-09T10:23:25.610682+0000","last_became_peered":"2023-10-09T10:23:25.610682+0000","last_unstale":"2023-10-09T10:23:54.630693+0000","last_undegraded":"2023-10-09T10:23:54.630693+0000","last_fullsized":"2023-10-09T10:23:54.630693+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"498'503","reported_seq":516,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:02.565258+0000","last_change":"2023-10-09T10:23:53.842397+0000","last_active":"2023-10-09T10:24:02.565258+0000","last_peered":"2023-10-09T10:24:02.565258+0000","last_clean":"2023-10-09T10:24:02.565258+0000","last_became_active":"2023-10-09T10:23:53.842058+0000","last_became_peered":"2023-10-09T10:23:53.842058+0000","last_unstale":"2023-10-09T10:24:02.565258+0000","last_undegraded":"2023-10-09T10:24:02.565258+0000","last_fullsized":"2023-10-09T10:24:02.565258+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":503,"log_dups_size":0,"ondisk_log_size":503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:01.633261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32964608,"num_objects":503,"num_object_clones":0,"num_object_copies":1509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":503,"num_write_kb":32192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":163,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630385+0000","last_change":"2023-10-09T10:22:27.394423+0000","last_active":"2023-10-09T10:23:54.630385+0000","last_peered":"2023-10-09T10:23:54.630385+0000","last_clean":"2023-10-09T10:23:54.630385+0000","last_became_active":"2023-10-09T10:22:27.394040+0000","last_became_peered":"2023-10-09T10:22:27.394040+0000","last_unstale":"2023-10-09T10:23:54.630385+0000","last_undegraded":"2023-10-09T10:23:54.630385+0000","last_fullsized":"2023-10-09T10:23:54.630385+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":397,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146897+0000","last_change":"2023-10-09T10:22:13.366463+0000","last_active":"2023-10-09T10:23:59.146897+0000","last_peered":"2023-10-09T10:23:59.146897+0000","last_clean":"2023-10-09T10:23:59.146897+0000","last_became_active":"2023-10-09T10:22:13.366125+0000","last_became_peered":"2023-10-09T10:22:13.366125+0000","last_unstale":"2023-10-09T10:23:59.146897+0000","last_undegraded":"2023-10-09T10:23:59.146897+0000","last_fullsized":"2023-10-09T10:23:59.146897+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:41.113925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":11,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630347+0000","last_change":"2023-10-09T10:23:53.842233+0000","last_active":"2023-10-09T10:23:54.630347+0000","last_peered":"2023-10-09T10:23:54.630347+0000","last_clean":"2023-10-09T10:23:54.630347+0000","last_became_active":"2023-10-09T10:23:53.841967+0000","last_became_peered":"2023-10-09T10:23:53.841967+0000","last_unstale":"2023-10-09T10:23:54.630347+0000","last_undegraded":"2023-10-09T10:23:54.630347+0000","last_fullsized":"2023-10-09T10:23:54.630347+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":94,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146814+0000","last_change":"2023-10-09T10:23:25.554673+0000","last_active":"2023-10-09T10:23:59.146814+0000","last_peered":"2023-10-09T10:23:59.146814+0000","last_clean":"2023-10-09T10:23:59.146814+0000","last_became_active":"2023-10-09T10:23:25.436984+0000","last_became_peered":"2023-10-09T10:23:25.436984+0000","last_unstale":"2023-10-09T10:23:59.146814+0000","last_undegraded":"2023-10-09T10:23:59.146814+0000","last_fullsized":"2023-10-09T10:23:59.146814+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:00.646924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311006+0000","last_change":"2023-10-09T10:23:25.462246+0000","last_active":"2023-10-09T10:23:59.311006+0000","last_peered":"2023-10-09T10:23:59.311006+0000","last_clean":"2023-10-09T10:23:59.311006+0000","last_became_active":"2023-10-09T10:23:25.461706+0000","last_became_peered":"2023-10-09T10:23:25.461706+0000","last_unstale":"2023-10-09T10:23:59.311006+0000","last_undegraded":"2023-10-09T10:23:59.311006+0000","last_fullsized":"2023-10-09T10:23:59.311006+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"498'499","reported_seq":512,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:02.571612+0000","last_change":"2023-10-09T10:23:53.789814+0000","last_active":"2023-10-09T10:24:02.571612+0000","last_peered":"2023-10-09T10:24:02.571612+0000","last_clean":"2023-10-09T10:24:02.571612+0000","last_became_active":"2023-10-09T10:23:53.789476+0000","last_became_peered":"2023-10-09T10:23:53.789476+0000","last_unstale":"2023-10-09T10:24:02.571612+0000","last_undegraded":"2023-10-09T10:24:02.571612+0000","last_fullsized":"2023-10-09T10:24:02.571612+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":499,"log_dups_size":0,"ondisk_log_size":499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:04:59.746866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32702464,"num_objects":499,"num_object_clones":0,"num_object_copies":1497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":499,"num_write_kb":31936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":100,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.310647+0000","last_change":"2023-10-09T10:22:27.444402+0000","last_active":"2023-10-09T10:23:59.310647+0000","last_peered":"2023-10-09T10:23:59.310647+0000","last_clean":"2023-10-09T10:23:59.310647+0000","last_became_active":"2023-10-09T10:22:27.443887+0000","last_became_peered":"2023-10-09T10:22:27.443887+0000","last_unstale":"2023-10-09T10:23:59.310647+0000","last_undegraded":"2023-10-09T10:23:59.310647+0000","last_fullsized":"2023-10-09T10:23:59.310647+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":96,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630469+0000","last_change":"2023-10-09T10:22:40.529911+0000","last_active":"2023-10-09T10:23:54.630469+0000","last_peered":"2023-10-09T10:23:54.630469+0000","last_clean":"2023-10-09T10:23:54.630469+0000","last_became_active":"2023-10-09T10:22:40.529496+0000","last_became_peered":"2023-10-09T10:22:40.529496+0000","last_unstale":"2023-10-09T10:23:54.630469+0000","last_undegraded":"2023-10-09T10:23:54.630469+0000","last_fullsized":"2023-10-09T10:23:54.630469+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"498'508","reported_seq":521,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:02.590690+0000","last_change":"2023-10-09T10:23:53.788743+0000","last_active":"2023-10-09T10:24:02.590690+0000","last_peered":"2023-10-09T10:24:02.590690+0000","last_clean":"2023-10-09T10:24:02.590690+0000","last_became_active":"2023-10-09T10:23:53.788402+0000","last_became_peered":"2023-10-09T10:23:53.788402+0000","last_unstale":"2023-10-09T10:24:02.590690+0000","last_undegraded":"2023-10-09T10:24:02.590690+0000","last_fullsized":"2023-10-09T10:24:02.590690+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":508,"log_dups_size":0,"ondisk_log_size":508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:10:53.967581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33292288,"num_objects":508,"num_object_clones":0,"num_object_copies":1524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":508,"num_write_kb":32512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":91,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.308466+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:23:59.308466+0000","last_peered":"2023-10-09T10:23:59.308466+0000","last_clean":"2023-10-09T10:23:59.308466+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:23:59.308466+0000","last_undegraded":"2023-10-09T10:23:59.308466+0000","last_fullsized":"2023-10-09T10:23:59.308466+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":109,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.312085+0000","last_change":"2023-10-09T10:22:27.429388+0000","last_active":"2023-10-09T10:23:59.312085+0000","last_peered":"2023-10-09T10:23:59.312085+0000","last_clean":"2023-10-09T10:23:59.312085+0000","last_became_active":"2023-10-09T10:22:27.429067+0000","last_became_peered":"2023-10-09T10:22:27.429067+0000","last_unstale":"2023-10-09T10:23:59.312085+0000","last_undegraded":"2023-10-09T10:23:59.312085+0000","last_fullsized":"2023-10-09T10:23:59.312085+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"498'460","reported_seq":473,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:02.592517+0000","last_change":"2023-10-09T10:23:53.842520+0000","last_active":"2023-10-09T10:24:02.592517+0000","last_peered":"2023-10-09T10:24:02.592517+0000","last_clean":"2023-10-09T10:24:02.592517+0000","last_became_active":"2023-10-09T10:23:53.842261+0000","last_became_peered":"2023-10-09T10:23:53.842261+0000","last_unstale":"2023-10-09T10:24:02.592517+0000","last_undegraded":"2023-10-09T10:24:02.592517+0000","last_fullsized":"2023-10-09T10:24:02.592517+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":460,"log_dups_size":0,"ondisk_log_size":460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:36.091400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30146560,"num_objects":460,"num_object_clones":0,"num_object_copies":1380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":460,"num_write_kb":29440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":431,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630295+0000","last_change":"2023-10-09T10:23:25.518667+0000","last_active":"2023-10-09T10:23:54.630295+0000","last_peered":"2023-10-09T10:23:54.630295+0000","last_clean":"2023-10-09T10:23:54.630295+0000","last_became_active":"2023-10-09T10:23:25.518163+0000","last_became_peered":"2023-10-09T10:23:25.518163+0000","last_unstale":"2023-10-09T10:23:54.630295+0000","last_undegraded":"2023-10-09T10:23:54.630295+0000","last_fullsized":"2023-10-09T10:23:54.630295+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":512,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311260+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:23:59.311260+0000","last_peered":"2023-10-09T10:23:59.311260+0000","last_clean":"2023-10-09T10:23:59.311260+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:23:59.311260+0000","last_undegraded":"2023-10-09T10:23:59.311260+0000","last_fullsized":"2023-10-09T10:23:59.311260+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"498'567","reported_seq":576,"reported_epoch":498,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:24:03.062880+0000","last_change":"2023-10-09T10:24:00.142839+0000","last_active":"2023-10-09T10:24:03.062880+0000","last_peered":"2023-10-09T10:24:03.062880+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:23:53.626970+0000","last_became_peered":"2023-10-09T10:23:53.626970+0000","last_unstale":"2023-10-09T10:24:03.062880+0000","last_undegraded":"2023-10-09T10:24:00.139747+0000","last_fullsized":"2023-10-09T10:23:53.616547+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":567,"log_dups_size":0,"ondisk_log_size":567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158912,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":567,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":567,"num_write_kb":36288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":567}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":208,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146585+0000","last_change":"2023-10-09T10:23:25.462977+0000","last_active":"2023-10-09T10:23:59.146585+0000","last_peered":"2023-10-09T10:23:59.146585+0000","last_clean":"2023-10-09T10:23:59.146585+0000","last_became_active":"2023-10-09T10:23:25.462666+0000","last_became_peered":"2023-10-09T10:23:25.462666+0000","last_unstale":"2023-10-09T10:23:59.146585+0000","last_undegraded":"2023-10-09T10:23:59.146585+0000","last_fullsized":"2023-10-09T10:23:59.146585+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:25.535385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":163,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630989+0000","last_change":"2023-10-09T10:23:25.452505+0000","last_active":"2023-10-09T10:23:54.630989+0000","last_peered":"2023-10-09T10:23:54.630989+0000","last_clean":"2023-10-09T10:23:54.630989+0000","last_became_active":"2023-10-09T10:23:25.452190+0000","last_became_peered":"2023-10-09T10:23:25.452190+0000","last_unstale":"2023-10-09T10:23:54.630989+0000","last_undegraded":"2023-10-09T10:23:54.630989+0000","last_fullsized":"2023-10-09T10:23:54.630989+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":461,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630797+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:23:54.630797+0000","last_peered":"2023-10-09T10:23:54.630797+0000","last_clean":"2023-10-09T10:23:54.630797+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:23:54.630797+0000","last_undegraded":"2023-10-09T10:23:54.630797+0000","last_fullsized":"2023-10-09T10:23:54.630797+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"498'475","reported_seq":488,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:02.596138+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:24:02.596138+0000","last_peered":"2023-10-09T10:24:02.596138+0000","last_clean":"2023-10-09T10:24:02.596138+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:24:02.596138+0000","last_undegraded":"2023-10-09T10:24:02.596138+0000","last_fullsized":"2023-10-09T10:24:02.596138+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":475,"log_dups_size":0,"ondisk_log_size":475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31129600,"num_objects":475,"num_object_clones":0,"num_object_copies":1425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":475,"num_write_kb":30400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":172,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630598+0000","last_change":"2023-10-09T10:23:25.444540+0000","last_active":"2023-10-09T10:23:54.630598+0000","last_peered":"2023-10-09T10:23:54.630598+0000","last_clean":"2023-10-09T10:23:54.630598+0000","last_became_active":"2023-10-09T10:23:25.444038+0000","last_became_peered":"2023-10-09T10:23:25.444038+0000","last_unstale":"2023-10-09T10:23:54.630598+0000","last_undegraded":"2023-10-09T10:23:54.630598+0000","last_fullsized":"2023-10-09T10:23:54.630598+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":91,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.310907+0000","last_change":"2023-10-09T10:22:27.502663+0000","last_active":"2023-10-09T10:23:59.310907+0000","last_peered":"2023-10-09T10:23:59.310907+0000","last_clean":"2023-10-09T10:23:59.310907+0000","last_became_active":"2023-10-09T10:22:27.501953+0000","last_became_peered":"2023-10-09T10:22:27.501953+0000","last_unstale":"2023-10-09T10:23:59.310907+0000","last_undegraded":"2023-10-09T10:23:59.310907+0000","last_fullsized":"2023-10-09T10:23:59.310907+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":49,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630834+0000","last_change":"2023-10-09T10:23:25.522457+0000","last_active":"2023-10-09T10:23:54.630834+0000","last_peered":"2023-10-09T10:23:54.630834+0000","last_clean":"2023-10-09T10:23:54.630834+0000","last_became_active":"2023-10-09T10:23:25.522190+0000","last_became_peered":"2023-10-09T10:23:25.522190+0000","last_unstale":"2023-10-09T10:23:54.630834+0000","last_undegraded":"2023-10-09T10:23:54.630834+0000","last_fullsized":"2023-10-09T10:23:54.630834+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":11,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630052+0000","last_change":"2023-10-09T10:23:53.789564+0000","last_active":"2023-10-09T10:23:54.630052+0000","last_peered":"2023-10-09T10:23:54.630052+0000","last_clean":"2023-10-09T10:23:54.630052+0000","last_became_active":"2023-10-09T10:23:53.789275+0000","last_became_peered":"2023-10-09T10:23:53.789275+0000","last_unstale":"2023-10-09T10:23:54.630052+0000","last_undegraded":"2023-10-09T10:23:54.630052+0000","last_fullsized":"2023-10-09T10:23:54.630052+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":429,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146703+0000","last_change":"2023-10-09T10:22:27.441855+0000","last_active":"2023-10-09T10:23:59.146703+0000","last_peered":"2023-10-09T10:23:59.146703+0000","last_clean":"2023-10-09T10:23:59.146703+0000","last_became_active":"2023-10-09T10:22:27.441539+0000","last_became_peered":"2023-10-09T10:22:27.441539+0000","last_unstale":"2023-10-09T10:23:59.146703+0000","last_undegraded":"2023-10-09T10:23:59.146703+0000","last_fullsized":"2023-10-09T10:23:59.146703+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:46.202562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146174+0000","last_change":"2023-10-09T10:22:27.392415+0000","last_active":"2023-10-09T10:23:59.146174+0000","last_peered":"2023-10-09T10:23:59.146174+0000","last_clean":"2023-10-09T10:23:59.146174+0000","last_became_active":"2023-10-09T10:22:27.392069+0000","last_became_peered":"2023-10-09T10:22:27.392069+0000","last_unstale":"2023-10-09T10:23:59.146174+0000","last_undegraded":"2023-10-09T10:23:59.146174+0000","last_fullsized":"2023-10-09T10:23:59.146174+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:51:06.172359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":492,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.631024+0000","last_change":"2023-10-09T10:23:25.521874+0000","last_active":"2023-10-09T10:23:54.631024+0000","last_peered":"2023-10-09T10:23:54.631024+0000","last_clean":"2023-10-09T10:23:54.631024+0000","last_became_active":"2023-10-09T10:23:25.521446+0000","last_became_peered":"2023-10-09T10:23:25.521446+0000","last_unstale":"2023-10-09T10:23:54.631024+0000","last_undegraded":"2023-10-09T10:23:54.631024+0000","last_fullsized":"2023-10-09T10:23:54.631024+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":11,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630308+0000","last_change":"2023-10-09T10:23:53.842189+0000","last_active":"2023-10-09T10:23:54.630308+0000","last_peered":"2023-10-09T10:23:54.630308+0000","last_clean":"2023-10-09T10:23:54.630308+0000","last_became_active":"2023-10-09T10:23:53.841927+0000","last_became_peered":"2023-10-09T10:23:53.841927+0000","last_unstale":"2023-10-09T10:23:54.630308+0000","last_undegraded":"2023-10-09T10:23:54.630308+0000","last_fullsized":"2023-10-09T10:23:54.630308+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":151,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.631066+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:23:54.631066+0000","last_peered":"2023-10-09T10:23:54.631066+0000","last_clean":"2023-10-09T10:23:54.631066+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:23:54.631066+0000","last_undegraded":"2023-10-09T10:23:54.631066+0000","last_fullsized":"2023-10-09T10:23:54.631066+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":678,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.798774+0000","last_change":"2023-10-09T10:23:46.972678+0000","last_active":"2023-10-09T10:23:54.798774+0000","last_peered":"2023-10-09T10:23:54.798774+0000","last_clean":"2023-10-09T10:23:54.798774+0000","last_became_active":"2023-10-09T10:23:46.972311+0000","last_became_peered":"2023-10-09T10:23:46.972311+0000","last_unstale":"2023-10-09T10:23:54.798774+0000","last_undegraded":"2023-10-09T10:23:54.798774+0000","last_fullsized":"2023-10-09T10:23:54.798774+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:15.713716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":181,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630440+0000","last_change":"2023-10-09T10:23:25.681332+0000","last_active":"2023-10-09T10:23:54.630440+0000","last_peered":"2023-10-09T10:23:54.630440+0000","last_clean":"2023-10-09T10:23:54.630440+0000","last_became_active":"2023-10-09T10:23:25.610272+0000","last_became_peered":"2023-10-09T10:23:25.610272+0000","last_unstale":"2023-10-09T10:23:54.630440+0000","last_undegraded":"2023-10-09T10:23:54.630440+0000","last_fullsized":"2023-10-09T10:23:54.630440+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":100,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311378+0000","last_change":"2023-10-09T10:23:25.683944+0000","last_active":"2023-10-09T10:23:59.311378+0000","last_peered":"2023-10-09T10:23:59.311378+0000","last_clean":"2023-10-09T10:23:59.311378+0000","last_became_active":"2023-10-09T10:23:25.683505+0000","last_became_peered":"2023-10-09T10:23:25.683505+0000","last_unstale":"2023-10-09T10:23:59.311378+0000","last_undegraded":"2023-10-09T10:23:59.311378+0000","last_fullsized":"2023-10-09T10:23:59.311378+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":8,"reported_epoch":496,"state":"active+undersized","last_fresh":"2023-10-09T10:23:54.630561+0000","last_change":"2023-10-09T10:23:53.841867+0000","last_active":"2023-10-09T10:23:54.630561+0000","last_peered":"2023-10-09T10:23:54.630561+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:23:53.841867+0000","last_became_peered":"2023-10-09T10:23:53.841867+0000","last_unstale":"2023-10-09T10:23:54.630561+0000","last_undegraded":"2023-10-09T10:23:54.630561+0000","last_fullsized":"2023-10-09T10:23:53.624956+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":181,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146280+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:23:59.146280+0000","last_peered":"2023-10-09T10:23:59.146280+0000","last_clean":"2023-10-09T10:23:59.146280+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:23:59.146280+0000","last_undegraded":"2023-10-09T10:23:59.146280+0000","last_fullsized":"2023-10-09T10:23:59.146280+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":432,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146771+0000","last_change":"2023-10-09T10:19:43.292390+0000","last_active":"2023-10-09T10:23:59.146771+0000","last_peered":"2023-10-09T10:23:59.146771+0000","last_clean":"2023-10-09T10:23:59.146771+0000","last_became_active":"2023-10-09T10:19:43.290733+0000","last_became_peered":"2023-10-09T10:19:43.290733+0000","last_unstale":"2023-10-09T10:23:59.146771+0000","last_undegraded":"2023-10-09T10:23:59.146771+0000","last_fullsized":"2023-10-09T10:23:59.146771+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:53.407598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":53,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.631171+0000","last_change":"2023-10-09T10:23:25.531957+0000","last_active":"2023-10-09T10:23:54.631171+0000","last_peered":"2023-10-09T10:23:54.631171+0000","last_clean":"2023-10-09T10:23:54.631171+0000","last_became_active":"2023-10-09T10:23:25.531614+0000","last_became_peered":"2023-10-09T10:23:25.531614+0000","last_unstale":"2023-10-09T10:23:54.631171+0000","last_undegraded":"2023-10-09T10:23:54.631171+0000","last_fullsized":"2023-10-09T10:23:54.631171+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.c","version":"498'570","reported_seq":582,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:03.065808+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:24:03.065808+0000","last_peered":"2023-10-09T10:24:03.065808+0000","last_clean":"2023-10-09T10:24:03.065808+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:24:03.065808+0000","last_undegraded":"2023-10-09T10:24:03.065808+0000","last_fullsized":"2023-10-09T10:24:03.065808+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":570,"log_dups_size":0,"ondisk_log_size":570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37355520,"num_objects":570,"num_object_clones":0,"num_object_copies":1710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":570,"num_write_kb":36480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":109,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311463+0000","last_change":"2023-10-09T10:23:25.613084+0000","last_active":"2023-10-09T10:23:59.311463+0000","last_peered":"2023-10-09T10:23:59.311463+0000","last_clean":"2023-10-09T10:23:59.311463+0000","last_became_active":"2023-10-09T10:23:25.612722+0000","last_became_peered":"2023-10-09T10:23:25.612722+0000","last_unstale":"2023-10-09T10:23:59.311463+0000","last_undegraded":"2023-10-09T10:23:59.311463+0000","last_fullsized":"2023-10-09T10:23:59.311463+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":460,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630783+0000","last_change":"2023-10-09T10:22:27.429171+0000","last_active":"2023-10-09T10:23:54.630783+0000","last_peered":"2023-10-09T10:23:54.630783+0000","last_clean":"2023-10-09T10:23:54.630783+0000","last_became_active":"2023-10-09T10:22:27.428849+0000","last_became_peered":"2023-10-09T10:22:27.428849+0000","last_unstale":"2023-10-09T10:23:54.630783+0000","last_undegraded":"2023-10-09T10:23:54.630783+0000","last_fullsized":"2023-10-09T10:23:54.630783+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":400,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.147135+0000","last_change":"2023-10-09T10:23:25.458876+0000","last_active":"2023-10-09T10:23:59.147135+0000","last_peered":"2023-10-09T10:23:59.147135+0000","last_clean":"2023-10-09T10:23:59.147135+0000","last_became_active":"2023-10-09T10:23:25.458037+0000","last_became_peered":"2023-10-09T10:23:25.458037+0000","last_unstale":"2023-10-09T10:23:59.147135+0000","last_undegraded":"2023-10-09T10:23:59.147135+0000","last_fullsized":"2023-10-09T10:23:59.147135+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:34.186003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"498'487","reported_seq":500,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:02.573247+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:24:02.573247+0000","last_peered":"2023-10-09T10:24:02.573247+0000","last_clean":"2023-10-09T10:24:02.573247+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:24:02.573247+0000","last_undegraded":"2023-10-09T10:24:02.573247+0000","last_fullsized":"2023-10-09T10:24:02.573247+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":487,"log_dups_size":0,"ondisk_log_size":487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31916032,"num_objects":487,"num_object_clones":0,"num_object_copies":1461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":487,"num_write_kb":31168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":297,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.147581+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:23:59.147581+0000","last_peered":"2023-10-09T10:23:59.147581+0000","last_clean":"2023-10-09T10:23:59.147581+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:23:59.147581+0000","last_undegraded":"2023-10-09T10:23:59.147581+0000","last_fullsized":"2023-10-09T10:23:59.147581+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":470,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.312072+0000","last_change":"2023-10-09T10:22:27.442726+0000","last_active":"2023-10-09T10:23:59.312072+0000","last_peered":"2023-10-09T10:23:59.312072+0000","last_clean":"2023-10-09T10:23:59.312072+0000","last_became_active":"2023-10-09T10:22:27.442396+0000","last_became_peered":"2023-10-09T10:22:27.442396+0000","last_unstale":"2023-10-09T10:23:59.312072+0000","last_undegraded":"2023-10-09T10:23:59.312072+0000","last_fullsized":"2023-10-09T10:23:59.312072+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":363,"reported_epoch":496,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:23:54.630525+0000","last_change":"2023-10-09T10:23:26.562884+0000","last_active":"2023-10-09T10:23:54.630525+0000","last_peered":"2023-10-09T10:23:54.630525+0000","last_clean":"2023-10-09T10:23:54.630525+0000","last_became_active":"2023-10-09T10:23:26.483114+0000","last_became_peered":"2023-10-09T10:23:26.483114+0000","last_unstale":"2023-10-09T10:23:54.630525+0000","last_undegraded":"2023-10-09T10:23:54.630525+0000","last_fullsized":"2023-10-09T10:23:54.630525+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,7],"avail_no_missing":["5","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.e","version":"498'570","reported_seq":582,"reported_epoch":498,"state":"active+clean","last_fresh":"2023-10-09T10:24:03.086613+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:24:03.086613+0000","last_peered":"2023-10-09T10:24:03.086613+0000","last_clean":"2023-10-09T10:24:03.086613+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:24:03.086613+0000","last_undegraded":"2023-10-09T10:24:03.086613+0000","last_fullsized":"2023-10-09T10:24:03.086613+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":570,"log_dups_size":0,"ondisk_log_size":570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37355520,"num_objects":570,"num_object_clones":0,"num_object_copies":1710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":570,"num_write_kb":36480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":23,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311376+0000","last_change":"2023-10-09T10:23:25.433377+0000","last_active":"2023-10-09T10:23:59.311376+0000","last_peered":"2023-10-09T10:23:59.311376+0000","last_clean":"2023-10-09T10:23:59.311376+0000","last_became_active":"2023-10-09T10:23:25.422009+0000","last_became_peered":"2023-10-09T10:23:25.422009+0000","last_unstale":"2023-10-09T10:23:59.311376+0000","last_undegraded":"2023-10-09T10:23:59.311376+0000","last_fullsized":"2023-10-09T10:23:59.311376+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":172,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.312281+0000","last_change":"2023-10-09T10:23:25.670521+0000","last_active":"2023-10-09T10:23:59.312281+0000","last_peered":"2023-10-09T10:23:59.312281+0000","last_clean":"2023-10-09T10:23:59.312281+0000","last_became_active":"2023-10-09T10:23:25.525855+0000","last_became_peered":"2023-10-09T10:23:25.525855+0000","last_unstale":"2023-10-09T10:23:59.312281+0000","last_undegraded":"2023-10-09T10:23:59.312281+0000","last_fullsized":"2023-10-09T10:23:59.312281+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":168,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630148+0000","last_change":"2023-10-09T10:22:40.513566+0000","last_active":"2023-10-09T10:23:54.630148+0000","last_peered":"2023-10-09T10:23:54.630148+0000","last_clean":"2023-10-09T10:23:54.630148+0000","last_became_active":"2023-10-09T10:22:40.513228+0000","last_became_peered":"2023-10-09T10:22:40.513228+0000","last_unstale":"2023-10-09T10:23:54.630148+0000","last_undegraded":"2023-10-09T10:23:54.630148+0000","last_fullsized":"2023-10-09T10:23:54.630148+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":106,"reported_epoch":496,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:23:54.630981+0000","last_change":"2023-10-09T10:23:26.551630+0000","last_active":"2023-10-09T10:23:54.630981+0000","last_peered":"2023-10-09T10:23:54.630981+0000","last_clean":"2023-10-09T10:23:54.630981+0000","last_became_active":"2023-10-09T10:23:26.551134+0000","last_became_peered":"2023-10-09T10:23:26.551134+0000","last_unstale":"2023-10-09T10:23:54.630981+0000","last_undegraded":"2023-10-09T10:23:54.630981+0000","last_fullsized":"2023-10-09T10:23:54.630981+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,7],"avail_no_missing":["5","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":474,"reported_epoch":497,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:23:59.311009+0000","last_change":"2023-10-09T10:23:26.552568+0000","last_active":"2023-10-09T10:23:59.311009+0000","last_peered":"2023-10-09T10:23:59.311009+0000","last_clean":"2023-10-09T10:23:59.311009+0000","last_became_active":"2023-10-09T10:23:26.551999+0000","last_became_peered":"2023-10-09T10:23:26.551999+0000","last_unstale":"2023-10-09T10:23:59.311009+0000","last_undegraded":"2023-10-09T10:23:59.311009+0000","last_fullsized":"2023-10-09T10:23:59.311009+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,7],"avail_no_missing":["4","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":214,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630523+0000","last_change":"2023-10-09T10:23:25.539343+0000","last_active":"2023-10-09T10:23:54.630523+0000","last_peered":"2023-10-09T10:23:54.630523+0000","last_clean":"2023-10-09T10:23:54.630523+0000","last_became_active":"2023-10-09T10:23:25.514039+0000","last_became_peered":"2023-10-09T10:23:25.514039+0000","last_unstale":"2023-10-09T10:23:54.630523+0000","last_undegraded":"2023-10-09T10:23:54.630523+0000","last_fullsized":"2023-10-09T10:23:54.630523+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":91,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.311009+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:23:59.311009+0000","last_peered":"2023-10-09T10:23:59.311009+0000","last_clean":"2023-10-09T10:23:59.311009+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:23:59.311009+0000","last_undegraded":"2023-10-09T10:23:59.311009+0000","last_fullsized":"2023-10-09T10:23:59.311009+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":197,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T10:23:54.630779+0000","last_change":"2023-10-09T10:23:26.112703+0000","last_active":"2023-10-09T10:23:54.630779+0000","last_peered":"2023-10-09T10:23:54.630779+0000","last_clean":"2023-10-09T10:23:54.630779+0000","last_became_active":"2023-10-09T10:23:25.617862+0000","last_became_peered":"2023-10-09T10:23:25.617862+0000","last_unstale":"2023-10-09T10:23:54.630779+0000","last_undegraded":"2023-10-09T10:23:54.630779+0000","last_fullsized":"2023-10-09T10:23:54.630779+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":320,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146506+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:23:59.146506+0000","last_peered":"2023-10-09T10:23:59.146506+0000","last_clean":"2023-10-09T10:23:59.146506+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:23:59.146506+0000","last_undegraded":"2023-10-09T10:23:59.146506+0000","last_fullsized":"2023-10-09T10:23:59.146506+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":51,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146590+0000","last_change":"2023-10-09T10:23:25.438780+0000","last_active":"2023-10-09T10:23:59.146590+0000","last_peered":"2023-10-09T10:23:59.146590+0000","last_clean":"2023-10-09T10:23:59.146590+0000","last_became_active":"2023-10-09T10:23:25.438407+0000","last_became_peered":"2023-10-09T10:23:25.438407+0000","last_unstale":"2023-10-09T10:23:59.146590+0000","last_undegraded":"2023-10-09T10:23:59.146590+0000","last_fullsized":"2023-10-09T10:23:59.146590+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:08:16.664151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":540,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.146602+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:23:59.146602+0000","last_peered":"2023-10-09T10:23:59.146602+0000","last_clean":"2023-10-09T10:23:59.146602+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:23:59.146602+0000","last_undegraded":"2023-10-09T10:23:59.146602+0000","last_fullsized":"2023-10-09T10:23:59.146602+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":496,"reported_epoch":496,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:23:54.630963+0000","last_change":"2023-10-09T10:23:26.567480+0000","last_active":"2023-10-09T10:23:54.630963+0000","last_peered":"2023-10-09T10:23:54.630963+0000","last_clean":"2023-10-09T10:23:54.630963+0000","last_became_active":"2023-10-09T10:23:26.567054+0000","last_became_peered":"2023-10-09T10:23:26.567054+0000","last_unstale":"2023-10-09T10:23:54.630963+0000","last_undegraded":"2023-10-09T10:23:54.630963+0000","last_fullsized":"2023-10-09T10:23:54.630963+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":100,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T10:23:59.308263+0000","last_change":"2023-10-09T10:22:27.441829+0000","last_active":"2023-10-09T10:23:59.308263+0000","last_peered":"2023-10-09T10:23:59.308263+0000","last_clean":"2023-10-09T10:23:59.308263+0000","last_became_active":"2023-10-09T10:22:27.441479+0000","last_became_peered":"2023-10-09T10:22:27.441479+0000","last_unstale":"2023-10-09T10:23:59.308263+0000","last_undegraded":"2023-10-09T10:23:59.308263+0000","last_fullsized":"2023-10-09T10:23:59.308263+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":341770240,"num_objects":5215,"num_object_clones":0,"num_object_copies":15645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":567,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5215,"num_write_kb":333760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":708640768,"data_stored":708640768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5215,"ondisk_log_size":5215,"up":46,"acting":46,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":26954,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":94,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430652,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626880,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":360777253193,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416718,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":956260,"kb_used_data":190748,"kb_used_omap":28,"kb_used_meta":765475,"kb_avail":92788892,"statfs":{"total":95995035648,"available":95015825408,"internally_reserved":0,"allocated":195325952,"data_stored":193474093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":783847248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:23:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.0840000000000001,"15min":1.97},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.7290000000000001,"5min":8.8699999999999992,"15min":54.518000000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.5229999999999999,"5min":1.3240000000000001,"15min":2.718},"min":{"1min":0.214,"5min":0.192,"15min":0.192},"max":{"1min":4.9509999999999996,"5min":17.033000000000001,"15min":56.950000000000003},"last":0.41799999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:23:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":1.6719999999999999,"15min":2.153},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.7080000000000002,"5min":47.215000000000003,"15min":53.988999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.794,"15min":2.1549999999999998},"min":{"1min":0.28699999999999998,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":5.6790000000000003,"5min":47.000999999999998,"15min":54.119},"last":0.66500000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.3640000000000001,"15min":1.401},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":11.933,"5min":16.879999999999999,"15min":56.984999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.581,"5min":1.3400000000000001,"15min":1.345},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.0020000000000007,"5min":8.0020000000000007,"15min":56.475999999999999},"last":0.498}]},{"osd":4,"last update":"Mon Oct 9 10:23:10 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":1.9350000000000001,"15min":2.355},"min":{"1min":0.33700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":28.289000000000001,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.91900000000000004},{"interface":"front","average":{"1min":3.5390000000000001,"5min":1.887,"15min":2.2730000000000001},"min":{"1min":0.34499999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":28.331,"5min":28.331,"15min":56.378},"last":0.88500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:23:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5339999999999998,"5min":1.792,"15min":3.1909999999999998},"min":{"1min":0.38200000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.775,"5min":11.775,"15min":229.589},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.573,"15min":2.7269999999999999},"min":{"1min":0.38600000000000001,"5min":0.31900000000000001,"15min":0.27000000000000002},"max":{"1min":5.7409999999999997,"5min":8.7530000000000001,"15min":228.88900000000001},"last":0.86199999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:23:10 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.7470000000000001,"15min":2.1629999999999998},"min":{"1min":0.48599999999999999,"5min":0.34899999999999998,"15min":0.33700000000000002},"max":{"1min":4.5410000000000004,"5min":8.7810000000000006,"15min":54.482999999999997},"last":0.90500000000000003},{"interface":"front","average":{"1min":2.931,"5min":2.1070000000000002,"15min":2.742},"min":{"1min":0.45100000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.8639999999999999,"5min":16.954999999999998,"15min":57.753},"last":0.98999999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:23:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.7529999999999999,"15min":2.4550000000000001},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.1829999999999998,"5min":7.9880000000000004,"15min":56.411999999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":2.032,"5min":1.845,"15min":2.1840000000000002},"min":{"1min":0.45400000000000001,"5min":0.38600000000000001,"15min":0.36499999999999999},"max":{"1min":11.175000000000001,"5min":11.175000000000001,"15min":54.451999999999998},"last":0.82799999999999996}]}]},{"osd":2,"up_from":217,"seq":932007903453,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":677288,"kb_used_data":5984,"kb_used_omap":83,"kb_used_meta":671212,"kb_avail":93067864,"statfs":{"total":95995035648,"available":95301492736,"internally_reserved":0,"allocated":6127616,"data_stored":4442351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":687321583},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:23:13 2023","interfaces":[{"interface":"back","average":{"1min":3.847,"5min":2.129,"15min":2.032},"min":{"1min":0.20899999999999999,"5min":0.19,"15min":0.185},"max":{"1min":48.990000000000002,"5min":48.990000000000002,"15min":48.990000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":4.1230000000000002,"5min":2.3010000000000002,"15min":2.6880000000000002},"min":{"1min":0.24199999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":49.412999999999997,"5min":49.412999999999997,"15min":49.412999999999997},"last":0.26800000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.26,"15min":2.194},"min":{"1min":0.26500000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.2010000000000005,"5min":8.2010000000000005,"15min":34.893999999999998},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.2789999999999999,"15min":2.335},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":6.9710000000000001,"5min":6.9710000000000001,"15min":34.962000000000003},"last":0.39800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.2310000000000001,"15min":1.121},"min":{"1min":0.27700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":5.3010000000000002,"5min":7.54,"15min":35.043999999999997},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.089,"5min":0.92000000000000004,"15min":1.2030000000000001},"min":{"1min":0.32700000000000001,"5min":0.246,"15min":0.246},"max":{"1min":5.181,"5min":6.617,"15min":50.406999999999996},"last":0.38100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:23:07 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":1.155,"15min":1.5},"min":{"1min":0.34100000000000003,"5min":0.29799999999999999,"15min":0.26600000000000001},"max":{"1min":5.351,"5min":6.4349999999999996,"15min":18.873999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.575,"5min":1.355,"15min":1.6619999999999999},"min":{"1min":0.32700000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.7050000000000001,"5min":9.0069999999999997,"15min":24.352},"last":0.41399999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4569999999999999,"5min":1.651,"15min":2.1070000000000002},"min":{"1min":0.39200000000000002,"5min":0.32700000000000001,"15min":0.30199999999999999},"max":{"1min":10.448,"5min":10.448,"15min":57.298000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.73,"15min":1.948},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":8.1630000000000003,"5min":8.1630000000000003,"15min":57.828000000000003},"last":0.47099999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:23:13 2023","interfaces":[{"interface":"back","average":{"1min":3.3460000000000001,"5min":1.9430000000000001,"15min":2.0659999999999998},"min":{"1min":0.40799999999999997,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":12.411,"5min":12.411,"15min":18.943999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.8160000000000001,"15min":2.0470000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.36599999999999999},"max":{"1min":8.8689999999999998,"5min":9.8399999999999999,"15min":24.379000000000001},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:23:13 2023","interfaces":[{"interface":"back","average":{"1min":3.532,"5min":2.1699999999999999,"15min":2.331},"min":{"1min":0.47499999999999998,"5min":0.41699999999999998,"15min":0.308},"max":{"1min":11.704000000000001,"5min":13.821,"15min":35.008000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.0139999999999998,"5min":1.4910000000000001,"15min":2.1930000000000001},"min":{"1min":0.42799999999999999,"5min":0.41799999999999998,"15min":0.35899999999999999},"max":{"1min":6.1459999999999999,"5min":6.6399999999999997,"15min":24.323},"last":0.50700000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488293,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":300696,"kb_used_data":14416,"kb_used_omap":15,"kb_used_meta":286256,"kb_avail":93444456,"statfs":{"total":95995035648,"available":95687122944,"internally_reserved":0,"allocated":14761984,"data_stored":12632662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":293126598},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.7909999999999999,"15min":1.3100000000000001},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":3.0209999999999999,"5min":65.674999999999997,"15min":65.674999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.81699999999999995,"5min":1.78,"15min":1.288},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.19900000000000001},"max":{"1min":5.7519999999999998,"5min":21.149999999999999,"15min":21.149999999999999},"last":0.26700000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":3.052,"15min":1.8169999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6779999999999999,"5min":105.401,"15min":105.401},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.80000000000000004,"5min":3.165,"15min":1.5880000000000001},"min":{"1min":0.32900000000000001,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":2.8010000000000002,"5min":104.914,"15min":104.914},"last":0.99199999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":1.8520000000000001,"15min":1.9610000000000001},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":9.4369999999999994,"5min":9.4369999999999994,"15min":9.4369999999999994},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.109,"5min":1.383,"15min":1.298},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":3.8359999999999999,"5min":25.338999999999999,"15min":25.338999999999999},"last":0.51300000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":2.3069999999999999,"15min":1.2649999999999999},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":2.2629999999999999,"5min":21.831,"15min":21.831},"last":1.095},{"interface":"front","average":{"1min":2.8490000000000002,"5min":2.8300000000000001,"15min":1.8280000000000001},"min":{"1min":0.379,"5min":0.35699999999999998,"15min":0.30199999999999999},"max":{"1min":9.2599999999999998,"5min":22.097999999999999,"15min":22.097999999999999},"last":0.71199999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.782,"15min":1.157},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":2.629,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.617,"5min":2.4129999999999998,"15min":1.361},"min":{"1min":0.45400000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8099999999999996,"5min":13.394,"15min":13.394},"last":1.0509999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":2.1669999999999998,"15min":1.6419999999999999},"min":{"1min":0.41299999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.3719999999999999,"5min":19.114000000000001,"15min":19.114000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.262,"5min":1.966,"15min":1.3300000000000001},"min":{"1min":0.434,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":3.4209999999999998,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":2.2799999999999998,"15min":1.371},"min":{"1min":0.40999999999999998,"5min":0.39900000000000002,"15min":0.38400000000000001},"max":{"1min":4.6790000000000003,"5min":9.7880000000000003,"15min":10.459},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.744,"5min":2.1760000000000002,"15min":1.7130000000000001},"min":{"1min":0.443,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":11.413,"5min":19.280000000000001,"15min":19.280000000000001},"last":0.57599999999999996}]}]},{"osd":4,"up_from":266,"seq":1142461300923,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1308240,"kb_used_data":379208,"kb_used_omap":35,"kb_used_meta":928988,"kb_avail":92436912,"statfs":{"total":95995035648,"available":94655397888,"internally_reserved":0,"allocated":388308992,"data_stored":387479756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":951283950},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":3.8260000000000001,"15min":2.0110000000000001},"min":{"1min":0.222,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":6.5789999999999997,"5min":97.721999999999994,"15min":97.721999999999994},"last":0.66000000000000003},{"interface":"front","average":{"1min":2.2450000000000001,"5min":4.6870000000000003,"15min":2.6709999999999998},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":5.6239999999999997,"5min":97.915999999999997,"15min":97.915999999999997},"last":0.442}]},{"osd":1,"last update":"Mon Oct 9 10:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":3.1179999999999999,"15min":2.2469999999999999},"min":{"1min":0.32300000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.375,"5min":71.457999999999998,"15min":71.457999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.0059999999999998,"5min":2.5579999999999998,"15min":1.9550000000000001},"min":{"1min":0.36299999999999999,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":7.9210000000000003,"5min":71.658000000000001,"15min":71.658000000000001},"last":0.621}]},{"osd":2,"last update":"Mon Oct 9 10:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.8700000000000001,"15min":1.663},"min":{"1min":0.32500000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.9530000000000003,"5min":7.9530000000000003,"15min":9.2469999999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.8799999999999999,"5min":1.7929999999999999,"15min":1.6639999999999999},"min":{"1min":0.317,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":6.5469999999999997,"5min":11.962999999999999,"15min":11.962999999999999},"last":0.69999999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:23:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":2.5299999999999998,"15min":1.885},"min":{"1min":0.32300000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.093999999999999,"5min":12.074,"15min":12.074},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.4489999999999998,"5min":2.488,"15min":1.53},"min":{"1min":0.33800000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.067,"5min":16.067,"15min":16.067},"last":0.78400000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.609,"5min":2.3999999999999999,"15min":1.8080000000000001},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.23799999999999999},"max":{"1min":8.3849999999999998,"5min":14.162000000000001,"15min":99.585999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.3519999999999999,"5min":1.929,"15min":1.484},"min":{"1min":0.36199999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.633,"5min":11.994,"15min":99.384},"last":0.92700000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.9670000000000001,"15min":1.2290000000000001},"min":{"1min":0.35399999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.9710000000000001,"5min":12.019,"15min":12.019},"last":0.83599999999999997},{"interface":"front","average":{"1min":2.7469999999999999,"5min":2.419,"15min":1.464},"min":{"1min":0.36399999999999999,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":8.2560000000000002,"5min":13.016,"15min":13.016},"last":0.45600000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:23:02 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":2.548,"15min":1.5589999999999999},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.5700000000000003,"5min":17.369,"15min":17.369},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.9670000000000001,"5min":2.0960000000000001,"15min":1.4119999999999999},"min":{"1min":0.379,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":11.550000000000001,"5min":11.550000000000001,"15min":14.964},"last":0.45100000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823107,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909880,"kb_used_data":4464,"kb_used_omap":0,"kb_used_meta":905408,"kb_avail":92835272,"statfs":{"total":95995035648,"available":95063318528,"internally_reserved":0,"allocated":4571136,"data_stored":3749832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927137792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8359999999999999,"5min":2.2890000000000001,"15min":1.8480000000000001},"min":{"1min":0.35899999999999999,"5min":0.23899999999999999,"15min":0.21099999999999999},"max":{"1min":10.254,"5min":22.469999999999999,"15min":22.469999999999999},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.5669999999999999,"5min":2.0619999999999998,"15min":1.6739999999999999},"min":{"1min":0.309,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":8.7929999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.85899999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:23:36 2023","interfaces":[{"interface":"back","average":{"1min":2.7050000000000001,"5min":2.637,"15min":2.1280000000000001},"min":{"1min":0.432,"5min":0.28999999999999998,"15min":0.23999999999999999},"max":{"1min":7.3970000000000002,"5min":29.370000000000001,"15min":29.370000000000001},"last":0.82299999999999995},{"interface":"front","average":{"1min":2.04,"5min":2.2730000000000001,"15min":1.909},"min":{"1min":0.371,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.859,"5min":29.065000000000001,"15min":29.065000000000001},"last":0.876}]},{"osd":2,"last update":"Mon Oct 9 10:23:18 2023","interfaces":[{"interface":"back","average":{"1min":3.8319999999999999,"5min":2.2069999999999999,"15min":2.0179999999999998},"min":{"1min":0.46899999999999997,"5min":0.28999999999999998,"15min":0.27200000000000002},"max":{"1min":13.965999999999999,"5min":13.965999999999999,"15min":15.973000000000001},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.8959999999999999,"5min":2.0219999999999998,"15min":1.667},"min":{"1min":0.34499999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.8859999999999992,"5min":13.095000000000001,"15min":15.929},"last":0.70099999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":2.3479999999999999,"15min":1.7370000000000001},"min":{"1min":0.32300000000000001,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":7.04,"5min":10.561999999999999,"15min":10.561999999999999},"last":0.79400000000000004},{"interface":"front","average":{"1min":2.1200000000000001,"5min":2.0870000000000002,"15min":1.7010000000000001},"min":{"1min":0.48599999999999999,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9169999999999998,"5min":13.624000000000001,"15min":13.624000000000001},"last":0.81100000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":2.355,"15min":1.4830000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":12.023,"5min":12.023,"15min":12.023},"last":0.434},{"interface":"front","average":{"1min":2.5739999999999998,"5min":2.2650000000000001,"15min":1.333},"min":{"1min":0.28899999999999998,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":8.8350000000000009,"5min":11.956,"15min":11.956},"last":0.65600000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":2.653,"15min":1.6459999999999999},"min":{"1min":0.38900000000000001,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":7.069,"5min":12.772,"15min":12.772},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.887,"5min":2.1200000000000001,"15min":1.597},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.26300000000000001},"max":{"1min":7.3609999999999998,"5min":10.403,"15min":15.997},"last":0.56399999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:23:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1909999999999998,"5min":2.4129999999999998,"15min":1.6599999999999999},"min":{"1min":0.47699999999999998,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":21.109000000000002,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.609,"5min":1.7030000000000001,"15min":1.115},"min":{"1min":0.39900000000000002,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":9.0739999999999998,"5min":11.994,"15min":11.994},"last":0.51100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":290717696,"data_stored":290717696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":40042496,"data_stored":40042496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":377880576,"data_stored":377880576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:24:04.666 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:24:04.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-10-09T10:24:04.667 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:04.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:04 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 567/15717 objects degraded (3.608%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:24:04.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:04 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 567/15717 objects degraded (3.608%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:24:04.697 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:04 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 567/15717 objects degraded (3.608%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:24:04.700 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:04.748 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:04.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:04.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:04.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:04.908 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:04.955 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:04.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:04.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:05.059 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:05.129 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 5 16 14250 14234 177.735 151.5 0.00430648 0.00561877 2023-10-09T10:24:05.129 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:05.344 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:05.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:05.451 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:05.541 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:05.631 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:05 smithi012 ceph-mon[147926]: pgmap v1388: 89 pgs: 1 active+undersized+degraded, 1 active+undersized, 4 active+clean+remapped, 83 active+clean; 327 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 41 MiB/s wr, 651 op/s; 567/15717 objects degraded (3.608%); 2/15717 objects misplaced (0.013%) 2023-10-09T10:24:05.631 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:05 smithi012 ceph-mon[147926]: from='client.46874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:05.631 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:05 smithi012 ceph-mon[152263]: pgmap v1388: 89 pgs: 1 active+undersized+degraded, 1 active+undersized, 4 active+clean+remapped, 83 active+clean; 327 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 41 MiB/s wr, 651 op/s; 567/15717 objects degraded (3.608%); 2/15717 objects misplaced (0.013%) 2023-10-09T10:24:05.632 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:05 smithi012 ceph-mon[152263]: from='client.46874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:05.776 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:05.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:05.882 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:05.918 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:05 smithi110 ceph-mon[136032]: pgmap v1388: 89 pgs: 1 active+undersized+degraded, 1 active+undersized, 4 active+clean+remapped, 83 active+clean; 327 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 41 MiB/s wr, 651 op/s; 567/15717 objects degraded (3.608%); 2/15717 objects misplaced (0.013%) 2023-10-09T10:24:05.918 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:05 smithi110 ceph-mon[136032]: from='client.46874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:05.926 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:05.935 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:24:05.935 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:24:05.935 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.935 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.36870.0:16208 11.4 e498/494)", 2023-10-09T10:24:05.935 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:05.926967+0000", 2023-10-09T10:24:05.935 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00056747700000000004, 2023-10-09T10:24:05.935 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00059422700000000001, 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926967+0000", 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926967+0000", 2023-10-09T10:24:05.936 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926967+0000", 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926970+0000", 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7499999999999999e-06 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.937 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926977+0000", 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.306e-06 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926982+0000", 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6450000000000004e-06 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.938 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop_reply(client.36870.0:16208 11.4 e498/494)", 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:05.927384+0000", 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.00015122099999999999, 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023749400000000001, 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927384+0000", 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.939 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927384+0000", 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927383+0000", 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.940 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927388+0000", 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8319999999999996e-06 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927402+0000", 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4113e-05 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:05.941 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927413+0000", 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1282e-05 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:16214 11.4 11.d5772454 (undecoded) ondisk+write+known_if_redirected e498)", 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:05.927549+0000", 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9999862, 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128949, 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:24:05.942 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16214 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927549+0000", 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:05.943 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927549+0000", 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927547+0000", 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927571+0000", 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3224999999999999e-05 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.944 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927587+0000", 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6184999999999998e-05 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927596+0000", 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7709999999999995e-06 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.945 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:16219 11.4 11.8798cf94 (undecoded) ondisk+write+known_if_redirected e498)", 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:05.927604+0000", 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 4294967295.9999304, 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000140951, 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16219 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.946 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927604+0000", 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927604+0000", 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.947 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927604+0000", 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927622+0000", 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7472000000000001e-05 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927629+0000", 2023-10-09T10:24:05.948 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.221e-06 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.927637+0000", 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.67e-06 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:16208 11.4 11:24104704:::benchmark_data_smithi012_404402_object16207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e498)", 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:05.923723+0000", 2023-10-09T10:24:05.949 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0038113380000000001, 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040936890000000002, 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16208 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:05.950 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923723+0000", 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923723+0000", 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923723+0000", 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:05.951 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923745+0000", 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2648999999999999e-05 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923756+0000", 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0465999999999999e-05 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:05.952 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923762+0000", 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4500000000000001e-06 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923827+0000", 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4617999999999997e-05 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923975+0000", 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000148054 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.953 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.924108+0000", 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000132967 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926706+0000", 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025982150000000001 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.954 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:16209 11.0 11:0fb5012f:::benchmark_data_smithi012_404402_object16208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e498)", 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:05.923777+0000", 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0037577159999999999, 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041386840000000001, 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16209 2023-10-09T10:24:05.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923777+0000", 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923777+0000", 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923776+0000", 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923798+0000", 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1854000000000001e-05 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:05.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923806+0000", 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9070000000000007e-06 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923812+0000", 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9320000000000001e-06 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.924187+0000", 2023-10-09T10:24:05.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00037512100000000002 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.924323+0000", 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135804 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.924461+0000", 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137517 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926763+0000", 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023024769999999998 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:16212 11.b 11:d35e3733:::benchmark_data_smithi012_404402_object16211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e498)", 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:05.923867+0000", 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0036680480000000001, 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041414329999999999, 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:24:05.960 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 16212 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923867+0000", 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.961 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923867+0000", 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923866+0000", 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923888+0000", 2023-10-09T10:24:05.962 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1404999999999999e-05 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923895+0000", 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5800000000000003e-06 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.923901+0000", 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8100000000000003e-06 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.963 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.925000+0000", 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001098578 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.925131+0000", 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013135499999999999 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.925278+0000", 2023-10-09T10:24:05.964 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014738300000000001 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:05.926899+0000", 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016202670000000001 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 7 2023-10-09T10:24:05.965 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:24:05.973 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:05.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:05.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:06.079 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:06.113 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:06.128 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 6 16 16614 16598 172.739 147.75 0.00518729 0.00578078 2023-10-09T10:24:06.159 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:06.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:06.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:06.307 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:06.360 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:06.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:06.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:06.450 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:06.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:24:06.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:06.473 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:06.500 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:06.544 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:06.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:06.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:06.649 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:06.690 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:06.742 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:06.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:06.848 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:06.885 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:06.938 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:06.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:07.045 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:07.086 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:07.128 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 7 16 18971 18955 169.107 147.312 0.00811014 0.00590793 2023-10-09T10:24:07.130 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:07.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:07.235 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:07.276 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:07.330 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:07.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:07.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:07.435 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:07.480 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:24:07.481 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:07.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:07 smithi012 ceph-mon[147926]: pgmap v1389: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 658 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 1427/31626 objects degraded (4.512%); 2/31626 objects misplaced (0.006%) 2023-10-09T10:24:07.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3585799360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:07.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:07.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:07 smithi012 ceph-mon[152263]: pgmap v1389: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 658 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 1427/31626 objects degraded (4.512%); 2/31626 objects misplaced (0.006%) 2023-10-09T10:24:07.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3585799360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:07.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:07.531 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:07.535 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:07 smithi110 ceph-mon[136032]: pgmap v1389: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 658 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 1427/31626 objects degraded (4.512%); 2/31626 objects misplaced (0.006%) 2023-10-09T10:24:07.535 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3585799360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:07.535 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:07.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:07.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:07.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:07.715 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:08.130 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 8 16 21396 21380 166.887 151.562 0.0152287 0.00598434 2023-10-09T10:24:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:24:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:24:07] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.33.4" 2023-10-09T10:24:08.450 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:08.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:08.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:08.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:08.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[152263]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[152263]: osdmap e499: 8 total, 8 up, 5 in 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[152263]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[152263]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[152263]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[147926]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:24:08.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[147926]: osdmap e499: 8 total, 8 up, 5 in 2023-10-09T10:24:08.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[147926]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[147926]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:08 smithi012 ceph-mon[147926]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.632 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:08.700 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:08.702 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:08.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:08.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:08.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:08.882 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:08 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:08 smithi110 ceph-mon[136032]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:24:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:08 smithi110 ceph-mon[136032]: osdmap e499: 8 total, 8 up, 5 in 2023-10-09T10:24:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:08 smithi110 ceph-mon[136032]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:08 smithi110 ceph-mon[136032]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:08 smithi110 ceph-mon[136032]: failed to encode map e499 with expected crc 2023-10-09T10:24:08.958 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:08.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:09.054 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:24:09.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:09.104 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:24:09.129 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 9 16 23340 23324 161.846 121.5 0.0022025 0.00616963 2023-10-09T10:24:09.143 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:09.151 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:24:09.196 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:24:09.229 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:24:09.238 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:09.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:09.269 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:24:09.311 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:24:09.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:09.350 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:24:09.362 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:24:09.363 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:24:09.363 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:24:09.363 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:24:09.363 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:24:09.363 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:24:09.364 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:24:09.364 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:24:09.436 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:09.509 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:09.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:09.615 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:09.666 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:09 smithi012 ceph-mon[147926]: pgmap v1391: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 938 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 117 MiB/s wr, 1.87k op/s; 1427/45060 objects degraded (3.167%); 2/45060 objects misplaced (0.004%) 2023-10-09T10:24:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:09 smithi012 ceph-mon[147926]: failed to encode map e499 with expected crc 2023-10-09T10:24:09.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/612241710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:09 smithi012 ceph-mon[152263]: pgmap v1391: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 938 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 117 MiB/s wr, 1.87k op/s; 1427/45060 objects degraded (3.167%); 2/45060 objects misplaced (0.004%) 2023-10-09T10:24:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:09 smithi012 ceph-mon[152263]: failed to encode map e499 with expected crc 2023-10-09T10:24:09.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/612241710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:09.721 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:09.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:09.827 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:09.866 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:09 smithi110 ceph-mon[136032]: pgmap v1391: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 938 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 117 MiB/s wr, 1.87k op/s; 1427/45060 objects degraded (3.167%); 2/45060 objects misplaced (0.004%) 2023-10-09T10:24:09.866 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:09 smithi110 ceph-mon[136032]: failed to encode map e499 with expected crc 2023-10-09T10:24:09.866 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/612241710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:09.868 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:09.921 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:09.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:09.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:10.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:10.090 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:10.129 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 10 16 25361 25345 158.294 126.312 0.00723817 0.00631177 2023-10-09T10:24:10.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:24:10.156 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:10.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:10.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:10.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:10.372 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:10.443 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:10.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:10.536 INFO:teuthology.orchestra.run.smithi012.stderr:set pool 11 pg_num to 26 2023-10-09T10:24:10.553 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:10.571 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1864501650' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:24:10.571 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:10 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:24:10.572 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1864501650' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:24:10.572 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:10 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:24:10.590 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:10.639 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:10.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:10.676 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:24:10.676 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:24:10.746 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:10.781 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1864501650' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:24:10.781 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:10 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:24:10.785 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:10.833 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:10.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:10.938 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:10.995 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:11.113 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:11.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:11.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:11.130 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 11 16 27305 27289 154.945 121.5 0.00557893 0.00644891 2023-10-09T10:24:11.219 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:11.268 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:11.324 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:11.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:11.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:11.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:24:11.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:11.509 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:11.512 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:11 smithi110 ceph-mon[136032]: pgmap v1392: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 1.1 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 115 MiB/s wr, 1.84k op/s; 2163/54048 objects degraded (4.002%); 2/54048 objects misplaced (0.004%) 2023-10-09T10:24:11.513 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:11 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 2163/54048 objects degraded (4.002%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:11.513 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:11 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:24:11.513 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:11 smithi110 ceph-mon[136032]: osdmap e500: 8 total, 8 up, 5 in 2023-10-09T10:24:11.513 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:11 smithi110 ceph-mon[136032]: failed to encode map e500 with expected crc 2023-10-09T10:24:11.569 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:11.579 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:11.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[147926]: pgmap v1392: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 1.1 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 115 MiB/s wr, 1.84k op/s; 2163/54048 objects degraded (4.002%); 2/54048 objects misplaced (0.004%) 2023-10-09T10:24:11.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 2163/54048 objects degraded (4.002%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:11.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:24:11.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[147926]: osdmap e500: 8 total, 8 up, 5 in 2023-10-09T10:24:11.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[147926]: failed to encode map e500 with expected crc 2023-10-09T10:24:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[152263]: pgmap v1392: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 1.1 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 115 MiB/s wr, 1.84k op/s; 2163/54048 objects degraded (4.002%); 2/54048 objects misplaced (0.004%) 2023-10-09T10:24:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 2163/54048 objects degraded (4.002%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:24:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[152263]: osdmap e500: 8 total, 8 up, 5 in 2023-10-09T10:24:11.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:11 smithi012 ceph-mon[152263]: failed to encode map e500 with expected crc 2023-10-09T10:24:11.695 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:11.741 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:11.795 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:11.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:11.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:11.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:11.975 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:12.114 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:12.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:12.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:12.130 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 12 16 29256 29240 152.195 121.938 0.00836498 0.00656486 2023-10-09T10:24:12.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:24:12.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:12.311 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:12.435 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:12.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:12.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:12.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:12.655 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:12 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:24:12.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:24:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:12 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:24:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:24:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:12 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:24:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:24:13.130 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 13 16 30007 29991 144.102 46.9375 0.108495 0.00663574 2023-10-09T10:24:13.588 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:13.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:13.695 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:13.741 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:13.801 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:13.805 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:13 smithi110 ceph-mon[136032]: pgmap v1394: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 132 MiB/s wr, 2.11k op/s; 2826/66309 objects degraded (4.262%); 2/66309 objects misplaced (0.003%) 2023-10-09T10:24:13.805 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:13 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:24:13.806 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:13 smithi110 ceph-mon[136032]: osdmap e501: 8 total, 8 up, 5 in 2023-10-09T10:24:13.806 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:13 smithi110 ceph-mon[136032]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.806 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:13 smithi110 ceph-mon[136032]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.806 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:13 smithi110 ceph-mon[136032]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:13.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:13.854 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:13.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:13.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[147926]: pgmap v1394: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 132 MiB/s wr, 2.11k op/s; 2826/66309 objects degraded (4.262%); 2/66309 objects misplaced (0.003%) 2023-10-09T10:24:13.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:24:13.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[147926]: osdmap e501: 8 total, 8 up, 5 in 2023-10-09T10:24:13.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[147926]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[147926]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[147926]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[152263]: pgmap v1394: 89 pgs: 2 active+undersized+degraded, 4 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 132 MiB/s wr, 2.11k op/s; 2826/66309 objects degraded (4.262%); 2/66309 objects misplaced (0.003%) 2023-10-09T10:24:13.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:24:13.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[152263]: osdmap e501: 8 total, 8 up, 5 in 2023-10-09T10:24:13.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[152263]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[152263]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:13 smithi012 ceph-mon[152263]: failed to encode map e501 with expected crc 2023-10-09T10:24:13.938 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:13.992 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:13.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:13.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:14.096 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:14.130 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 14 16 31107 31091 138.722 68.75 0.00382101 0.00720264 2023-10-09T10:24:14.137 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:14.195 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:14.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:14.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:14.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:14.380 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[152263]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[152263]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[152263]: osdmap e502: 8 total, 8 up, 5 in 2023-10-09T10:24:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[152263]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[152263]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:24:14.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:14.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[147926]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[147926]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[147926]: osdmap e502: 8 total, 8 up, 5 in 2023-10-09T10:24:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[147926]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[147926]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:24:14.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:14.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:14.758 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:14.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:14.864 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:14.925 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:14 smithi110 ceph-mon[136032]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:14 smithi110 ceph-mon[136032]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:14 smithi110 ceph-mon[136032]: osdmap e502: 8 total, 8 up, 5 in 2023-10-09T10:24:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:14 smithi110 ceph-mon[136032]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:14 smithi110 ceph-mon[136032]: failed to encode map e502 with expected crc 2023-10-09T10:24:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:24:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:14.994 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:15.000 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:15.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:15.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:15.104 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:15.131 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 15 16 33042 33026 137.526 120.938 0.00584683 0.00726651 2023-10-09T10:24:15.145 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:15.453 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:15.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:15.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:15.558 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:15.570 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:24:15.599 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:15.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:15 smithi012 ceph-mon[147926]: pgmap v1397: 99 pgs: 5 peering, 10 unknown, 2 active+undersized+degraded, 4 active+clean+remapped, 78 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 127 MiB/s wr, 2.03k op/s; 3367/81585 objects degraded (4.127%); 2/81585 objects misplaced (0.002%) 2023-10-09T10:24:15.653 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/259992403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:15.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:15 smithi012 ceph-mon[152263]: pgmap v1397: 99 pgs: 5 peering, 10 unknown, 2 active+undersized+degraded, 4 active+clean+remapped, 78 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 127 MiB/s wr, 2.03k op/s; 3367/81585 objects degraded (4.127%); 2/81585 objects misplaced (0.002%) 2023-10-09T10:24:15.655 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/259992403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:15.656 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:15.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:15.761 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:15.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:15 smithi110 ceph-mon[136032]: pgmap v1397: 99 pgs: 5 peering, 10 unknown, 2 active+undersized+degraded, 4 active+clean+remapped, 78 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 127 MiB/s wr, 2.03k op/s; 3367/81585 objects degraded (4.127%); 2/81585 objects misplaced (0.002%) 2023-10-09T10:24:15.811 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/259992403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:15.813 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:15.867 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:15.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:15.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:15.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:16.022 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4] out_osds: [1, 6, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:24:16.022 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:24:16.023 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T10:24:16.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:24:16.061 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:16.132 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 16 16 35147 35131 137.152 131.562 0.00567963 0.00728357 2023-10-09T10:24:16.134 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:16.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:16.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:16.254 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:16.297 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:16.351 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:16.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:16.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:16.462 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:24:16.462 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:24:16.500 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:16.546 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:16.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:16.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:16.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:16.714 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:16.835 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:16.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:16.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:16.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/259992403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:16.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[147926]: osdmap e503: 8 total, 8 up, 5 in 2023-10-09T10:24:16.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[147926]: failed to encode map e503 with expected crc 2023-10-09T10:24:16.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[147926]: failed to encode map e503 with expected crc 2023-10-09T10:24:16.841 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:16.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/259992403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:16.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[152263]: osdmap e503: 8 total, 8 up, 5 in 2023-10-09T10:24:16.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[152263]: failed to encode map e503 with expected crc 2023-10-09T10:24:16.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:16 smithi012 ceph-mon[152263]: failed to encode map e503 with expected crc 2023-10-09T10:24:16.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:16 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:16 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/259992403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:16 smithi110 ceph-mon[136032]: osdmap e503: 8 total, 8 up, 5 in 2023-10-09T10:24:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:16 smithi110 ceph-mon[136032]: failed to encode map e503 with expected crc 2023-10-09T10:24:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:16 smithi110 ceph-mon[136032]: failed to encode map e503 with expected crc 2023-10-09T10:24:16.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:24:16.999 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:17.132 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 17 16 37009 36993 135.93 116.375 0.00474268 0.00735085 2023-10-09T10:24:17.147 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:17.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:17.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:17.203 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:17.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:17.369 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:17.500 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:17.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:17.608 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:17.656 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:17.659 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:24:17.697 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:24:17.699 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:17.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:17.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:17.735 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:24:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:17 smithi012 ceph-mon[147926]: pgmap v1399: 99 pgs: 5 peering, 5 unknown, 3 active+undersized+degraded, 4 active+clean+remapped, 82 active+clean; 1.8 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s; 3878/90255 objects degraded (4.297%); 2/90255 objects misplaced (0.002%) 2023-10-09T10:24:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:17 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 3878/90255 objects degraded (4.297%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:17.779 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:17 smithi012 ceph-mon[152263]: pgmap v1399: 99 pgs: 5 peering, 5 unknown, 3 active+undersized+degraded, 4 active+clean+remapped, 82 active+clean; 1.8 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s; 3878/90255 objects degraded (4.297%); 2/90255 objects misplaced (0.002%) 2023-10-09T10:24:17.779 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:17 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 3878/90255 objects degraded (4.297%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:17.780 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:24:17.804 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:17.817 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:24:17.835 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:17 smithi110 ceph-mon[136032]: pgmap v1399: 99 pgs: 5 peering, 5 unknown, 3 active+undersized+degraded, 4 active+clean+remapped, 82 active+clean; 1.8 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s; 3878/90255 objects degraded (4.297%); 2/90255 objects misplaced (0.002%) 2023-10-09T10:24:17.836 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:17 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 3878/90255 objects degraded (4.297%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:17.839 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:17.851 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:24:17.887 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:17.889 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:24:17.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:17.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:17.928 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:24:17.950 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:24:17.950 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:24:17.950 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:24:17.950 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:24:17.950 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:24:17.950 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:24:17.951 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:24:17.951 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:24:17.996 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:18.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:24:18.059 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:24:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:24:17] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.33.4" 2023-10-09T10:24:18.067 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:18.132 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 18 16 38633 38617 134.017 101.5 0.0038306 0.00745635 2023-10-09T10:24:18.154 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:18.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:18.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:18.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:18.310 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:18.362 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:18.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:18.469 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:18.513 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:18.566 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:18.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:18.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:18.671 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:18.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2711664797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:18.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2711664797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:18.723 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:18.784 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:18.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:18.833 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2711664797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:18.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:18.936 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:18.981 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:19.077 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:19.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:19.134 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 19 16 40594 40578 133.403 122.562 0.015219 0.00749081 2023-10-09T10:24:19.186 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:19.222 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:19.280 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:19.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:19.385 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:19.464 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:19.878 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:19 smithi012 ceph-mon[147926]: pgmap v1400: 99 pgs: 1 peering, 1 unknown, 3 active+undersized+degraded, 4 active+clean+remapped, 90 active+clean; 2.0 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s; 3878/98817 objects degraded (3.924%); 2/98817 objects misplaced (0.002%); 11 KiB/s, 0 objects/s recovering 2023-10-09T10:24:19.879 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:19 smithi012 ceph-mon[152263]: pgmap v1400: 99 pgs: 1 peering, 1 unknown, 3 active+undersized+degraded, 4 active+clean+remapped, 90 active+clean; 2.0 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s; 3878/98817 objects degraded (3.924%); 2/98817 objects misplaced (0.002%); 11 KiB/s, 0 objects/s recovering 2023-10-09T10:24:19.892 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:24:19.894 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:19.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:19.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:19.899 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:24:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:19 smithi110 ceph-mon[136032]: pgmap v1400: 99 pgs: 1 peering, 1 unknown, 3 active+undersized+degraded, 4 active+clean+remapped, 90 active+clean; 2.0 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s; 3878/98817 objects degraded (3.924%); 2/98817 objects misplaced (0.002%); 11 KiB/s, 0 objects/s recovering 2023-10-09T10:24:20.000 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:20.038 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:20.096 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:20.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:20.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:20.134 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:24:20.134207+0000 min lat: 0.00117322 max lat: 1.03449 avg lat: 0.0075524 2023-10-09T10:24:20.134 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:24:20.134 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 20 16 42380 42364 132.313 111.625 0.00572985 0.0075524 2023-10-09T10:24:20.200 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:20.262 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:20.448 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:24:21.136 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 21 16 44409 44393 132.037 126.812 0.00297945 0.00756921 2023-10-09T10:24:21.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:21 smithi012 ceph-mon[147926]: pgmap v1401: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 121 MiB/s wr, 1.94k op/s; 4396/108141 objects degraded (4.065%); 2/108141 objects misplaced (0.002%); 8.9 KiB/s, 0 objects/s recovering 2023-10-09T10:24:21.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:21 smithi012 ceph-mon[147926]: from='client.56920 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:21.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:21 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:21.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:21 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:21.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:21 smithi012 ceph-mon[152263]: pgmap v1401: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 121 MiB/s wr, 1.94k op/s; 4396/108141 objects degraded (4.065%); 2/108141 objects misplaced (0.002%); 8.9 KiB/s, 0 objects/s recovering 2023-10-09T10:24:21.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:21 smithi012 ceph-mon[152263]: from='client.56920 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:21.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:21 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:21.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:21 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:21 smithi110 ceph-mon[136032]: pgmap v1401: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 121 MiB/s wr, 1.94k op/s; 4396/108141 objects degraded (4.065%); 2/108141 objects misplaced (0.002%); 8.9 KiB/s, 0 objects/s recovering 2023-10-09T10:24:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:21 smithi110 ceph-mon[136032]: from='client.56920 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:21 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:21 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:22.136 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 22 16 46503 46487 131.982 130.875 0.00309382 0.00757182 2023-10-09T10:24:22.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:22 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4999/118674 objects degraded (4.212%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:22.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:22 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4999/118674 objects degraded (4.212%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:22 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4999/118674 objects degraded (4.212%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:22.696 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:22.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:22.804 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:22.861 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:23.137 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 23 16 48622 48606 132.001 132.438 0.00703536 0.00757088 2023-10-09T10:24:23.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:23 smithi012 ceph-mon[147926]: pgmap v1402: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 4999/118674 objects degraded (4.212%); 2/118674 objects misplaced (0.002%); 7.8 KiB/s, 0 objects/s recovering 2023-10-09T10:24:23.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:23 smithi012 ceph-mon[152263]: pgmap v1402: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 4999/118674 objects degraded (4.212%); 2/118674 objects misplaced (0.002%); 7.8 KiB/s, 0 objects/s recovering 2023-10-09T10:24:23.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:23 smithi110 ceph-mon[136032]: pgmap v1402: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 4999/118674 objects degraded (4.212%); 2/118674 objects misplaced (0.002%); 7.8 KiB/s, 0 objects/s recovering 2023-10-09T10:24:23.606 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:23.674 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1401,"stamp":"2023-10-09T10:24:19.744105+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2361399200,"num_objects":36047,"num_object_clones":0,"num_object_copies":108141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4396,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":36047,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":36283,"num_write_kb":2307917,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":1863594,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":54983,"ondisk_log_size":54983,"up":290,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":290,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":10215340,"kb_used_data":6581892,"kb_used_omap":163,"kb_used_meta":3633244,"kb_avail":458510420,"statfs":{"total":479975178240,"available":469514670080,"internally_reserved":0,"allocated":6739857408,"data_stored":6732499059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":3720442867},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":913833984,"num_objects":13944,"num_object_clones":0,"num_object_copies":41832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13944,"num_write_kb":892416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.204003"},"pg_stats":[{"pgid":"11.12","version":"503'2123","reported_seq":2150,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.637253+0000","last_change":"2023-10-09T10:24:13.569654+0000","last_active":"2023-10-09T10:24:17.637253+0000","last_peered":"2023-10-09T10:24:17.637253+0000","last_clean":"2023-10-09T10:24:17.637253+0000","last_became_active":"2023-10-09T10:24:13.568512+0000","last_became_peered":"2023-10-09T10:24:13.568512+0000","last_unstale":"2023-10-09T10:24:17.637253+0000","last_undegraded":"2023-10-09T10:24:17.637253+0000","last_fullsized":"2023-10-09T10:24:17.637253+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2123,"log_dups_size":0,"ondisk_log_size":2123,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76972032,"num_objects":1175,"num_object_clones":0,"num_object_copies":3525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1175,"num_write_kb":75168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":96,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537272+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:24:13.537272+0000","last_peered":"2023-10-09T10:24:13.537272+0000","last_clean":"2023-10-09T10:24:13.537272+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:24:13.537272+0000","last_undegraded":"2023-10-09T10:24:13.537272+0000","last_fullsized":"2023-10-09T10:24:13.537272+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":448,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537872+0000","last_change":"2023-10-09T10:22:27.502142+0000","last_active":"2023-10-09T10:24:13.537872+0000","last_peered":"2023-10-09T10:24:13.537872+0000","last_clean":"2023-10-09T10:24:13.537872+0000","last_became_active":"2023-10-09T10:22:27.501338+0000","last_became_peered":"2023-10-09T10:22:27.501338+0000","last_unstale":"2023-10-09T10:24:13.537872+0000","last_undegraded":"2023-10-09T10:24:13.537872+0000","last_fullsized":"2023-10-09T10:24:13.537872+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"503'2111","reported_seq":2139,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.580225+0000","last_change":"2023-10-09T10:24:13.569890+0000","last_active":"2023-10-09T10:24:17.580225+0000","last_peered":"2023-10-09T10:24:17.580225+0000","last_clean":"2023-10-09T10:24:17.580225+0000","last_became_active":"2023-10-09T10:24:13.568636+0000","last_became_peered":"2023-10-09T10:24:13.568636+0000","last_unstale":"2023-10-09T10:24:17.580225+0000","last_undegraded":"2023-10-09T10:24:17.580225+0000","last_fullsized":"2023-10-09T10:24:17.580225+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2111,"log_dups_size":0,"ondisk_log_size":2111,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76414976,"num_objects":1166,"num_object_clones":0,"num_object_copies":3498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1166,"num_write_kb":74624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":525,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537235+0000","last_change":"2023-10-09T10:23:25.445976+0000","last_active":"2023-10-09T10:24:13.537235+0000","last_peered":"2023-10-09T10:24:13.537235+0000","last_clean":"2023-10-09T10:24:13.537235+0000","last_became_active":"2023-10-09T10:23:25.424379+0000","last_became_peered":"2023-10-09T10:23:25.424379+0000","last_unstale":"2023-10-09T10:24:13.537235+0000","last_undegraded":"2023-10-09T10:24:13.537235+0000","last_fullsized":"2023-10-09T10:24:13.537235+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":417,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538633+0000","last_change":"2023-10-09T10:23:25.438653+0000","last_active":"2023-10-09T10:24:13.538633+0000","last_peered":"2023-10-09T10:24:13.538633+0000","last_clean":"2023-10-09T10:24:13.538633+0000","last_became_active":"2023-10-09T10:23:25.438243+0000","last_became_peered":"2023-10-09T10:23:25.438243+0000","last_unstale":"2023-10-09T10:24:13.538633+0000","last_undegraded":"2023-10-09T10:24:13.538633+0000","last_fullsized":"2023-10-09T10:24:13.538633+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:59.024327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"503'2098","reported_seq":2125,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.598336+0000","last_change":"2023-10-09T10:24:13.571864+0000","last_active":"2023-10-09T10:24:17.598336+0000","last_peered":"2023-10-09T10:24:17.598336+0000","last_clean":"2023-10-09T10:24:17.598336+0000","last_became_active":"2023-10-09T10:24:13.570664+0000","last_became_peered":"2023-10-09T10:24:13.570664+0000","last_unstale":"2023-10-09T10:24:17.598336+0000","last_undegraded":"2023-10-09T10:24:17.598336+0000","last_fullsized":"2023-10-09T10:24:17.598336+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2098,"log_dups_size":0,"ondisk_log_size":2098,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76054528,"num_objects":1161,"num_object_clones":0,"num_object_copies":3483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1161,"num_write_kb":74272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":27,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.548054+0000","last_change":"2023-10-09T10:23:25.411688+0000","last_active":"2023-10-09T10:24:12.548054+0000","last_peered":"2023-10-09T10:24:12.548054+0000","last_clean":"2023-10-09T10:24:12.548054+0000","last_became_active":"2023-10-09T10:23:25.411340+0000","last_became_peered":"2023-10-09T10:23:25.411340+0000","last_unstale":"2023-10-09T10:24:12.548054+0000","last_undegraded":"2023-10-09T10:24:12.548054+0000","last_fullsized":"2023-10-09T10:24:12.548054+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538975+0000","last_change":"2023-10-09T10:23:25.433215+0000","last_active":"2023-10-09T10:24:13.538975+0000","last_peered":"2023-10-09T10:24:13.538975+0000","last_clean":"2023-10-09T10:24:13.538975+0000","last_became_active":"2023-10-09T10:23:25.432369+0000","last_became_peered":"2023-10-09T10:23:25.432369+0000","last_unstale":"2023-10-09T10:24:13.538975+0000","last_undegraded":"2023-10-09T10:24:13.538975+0000","last_fullsized":"2023-10-09T10:24:13.538975+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:00.938191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":443,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:15.571132+0000","last_change":"2023-10-09T10:22:27.392579+0000","last_active":"2023-10-09T10:24:15.571132+0000","last_peered":"2023-10-09T10:24:15.571132+0000","last_clean":"2023-10-09T10:24:15.571132+0000","last_became_active":"2023-10-09T10:22:27.392040+0000","last_became_peered":"2023-10-09T10:22:27.392040+0000","last_unstale":"2023-10-09T10:24:15.571132+0000","last_undegraded":"2023-10-09T10:24:15.571132+0000","last_fullsized":"2023-10-09T10:24:15.571132+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:20.831225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":55,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.549574+0000","last_change":"2023-10-09T10:22:40.543504+0000","last_active":"2023-10-09T10:24:12.549574+0000","last_peered":"2023-10-09T10:24:12.549574+0000","last_clean":"2023-10-09T10:24:12.549574+0000","last_became_active":"2023-10-09T10:22:40.542810+0000","last_became_peered":"2023-10-09T10:22:40.542810+0000","last_unstale":"2023-10-09T10:24:12.549574+0000","last_undegraded":"2023-10-09T10:24:12.549574+0000","last_fullsized":"2023-10-09T10:24:12.549574+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":168,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.543879+0000","last_change":"2023-10-09T10:23:25.517220+0000","last_active":"2023-10-09T10:24:12.543879+0000","last_peered":"2023-10-09T10:24:12.543879+0000","last_clean":"2023-10-09T10:24:12.543879+0000","last_became_active":"2023-10-09T10:23:25.516909+0000","last_became_peered":"2023-10-09T10:23:25.516909+0000","last_unstale":"2023-10-09T10:24:12.543879+0000","last_undegraded":"2023-10-09T10:24:12.543879+0000","last_fullsized":"2023-10-09T10:24:12.543879+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":29,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537619+0000","last_change":"2023-10-09T10:23:25.414305+0000","last_active":"2023-10-09T10:24:13.537619+0000","last_peered":"2023-10-09T10:24:13.537619+0000","last_clean":"2023-10-09T10:24:13.537619+0000","last_became_active":"2023-10-09T10:23:25.413716+0000","last_became_peered":"2023-10-09T10:23:25.413716+0000","last_unstale":"2023-10-09T10:24:13.537619+0000","last_undegraded":"2023-10-09T10:24:13.537619+0000","last_fullsized":"2023-10-09T10:24:13.537619+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:52.696335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":108,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538148+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:24:13.538148+0000","last_peered":"2023-10-09T10:24:13.538148+0000","last_clean":"2023-10-09T10:24:13.538148+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:24:13.538148+0000","last_undegraded":"2023-10-09T10:24:13.538148+0000","last_fullsized":"2023-10-09T10:24:13.538148+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":67,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.553381+0000","last_change":"2023-10-09T10:23:25.681695+0000","last_active":"2023-10-09T10:24:12.553381+0000","last_peered":"2023-10-09T10:24:12.553381+0000","last_clean":"2023-10-09T10:24:12.553381+0000","last_became_active":"2023-10-09T10:23:25.609955+0000","last_became_peered":"2023-10-09T10:23:25.609955+0000","last_unstale":"2023-10-09T10:24:12.553381+0000","last_undegraded":"2023-10-09T10:24:12.553381+0000","last_fullsized":"2023-10-09T10:24:12.553381+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":135,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538139+0000","last_change":"2023-10-09T10:23:26.105223+0000","last_active":"2023-10-09T10:24:13.538139+0000","last_peered":"2023-10-09T10:24:13.538139+0000","last_clean":"2023-10-09T10:24:13.538139+0000","last_became_active":"2023-10-09T10:23:25.529288+0000","last_became_peered":"2023-10-09T10:23:25.529288+0000","last_unstale":"2023-10-09T10:24:13.538139+0000","last_undegraded":"2023-10-09T10:24:13.538139+0000","last_fullsized":"2023-10-09T10:24:13.538139+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":105,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537349+0000","last_change":"2023-10-09T10:22:27.502829+0000","last_active":"2023-10-09T10:24:13.537349+0000","last_peered":"2023-10-09T10:24:13.537349+0000","last_clean":"2023-10-09T10:24:13.537349+0000","last_became_active":"2023-10-09T10:22:27.502361+0000","last_became_peered":"2023-10-09T10:22:27.502361+0000","last_unstale":"2023-10-09T10:24:13.537349+0000","last_undegraded":"2023-10-09T10:24:13.537349+0000","last_fullsized":"2023-10-09T10:24:13.537349+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":35,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537652+0000","last_change":"2023-10-09T10:23:26.091433+0000","last_active":"2023-10-09T10:24:13.537652+0000","last_peered":"2023-10-09T10:24:13.537652+0000","last_clean":"2023-10-09T10:24:13.537652+0000","last_became_active":"2023-10-09T10:23:25.437180+0000","last_became_peered":"2023-10-09T10:23:25.437180+0000","last_unstale":"2023-10-09T10:24:13.537652+0000","last_undegraded":"2023-10-09T10:24:13.537652+0000","last_fullsized":"2023-10-09T10:24:13.537652+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:18.902210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":27,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.542498+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:24:12.542498+0000","last_peered":"2023-10-09T10:24:12.542498+0000","last_clean":"2023-10-09T10:24:12.542498+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:24:12.542498+0000","last_undegraded":"2023-10-09T10:24:12.542498+0000","last_fullsized":"2023-10-09T10:24:12.542498+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":416,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538505+0000","last_change":"2023-10-09T10:23:25.421838+0000","last_active":"2023-10-09T10:24:13.538505+0000","last_peered":"2023-10-09T10:24:13.538505+0000","last_clean":"2023-10-09T10:24:13.538505+0000","last_became_active":"2023-10-09T10:23:25.421388+0000","last_became_peered":"2023-10-09T10:23:25.421388+0000","last_unstale":"2023-10-09T10:24:13.538505+0000","last_undegraded":"2023-10-09T10:24:13.538505+0000","last_fullsized":"2023-10-09T10:24:13.538505+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:56.171644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":159,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.554088+0000","last_change":"2023-10-09T10:23:25.449285+0000","last_active":"2023-10-09T10:24:12.554088+0000","last_peered":"2023-10-09T10:24:12.554088+0000","last_clean":"2023-10-09T10:24:12.554088+0000","last_became_active":"2023-10-09T10:23:25.448967+0000","last_became_peered":"2023-10-09T10:23:25.448967+0000","last_unstale":"2023-10-09T10:24:12.554088+0000","last_undegraded":"2023-10-09T10:24:12.554088+0000","last_fullsized":"2023-10-09T10:24:12.554088+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"502'1942","reported_seq":1961,"reported_epoch":502,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:24:14.850163+0000","last_change":"2023-10-09T10:24:13.558050+0000","last_active":"2023-10-09T10:24:14.850163+0000","last_peered":"2023-10-09T10:24:14.850163+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:24:13.558050+0000","last_became_peered":"2023-10-09T10:24:13.558050+0000","last_unstale":"2023-10-09T10:24:14.850163+0000","last_undegraded":"2023-10-09T10:24:13.553724+0000","last_fullsized":"2023-10-09T10:24:13.543917+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":1942,"log_dups_size":0,"ondisk_log_size":1942,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66748416,"num_objects":1019,"num_object_clones":0,"num_object_copies":3057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1019,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1019,"num_write_kb":65184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1019}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":105,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537351+0000","last_change":"2023-10-09T10:23:25.447085+0000","last_active":"2023-10-09T10:24:13.537351+0000","last_peered":"2023-10-09T10:24:13.537351+0000","last_clean":"2023-10-09T10:24:13.537351+0000","last_became_active":"2023-10-09T10:23:25.442003+0000","last_became_peered":"2023-10-09T10:23:25.442003+0000","last_unstale":"2023-10-09T10:24:13.537351+0000","last_undegraded":"2023-10-09T10:24:13.537351+0000","last_fullsized":"2023-10-09T10:24:13.537351+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":27,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.541339+0000","last_change":"2023-10-09T10:23:25.452755+0000","last_active":"2023-10-09T10:24:12.541339+0000","last_peered":"2023-10-09T10:24:12.541339+0000","last_clean":"2023-10-09T10:24:12.541339+0000","last_became_active":"2023-10-09T10:23:25.452438+0000","last_became_peered":"2023-10-09T10:23:25.452438+0000","last_unstale":"2023-10-09T10:24:12.541339+0000","last_undegraded":"2023-10-09T10:24:12.541339+0000","last_fullsized":"2023-10-09T10:24:12.541339+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.18","version":"502'2021","reported_seq":2046,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.830554+0000","last_change":"2023-10-09T10:24:13.559969+0000","last_active":"2023-10-09T10:24:14.830554+0000","last_peered":"2023-10-09T10:24:14.830554+0000","last_clean":"2023-10-09T10:24:14.830554+0000","last_became_active":"2023-10-09T10:24:13.558665+0000","last_became_peered":"2023-10-09T10:24:13.558665+0000","last_unstale":"2023-10-09T10:24:14.830554+0000","last_undegraded":"2023-10-09T10:24:14.830554+0000","last_fullsized":"2023-10-09T10:24:14.830554+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2021,"log_dups_size":0,"ondisk_log_size":2021,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69173248,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1056,"num_write_kb":67552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":455,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.550953+0000","last_change":"2023-10-09T10:23:26.100317+0000","last_active":"2023-10-09T10:24:12.550953+0000","last_peered":"2023-10-09T10:24:12.550953+0000","last_clean":"2023-10-09T10:24:12.550953+0000","last_became_active":"2023-10-09T10:23:25.558059+0000","last_became_peered":"2023-10-09T10:23:25.558059+0000","last_unstale":"2023-10-09T10:24:12.550953+0000","last_undegraded":"2023-10-09T10:24:12.550953+0000","last_fullsized":"2023-10-09T10:24:12.550953+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":54,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.545721+0000","last_change":"2023-10-09T10:23:25.521718+0000","last_active":"2023-10-09T10:24:12.545721+0000","last_peered":"2023-10-09T10:24:12.545721+0000","last_clean":"2023-10-09T10:24:12.545721+0000","last_became_active":"2023-10-09T10:23:25.521075+0000","last_became_peered":"2023-10-09T10:23:25.521075+0000","last_unstale":"2023-10-09T10:24:12.545721+0000","last_undegraded":"2023-10-09T10:24:12.545721+0000","last_fullsized":"2023-10-09T10:24:12.545721+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.7","version":"502'1997","reported_seq":2021,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.761688+0000","last_change":"2023-10-09T10:24:13.549409+0000","last_active":"2023-10-09T10:24:14.761688+0000","last_peered":"2023-10-09T10:24:14.761688+0000","last_clean":"2023-10-09T10:24:14.761688+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:24:14.761688+0000","last_undegraded":"2023-10-09T10:24:14.761688+0000","last_fullsized":"2023-10-09T10:24:14.761688+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":1997,"log_dups_size":0,"ondisk_log_size":1997,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68091904,"num_objects":1039,"num_object_clones":0,"num_object_copies":3117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1039,"num_write_kb":66496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":114,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.539217+0000","last_change":"2023-10-09T10:23:25.461650+0000","last_active":"2023-10-09T10:24:13.539217+0000","last_peered":"2023-10-09T10:24:13.539217+0000","last_clean":"2023-10-09T10:24:13.539217+0000","last_became_active":"2023-10-09T10:23:25.432695+0000","last_became_peered":"2023-10-09T10:23:25.432695+0000","last_unstale":"2023-10-09T10:24:13.539217+0000","last_undegraded":"2023-10-09T10:24:13.539217+0000","last_fullsized":"2023-10-09T10:24:13.539217+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:15.291716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":168,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.549788+0000","last_change":"2023-10-09T10:23:25.518495+0000","last_active":"2023-10-09T10:24:12.549788+0000","last_peered":"2023-10-09T10:24:12.549788+0000","last_clean":"2023-10-09T10:24:12.549788+0000","last_became_active":"2023-10-09T10:23:25.517781+0000","last_became_peered":"2023-10-09T10:23:25.517781+0000","last_unstale":"2023-10-09T10:24:12.549788+0000","last_undegraded":"2023-10-09T10:24:12.549788+0000","last_fullsized":"2023-10-09T10:24:12.549788+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"503'2085","reported_seq":2110,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:18.077427+0000","last_change":"2023-10-09T10:24:13.550357+0000","last_active":"2023-10-09T10:24:18.077427+0000","last_peered":"2023-10-09T10:24:18.077427+0000","last_clean":"2023-10-09T10:24:18.077427+0000","last_became_active":"2023-10-09T10:24:13.547606+0000","last_became_peered":"2023-10-09T10:24:13.547606+0000","last_unstale":"2023-10-09T10:24:18.077427+0000","last_undegraded":"2023-10-09T10:24:18.077427+0000","last_fullsized":"2023-10-09T10:24:18.077427+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2085,"log_dups_size":0,"ondisk_log_size":2085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76644352,"num_objects":1169,"num_object_clones":0,"num_object_copies":3507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1169,"num_write_kb":74848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":117,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538912+0000","last_change":"2023-10-09T10:23:25.742454+0000","last_active":"2023-10-09T10:24:13.538912+0000","last_peered":"2023-10-09T10:24:13.538912+0000","last_clean":"2023-10-09T10:24:13.538912+0000","last_became_active":"2023-10-09T10:23:25.526430+0000","last_became_peered":"2023-10-09T10:23:25.526430+0000","last_unstale":"2023-10-09T10:24:13.538912+0000","last_undegraded":"2023-10-09T10:24:13.538912+0000","last_fullsized":"2023-10-09T10:24:13.538912+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":111,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538117+0000","last_change":"2023-10-09T10:23:25.446111+0000","last_active":"2023-10-09T10:24:13.538117+0000","last_peered":"2023-10-09T10:24:13.538117+0000","last_clean":"2023-10-09T10:24:13.538117+0000","last_became_active":"2023-10-09T10:23:25.445584+0000","last_became_peered":"2023-10-09T10:23:25.445584+0000","last_unstale":"2023-10-09T10:24:13.538117+0000","last_undegraded":"2023-10-09T10:24:13.538117+0000","last_fullsized":"2023-10-09T10:24:13.538117+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:40:04.357388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.5","version":"502'1963","reported_seq":1985,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.869603+0000","last_change":"2023-10-09T10:24:13.555931+0000","last_active":"2023-10-09T10:24:14.869603+0000","last_peered":"2023-10-09T10:24:14.869603+0000","last_clean":"2023-10-09T10:24:14.869603+0000","last_became_active":"2023-10-09T10:24:13.554485+0000","last_became_peered":"2023-10-09T10:24:13.554485+0000","last_unstale":"2023-10-09T10:24:14.869603+0000","last_undegraded":"2023-10-09T10:24:14.869603+0000","last_fullsized":"2023-10-09T10:24:14.869603+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":1963,"log_dups_size":0,"ondisk_log_size":1963,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66748416,"num_objects":1018,"num_object_clones":0,"num_object_copies":3054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1018,"num_write_kb":65184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":180,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.553776+0000","last_change":"2023-10-09T10:23:26.107824+0000","last_active":"2023-10-09T10:24:12.553776+0000","last_peered":"2023-10-09T10:24:12.553776+0000","last_clean":"2023-10-09T10:24:12.553776+0000","last_became_active":"2023-10-09T10:23:25.617347+0000","last_became_peered":"2023-10-09T10:23:25.617347+0000","last_unstale":"2023-10-09T10:24:12.553776+0000","last_undegraded":"2023-10-09T10:24:12.553776+0000","last_fullsized":"2023-10-09T10:24:12.553776+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":67,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.554225+0000","last_change":"2023-10-09T10:23:25.681497+0000","last_active":"2023-10-09T10:24:12.554225+0000","last_peered":"2023-10-09T10:24:12.554225+0000","last_clean":"2023-10-09T10:24:12.554225+0000","last_became_active":"2023-10-09T10:23:25.610682+0000","last_became_peered":"2023-10-09T10:23:25.610682+0000","last_unstale":"2023-10-09T10:24:12.554225+0000","last_undegraded":"2023-10-09T10:24:12.554225+0000","last_fullsized":"2023-10-09T10:24:12.554225+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"503'2095","reported_seq":2123,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.629223+0000","last_change":"2023-10-09T10:24:13.557004+0000","last_active":"2023-10-09T10:24:17.629223+0000","last_peered":"2023-10-09T10:24:17.629223+0000","last_clean":"2023-10-09T10:24:17.629223+0000","last_became_active":"2023-10-09T10:24:13.549930+0000","last_became_peered":"2023-10-09T10:24:13.549930+0000","last_unstale":"2023-10-09T10:24:17.629223+0000","last_undegraded":"2023-10-09T10:24:17.629223+0000","last_fullsized":"2023-10-09T10:24:17.629223+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2095,"log_dups_size":0,"ondisk_log_size":2095,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76709888,"num_objects":1170,"num_object_clones":0,"num_object_copies":3510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1170,"num_write_kb":74912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.542042+0000","last_change":"2023-10-09T10:22:27.394423+0000","last_active":"2023-10-09T10:24:12.542042+0000","last_peered":"2023-10-09T10:24:12.542042+0000","last_clean":"2023-10-09T10:24:12.542042+0000","last_became_active":"2023-10-09T10:22:27.394040+0000","last_became_peered":"2023-10-09T10:22:27.394040+0000","last_unstale":"2023-10-09T10:24:12.542042+0000","last_undegraded":"2023-10-09T10:24:12.542042+0000","last_fullsized":"2023-10-09T10:24:12.542042+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":402,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537923+0000","last_change":"2023-10-09T10:22:13.366463+0000","last_active":"2023-10-09T10:24:13.537923+0000","last_peered":"2023-10-09T10:24:13.537923+0000","last_clean":"2023-10-09T10:24:13.537923+0000","last_became_active":"2023-10-09T10:22:13.366125+0000","last_became_peered":"2023-10-09T10:22:13.366125+0000","last_unstale":"2023-10-09T10:24:13.537923+0000","last_undegraded":"2023-10-09T10:24:13.537923+0000","last_fullsized":"2023-10-09T10:24:13.537923+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:41.113925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"502'1924","reported_seq":1950,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.818121+0000","last_change":"2023-10-09T10:24:13.550441+0000","last_active":"2023-10-09T10:24:14.818121+0000","last_peered":"2023-10-09T10:24:14.818121+0000","last_clean":"2023-10-09T10:24:14.818121+0000","last_became_active":"2023-10-09T10:24:13.549153+0000","last_became_peered":"2023-10-09T10:24:13.549153+0000","last_unstale":"2023-10-09T10:24:14.818121+0000","last_undegraded":"2023-10-09T10:24:14.818121+0000","last_fullsized":"2023-10-09T10:24:14.818121+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":1924,"log_dups_size":0,"ondisk_log_size":1924,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65765376,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1003,"num_write_kb":64224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":99,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537787+0000","last_change":"2023-10-09T10:23:25.554673+0000","last_active":"2023-10-09T10:24:13.537787+0000","last_peered":"2023-10-09T10:24:13.537787+0000","last_clean":"2023-10-09T10:24:13.537787+0000","last_became_active":"2023-10-09T10:23:25.436984+0000","last_became_peered":"2023-10-09T10:23:25.436984+0000","last_unstale":"2023-10-09T10:24:13.537787+0000","last_undegraded":"2023-10-09T10:24:13.537787+0000","last_fullsized":"2023-10-09T10:24:13.537787+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:00.646924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":105,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538068+0000","last_change":"2023-10-09T10:23:25.462246+0000","last_active":"2023-10-09T10:24:13.538068+0000","last_peered":"2023-10-09T10:24:13.538068+0000","last_clean":"2023-10-09T10:24:13.538068+0000","last_became_active":"2023-10-09T10:23:25.461706+0000","last_became_peered":"2023-10-09T10:23:25.461706+0000","last_unstale":"2023-10-09T10:24:13.538068+0000","last_undegraded":"2023-10-09T10:24:13.538068+0000","last_fullsized":"2023-10-09T10:24:13.538068+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"503'2137","reported_seq":2165,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.593158+0000","last_change":"2023-10-09T10:24:13.551568+0000","last_active":"2023-10-09T10:24:17.593158+0000","last_peered":"2023-10-09T10:24:17.593158+0000","last_clean":"2023-10-09T10:24:17.593158+0000","last_became_active":"2023-10-09T10:24:13.548575+0000","last_became_peered":"2023-10-09T10:24:13.548575+0000","last_unstale":"2023-10-09T10:24:17.593158+0000","last_undegraded":"2023-10-09T10:24:17.593158+0000","last_fullsized":"2023-10-09T10:24:17.593158+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2137,"log_dups_size":0,"ondisk_log_size":2137,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77889536,"num_objects":1188,"num_object_clones":0,"num_object_copies":3564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1188,"num_write_kb":76064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":105,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537026+0000","last_change":"2023-10-09T10:22:27.444402+0000","last_active":"2023-10-09T10:24:13.537026+0000","last_peered":"2023-10-09T10:24:13.537026+0000","last_clean":"2023-10-09T10:24:13.537026+0000","last_became_active":"2023-10-09T10:22:27.443887+0000","last_became_peered":"2023-10-09T10:22:27.443887+0000","last_unstale":"2023-10-09T10:24:13.537026+0000","last_undegraded":"2023-10-09T10:24:13.537026+0000","last_fullsized":"2023-10-09T10:24:13.537026+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":101,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.553650+0000","last_change":"2023-10-09T10:22:40.529911+0000","last_active":"2023-10-09T10:24:12.553650+0000","last_peered":"2023-10-09T10:24:12.553650+0000","last_clean":"2023-10-09T10:24:12.553650+0000","last_became_active":"2023-10-09T10:22:40.529496+0000","last_became_peered":"2023-10-09T10:22:40.529496+0000","last_unstale":"2023-10-09T10:24:12.553650+0000","last_undegraded":"2023-10-09T10:24:12.553650+0000","last_fullsized":"2023-10-09T10:24:12.553650+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"503'2152","reported_seq":2180,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.553487+0000","last_change":"2023-10-09T10:24:13.549850+0000","last_active":"2023-10-09T10:24:17.553487+0000","last_peered":"2023-10-09T10:24:17.553487+0000","last_clean":"2023-10-09T10:24:17.553487+0000","last_became_active":"2023-10-09T10:24:13.548610+0000","last_became_peered":"2023-10-09T10:24:13.548610+0000","last_unstale":"2023-10-09T10:24:17.553487+0000","last_undegraded":"2023-10-09T10:24:17.553487+0000","last_fullsized":"2023-10-09T10:24:17.553487+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2152,"log_dups_size":0,"ondisk_log_size":2152,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79101952,"num_objects":1207,"num_object_clones":0,"num_object_copies":3621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1207,"num_write_kb":77248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":96,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537678+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:24:13.537678+0000","last_peered":"2023-10-09T10:24:13.537678+0000","last_clean":"2023-10-09T10:24:13.537678+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:24:13.537678+0000","last_undegraded":"2023-10-09T10:24:13.537678+0000","last_fullsized":"2023-10-09T10:24:13.537678+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":114,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538334+0000","last_change":"2023-10-09T10:22:27.429388+0000","last_active":"2023-10-09T10:24:13.538334+0000","last_peered":"2023-10-09T10:24:13.538334+0000","last_clean":"2023-10-09T10:24:13.538334+0000","last_became_active":"2023-10-09T10:22:27.429067+0000","last_became_peered":"2023-10-09T10:22:27.429067+0000","last_unstale":"2023-10-09T10:24:13.538334+0000","last_undegraded":"2023-10-09T10:24:13.538334+0000","last_fullsized":"2023-10-09T10:24:13.538334+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"503'2112","reported_seq":2143,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.569350+0000","last_change":"2023-10-09T10:24:13.566288+0000","last_active":"2023-10-09T10:24:17.569350+0000","last_peered":"2023-10-09T10:24:17.569350+0000","last_clean":"2023-10-09T10:24:17.569350+0000","last_became_active":"2023-10-09T10:24:13.550926+0000","last_became_peered":"2023-10-09T10:24:13.550926+0000","last_unstale":"2023-10-09T10:24:17.569350+0000","last_undegraded":"2023-10-09T10:24:17.569350+0000","last_fullsized":"2023-10-09T10:24:17.569350+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2112,"log_dups_size":0,"ondisk_log_size":2112,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76972032,"num_objects":1174,"num_object_clones":0,"num_object_copies":3522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1174,"num_write_kb":75168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":436,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.549711+0000","last_change":"2023-10-09T10:23:25.518667+0000","last_active":"2023-10-09T10:24:12.549711+0000","last_peered":"2023-10-09T10:24:12.549711+0000","last_clean":"2023-10-09T10:24:12.549711+0000","last_became_active":"2023-10-09T10:23:25.518163+0000","last_became_peered":"2023-10-09T10:23:25.518163+0000","last_unstale":"2023-10-09T10:24:12.549711+0000","last_undegraded":"2023-10-09T10:24:12.549711+0000","last_fullsized":"2023-10-09T10:24:12.549711+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":517,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538274+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:24:13.538274+0000","last_peered":"2023-10-09T10:24:13.538274+0000","last_clean":"2023-10-09T10:24:13.538274+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:24:13.538274+0000","last_undegraded":"2023-10-09T10:24:13.538274+0000","last_fullsized":"2023-10-09T10:24:13.538274+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"503'2362","reported_seq":2376,"reported_epoch":503,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:24:18.023966+0000","last_change":"2023-10-09T10:24:00.142839+0000","last_active":"2023-10-09T10:24:18.023966+0000","last_peered":"2023-10-09T10:24:18.023966+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:23:53.626970+0000","last_became_peered":"2023-10-09T10:23:53.626970+0000","last_unstale":"2023-10-09T10:24:18.023966+0000","last_undegraded":"2023-10-09T10:24:00.139747+0000","last_fullsized":"2023-10-09T10:23:53.616547+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2362,"log_dups_size":0,"ondisk_log_size":2362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154796032,"num_objects":2362,"num_object_clones":0,"num_object_copies":7086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2362,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2362,"num_write_kb":151168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":2362}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":213,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538353+0000","last_change":"2023-10-09T10:23:25.462977+0000","last_active":"2023-10-09T10:24:13.538353+0000","last_peered":"2023-10-09T10:24:13.538353+0000","last_clean":"2023-10-09T10:24:13.538353+0000","last_became_active":"2023-10-09T10:23:25.462666+0000","last_became_peered":"2023-10-09T10:23:25.462666+0000","last_unstale":"2023-10-09T10:24:13.538353+0000","last_undegraded":"2023-10-09T10:24:13.538353+0000","last_fullsized":"2023-10-09T10:24:13.538353+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:25.535385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":168,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.554020+0000","last_change":"2023-10-09T10:23:25.452505+0000","last_active":"2023-10-09T10:24:12.554020+0000","last_peered":"2023-10-09T10:24:12.554020+0000","last_clean":"2023-10-09T10:24:12.554020+0000","last_became_active":"2023-10-09T10:23:25.452190+0000","last_became_peered":"2023-10-09T10:23:25.452190+0000","last_unstale":"2023-10-09T10:24:12.554020+0000","last_undegraded":"2023-10-09T10:24:12.554020+0000","last_fullsized":"2023-10-09T10:24:12.554020+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":466,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.554292+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:24:12.554292+0000","last_peered":"2023-10-09T10:24:12.554292+0000","last_clean":"2023-10-09T10:24:12.554292+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:24:12.554292+0000","last_undegraded":"2023-10-09T10:24:12.554292+0000","last_fullsized":"2023-10-09T10:24:12.554292+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"503'2366","reported_seq":2384,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.644751+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:24:17.644751+0000","last_peered":"2023-10-09T10:24:17.644751+0000","last_clean":"2023-10-09T10:24:17.644751+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:24:17.644751+0000","last_undegraded":"2023-10-09T10:24:17.644751+0000","last_fullsized":"2023-10-09T10:24:17.644751+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2366,"log_dups_size":0,"ondisk_log_size":2366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155058176,"num_objects":2366,"num_object_clones":0,"num_object_copies":7098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2366,"num_write_kb":151424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":177,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.554146+0000","last_change":"2023-10-09T10:23:25.444540+0000","last_active":"2023-10-09T10:24:12.554146+0000","last_peered":"2023-10-09T10:24:12.554146+0000","last_clean":"2023-10-09T10:24:12.554146+0000","last_became_active":"2023-10-09T10:23:25.444038+0000","last_became_peered":"2023-10-09T10:23:25.444038+0000","last_unstale":"2023-10-09T10:24:12.554146+0000","last_undegraded":"2023-10-09T10:24:12.554146+0000","last_fullsized":"2023-10-09T10:24:12.554146+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":96,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537264+0000","last_change":"2023-10-09T10:22:27.502663+0000","last_active":"2023-10-09T10:24:13.537264+0000","last_peered":"2023-10-09T10:24:13.537264+0000","last_clean":"2023-10-09T10:24:13.537264+0000","last_became_active":"2023-10-09T10:22:27.501953+0000","last_became_peered":"2023-10-09T10:22:27.501953+0000","last_unstale":"2023-10-09T10:24:13.537264+0000","last_undegraded":"2023-10-09T10:24:13.537264+0000","last_fullsized":"2023-10-09T10:24:13.537264+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":54,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.542431+0000","last_change":"2023-10-09T10:23:25.522457+0000","last_active":"2023-10-09T10:24:12.542431+0000","last_peered":"2023-10-09T10:24:12.542431+0000","last_clean":"2023-10-09T10:24:12.542431+0000","last_became_active":"2023-10-09T10:23:25.522190+0000","last_became_peered":"2023-10-09T10:23:25.522190+0000","last_unstale":"2023-10-09T10:24:12.542431+0000","last_undegraded":"2023-10-09T10:24:12.542431+0000","last_fullsized":"2023-10-09T10:24:12.542431+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"502'2007","reported_seq":2033,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.849636+0000","last_change":"2023-10-09T10:24:13.549781+0000","last_active":"2023-10-09T10:24:14.849636+0000","last_peered":"2023-10-09T10:24:14.849636+0000","last_clean":"2023-10-09T10:24:14.849636+0000","last_became_active":"2023-10-09T10:24:13.548389+0000","last_became_peered":"2023-10-09T10:24:13.548389+0000","last_unstale":"2023-10-09T10:24:14.849636+0000","last_undegraded":"2023-10-09T10:24:14.849636+0000","last_fullsized":"2023-10-09T10:24:14.849636+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2007,"log_dups_size":0,"ondisk_log_size":2007,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68255744,"num_objects":1041,"num_object_clones":0,"num_object_copies":3123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1041,"num_write_kb":66656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":434,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538247+0000","last_change":"2023-10-09T10:22:27.441855+0000","last_active":"2023-10-09T10:24:13.538247+0000","last_peered":"2023-10-09T10:24:13.538247+0000","last_clean":"2023-10-09T10:24:13.538247+0000","last_became_active":"2023-10-09T10:22:27.441539+0000","last_became_peered":"2023-10-09T10:22:27.441539+0000","last_unstale":"2023-10-09T10:24:13.538247+0000","last_undegraded":"2023-10-09T10:24:13.538247+0000","last_fullsized":"2023-10-09T10:24:13.538247+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:46.202562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":61,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:15.569334+0000","last_change":"2023-10-09T10:22:27.392415+0000","last_active":"2023-10-09T10:24:15.569334+0000","last_peered":"2023-10-09T10:24:15.569334+0000","last_clean":"2023-10-09T10:24:15.569334+0000","last_became_active":"2023-10-09T10:22:27.392069+0000","last_became_peered":"2023-10-09T10:22:27.392069+0000","last_unstale":"2023-10-09T10:24:15.569334+0000","last_undegraded":"2023-10-09T10:24:15.569334+0000","last_fullsized":"2023-10-09T10:24:15.569334+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:51:06.172359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":497,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.587427+0000","last_change":"2023-10-09T10:23:25.521874+0000","last_active":"2023-10-09T10:24:12.587427+0000","last_peered":"2023-10-09T10:24:12.587427+0000","last_clean":"2023-10-09T10:24:12.587427+0000","last_became_active":"2023-10-09T10:23:25.521446+0000","last_became_peered":"2023-10-09T10:23:25.521446+0000","last_unstale":"2023-10-09T10:24:12.587427+0000","last_undegraded":"2023-10-09T10:24:12.587427+0000","last_fullsized":"2023-10-09T10:24:12.587427+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"502'2034","reported_seq":2050,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.839950+0000","last_change":"2023-10-09T10:23:53.842189+0000","last_active":"2023-10-09T10:24:14.839950+0000","last_peered":"2023-10-09T10:24:14.839950+0000","last_clean":"2023-10-09T10:24:14.839950+0000","last_became_active":"2023-10-09T10:23:53.841927+0000","last_became_peered":"2023-10-09T10:23:53.841927+0000","last_unstale":"2023-10-09T10:24:14.839950+0000","last_undegraded":"2023-10-09T10:24:14.839950+0000","last_fullsized":"2023-10-09T10:24:14.839950+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2034,"log_dups_size":0,"ondisk_log_size":2034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133300224,"num_objects":2034,"num_object_clones":0,"num_object_copies":6102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2034,"num_write_kb":130176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":156,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.547967+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:24:12.547967+0000","last_peered":"2023-10-09T10:24:12.547967+0000","last_clean":"2023-10-09T10:24:12.547967+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:24:12.547967+0000","last_undegraded":"2023-10-09T10:24:12.547967+0000","last_fullsized":"2023-10-09T10:24:12.547967+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":684,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.540838+0000","last_change":"2023-10-09T10:23:46.972678+0000","last_active":"2023-10-09T10:24:13.540838+0000","last_peered":"2023-10-09T10:24:13.540838+0000","last_clean":"2023-10-09T10:24:13.540838+0000","last_became_active":"2023-10-09T10:23:46.972311+0000","last_became_peered":"2023-10-09T10:23:46.972311+0000","last_unstale":"2023-10-09T10:24:13.540838+0000","last_undegraded":"2023-10-09T10:24:13.540838+0000","last_fullsized":"2023-10-09T10:24:13.540838+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:15.713716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":186,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.553307+0000","last_change":"2023-10-09T10:23:25.681332+0000","last_active":"2023-10-09T10:24:12.553307+0000","last_peered":"2023-10-09T10:24:12.553307+0000","last_clean":"2023-10-09T10:24:12.553307+0000","last_became_active":"2023-10-09T10:23:25.610272+0000","last_became_peered":"2023-10-09T10:23:25.610272+0000","last_unstale":"2023-10-09T10:24:12.553307+0000","last_undegraded":"2023-10-09T10:24:12.553307+0000","last_fullsized":"2023-10-09T10:24:12.553307+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538023+0000","last_change":"2023-10-09T10:23:25.683944+0000","last_active":"2023-10-09T10:24:13.538023+0000","last_peered":"2023-10-09T10:24:13.538023+0000","last_clean":"2023-10-09T10:24:13.538023+0000","last_became_active":"2023-10-09T10:23:25.683505+0000","last_became_peered":"2023-10-09T10:23:25.683505+0000","last_unstale":"2023-10-09T10:24:13.538023+0000","last_undegraded":"2023-10-09T10:24:13.538023+0000","last_fullsized":"2023-10-09T10:24:13.538023+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"502'1939","reported_seq":1959,"reported_epoch":502,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:24:14.856379+0000","last_change":"2023-10-09T10:24:13.545897+0000","last_active":"2023-10-09T10:24:14.856379+0000","last_peered":"2023-10-09T10:24:14.856379+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:24:13.545897+0000","last_became_peered":"2023-10-09T10:24:13.545897+0000","last_unstale":"2023-10-09T10:24:14.856379+0000","last_undegraded":"2023-10-09T10:24:13.541527+0000","last_fullsized":"2023-10-09T10:24:13.541207+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":1939,"log_dups_size":0,"ondisk_log_size":1939,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66551808,"num_objects":1015,"num_object_clones":0,"num_object_copies":3045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1015,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1015,"num_write_kb":64992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1015}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":187,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:15.569438+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:24:15.569438+0000","last_peered":"2023-10-09T10:24:15.569438+0000","last_clean":"2023-10-09T10:24:15.569438+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:24:15.569438+0000","last_undegraded":"2023-10-09T10:24:15.569438+0000","last_fullsized":"2023-10-09T10:24:15.569438+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":437,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538318+0000","last_change":"2023-10-09T10:19:43.292390+0000","last_active":"2023-10-09T10:24:13.538318+0000","last_peered":"2023-10-09T10:24:13.538318+0000","last_clean":"2023-10-09T10:24:13.538318+0000","last_became_active":"2023-10-09T10:19:43.290733+0000","last_became_peered":"2023-10-09T10:19:43.290733+0000","last_unstale":"2023-10-09T10:24:13.538318+0000","last_undegraded":"2023-10-09T10:24:13.538318+0000","last_fullsized":"2023-10-09T10:24:13.538318+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:53.407598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":58,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.543800+0000","last_change":"2023-10-09T10:23:25.531957+0000","last_active":"2023-10-09T10:24:12.543800+0000","last_peered":"2023-10-09T10:24:12.543800+0000","last_clean":"2023-10-09T10:24:12.543800+0000","last_became_active":"2023-10-09T10:23:25.531614+0000","last_became_peered":"2023-10-09T10:23:25.531614+0000","last_unstale":"2023-10-09T10:24:12.543800+0000","last_undegraded":"2023-10-09T10:24:12.543800+0000","last_fullsized":"2023-10-09T10:24:12.543800+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.c","version":"503'2459","reported_seq":2476,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:18.059867+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:24:18.059867+0000","last_peered":"2023-10-09T10:24:18.059867+0000","last_clean":"2023-10-09T10:24:18.059867+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:24:18.059867+0000","last_undegraded":"2023-10-09T10:24:18.059867+0000","last_fullsized":"2023-10-09T10:24:18.059867+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2459,"log_dups_size":0,"ondisk_log_size":2459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161153024,"num_objects":2459,"num_object_clones":0,"num_object_copies":7377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2459,"num_write_kb":157376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":114,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538259+0000","last_change":"2023-10-09T10:23:25.613084+0000","last_active":"2023-10-09T10:24:13.538259+0000","last_peered":"2023-10-09T10:24:13.538259+0000","last_clean":"2023-10-09T10:24:13.538259+0000","last_became_active":"2023-10-09T10:23:25.612722+0000","last_became_peered":"2023-10-09T10:23:25.612722+0000","last_unstale":"2023-10-09T10:24:13.538259+0000","last_undegraded":"2023-10-09T10:24:13.538259+0000","last_fullsized":"2023-10-09T10:24:13.538259+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":465,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.542350+0000","last_change":"2023-10-09T10:22:27.429171+0000","last_active":"2023-10-09T10:24:12.542350+0000","last_peered":"2023-10-09T10:24:12.542350+0000","last_clean":"2023-10-09T10:24:12.542350+0000","last_became_active":"2023-10-09T10:22:27.428849+0000","last_became_peered":"2023-10-09T10:22:27.428849+0000","last_unstale":"2023-10-09T10:24:12.542350+0000","last_undegraded":"2023-10-09T10:24:12.542350+0000","last_fullsized":"2023-10-09T10:24:12.542350+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":405,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538284+0000","last_change":"2023-10-09T10:23:25.458876+0000","last_active":"2023-10-09T10:24:13.538284+0000","last_peered":"2023-10-09T10:24:13.538284+0000","last_clean":"2023-10-09T10:24:13.538284+0000","last_became_active":"2023-10-09T10:23:25.458037+0000","last_became_peered":"2023-10-09T10:23:25.458037+0000","last_unstale":"2023-10-09T10:24:13.538284+0000","last_undegraded":"2023-10-09T10:24:13.538284+0000","last_fullsized":"2023-10-09T10:24:13.538284+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:34.186003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"503'2288","reported_seq":2306,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.645233+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:24:17.645233+0000","last_peered":"2023-10-09T10:24:17.645233+0000","last_clean":"2023-10-09T10:24:17.645233+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:24:17.645233+0000","last_undegraded":"2023-10-09T10:24:17.645233+0000","last_fullsized":"2023-10-09T10:24:17.645233+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2288,"log_dups_size":0,"ondisk_log_size":2288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149946368,"num_objects":2288,"num_object_clones":0,"num_object_copies":6864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2288,"num_write_kb":146432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":302,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.539760+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:24:13.539760+0000","last_peered":"2023-10-09T10:24:13.539760+0000","last_clean":"2023-10-09T10:24:13.539760+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:24:13.539760+0000","last_undegraded":"2023-10-09T10:24:13.539760+0000","last_fullsized":"2023-10-09T10:24:13.539760+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":475,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538225+0000","last_change":"2023-10-09T10:22:27.442726+0000","last_active":"2023-10-09T10:24:13.538225+0000","last_peered":"2023-10-09T10:24:13.538225+0000","last_clean":"2023-10-09T10:24:13.538225+0000","last_became_active":"2023-10-09T10:22:27.442396+0000","last_became_peered":"2023-10-09T10:22:27.442396+0000","last_unstale":"2023-10-09T10:24:13.538225+0000","last_undegraded":"2023-10-09T10:24:13.538225+0000","last_fullsized":"2023-10-09T10:24:13.538225+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":368,"reported_epoch":501,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:24:12.551062+0000","last_change":"2023-10-09T10:23:26.562884+0000","last_active":"2023-10-09T10:24:12.551062+0000","last_peered":"2023-10-09T10:24:12.551062+0000","last_clean":"2023-10-09T10:24:12.551062+0000","last_became_active":"2023-10-09T10:23:26.483114+0000","last_became_peered":"2023-10-09T10:23:26.483114+0000","last_unstale":"2023-10-09T10:24:12.551062+0000","last_undegraded":"2023-10-09T10:24:12.551062+0000","last_fullsized":"2023-10-09T10:24:12.551062+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,7],"avail_no_missing":["5","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.e","version":"503'2493","reported_seq":2510,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:18.051961+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:24:18.051961+0000","last_peered":"2023-10-09T10:24:18.051961+0000","last_clean":"2023-10-09T10:24:18.051961+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:24:18.051961+0000","last_undegraded":"2023-10-09T10:24:18.051961+0000","last_fullsized":"2023-10-09T10:24:18.051961+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2493,"log_dups_size":0,"ondisk_log_size":2493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163381248,"num_objects":2493,"num_object_clones":0,"num_object_copies":7479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2493,"num_write_kb":159552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538543+0000","last_change":"2023-10-09T10:23:25.433377+0000","last_active":"2023-10-09T10:24:13.538543+0000","last_peered":"2023-10-09T10:24:13.538543+0000","last_clean":"2023-10-09T10:24:13.538543+0000","last_became_active":"2023-10-09T10:23:25.422009+0000","last_became_peered":"2023-10-09T10:23:25.422009+0000","last_unstale":"2023-10-09T10:24:13.538543+0000","last_undegraded":"2023-10-09T10:24:13.538543+0000","last_fullsized":"2023-10-09T10:24:13.538543+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":177,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.539119+0000","last_change":"2023-10-09T10:23:25.670521+0000","last_active":"2023-10-09T10:24:13.539119+0000","last_peered":"2023-10-09T10:24:13.539119+0000","last_clean":"2023-10-09T10:24:13.539119+0000","last_became_active":"2023-10-09T10:23:25.525855+0000","last_became_peered":"2023-10-09T10:23:25.525855+0000","last_unstale":"2023-10-09T10:24:13.539119+0000","last_undegraded":"2023-10-09T10:24:13.539119+0000","last_fullsized":"2023-10-09T10:24:13.539119+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":173,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.552939+0000","last_change":"2023-10-09T10:22:40.513566+0000","last_active":"2023-10-09T10:24:12.552939+0000","last_peered":"2023-10-09T10:24:12.552939+0000","last_clean":"2023-10-09T10:24:12.552939+0000","last_became_active":"2023-10-09T10:22:40.513228+0000","last_became_peered":"2023-10-09T10:22:40.513228+0000","last_unstale":"2023-10-09T10:24:12.552939+0000","last_undegraded":"2023-10-09T10:24:12.552939+0000","last_fullsized":"2023-10-09T10:24:12.552939+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"502'1927","reported_seq":1952,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.804258+0000","last_change":"2023-10-09T10:24:13.566102+0000","last_active":"2023-10-09T10:24:14.804258+0000","last_peered":"2023-10-09T10:24:14.804258+0000","last_clean":"2023-10-09T10:24:14.804258+0000","last_became_active":"2023-10-09T10:24:13.564107+0000","last_became_peered":"2023-10-09T10:24:13.564107+0000","last_unstale":"2023-10-09T10:24:14.804258+0000","last_undegraded":"2023-10-09T10:24:14.804258+0000","last_fullsized":"2023-10-09T10:24:14.804258+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":1927,"log_dups_size":0,"ondisk_log_size":1927,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65961984,"num_objects":1007,"num_object_clones":0,"num_object_copies":3021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1007,"num_write_kb":64416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":111,"reported_epoch":501,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:24:12.547789+0000","last_change":"2023-10-09T10:23:26.551630+0000","last_active":"2023-10-09T10:24:12.547789+0000","last_peered":"2023-10-09T10:24:12.547789+0000","last_clean":"2023-10-09T10:24:12.547789+0000","last_became_active":"2023-10-09T10:23:26.551134+0000","last_became_peered":"2023-10-09T10:23:26.551134+0000","last_unstale":"2023-10-09T10:24:12.547789+0000","last_undegraded":"2023-10-09T10:24:12.547789+0000","last_fullsized":"2023-10-09T10:24:12.547789+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,7],"avail_no_missing":["5","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":479,"reported_epoch":502,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:24:13.537908+0000","last_change":"2023-10-09T10:23:26.552568+0000","last_active":"2023-10-09T10:24:13.537908+0000","last_peered":"2023-10-09T10:24:13.537908+0000","last_clean":"2023-10-09T10:24:13.537908+0000","last_became_active":"2023-10-09T10:23:26.551999+0000","last_became_peered":"2023-10-09T10:23:26.551999+0000","last_unstale":"2023-10-09T10:24:13.537908+0000","last_undegraded":"2023-10-09T10:24:13.537908+0000","last_fullsized":"2023-10-09T10:24:13.537908+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,7],"avail_no_missing":["4","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.14","version":"503'2111","reported_seq":2139,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:17.622977+0000","last_change":"2023-10-09T10:24:13.574666+0000","last_active":"2023-10-09T10:24:17.622977+0000","last_peered":"2023-10-09T10:24:17.622977+0000","last_clean":"2023-10-09T10:24:17.622977+0000","last_became_active":"2023-10-09T10:24:13.571081+0000","last_became_peered":"2023-10-09T10:24:13.571081+0000","last_unstale":"2023-10-09T10:24:17.622977+0000","last_undegraded":"2023-10-09T10:24:17.622977+0000","last_fullsized":"2023-10-09T10:24:17.622977+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2111,"log_dups_size":0,"ondisk_log_size":2111,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77758464,"num_objects":1187,"num_object_clones":0,"num_object_copies":3561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1187,"num_write_kb":75936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":219,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.542130+0000","last_change":"2023-10-09T10:23:25.539343+0000","last_active":"2023-10-09T10:24:12.542130+0000","last_peered":"2023-10-09T10:24:12.542130+0000","last_clean":"2023-10-09T10:24:12.542130+0000","last_became_active":"2023-10-09T10:23:25.514039+0000","last_became_peered":"2023-10-09T10:23:25.514039+0000","last_unstale":"2023-10-09T10:24:12.542130+0000","last_undegraded":"2023-10-09T10:24:12.542130+0000","last_fullsized":"2023-10-09T10:24:12.542130+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":96,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.537804+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:24:13.537804+0000","last_peered":"2023-10-09T10:24:13.537804+0000","last_clean":"2023-10-09T10:24:13.537804+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:24:13.537804+0000","last_undegraded":"2023-10-09T10:24:13.537804+0000","last_fullsized":"2023-10-09T10:24:13.537804+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"502'1962","reported_seq":1985,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.799929+0000","last_change":"2023-10-09T10:24:13.558660+0000","last_active":"2023-10-09T10:24:14.799929+0000","last_peered":"2023-10-09T10:24:14.799929+0000","last_clean":"2023-10-09T10:24:14.799929+0000","last_became_active":"2023-10-09T10:24:13.557144+0000","last_became_peered":"2023-10-09T10:24:13.557144+0000","last_unstale":"2023-10-09T10:24:14.799929+0000","last_undegraded":"2023-10-09T10:24:14.799929+0000","last_fullsized":"2023-10-09T10:24:14.799929+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":1962,"log_dups_size":0,"ondisk_log_size":1962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66682880,"num_objects":1018,"num_object_clones":0,"num_object_copies":3054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1018,"num_write_kb":65120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":202,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-09T10:24:12.553856+0000","last_change":"2023-10-09T10:23:26.112703+0000","last_active":"2023-10-09T10:24:12.553856+0000","last_peered":"2023-10-09T10:24:12.553856+0000","last_clean":"2023-10-09T10:24:12.553856+0000","last_became_active":"2023-10-09T10:23:25.617862+0000","last_became_peered":"2023-10-09T10:23:25.617862+0000","last_unstale":"2023-10-09T10:24:12.553856+0000","last_undegraded":"2023-10-09T10:24:12.553856+0000","last_fullsized":"2023-10-09T10:24:12.553856+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":325,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538590+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:24:13.538590+0000","last_peered":"2023-10-09T10:24:13.538590+0000","last_clean":"2023-10-09T10:24:13.538590+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:24:13.538590+0000","last_undegraded":"2023-10-09T10:24:13.538590+0000","last_fullsized":"2023-10-09T10:24:13.538590+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"503'2093","reported_seq":2117,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T10:24:18.046875+0000","last_change":"2023-10-09T10:24:13.568064+0000","last_active":"2023-10-09T10:24:18.046875+0000","last_peered":"2023-10-09T10:24:18.046875+0000","last_clean":"2023-10-09T10:24:18.046875+0000","last_became_active":"2023-10-09T10:24:13.564880+0000","last_became_peered":"2023-10-09T10:24:13.564880+0000","last_unstale":"2023-10-09T10:24:18.046875+0000","last_undegraded":"2023-10-09T10:24:18.046875+0000","last_fullsized":"2023-10-09T10:24:18.046875+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":2093,"log_dups_size":0,"ondisk_log_size":2093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77168640,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1178,"num_write_kb":75360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":56,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538635+0000","last_change":"2023-10-09T10:23:25.438780+0000","last_active":"2023-10-09T10:24:13.538635+0000","last_peered":"2023-10-09T10:24:13.538635+0000","last_clean":"2023-10-09T10:24:13.538635+0000","last_became_active":"2023-10-09T10:23:25.438407+0000","last_became_peered":"2023-10-09T10:23:25.438407+0000","last_unstale":"2023-10-09T10:24:13.538635+0000","last_undegraded":"2023-10-09T10:24:13.538635+0000","last_fullsized":"2023-10-09T10:24:13.538635+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:08:16.664151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":545,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.538171+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:24:13.538171+0000","last_peered":"2023-10-09T10:24:13.538171+0000","last_clean":"2023-10-09T10:24:13.538171+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:24:13.538171+0000","last_undegraded":"2023-10-09T10:24:13.538171+0000","last_fullsized":"2023-10-09T10:24:13.538171+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"502'1988","reported_seq":2012,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:14.793501+0000","last_change":"2023-10-09T10:24:13.558001+0000","last_active":"2023-10-09T10:24:14.793501+0000","last_peered":"2023-10-09T10:24:14.793501+0000","last_clean":"2023-10-09T10:24:14.793501+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:24:14.793501+0000","last_undegraded":"2023-10-09T10:24:14.793501+0000","last_fullsized":"2023-10-09T10:24:14.793501+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":1988,"log_dups_size":0,"ondisk_log_size":1988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67502080,"num_objects":1030,"num_object_clones":0,"num_object_copies":3090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1030,"num_write_kb":65920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":501,"reported_epoch":501,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:24:12.587237+0000","last_change":"2023-10-09T10:23:26.567480+0000","last_active":"2023-10-09T10:24:12.587237+0000","last_peered":"2023-10-09T10:24:12.587237+0000","last_clean":"2023-10-09T10:24:12.587237+0000","last_became_active":"2023-10-09T10:23:26.567054+0000","last_became_peered":"2023-10-09T10:23:26.567054+0000","last_unstale":"2023-10-09T10:24:12.587237+0000","last_undegraded":"2023-10-09T10:24:12.587237+0000","last_fullsized":"2023-10-09T10:24:12.587237+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,7],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"4,5,7","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":105,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-10-09T10:24:13.536989+0000","last_change":"2023-10-09T10:22:27.441829+0000","last_active":"2023-10-09T10:24:13.536989+0000","last_peered":"2023-10-09T10:24:13.536989+0000","last_clean":"2023-10-09T10:24:13.536989+0000","last_became_active":"2023-10-09T10:22:27.441479+0000","last_became_peered":"2023-10-09T10:22:27.441479+0000","last_unstale":"2023-10-09T10:24:13.536989+0000","last_undegraded":"2023-10-09T10:24:13.536989+0000","last_fullsized":"2023-10-09T10:24:13.536989+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":2360803328,"num_objects":36023,"num_object_clones":0,"num_object_copies":108069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":36023,"num_write_kb":2305472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6961299456,"data_stored":6961299456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":54789,"ondisk_log_size":54789,"up":75,"acting":75,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":26954,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":94,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430655,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626883,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":360777253197,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416721,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2118364,"kb_used_data":1326996,"kb_used_omap":28,"kb_used_meta":791331,"kb_avail":91626788,"statfs":{"total":95995035648,"available":93825830912,"internally_reserved":0,"allocated":1358843904,"data_stored":1356976616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":810323792},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:23:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.0840000000000001,"15min":1.97},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.7290000000000001,"5min":8.8699999999999992,"15min":54.518000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.5229999999999999,"5min":1.3240000000000001,"15min":2.718},"min":{"1min":0.214,"5min":0.192,"15min":0.192},"max":{"1min":4.9509999999999996,"5min":17.033000000000001,"15min":56.950000000000003},"last":0.52700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:23:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":1.6719999999999999,"15min":2.153},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.7080000000000002,"5min":47.215000000000003,"15min":53.988999999999997},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.794,"15min":2.1549999999999998},"min":{"1min":0.28699999999999998,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":5.6790000000000003,"5min":47.000999999999998,"15min":54.119},"last":0.57099999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.3640000000000001,"15min":1.401},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":11.933,"5min":16.879999999999999,"15min":56.984999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.581,"5min":1.3400000000000001,"15min":1.345},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.0020000000000007,"5min":8.0020000000000007,"15min":56.475999999999999},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:24:13 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.857,"15min":2.3399999999999999},"min":{"1min":0.38800000000000001,"5min":0.251,"15min":0.251},"max":{"1min":3.827,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.901,"15min":2.2349999999999999},"min":{"1min":0.36799999999999999,"5min":0.34000000000000002,"15min":0.32400000000000001},"max":{"1min":8.0640000000000001,"5min":28.331,"15min":56.378},"last":0.88300000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:24:13 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.8340000000000001,"15min":3.137},"min":{"1min":0.312,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":8.0920000000000005,"5min":11.775,"15min":229.589},"last":0.80100000000000005},{"interface":"front","average":{"1min":1.446,"5min":1.5129999999999999,"15min":2.742},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":5.7709999999999999,"5min":8.7530000000000001,"15min":228.88900000000001},"last":0.90600000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:24:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6080000000000001,"5min":1.78,"15min":2.1709999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.33700000000000002},"max":{"1min":5.8090000000000002,"5min":8.7810000000000006,"15min":54.482999999999997},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.4410000000000001,"5min":2.0899999999999999,"15min":2.6539999999999999},"min":{"1min":0.42199999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.8849999999999998,"5min":16.954999999999998,"15min":57.753},"last":0.78600000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:24:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":1.8360000000000001,"15min":2.4009999999999998},"min":{"1min":0.40500000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.0329999999999995,"5min":8.0329999999999995,"15min":56.411999999999999},"last":0.93300000000000005},{"interface":"front","average":{"1min":2.1000000000000001,"5min":1.944,"15min":2.1970000000000001},"min":{"1min":0.39900000000000002,"5min":0.38600000000000001,"15min":0.36499999999999999},"max":{"1min":10.215999999999999,"5min":11.175000000000001,"15min":54.451999999999998},"last":0.97999999999999998}]}]},{"osd":2,"up_from":217,"seq":932007903457,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2299112,"kb_used_data":1591712,"kb_used_omap":83,"kb_used_meta":707308,"kb_avail":91446040,"statfs":{"total":95995035648,"available":93640744960,"internally_reserved":0,"allocated":1629913088,"data_stored":1628219232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":724283887},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.8400000000000001,"15min":2.032},"min":{"1min":0.21299999999999999,"5min":0.19,"15min":0.185},"max":{"1min":4.2930000000000001,"5min":48.990000000000002,"15min":48.990000000000002},"last":2.3639999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.8999999999999999,"15min":2.431},"min":{"1min":0.248,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.1139999999999999,"5min":49.412999999999997,"15min":49.412999999999997},"last":0.73799999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":1.353,"15min":2.1920000000000002},"min":{"1min":0.22900000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.8220000000000001,"5min":8.2010000000000005,"15min":34.893999999999998},"last":2.1859999999999999},{"interface":"front","average":{"1min":1.109,"5min":1.357,"15min":2.1749999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":3.4489999999999998,"5min":6.9710000000000001,"15min":34.962000000000003},"last":0.79500000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.2310000000000001,"15min":1.121},"min":{"1min":0.27700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":5.3010000000000002,"5min":7.54,"15min":35.043999999999997},"last":2.4529999999999998},{"interface":"front","average":{"1min":1.089,"5min":0.92000000000000004,"15min":1.2030000000000001},"min":{"1min":0.32700000000000001,"5min":0.246,"15min":0.246},"max":{"1min":5.181,"5min":6.617,"15min":50.406999999999996},"last":0.33200000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:24:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":1.6220000000000001,"15min":1.663},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.26600000000000001},"max":{"1min":47.503999999999998,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.95199999999999996},{"interface":"front","average":{"1min":3.0169999999999999,"5min":1.7669999999999999,"15min":1.8080000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":47.704000000000001,"5min":47.704000000000001,"15min":47.704000000000001},"last":1.002}]},{"osd":5,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.5920000000000001,"15min":2.008},"min":{"1min":0.308,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":5.2809999999999997,"5min":10.448,"15min":57.298000000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.7,"5min":1.764,"15min":2.0089999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.31},"max":{"1min":6.2469999999999999,"5min":8.1630000000000003,"15min":57.828000000000003},"last":1.881}]},{"osd":6,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":2.004,"15min":1.9510000000000001},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":4.9180000000000001,"5min":12.411,"15min":18.943999999999999},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.7290000000000001,"15min":1.944},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.36599999999999999},"max":{"1min":5.3159999999999998,"5min":9.8399999999999999,"15min":24.379000000000001},"last":1.036}]},{"osd":7,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":2.2080000000000002,"15min":2.25},"min":{"1min":0.41799999999999998,"5min":0.41699999999999998,"15min":0.308},"max":{"1min":10.17,"5min":13.821,"15min":35.008000000000003},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.5569999999999999,"5min":1.5109999999999999,"15min":2.1190000000000002},"min":{"1min":0.45300000000000001,"5min":0.41799999999999998,"15min":0.35899999999999999},"max":{"1min":10.507,"5min":10.507,"15min":24.323},"last":0.45300000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488296,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1219144,"kb_used_data":923008,"kb_used_omap":15,"kb_used_meta":296112,"kb_avail":92526008,"statfs":{"total":95995035648,"available":94746632192,"internally_reserved":0,"allocated":945160192,"data_stored":943015634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":303219142},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.7909999999999999,"15min":1.3100000000000001},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":3.0209999999999999,"5min":65.674999999999997,"15min":65.674999999999997},"last":0.253},{"interface":"front","average":{"1min":0.81699999999999995,"5min":1.78,"15min":1.288},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.19900000000000001},"max":{"1min":5.7519999999999998,"5min":21.149999999999999,"15min":21.149999999999999},"last":0.78400000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":3.052,"15min":1.8169999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6779999999999999,"5min":105.401,"15min":105.401},"last":5.3760000000000003},{"interface":"front","average":{"1min":0.80000000000000004,"5min":3.165,"15min":1.5880000000000001},"min":{"1min":0.32900000000000001,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":2.8010000000000002,"5min":104.914,"15min":104.914},"last":6.7050000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":1.8520000000000001,"15min":1.9610000000000001},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":9.4369999999999994,"5min":9.4369999999999994,"15min":9.4369999999999994},"last":0.70499999999999996},{"interface":"front","average":{"1min":1.109,"5min":1.383,"15min":1.298},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":3.8359999999999999,"5min":25.338999999999999,"15min":25.338999999999999},"last":1.129}]},{"osd":4,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":2.3069999999999999,"15min":1.2649999999999999},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":2.2629999999999999,"5min":21.831,"15min":21.831},"last":0.84399999999999997},{"interface":"front","average":{"1min":2.8490000000000002,"5min":2.8300000000000001,"15min":1.8280000000000001},"min":{"1min":0.379,"5min":0.35699999999999998,"15min":0.30199999999999999},"max":{"1min":9.2599999999999998,"5min":22.097999999999999,"15min":22.097999999999999},"last":0.57899999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.782,"15min":1.157},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":2.629,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.44},{"interface":"front","average":{"1min":1.617,"5min":2.4129999999999998,"15min":1.361},"min":{"1min":0.45400000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8099999999999996,"5min":13.394,"15min":13.394},"last":0.875}]},{"osd":6,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":2.1669999999999998,"15min":1.6419999999999999},"min":{"1min":0.41299999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.3719999999999999,"5min":19.114000000000001,"15min":19.114000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.262,"5min":1.966,"15min":1.3300000000000001},"min":{"1min":0.434,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":3.4209999999999998,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.47099999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":2.2799999999999998,"15min":1.371},"min":{"1min":0.40999999999999998,"5min":0.39900000000000002,"15min":0.38400000000000001},"max":{"1min":4.6790000000000003,"5min":9.7880000000000003,"15min":10.459},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.744,"5min":2.1760000000000002,"15min":1.7130000000000001},"min":{"1min":0.443,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":11.413,"5min":19.280000000000001,"15min":19.280000000000001},"last":0.75800000000000001}]}]},{"osd":4,"up_from":266,"seq":1142461300926,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2428792,"kb_used_data":1491568,"kb_used_omap":35,"kb_used_meta":937180,"kb_avail":91316360,"statfs":{"total":95995035648,"available":93507952640,"internally_reserved":0,"allocated":1527365632,"data_stored":1526535439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":959672558},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.3039999999999998,"15min":2.0939999999999999},"min":{"1min":0.224,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.548,"5min":97.721999999999994,"15min":97.721999999999994},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.1400000000000001,"5min":4.1319999999999997,"15min":2.75},"min":{"1min":0.32400000000000001,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":10.003,"5min":97.915999999999997,"15min":97.915999999999997},"last":2}]},{"osd":1,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":2.569,"15min":2.335},"min":{"1min":0.379,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.250999999999999,"5min":11.32,"15min":71.457999999999998},"last":2.032},{"interface":"front","average":{"1min":4.5140000000000002,"5min":2.577,"15min":2.1709999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":49.781999999999996,"5min":49.781999999999996,"15min":71.658000000000001},"last":1.976}]},{"osd":2,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":2.085,"15min":1.752},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.944000000000001,"5min":15.944000000000001,"15min":15.944000000000001},"last":2.0179999999999998},{"interface":"front","average":{"1min":2.0409999999999999,"5min":1.927,"15min":1.74},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":14.473000000000001,"5min":14.473000000000001,"15min":14.473000000000001},"last":1.736}]},{"osd":3,"last update":"Mon Oct 9 10:23:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":2.5299999999999998,"15min":1.885},"min":{"1min":0.32300000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.093999999999999,"5min":12.074,"15min":12.074},"last":2.2040000000000002},{"interface":"front","average":{"1min":2.4489999999999998,"5min":2.488,"15min":1.53},"min":{"1min":0.33800000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.067,"5min":16.067,"15min":16.067},"last":2.097}]},{"osd":5,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":2.633,"15min":1.9239999999999999},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":16.094000000000001,"5min":16.094000000000001,"15min":99.585999999999999},"last":4.5910000000000002},{"interface":"front","average":{"1min":4.9089999999999998,"5min":2.597,"15min":1.768},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":49.915999999999997,"5min":49.915999999999997,"15min":99.384},"last":0.71399999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.056,"15min":1.3220000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":14.499000000000001,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.617,"5min":2.657,"15min":1.599},"min":{"1min":0.312,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":11.096,"5min":13.016,"15min":13.016},"last":2.0590000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8889999999999998,"5min":2.7360000000000002,"15min":1.71},"min":{"1min":0.40300000000000002,"5min":0.315,"15min":0.307},"max":{"1min":10.343999999999999,"5min":17.369,"15min":17.369},"last":2.1070000000000002},{"interface":"front","average":{"1min":5.0679999999999996,"5min":2.8029999999999999,"15min":1.694},"min":{"1min":0.38400000000000001,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":51.026000000000003,"5min":51.026000000000003,"15min":51.026000000000003},"last":2.2629999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823110,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2149928,"kb_used_data":1248608,"kb_used_omap":0,"kb_used_meta":901312,"kb_avail":91595224,"statfs":{"total":95995035648,"available":93793509376,"internally_reserved":0,"allocated":1278574592,"data_stored":1277752138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922943488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8359999999999999,"5min":2.2890000000000001,"15min":1.8480000000000001},"min":{"1min":0.35899999999999999,"5min":0.23899999999999999,"15min":0.21099999999999999},"max":{"1min":10.254,"5min":22.469999999999999,"15min":22.469999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.5669999999999999,"5min":2.0619999999999998,"15min":1.6739999999999999},"min":{"1min":0.309,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":8.7929999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.61899999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:23:36 2023","interfaces":[{"interface":"back","average":{"1min":2.7050000000000001,"5min":2.637,"15min":2.1280000000000001},"min":{"1min":0.432,"5min":0.28999999999999998,"15min":0.23999999999999999},"max":{"1min":7.3970000000000002,"5min":29.370000000000001,"15min":29.370000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":2.04,"5min":2.2730000000000001,"15min":1.909},"min":{"1min":0.371,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.859,"5min":29.065000000000001,"15min":29.065000000000001},"last":0.67100000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:23:18 2023","interfaces":[{"interface":"back","average":{"1min":3.8319999999999999,"5min":2.2069999999999999,"15min":2.0179999999999998},"min":{"1min":0.46899999999999997,"5min":0.28999999999999998,"15min":0.27200000000000002},"max":{"1min":13.965999999999999,"5min":13.965999999999999,"15min":15.973000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.8959999999999999,"5min":2.0219999999999998,"15min":1.667},"min":{"1min":0.34499999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.8859999999999992,"5min":13.095000000000001,"15min":15.929},"last":0.56399999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":2.3479999999999999,"15min":1.7370000000000001},"min":{"1min":0.32300000000000001,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":7.04,"5min":10.561999999999999,"15min":10.561999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.1200000000000001,"5min":2.0870000000000002,"15min":1.7010000000000001},"min":{"1min":0.48599999999999999,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9169999999999998,"5min":13.624000000000001,"15min":13.624000000000001},"last":0.64600000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":2.355,"15min":1.4830000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":12.023,"5min":12.023,"15min":12.023},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.5739999999999998,"5min":2.2650000000000001,"15min":1.333},"min":{"1min":0.28899999999999998,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":8.8350000000000009,"5min":11.956,"15min":11.956},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":2.653,"15min":1.6459999999999999},"min":{"1min":0.38900000000000001,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":7.069,"5min":12.772,"15min":12.772},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.887,"5min":2.1200000000000001,"15min":1.597},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.26300000000000001},"max":{"1min":7.3609999999999998,"5min":10.403,"15min":15.997},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:23:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1909999999999998,"5min":2.4129999999999998,"15min":1.6599999999999999},"min":{"1min":0.47699999999999998,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":21.109000000000002,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.609,"5min":1.7030000000000001,"15min":1.115},"min":{"1min":0.39900000000000002,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":9.0739999999999998,"5min":11.994,"15min":11.994},"last":0.59799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1396113408,"data_stored":1396113408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1673854976,"data_stored":1673854976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":972292096,"data_stored":972292096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1583415296,"data_stored":1583415296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1335623680,"data_stored":1335623680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:24:23.681 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:24:23.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 31 2023-10-09T10:24:24.137 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 24 16 50517 50501 131.436 118.438 0.00889401 0.00760382 2023-10-09T10:24:24.250 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:24.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:24.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:24.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:24.413 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:24.662 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:24.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:24.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:24.821 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:24.872 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:24.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:24.976 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:25.021 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:25.068 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:25.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:25.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:25.100 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:25.137 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 25 16 52316 52300 130.676 112.438 0.0088791 0.00764774 2023-10-09T10:24:25.175 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:25.216 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:25.258 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:25.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:25.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:25.430 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:25.462 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:25 smithi012 ceph-mon[147926]: pgmap v1403: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 116 MiB/s wr, 1.85k op/s; 5585/137097 objects degraded (4.074%); 2/137097 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T10:24:25.462 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:25 smithi012 ceph-mon[152263]: pgmap v1403: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 116 MiB/s wr, 1.85k op/s; 5585/137097 objects degraded (4.074%); 2/137097 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T10:24:25.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:25 smithi110 ceph-mon[136032]: pgmap v1403: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 116 MiB/s wr, 1.85k op/s; 5585/137097 objects degraded (4.074%); 2/137097 objects misplaced (0.001%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T10:24:25.831 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:25.960 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:24:25.960 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:24:25.960 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:24:25.961 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:24:26.137 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 26 16 53430 53414 128.328 69.625 0.00800894 0.00778752 2023-10-09T10:24:26.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:24:27.137 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 27 16 55188 55172 127.645 109.875 0.0141654 0.00782737 2023-10-09T10:24:27.466 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:27 smithi012 ceph-mon[152263]: pgmap v1404: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 119 MiB/s wr, 1.90k op/s; 6217/148362 objects degraded (4.190%); 2/148362 objects misplaced (0.001%); 6.3 KiB/s, 0 objects/s recovering 2023-10-09T10:24:27.473 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:27 smithi012 ceph-mon[147926]: pgmap v1404: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 119 MiB/s wr, 1.90k op/s; 6217/148362 objects degraded (4.190%); 2/148362 objects misplaced (0.001%); 6.3 KiB/s, 0 objects/s recovering 2023-10-09T10:24:27.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:27 smithi110 ceph-mon[136032]: pgmap v1404: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 119 MiB/s wr, 1.90k op/s; 6217/148362 objects degraded (4.190%); 2/148362 objects misplaced (0.001%); 6.3 KiB/s, 0 objects/s recovering 2023-10-09T10:24:27.778 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:27.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:27.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:27.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:27.970 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:28.137 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 28 16 56627 56611 126.298 89.9375 0.00599269 0.00791346 2023-10-09T10:24:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:24:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:24:27] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.33.4" 2023-10-09T10:24:28.341 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:28.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:28.451 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:28.482 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:28.529 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:28.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:28.634 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:28.670 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:28.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 6217/148362 objects degraded (4.190%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:28.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 6217/148362 objects degraded (4.190%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:28.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 6217/148362 objects degraded (4.190%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:28.708 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:28.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:28.800 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:24:28.812 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:28.845 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:24:28.898 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:24:28.904 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:28.947 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:24:28.998 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:29.007 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:24:29.042 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:24:29.081 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:24:29.114 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:24:29.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:24:29.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:24:29.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:24:29.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:24:29.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:24:29.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:24:29.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:24:29.116 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:24:29.137 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 29 16 57898 57882 124.683 79.4375 0.00719573 0.00801628 2023-10-09T10:24:29.178 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:29.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:29.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:29.290 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:29.388 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:29.491 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:29.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:29.500 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:24:29.600 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:29.630 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:29.673 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:29.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:29.708 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[147926]: pgmap v1405: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 117 MiB/s wr, 1.87k op/s; 6217/157590 objects degraded (3.945%); 2/157590 objects misplaced (0.001%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T10:24:29.708 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2127809614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:29.708 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:29.709 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/319776853' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:24:29.709 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:24:29.709 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:29.709 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:29.709 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[152263]: pgmap v1405: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 117 MiB/s wr, 1.87k op/s; 6217/157590 objects degraded (3.945%); 2/157590 objects misplaced (0.001%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T10:24:29.709 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2127809614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:29.709 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:29.710 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/319776853' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:24:29.710 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:24:29.710 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:29.710 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:29.777 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:29.841 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:29.931 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T10:24:29.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:29 smithi110 ceph-mon[136032]: pgmap v1405: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 117 MiB/s wr, 1.87k op/s; 6217/157590 objects degraded (3.945%); 2/157590 objects misplaced (0.001%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T10:24:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2127809614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/319776853' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:24:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:24:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:30.137 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 30 16 59229 59213 123.3 83.1875 0.00464705 0.00810492 2023-10-09T10:24:30.329 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:30.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:30.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:30.494 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:30.620 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:30.620 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:24:30.620 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: osdmap e504: 8 total, 8 up, 5 in 2023-10-09T10:24:30.620 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: failed to encode map e504 with expected crc 2023-10-09T10:24:30.620 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: failed to encode map e504 with expected crc 2023-10-09T10:24:30.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1079777037' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:24:30.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:24:30.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:24:30.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:24:30.621 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:30.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: osdmap e504: 8 total, 8 up, 5 in 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: failed to encode map e504 with expected crc 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: failed to encode map e504 with expected crc 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1079777037' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:24:30.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:30 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T10:24:30.863 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:30.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:30.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:30.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: osdmap e504: 8 total, 8 up, 5 in 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: failed to encode map e504 with expected crc 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: failed to encode map e504 with expected crc 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1079777037' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:24:30.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:30 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:30.969 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:31.010 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:31.058 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:31.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:31.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:31.138 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 31 16 60920 60904 122.728 105.688 0.00886444 0.00814342 2023-10-09T10:24:31.167 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:31.217 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:31.270 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:31.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:31.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:31.379 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:31.429 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:31.462 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:24:31.462 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:24:31.483 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:31.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:31.589 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:31.655 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:31 smithi110 ceph-mon[136032]: pgmap v1407: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 116 MiB/s wr, 1.85k op/s; 6724/163626 objects degraded (4.109%); 2/163626 objects misplaced (0.001%) 2023-10-09T10:24:31.887 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:31.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:31.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:31 smithi012 ceph-mon[147926]: pgmap v1407: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 116 MiB/s wr, 1.85k op/s; 6724/163626 objects degraded (4.109%); 2/163626 objects misplaced (0.001%) 2023-10-09T10:24:31.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:31 smithi012 ceph-mon[152263]: pgmap v1407: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 116 MiB/s wr, 1.85k op/s; 6724/163626 objects degraded (4.109%); 2/163626 objects misplaced (0.001%) 2023-10-09T10:24:31.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:24:31.992 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:32.039 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:32.091 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:32.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:32.130 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:24:32.148 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 32 16 62536 62520 122.047 101 0.0148813 0.00818911 2023-10-09T10:24:32.198 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:32.235 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:32.284 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:32.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:32.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:32.391 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:32.431 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:32.487 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:32.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:32.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:32.594 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:32.655 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:32 smithi012 ceph-mon[152263]: 11.6 scrub starts 2023-10-09T10:24:32.656 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:32 smithi012 ceph-mon[152263]: 11.6 scrub ok 2023-10-09T10:24:32.656 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:32 smithi012 ceph-mon[152263]: 11.2 scrub starts 2023-10-09T10:24:32.656 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:32 smithi012 ceph-mon[147926]: 11.6 scrub starts 2023-10-09T10:24:32.656 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:32 smithi012 ceph-mon[147926]: 11.6 scrub ok 2023-10-09T10:24:32.656 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:32 smithi012 ceph-mon[147926]: 11.2 scrub starts 2023-10-09T10:24:32.665 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:32.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:32 smithi110 ceph-mon[136032]: 11.6 scrub starts 2023-10-09T10:24:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:32 smithi110 ceph-mon[136032]: 11.6 scrub ok 2023-10-09T10:24:32.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:32 smithi110 ceph-mon[136032]: 11.2 scrub starts 2023-10-09T10:24:32.734 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:32.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:32.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:32.905 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:32.906 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:32.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:24:32.980 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:32.985 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:32.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:32.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:33.100 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:33.134 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:33.139 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 33 16 63854 63838 120.845 82.375 0.0122176 0.00827072 2023-10-09T10:24:33.181 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:33.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:33.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:33.290 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:33.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[152263]: pgmap v1408: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 110 MiB/s wr, 1.76k op/s; 7155/171459 objects degraded (4.173%); 2/171459 objects misplaced (0.001%) 2023-10-09T10:24:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[152263]: 11.16 scrub starts 2023-10-09T10:24:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[152263]: 11.2 scrub ok 2023-10-09T10:24:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[152263]: 11.4 scrub starts 2023-10-09T10:24:33.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 7155/171459 objects degraded (4.173%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[147926]: pgmap v1408: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 110 MiB/s wr, 1.76k op/s; 7155/171459 objects degraded (4.173%); 2/171459 objects misplaced (0.001%) 2023-10-09T10:24:33.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[147926]: 11.16 scrub starts 2023-10-09T10:24:33.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[147926]: 11.2 scrub ok 2023-10-09T10:24:33.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[147926]: 11.4 scrub starts 2023-10-09T10:24:33.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 7155/171459 objects degraded (4.173%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:33.825 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:33.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:33.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:33 smithi110 ceph-mon[136032]: pgmap v1408: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 110 MiB/s wr, 1.76k op/s; 7155/171459 objects degraded (4.173%); 2/171459 objects misplaced (0.001%) 2023-10-09T10:24:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:33 smithi110 ceph-mon[136032]: 11.16 scrub starts 2023-10-09T10:24:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:33 smithi110 ceph-mon[136032]: 11.2 scrub ok 2023-10-09T10:24:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:33 smithi110 ceph-mon[136032]: 11.4 scrub starts 2023-10-09T10:24:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 7155/171459 objects degraded (4.173%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:33.981 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:34.021 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:34.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:34.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:34.125 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:34.139 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 34 16 65304 65288 119.956 90.625 0.00872788 0.00833198 2023-10-09T10:24:34.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:34.228 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:34.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:34.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:34.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:34.387 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:34.439 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:34.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:34.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:34.497 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:34 smithi012 ceph-mon[147926]: 11.4 scrub ok 2023-10-09T10:24:34.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:34 smithi012 ceph-mon[152263]: 11.4 scrub ok 2023-10-09T10:24:34.545 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:34.634 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:34.658 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:24:34.658 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:24:34.700 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:34.720 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:34.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:34.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:34.893 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:34 smithi110 ceph-mon[136032]: 11.4 scrub ok 2023-10-09T10:24:35.140 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 35 16 66651 66635 118.931 84.1875 0.0060857 0.00840406 2023-10-09T10:24:35.621 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:35.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:35.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:35.729 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:35.770 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4] out_osds: [1, 6, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:24:35.770 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:24:35.770 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T10:24:35.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T10:24:35.791 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:35 smithi012 ceph-mon[152263]: pgmap v1409: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 76 MiB/s wr, 1.22k op/s; 7640/185055 objects degraded (4.129%); 2/185055 objects misplaced (0.001%) 2023-10-09T10:24:35.791 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:35 smithi012 ceph-mon[152263]: 11.16 scrub starts 2023-10-09T10:24:35.791 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:35 smithi012 ceph-mon[152263]: 11.16 scrub ok 2023-10-09T10:24:35.791 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:35 smithi012 ceph-mon[147926]: pgmap v1409: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 76 MiB/s wr, 1.22k op/s; 7640/185055 objects degraded (4.129%); 2/185055 objects misplaced (0.001%) 2023-10-09T10:24:35.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:35 smithi012 ceph-mon[147926]: 11.16 scrub starts 2023-10-09T10:24:35.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:35 smithi012 ceph-mon[147926]: 11.16 scrub ok 2023-10-09T10:24:35.800 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:35 smithi110 ceph-mon[136032]: pgmap v1409: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 76 MiB/s wr, 1.22k op/s; 7640/185055 objects degraded (4.129%); 2/185055 objects misplaced (0.001%) 2023-10-09T10:24:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:35 smithi110 ceph-mon[136032]: 11.16 scrub starts 2023-10-09T10:24:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:35 smithi110 ceph-mon[136032]: 11.16 scrub ok 2023-10-09T10:24:35.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:24:36.083 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:36.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:36.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:36.140 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 36 16 68159 68143 118.245 94.25 0.00553429 0.00845225 2023-10-09T10:24:36.199 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:36.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:36.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:36.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:36.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:36.368 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:24:36.408 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:24:36.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:36.444 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:36.461 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:24:36.494 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:36.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:36.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:36.501 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:24:36.586 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:24:36.599 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:36.635 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:24:36.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2356768285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:36.653 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:36.669 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:24:36.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2356768285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:36.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2356768285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:36.701 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:36.706 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:24:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:36.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:36.713 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:24:36.713 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:24:36.713 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:24:36.713 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:24:36.713 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:24:36.713 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:24:36.713 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:24:36.713 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:24:36.767 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:24:36.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:36.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:36.966 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:37.140 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 37 16 69386 69370 117.122 76.6875 0.00916018 0.00853298 2023-10-09T10:24:37.583 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:24:37.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:37 smithi012 ceph-mon[152263]: pgmap v1410: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 8148/193779 objects degraded (4.205%); 2/193779 objects misplaced (0.001%) 2023-10-09T10:24:37.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:37 smithi012 ceph-mon[147926]: pgmap v1410: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 8148/193779 objects degraded (4.205%); 2/193779 objects misplaced (0.001%) 2023-10-09T10:24:37.709 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:37.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:37.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:37.877 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:37.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:37 smithi110 ceph-mon[136032]: pgmap v1410: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 8148/193779 objects degraded (4.205%); 2/193779 objects misplaced (0.001%) 2023-10-09T10:24:38.074 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:38.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:38.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 38 16 70651 70635 116.121 79.0625 0.00244012 0.00859841 2023-10-09T10:24:38.156 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:24:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:24:37] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.33.4" 2023-10-09T10:24:38.181 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:38.245 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:38.361 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:38.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:38.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:38.468 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:38.509 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:38.566 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:38.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:38.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:38.674 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:38.730 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:38.776 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:38.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:38.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:38.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2063647262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:38.782 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:38 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:38.783 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:38 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:24:38.783 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:38 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:38.783 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:38 smithi110 ceph-mon[136032]: osdmap e505: 8 total, 8 up, 5 in 2023-10-09T10:24:38.783 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:38 smithi110 ceph-mon[136032]: failed to encode map e505 with expected crc 2023-10-09T10:24:38.783 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:38 smithi110 ceph-mon[136032]: 11.0 scrub starts 2023-10-09T10:24:38.783 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 8148/193779 objects degraded (4.205%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:38.882 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2063647262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:24:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[152263]: osdmap e505: 8 total, 8 up, 5 in 2023-10-09T10:24:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[152263]: failed to encode map e505 with expected crc 2023-10-09T10:24:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[152263]: 11.0 scrub starts 2023-10-09T10:24:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 8148/193779 objects degraded (4.205%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2063647262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:24:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:24:38.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[147926]: osdmap e505: 8 total, 8 up, 5 in 2023-10-09T10:24:38.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[147926]: failed to encode map e505 with expected crc 2023-10-09T10:24:38.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[147926]: 11.0 scrub starts 2023-10-09T10:24:38.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 8148/193779 objects degraded (4.205%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:38.943 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:39.001 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:39.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:39.108 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:39.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 39 16 72078 72062 115.43 89.1875 0.0108538 0.00865836 2023-10-09T10:24:39.149 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:39.202 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:39.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:39.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:39.267 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:24:39.307 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:39.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:39.589 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:39.597 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:24:39.297+0000 7f85a99d3700 -1 mon.a@0(leader).osd e505 definitely_dead 0 2023-10-09T10:24:39.597 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[147926]: pgmap v1412: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 830 op/s; 8148/201417 objects degraded (4.045%); 2/201417 objects misplaced (0.001%) 2023-10-09T10:24:39.597 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[147926]: 11.5 scrub starts 2023-10-09T10:24:39.597 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[147926]: 11.0 scrub ok 2023-10-09T10:24:39.597 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2993909747' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T10:24:39.597 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T10:24:39.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:39.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:39.600 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.6. 2023-10-09T10:24:39.699 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:39.727 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:39.764 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:39.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:39.858 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:39 smithi110 ceph-mon[136032]: pgmap v1412: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 830 op/s; 8148/201417 objects degraded (4.045%); 2/201417 objects misplaced (0.001%) 2023-10-09T10:24:39.858 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:39 smithi110 ceph-mon[136032]: 11.5 scrub starts 2023-10-09T10:24:39.858 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:39 smithi110 ceph-mon[136032]: 11.0 scrub ok 2023-10-09T10:24:39.858 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2993909747' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T10:24:39.858 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:39 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T10:24:39.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:39.896 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[152263]: pgmap v1412: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 52 MiB/s wr, 830 op/s; 8148/201417 objects degraded (4.045%); 2/201417 objects misplaced (0.001%) 2023-10-09T10:24:39.896 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[152263]: 11.5 scrub starts 2023-10-09T10:24:39.896 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[152263]: 11.0 scrub ok 2023-10-09T10:24:39.896 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2993909747' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T10:24:39.896 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:39 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T10:24:39.899 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:39.933 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:39.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:39.940 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T10:24:40.038 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:40.065 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:40.100 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:40.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:40.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:40.127 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:40.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:24:40.141338+0000 min lat: 0.00117322 max lat: 1.03449 avg lat: 0.00868163 2023-10-09T10:24:40.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:24:40.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 40 16 72742 72726 113.582 41.5 0.00265328 0.00868163 2023-10-09T10:24:40.204 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:40.211 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:24:40.211 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:24:40.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.211 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72720 11.a 11:5b185df8:::benchmark_data_smithi012_404402_object72719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e505)", 2023-10-09T10:24:40.211 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.583141+0000", 2023-10-09T10:24:40.211 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.62710717800000004, 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.62713477200000001, 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72720 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583141+0000", 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583141+0000", 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583139+0000", 2023-10-09T10:24:40.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9317999999999997e-05 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583199+0000", 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9536999999999999e-05 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583218+0000", 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6407999999999999e-05 2023-10-09T10:24:40.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583234+0000", 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9909e-05 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583274+0000", 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011687 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:24:40.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583391+0000", 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000171979 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583563+0000", 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.016643749999999999 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.600207+0000", 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2649999999999996e-06 2023-10-09T10:24:40.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.600215+0000", 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017074400999999999 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.217 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72721 11.a 11:5f366b23:::benchmark_data_smithi012_404402_object72720:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e505)", 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.583248+0000", 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.62700037399999997, 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.62708636100000004, 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72721 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:40.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583248+0000", 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583248+0000", 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:40.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583247+0000", 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2159999999999997e-05 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583279+0000", 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3950000000000003e-06 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583286+0000", 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1819999999999997e-06 2023-10-09T10:24:40.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.583292+0000", 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040145750000000003 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.587306+0000", 2023-10-09T10:24:40.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014827699999999999 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.587455+0000", 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018874399999999999 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.587643+0000", 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012599445000000001 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.600243+0000", 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8680000000000001e-06 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.600250+0000", 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.017001924000000002 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72726 11.8 11.50fd47a8 (undecoded) ondisk+write+known_if_redirected e505)", 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.597314+0000", 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.61293390199999997, 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.61306772099999995, 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:40.224 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72726 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597314+0000", 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597314+0000", 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:24:40.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597312+0000", 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2837999999999999e-05 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597355+0000", 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3868e-05 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:40.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597369+0000", 2023-10-09T10:24:40.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0740000000000004e-06 2023-10-09T10:24:40.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:40.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597376+0000", 2023-10-09T10:24:40.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112883 2023-10-09T10:24:40.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597489+0000", 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3849999999999996e-06 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597496+0000", 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018199899999999999 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:40.228 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72727 11.5 11.ca065805 (undecoded) ondisk+write+known_if_redirected e505)", 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.597383+0000", 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.61286480099999996, 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.61303599399999997, 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:24:40.229 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72727 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597383+0000", 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597383+0000", 2023-10-09T10:24:40.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597383+0000", 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6319000000000002e-05 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597419+0000", 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9040000000000003e-06 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597426+0000", 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7140000000000001e-06 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597430+0000", 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000480099 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597911+0000", 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3690000000000007e-06 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597919+0000", 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00053572099999999996 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.233 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72729 11.13 11.3180d733 (undecoded) ondisk+write+known_if_redirected e505)", 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.597437+0000", 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.61281068800000005, 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.61301834600000005, 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72729 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:40.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597437+0000", 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597437+0000", 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597437+0000", 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4285000000000001e-05 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597471+0000", 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6329999999999999e-06 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597478+0000", 2023-10-09T10:24:40.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9060000000000001e-06 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.597483+0000", 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00056775599999999999 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598050+0000", 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0019e-05 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598060+0000", 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062306500000000001 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72730 11.18 11.b33e3738 (undecoded) ondisk+write+known_if_redirected e505)", 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.598083+0000", 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.612165242, 2023-10-09T10:24:40.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.61240909200000004, 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72730 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598083+0000", 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:40.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598083+0000", 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598082+0000", 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0417999999999998e-05 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598122+0000", 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0183999999999999e-05 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598132+0000", 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0839999999999997e-06 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598139+0000", 2023-10-09T10:24:40.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010014 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598239+0000", 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7350000000000008e-06 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.598248+0000", 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016558799999999999 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.242 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72735 11.13 11.26c98773 (undecoded) ondisk+write+known_if_redirected e506)", 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.618331+0000", 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.59191718299999996, 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.59223223999999997, 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:40.243 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72735 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.618331+0000", 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.618331+0000", 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:24:40.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.618329+0000", 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0661000000000002e-05 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.618380+0000", 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2721999999999999e-05 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.618393+0000", 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6790000000000002e-06 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.618398+0000", 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1766e-05 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.618440+0000", 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9119999999999993e-06 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.618450+0000", 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011917900000000001 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72742 11.a 11.7ef9e59a (undecoded) ondisk+write+known_if_redirected e506)", 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.628178+0000", 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.58207019800000004, 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.58242503700000003, 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "delayed", 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72742 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.248 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628178+0000", 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628178+0000", 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628177+0000", 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5803000000000002e-05 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628223+0000", 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7119999999999995e-06 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628232+0000", 2023-10-09T10:24:40.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9359999999999996e-06 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628238+0000", 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1192999999999997e-05 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628279+0000", 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4279999999999996e-06 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:40.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628288+0000", 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011003200000000001 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 8 2023-10-09T10:24:40.252 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:24:40.253 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:40.268 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:40.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:40.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:40.401 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:40.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:24:40.533 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:40.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:40.635 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:24:40.637 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:40.687 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:40.736 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:40.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:40.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:40.843 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:40.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:40 smithi012 ceph-mon[147926]: 11.5 scrub starts 2023-10-09T10:24:40.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:40 smithi012 ceph-mon[147926]: failed to encode map e506 with expected crc 2023-10-09T10:24:40.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:40 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T10:24:40.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:40 smithi012 ceph-mon[147926]: osdmap e506: 8 total, 8 up, 6 in 2023-10-09T10:24:40.847 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:40 smithi012 ceph-mon[152263]: 11.5 scrub starts 2023-10-09T10:24:40.847 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:40 smithi012 ceph-mon[152263]: failed to encode map e506 with expected crc 2023-10-09T10:24:40.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:40 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T10:24:40.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:40 smithi012 ceph-mon[152263]: osdmap e506: 8 total, 8 up, 6 in 2023-10-09T10:24:40.867 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:40 smithi110 ceph-mon[136032]: 11.5 scrub starts 2023-10-09T10:24:40.867 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:40 smithi110 ceph-mon[136032]: failed to encode map e506 with expected crc 2023-10-09T10:24:40.867 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:40 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T10:24:40.867 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:40 smithi110 ceph-mon[136032]: osdmap e506: 8 total, 8 up, 6 in 2023-10-09T10:24:40.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:40.907 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:40.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:40.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:41.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:41.041 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:41.077 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:41.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:41.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:41.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 41 16 72742 72726 110.813 0 - 0.00868163 2023-10-09T10:24:41.183 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:41.218 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:41.261 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:41.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:41.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:41.290 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:24:41.301 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:24:41.319 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:24:41.336 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:24:41.358 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:24:41.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:41.370 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:41.375 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:24:41.400 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:24:41.402 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:41.423 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:24:41.443 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:24:41.444 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:24:41.444 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:24:41.444 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:24:41.444 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:24:41.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:24:41.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:24:41.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:24:41.445 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:24:41.505 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:41.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:41.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:41.608 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:41.716 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:41.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: pgmap v1414: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 31 MiB/s wr, 493 op/s; 8548/206646 objects degraded (4.137%); 2/206646 objects misplaced (0.001%) 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: 11.7 scrub starts 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: osdmap e507: 8 total, 8 up, 6 in 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: 11.7 scrub ok 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/834874435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: pgmap v1414: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 31 MiB/s wr, 493 op/s; 8548/206646 objects degraded (4.137%); 2/206646 objects misplaced (0.001%) 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: 11.7 scrub starts 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: osdmap e507: 8 total, 8 up, 6 in 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: 11.7 scrub ok 2023-10-09T10:24:41.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/834874435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: pgmap v1414: 99 pgs: 3 active+undersized+degraded, 4 active+clean+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 31 MiB/s wr, 493 op/s; 8548/206646 objects degraded (4.137%); 2/206646 objects misplaced (0.001%) 2023-10-09T10:24:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: 11.7 scrub starts 2023-10-09T10:24:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: osdmap e507: 8 total, 8 up, 6 in 2023-10-09T10:24:41.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: failed to encode map e507 with expected crc 2023-10-09T10:24:41.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: 11.7 scrub ok 2023-10-09T10:24:41.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/834874435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:42.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 42 16 72898 72882 108.408 4.875 0.0127789 0.00921944 2023-10-09T10:24:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: 11.17 scrub starts 2023-10-09T10:24:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: osdmap e508: 8 total, 8 up, 6 in 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: 11.d starting backfill to osd.6 from (0'0,0'0] MAX to 505'4490 2023-10-09T10:24:42.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: 11.a starting backfill to osd.6 from (0'0,0'0] MAX to 505'4561 2023-10-09T10:24:42.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[147926]: 11.17 scrub ok 2023-10-09T10:24:42.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: 11.17 scrub starts 2023-10-09T10:24:42.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: osdmap e508: 8 total, 8 up, 6 in 2023-10-09T10:24:42.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: 11.d starting backfill to osd.6 from (0'0,0'0] MAX to 505'4490 2023-10-09T10:24:42.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: 11.a starting backfill to osd.6 from (0'0,0'0] MAX to 505'4561 2023-10-09T10:24:42.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:42 smithi012 ceph-mon[152263]: 11.17 scrub ok 2023-10-09T10:24:42.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: 11.17 scrub starts 2023-10-09T10:24:42.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: osdmap e508: 8 total, 8 up, 6 in 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: failed to encode map e508 with expected crc 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: 11.d starting backfill to osd.6 from (0'0,0'0] MAX to 505'4490 2023-10-09T10:24:42.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: 11.a starting backfill to osd.6 from (0'0,0'0] MAX to 505'4561 2023-10-09T10:24:42.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:42 smithi110 ceph-mon[136032]: 11.17 scrub ok 2023-10-09T10:24:43.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 43 16 74084 74068 107.61 74.125 0.0263634 0.00928821 2023-10-09T10:24:43.264 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:43.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:43.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:43.451 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:43.626 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:43 smithi110 ceph-mon[136032]: pgmap v1417: 99 pgs: 3 remapped+peering, 28 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 66 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 39 MiB/s wr, 622 op/s; 4340/213669 objects degraded (2.031%) 2023-10-09T10:24:43.626 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:43 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:24:43.627 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 4340/213669 objects degraded (2.031%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:24:43.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:43 smithi012 ceph-mon[147926]: pgmap v1417: 99 pgs: 3 remapped+peering, 28 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 66 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 39 MiB/s wr, 622 op/s; 4340/213669 objects degraded (2.031%) 2023-10-09T10:24:43.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:43 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:24:43.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 4340/213669 objects degraded (2.031%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:24:43.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:43 smithi012 ceph-mon[152263]: pgmap v1417: 99 pgs: 3 remapped+peering, 28 peering, 1 active+undersized+degraded, 1 active+clean+remapped, 66 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 39 MiB/s wr, 622 op/s; 4340/213669 objects degraded (2.031%) 2023-10-09T10:24:43.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:43 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:24:43.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 4340/213669 objects degraded (2.031%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:24:44.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:24:44.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 44 16 75258 75242 106.832 73.375 0.00525483 0.00935101 2023-10-09T10:24:44.823 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:44.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:44.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:44.941 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4, 6] out_osds: [1, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:24:44.941 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:24:44.941 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T10:24:44.941 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:24:45.020 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:45.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:44 smithi012 ceph-mon[147926]: pgmap v1418: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 3 remapped+peering, 28 peering, 5 active+recovery_wait+undersized+degraded+remapped, 61 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 21 MiB/s wr, 333 op/s; 13578/220095 objects degraded (6.169%); 4544/220095 objects misplaced (2.065%); 577 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T10:24:45.050 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:45.050 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:45.050 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:44 smithi012 ceph-mon[152263]: pgmap v1418: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 3 remapped+peering, 28 peering, 5 active+recovery_wait+undersized+degraded+remapped, 61 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 21 MiB/s wr, 333 op/s; 13578/220095 objects degraded (6.169%); 4544/220095 objects misplaced (2.065%); 577 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T10:24:45.050 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:45.050 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:45.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 45 16 76300 76284 105.906 65.125 0.00820537 0.00943822 2023-10-09T10:24:45.433 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:44 smithi110 ceph-mon[136032]: pgmap v1418: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 3 remapped+peering, 28 peering, 5 active+recovery_wait+undersized+degraded+remapped, 61 active+clean; 4.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 21 MiB/s wr, 333 op/s; 13578/220095 objects degraded (6.169%); 4544/220095 objects misplaced (2.065%); 577 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T10:24:45.433 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:45.433 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:45.717 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:45.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:45.724 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:45.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:45.877 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:45.947 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:45.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:45.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:46.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:46 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:24:46.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:46 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:24:46.031 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:46 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:24:46.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:46.101 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:46.141 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 46 16 77374 77358 105.063 67.125 0.0110208 0.00951223 2023-10-09T10:24:46.211 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:24:46.211 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:24:46.563 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:24:47.083 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:47.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:47.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:47.142 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 47 16 78434 78418 104.236 66.25 0.0189038 0.00958817 2023-10-09T10:24:47.189 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:47.248 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:47.306 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:47.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:47.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:47.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:47 smithi012 ceph-mon[147926]: pgmap v1419: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 31791/223782 objects degraded (14.206%); 9305/223782 objects misplaced (4.158%); 633 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T10:24:47.318 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:47 smithi012 ceph-mon[152263]: pgmap v1419: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 31791/223782 objects degraded (14.206%); 9305/223782 objects misplaced (4.158%); 633 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T10:24:47.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:47.442 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:24:47.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:47 smithi110 ceph-mon[136032]: pgmap v1419: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 31791/223782 objects degraded (14.206%); 9305/223782 objects misplaced (4.158%); 633 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T10:24:47.480 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:47.595 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:47.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:47.702 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:47.796 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:47.922 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:48.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:48.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:48.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 48 16 79486 79470 103.434 65.75 0.0185106 0.00966324 2023-10-09T10:24:48.182 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:24:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:24:47] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.33.4" 2023-10-09T10:24:48.226 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:48.277 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:48.294 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:48.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:48.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:48.458 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:48.960 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:24:48.960 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:24:49.078 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:24:49.142 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 49 16 80540 80524 102.668 65.875 0.00771418 0.00973453 2023-10-09T10:24:49.154 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:49.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:49.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:49.260 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:49.345 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:49 smithi110 ceph-mon[136032]: pgmap v1420: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 7.2 MiB/s wr, 115 op/s; 31791/229815 objects degraded (13.833%); 8954/229815 objects misplaced (3.896%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-09T10:24:49.345 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:49 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1864197474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:49.345 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:49 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 31791/229815 objects degraded (13.833%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:49.354 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:49.400 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:24:49.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:49 smithi012 ceph-mon[152263]: pgmap v1420: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 7.2 MiB/s wr, 115 op/s; 31791/229815 objects degraded (13.833%); 8954/229815 objects misplaced (3.896%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-09T10:24:49.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:49 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1864197474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:49.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:49 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 31791/229815 objects degraded (13.833%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:49 smithi012 ceph-mon[147926]: pgmap v1420: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 7.2 MiB/s wr, 115 op/s; 31791/229815 objects degraded (13.833%); 8954/229815 objects misplaced (3.896%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-09T10:24:49.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:49 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1864197474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:24:49.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:49 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 31791/229815 objects degraded (13.833%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:49.436 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:49.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:49.543 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:49.645 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:49.739 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:49.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:49.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:49.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:49.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:50.023 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:50.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:50.132 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:50.139 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:24:50.142 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 50 16 81488 81472 101.8 59.25 0.0214696 0.00981836 2023-10-09T10:24:50.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:50 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:50.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:50 smithi110 ceph-mon[136032]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1864197474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:24:50.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:50 smithi110 ceph-mon[136032]: osdmap e509: 8 total, 8 up, 6 in 2023-10-09T10:24:50.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:50 smithi110 ceph-mon[136032]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:50 smithi110 ceph-mon[136032]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.213 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:50.379 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:50.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:50.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:50.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:50.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[147926]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1864197474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:24:50.405 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[147926]: osdmap e509: 8 total, 8 up, 6 in 2023-10-09T10:24:50.405 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[147926]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.405 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[147926]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.406 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:24:50.406 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[152263]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.406 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1864197474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:24:50.406 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[152263]: osdmap e509: 8 total, 8 up, 6 in 2023-10-09T10:24:50.406 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[152263]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.406 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:50 smithi012 ceph-mon[152263]: failed to encode map e509 with expected crc 2023-10-09T10:24:50.484 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:50.546 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:50.608 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:50.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:50.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:50.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:50.796 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:51.106 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:51.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:51.142 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 51 16 82537 82521 101.089 65.5625 0.00778339 0.00988783 2023-10-09T10:24:51.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:51 smithi110 ceph-mon[136032]: failed to encode map e509 with expected crc 2023-10-09T10:24:51.172 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:51 smithi110 ceph-mon[136032]: pgmap v1422: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 32160/233952 objects degraded (13.746%); 8954/233952 objects misplaced (3.827%); 3.2 MiB/s, 1 keys/s, 52 objects/s recovering 2023-10-09T10:24:51.218 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:51.311 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:51.367 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:51.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:51.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:51.381 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:24:51.413 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:24:51.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:51 smithi012 ceph-mon[147926]: failed to encode map e509 with expected crc 2023-10-09T10:24:51.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:51 smithi012 ceph-mon[147926]: pgmap v1422: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 32160/233952 objects degraded (13.746%); 8954/233952 objects misplaced (3.827%); 3.2 MiB/s, 1 keys/s, 52 objects/s recovering 2023-10-09T10:24:51.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:51 smithi012 ceph-mon[152263]: failed to encode map e509 with expected crc 2023-10-09T10:24:51.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:51 smithi012 ceph-mon[152263]: pgmap v1422: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 32160/233952 objects degraded (13.746%); 8954/233952 objects misplaced (3.827%); 3.2 MiB/s, 1 keys/s, 52 objects/s recovering 2023-10-09T10:24:51.451 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:24:51.474 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:51.491 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:24:51.516 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:24:51.550 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:51.559 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:24:51.602 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:24:51.635 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:24:51.640 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:24:51.640 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:24:51.640 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:24:51.640 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:24:51.640 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:24:51.640 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:24:51.640 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:24:51.640 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:24:51.822 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:24:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:24:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.822 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:83219 11.d e509/507 11:b714c506:::benchmark_data_smithi012_404402_object83218:head v 509'5214, mlcod=509'5211)", 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:51.819147+0000", 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.002534438, 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025623780000000001, 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819147+0000", 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819147+0000", 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819145+0000", 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1534e-05 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819157+0000", 2023-10-09T10:24:51.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5761999999999999e-05 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819173+0000", 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9230000000000002e-06 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819181+0000", 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1707999999999998e-05 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819262+0000", 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3176999999999998e-05 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819295+0000", 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014896000000000001 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:83218 11.0 e509/507 11:0797ccdd:::benchmark_data_smithi012_404402_object83217:head v 509'3496, mlcod=0'0)", 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:51.819351+0000", 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0023298110000000002, 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024281289999999998, 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819351+0000", 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:51.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819351+0000", 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819350+0000", 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8589000000000003e-05 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:51.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819419+0000", 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1477e-05 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819430+0000", 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4320000000000004e-06 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819437+0000", 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2519000000000001e-05 2023-10-09T10:24:51.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819519+0000", 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3646e-05 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.819543+0000", 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000191967 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:83215 11.3 e509/507 11:c752144e:::benchmark_data_smithi012_404402_object83214:head v 509'3502, mlcod=0'0)", 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:51.821695+0000", 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4294967295.9999866, 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146983, 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821695+0000", 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:51.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821695+0000", 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821693+0000", 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9689999999999999e-05 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821723+0000", 2023-10-09T10:24:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0104e-05 2023-10-09T10:24:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821733+0000", 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8259999999999998e-06 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821739+0000", 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1425999999999999e-05 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821780+0000", 2023-10-09T10:24:51.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3907999999999999e-05 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821804+0000", 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010988399999999999 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:83220 11.a e509/507)", 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:51.821752+0000", 2023-10-09T10:24:51.835 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4294967295.9999285, 2023-10-09T10:24:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014545000000000001, 2023-10-09T10:24:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:24:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:24:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:24:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:24:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821752+0000", 2023-10-09T10:24:51.836 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821752+0000", 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821752+0000", 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8399999999999997e-06 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821756+0000", 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.468e-06 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821761+0000", 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3830000000000001e-06 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:51.821765+0000", 2023-10-09T10:24:51.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2336e-05 2023-10-09T10:24:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:24:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:24:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:24:51.839 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 4 2023-10-09T10:24:51.839 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:24:51.978 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:51.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:52.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:52.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:52 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1725739181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:52.142 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 52 16 83570 83554 100.387 64.5625 0.00543565 0.00995654 2023-10-09T10:24:52.154 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:52 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1725739181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:52.163 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:52.247 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:52.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:52.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:52.309 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:24:52.345 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:52 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1725739181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:52.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:52.420 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:52.480 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:52.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:52.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:52.594 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:52.685 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:52.763 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:52.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:52.873 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:24:52.874 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:52.883 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:24:52.954 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:53.022 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:53.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:53.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:53.085 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:24:53.130 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:24:53.142 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 53 16 84555 84539 99.6546 61.5625 0.00889015 0.010029 2023-10-09T10:24:53.173 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:53 smithi012 ceph-mon[152263]: pgmap v1423: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 12 MiB/s wr, 192 op/s; 32160/239847 objects degraded (13.409%); 9281/239847 objects misplaced (3.870%); 2.6 MiB/s, 0 keys/s, 42 objects/s recovering 2023-10-09T10:24:53.173 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:53 smithi012 ceph-mon[147926]: pgmap v1423: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 12 MiB/s wr, 192 op/s; 32160/239847 objects degraded (13.409%); 9281/239847 objects misplaced (3.870%); 2.6 MiB/s, 0 keys/s, 42 objects/s recovering 2023-10-09T10:24:53.179 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:53.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:24:53.298 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:24:53.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:53.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:24:53.307 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1423,"stamp":"2023-10-09T10:24:51.755988+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5238462368,"num_objects":79949,"num_object_clones":0,"num_object_copies":239847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32160,"num_objects_misplaced":9281,"num_objects_unfound":0,"num_objects_dirty":79949,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":65609,"num_write_kb":4184685,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":559,"num_bytes_recovered":28870362,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97605,"ondisk_log_size":97605,"up":297,"acting":283,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":19569944,"kb_used_data":14842024,"kb_used_omap":163,"kb_used_meta":4727708,"kb_avail":542900968,"statfs":{"total":575970213888,"available":555930591232,"internally_reserved":0,"allocated":15198232576,"data_stored":15189952367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":4841174003},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":571834368,"num_objects":8726,"num_object_clones":0,"num_object_copies":26178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27820,"num_objects_misplaced":9281,"num_objects_unfound":0,"num_objects_dirty":8726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1926,"num_write_kb":123232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":425,"num_bytes_recovered":27006768,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003199"},"pg_stats":[{"pgid":"11.12","version":"508'3435","reported_seq":3468,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:47.629053+0000","last_change":"2023-10-09T10:24:13.569654+0000","last_active":"2023-10-09T10:24:47.629053+0000","last_peered":"2023-10-09T10:24:47.629053+0000","last_clean":"2023-10-09T10:24:47.629053+0000","last_became_active":"2023-10-09T10:24:13.568512+0000","last_became_peered":"2023-10-09T10:24:13.568512+0000","last_unstale":"2023-10-09T10:24:47.629053+0000","last_undegraded":"2023-10-09T10:24:47.629053+0000","last_fullsized":"2023-10-09T10:24:47.629053+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3435,"log_dups_size":0,"ondisk_log_size":3435,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162955264,"num_objects":2487,"num_object_clones":0,"num_object_copies":7461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2487,"num_write_kb":159136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":101,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.635250+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:24:40.635250+0000","last_peered":"2023-10-09T10:24:40.635250+0000","last_clean":"2023-10-09T10:24:40.635250+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:24:40.635250+0000","last_undegraded":"2023-10-09T10:24:40.635250+0000","last_fullsized":"2023-10-09T10:24:40.635250+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":462,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.663828+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:24:40.663828+0000","last_peered":"2023-10-09T10:24:40.663828+0000","last_clean":"2023-10-09T10:24:40.663828+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:24:40.663828+0000","last_undegraded":"2023-10-09T10:24:40.663828+0000","last_fullsized":"2023-10-09T10:24:40.663828+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"508'3384","reported_seq":3425,"reported_epoch":508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:47.575280+0000","last_change":"2023-10-09T10:24:41.952909+0000","last_active":"2023-10-09T10:24:47.575280+0000","last_peered":"2023-10-09T10:24:47.575280+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:24:41.948939+0000","last_became_peered":"2023-10-09T10:24:41.948939+0000","last_unstale":"2023-10-09T10:24:47.575280+0000","last_undegraded":"2023-10-09T10:24:41.628593+0000","last_fullsized":"2023-10-09T10:24:41.606431+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3384,"log_dups_size":0,"ondisk_log_size":3384,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159842304,"num_objects":2439,"num_object_clones":0,"num_object_copies":7317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2264,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2439,"num_write_kb":156096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,2],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2","objects":175},{"shards":"0,2,4","objects":2264}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":539,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.660752+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:24:40.660752+0000","last_peered":"2023-10-09T10:24:40.660752+0000","last_clean":"2023-10-09T10:24:40.660752+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:24:40.660752+0000","last_undegraded":"2023-10-09T10:24:40.660752+0000","last_fullsized":"2023-10-09T10:24:40.660752+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":423,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.611660+0000","last_change":"2023-10-09T10:23:25.438653+0000","last_active":"2023-10-09T10:24:40.611660+0000","last_peered":"2023-10-09T10:24:40.611660+0000","last_clean":"2023-10-09T10:24:40.611660+0000","last_became_active":"2023-10-09T10:23:25.438243+0000","last_became_peered":"2023-10-09T10:23:25.438243+0000","last_unstale":"2023-10-09T10:24:40.611660+0000","last_undegraded":"2023-10-09T10:24:40.611660+0000","last_fullsized":"2023-10-09T10:24:40.611660+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:59.024327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"508'3358","reported_seq":3401,"reported_epoch":508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:47.602475+0000","last_change":"2023-10-09T10:24:41.977456+0000","last_active":"2023-10-09T10:24:47.602475+0000","last_peered":"2023-10-09T10:24:47.602475+0000","last_clean":"2023-10-09T10:24:39.547747+0000","last_became_active":"2023-10-09T10:24:41.974279+0000","last_became_peered":"2023-10-09T10:24:41.974279+0000","last_unstale":"2023-10-09T10:24:47.602475+0000","last_undegraded":"2023-10-09T10:24:41.714846+0000","last_fullsized":"2023-10-09T10:24:41.670066+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3358,"log_dups_size":0,"ondisk_log_size":3358,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158629888,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2232,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2421,"num_write_kb":154912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2232},{"shards":"0,3","objects":189}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":34,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.679091+0000","last_change":"2023-10-09T10:23:25.411688+0000","last_active":"2023-10-09T10:24:41.679091+0000","last_peered":"2023-10-09T10:24:41.679091+0000","last_clean":"2023-10-09T10:24:41.679091+0000","last_became_active":"2023-10-09T10:23:25.411340+0000","last_became_peered":"2023-10-09T10:23:25.411340+0000","last_unstale":"2023-10-09T10:24:41.679091+0000","last_undegraded":"2023-10-09T10:24:41.679091+0000","last_fullsized":"2023-10-09T10:24:41.679091+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":84,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.654299+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:24:40.654299+0000","last_peered":"2023-10-09T10:24:40.654299+0000","last_clean":"2023-10-09T10:24:40.654299+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:24:40.654299+0000","last_undegraded":"2023-10-09T10:24:40.654299+0000","last_fullsized":"2023-10-09T10:24:40.654299+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":457,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.654446+0000","last_change":"2023-10-09T10:24:40.654195+0000","last_active":"2023-10-09T10:24:40.654446+0000","last_peered":"2023-10-09T10:24:40.654446+0000","last_clean":"2023-10-09T10:24:40.654446+0000","last_became_active":"2023-10-09T10:24:40.653888+0000","last_became_peered":"2023-10-09T10:24:40.653888+0000","last_unstale":"2023-10-09T10:24:40.654446+0000","last_undegraded":"2023-10-09T10:24:40.654446+0000","last_fullsized":"2023-10-09T10:24:40.654446+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:44:04.586031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":416,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.603980+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:24:41.603980+0000","last_peered":"2023-10-09T10:24:41.603980+0000","last_clean":"2023-10-09T10:24:41.603980+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:24:41.603980+0000","last_undegraded":"2023-10-09T10:24:41.603980+0000","last_fullsized":"2023-10-09T10:24:41.603980+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":175,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.614382+0000","last_change":"2023-10-09T10:23:25.517220+0000","last_active":"2023-10-09T10:24:41.614382+0000","last_peered":"2023-10-09T10:24:41.614382+0000","last_clean":"2023-10-09T10:24:41.614382+0000","last_became_active":"2023-10-09T10:23:25.516909+0000","last_became_peered":"2023-10-09T10:23:25.516909+0000","last_unstale":"2023-10-09T10:24:41.614382+0000","last_undegraded":"2023-10-09T10:24:41.614382+0000","last_fullsized":"2023-10-09T10:24:41.614382+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":415,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.603785+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:24:41.603785+0000","last_peered":"2023-10-09T10:24:41.603785+0000","last_clean":"2023-10-09T10:24:41.603785+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:24:41.603785+0000","last_undegraded":"2023-10-09T10:24:41.603785+0000","last_fullsized":"2023-10-09T10:24:41.603785+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":113,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.638355+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:24:40.638355+0000","last_peered":"2023-10-09T10:24:40.638355+0000","last_clean":"2023-10-09T10:24:40.638355+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:24:40.638355+0000","last_undegraded":"2023-10-09T10:24:40.638355+0000","last_fullsized":"2023-10-09T10:24:40.638355+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.664759+0000","last_change":"2023-10-09T10:24:40.663438+0000","last_active":"2023-10-09T10:24:41.664759+0000","last_peered":"2023-10-09T10:24:41.664759+0000","last_clean":"2023-10-09T10:24:41.664759+0000","last_became_active":"2023-10-09T10:24:40.662672+0000","last_became_peered":"2023-10-09T10:24:40.662672+0000","last_unstale":"2023-10-09T10:24:41.664759+0000","last_undegraded":"2023-10-09T10:24:41.664759+0000","last_fullsized":"2023-10-09T10:24:41.664759+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":159,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.674426+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:24:40.674426+0000","last_peered":"2023-10-09T10:24:40.674426+0000","last_clean":"2023-10-09T10:24:40.674426+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:24:40.674426+0000","last_undegraded":"2023-10-09T10:24:40.674426+0000","last_fullsized":"2023-10-09T10:24:40.674426+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":119,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.673454+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:24:40.673454+0000","last_peered":"2023-10-09T10:24:40.673454+0000","last_clean":"2023-10-09T10:24:40.673454+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:24:40.673454+0000","last_undegraded":"2023-10-09T10:24:40.673454+0000","last_fullsized":"2023-10-09T10:24:40.673454+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":79,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.683771+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:24:40.683771+0000","last_peered":"2023-10-09T10:24:40.683771+0000","last_clean":"2023-10-09T10:24:40.683771+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:24:40.683771+0000","last_undegraded":"2023-10-09T10:24:40.683771+0000","last_fullsized":"2023-10-09T10:24:40.683771+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":34,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.614076+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:24:41.614076+0000","last_peered":"2023-10-09T10:24:41.614076+0000","last_clean":"2023-10-09T10:24:41.614076+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:24:41.614076+0000","last_undegraded":"2023-10-09T10:24:41.614076+0000","last_fullsized":"2023-10-09T10:24:41.614076+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":421,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.608689+0000","last_change":"2023-10-09T10:23:25.421838+0000","last_active":"2023-10-09T10:24:40.608689+0000","last_peered":"2023-10-09T10:24:40.608689+0000","last_clean":"2023-10-09T10:24:40.608689+0000","last_became_active":"2023-10-09T10:23:25.421388+0000","last_became_peered":"2023-10-09T10:23:25.421388+0000","last_unstale":"2023-10-09T10:24:40.608689+0000","last_undegraded":"2023-10-09T10:24:40.608689+0000","last_fullsized":"2023-10-09T10:24:40.608689+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:56.171644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":175,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.684796+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:24:41.684796+0000","last_peered":"2023-10-09T10:24:41.684796+0000","last_clean":"2023-10-09T10:24:41.684796+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:24:41.684796+0000","last_undegraded":"2023-10-09T10:24:41.684796+0000","last_fullsized":"2023-10-09T10:24:41.684796+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"509'3506","reported_seq":3540,"reported_epoch":509,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:49.879696+0000","last_change":"2023-10-09T10:24:41.963783+0000","last_active":"2023-10-09T10:24:49.879696+0000","last_peered":"2023-10-09T10:24:49.879696+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:24:41.957285+0000","last_became_peered":"2023-10-09T10:24:41.957285+0000","last_unstale":"2023-10-09T10:24:49.879696+0000","last_undegraded":"2023-10-09T10:24:41.705569+0000","last_fullsized":"2023-10-09T10:24:41.661643+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3506,"log_dups_size":0,"ondisk_log_size":3506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169246720,"num_objects":2583,"num_object_clones":0,"num_object_copies":7749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2309,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2583,"num_write_kb":165280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":274}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":119,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.660698+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:24:40.660698+0000","last_peered":"2023-10-09T10:24:40.660698+0000","last_clean":"2023-10-09T10:24:40.660698+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:24:40.660698+0000","last_undegraded":"2023-10-09T10:24:40.660698+0000","last_fullsized":"2023-10-09T10:24:40.660698+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":68,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.604845+0000","last_change":"2023-10-09T10:24:40.641755+0000","last_active":"2023-10-09T10:24:41.604845+0000","last_peered":"2023-10-09T10:24:41.604845+0000","last_clean":"2023-10-09T10:24:41.604845+0000","last_became_active":"2023-10-09T10:24:40.641382+0000","last_became_peered":"2023-10-09T10:24:40.641382+0000","last_unstale":"2023-10-09T10:24:41.604845+0000","last_undegraded":"2023-10-09T10:24:41.604845+0000","last_fullsized":"2023-10-09T10:24:41.604845+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:46.877379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"509'3525","reported_seq":3565,"reported_epoch":509,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:49.879471+0000","last_change":"2023-10-09T10:24:41.976069+0000","last_active":"2023-10-09T10:24:49.879471+0000","last_peered":"2023-10-09T10:24:49.879471+0000","last_clean":"2023-10-09T10:24:39.576387+0000","last_became_active":"2023-10-09T10:24:41.970833+0000","last_became_peered":"2023-10-09T10:24:41.970833+0000","last_unstale":"2023-10-09T10:24:49.879471+0000","last_undegraded":"2023-10-09T10:24:41.665651+0000","last_fullsized":"2023-10-09T10:24:41.611986+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3525,"log_dups_size":0,"ondisk_log_size":3525,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167739392,"num_objects":2560,"num_object_clones":0,"num_object_copies":7680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2560,"num_write_kb":163808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":2278},{"shards":"2,5","objects":282}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":481,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.687834+0000","last_change":"2023-10-09T10:24:40.679710+0000","last_active":"2023-10-09T10:24:41.687834+0000","last_peered":"2023-10-09T10:24:41.687834+0000","last_clean":"2023-10-09T10:24:41.687834+0000","last_became_active":"2023-10-09T10:24:40.664618+0000","last_became_peered":"2023-10-09T10:24:40.664618+0000","last_unstale":"2023-10-09T10:24:41.687834+0000","last_undegraded":"2023-10-09T10:24:41.687834+0000","last_fullsized":"2023-10-09T10:24:41.687834+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":70,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.620300+0000","last_change":"2023-10-09T10:24:40.654281+0000","last_active":"2023-10-09T10:24:41.620300+0000","last_peered":"2023-10-09T10:24:41.620300+0000","last_clean":"2023-10-09T10:24:41.620300+0000","last_became_active":"2023-10-09T10:24:40.654024+0000","last_became_peered":"2023-10-09T10:24:40.654024+0000","last_unstale":"2023-10-09T10:24:41.620300+0000","last_undegraded":"2023-10-09T10:24:41.620300+0000","last_fullsized":"2023-10-09T10:24:41.620300+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.7","version":"509'3574","reported_seq":3611,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-09T10:24:49.840286+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:24:49.840286+0000","last_peered":"2023-10-09T10:24:49.840286+0000","last_clean":"2023-10-09T10:24:49.840286+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:24:49.840286+0000","last_undegraded":"2023-10-09T10:24:49.840286+0000","last_fullsized":"2023-10-09T10:24:49.840286+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":3574,"log_dups_size":0,"ondisk_log_size":3574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172752896,"num_objects":2636,"num_object_clones":0,"num_object_copies":7908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":286,"num_write_kb":18304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":120,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.609108+0000","last_change":"2023-10-09T10:23:25.461650+0000","last_active":"2023-10-09T10:24:40.609108+0000","last_peered":"2023-10-09T10:24:40.609108+0000","last_clean":"2023-10-09T10:24:40.609108+0000","last_became_active":"2023-10-09T10:23:25.432695+0000","last_became_peered":"2023-10-09T10:23:25.432695+0000","last_unstale":"2023-10-09T10:24:40.609108+0000","last_undegraded":"2023-10-09T10:24:40.609108+0000","last_fullsized":"2023-10-09T10:24:40.609108+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:15.291716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":184,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.687700+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:24:41.687700+0000","last_peered":"2023-10-09T10:24:41.687700+0000","last_clean":"2023-10-09T10:24:41.687700+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:24:41.687700+0000","last_undegraded":"2023-10-09T10:24:41.687700+0000","last_fullsized":"2023-10-09T10:24:41.687700+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"508'3304","reported_seq":3339,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:48.099384+0000","last_change":"2023-10-09T10:24:31.689343+0000","last_active":"2023-10-09T10:24:48.099384+0000","last_peered":"2023-10-09T10:24:48.099384+0000","last_clean":"2023-10-09T10:24:48.099384+0000","last_became_active":"2023-10-09T10:24:13.547606+0000","last_became_peered":"2023-10-09T10:24:13.547606+0000","last_unstale":"2023-10-09T10:24:48.099384+0000","last_undegraded":"2023-10-09T10:24:48.099384+0000","last_fullsized":"2023-10-09T10:24:48.099384+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":3304,"log_dups_size":0,"ondisk_log_size":3304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154206208,"num_objects":2353,"num_object_clones":0,"num_object_copies":7059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":544,"num_write_kb":34816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":122,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.609048+0000","last_change":"2023-10-09T10:23:25.742454+0000","last_active":"2023-10-09T10:24:40.609048+0000","last_peered":"2023-10-09T10:24:40.609048+0000","last_clean":"2023-10-09T10:24:40.609048+0000","last_became_active":"2023-10-09T10:23:25.526430+0000","last_became_peered":"2023-10-09T10:23:25.526430+0000","last_unstale":"2023-10-09T10:24:40.609048+0000","last_undegraded":"2023-10-09T10:24:40.609048+0000","last_fullsized":"2023-10-09T10:24:40.609048+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":125,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.647570+0000","last_change":"2023-10-09T10:24:40.646971+0000","last_active":"2023-10-09T10:24:40.647570+0000","last_peered":"2023-10-09T10:24:40.647570+0000","last_clean":"2023-10-09T10:24:40.647570+0000","last_became_active":"2023-10-09T10:24:40.645867+0000","last_became_peered":"2023-10-09T10:24:40.645867+0000","last_unstale":"2023-10-09T10:24:40.647570+0000","last_undegraded":"2023-10-09T10:24:40.647570+0000","last_fullsized":"2023-10-09T10:24:40.647570+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:29.240439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.5","version":"509'3523","reported_seq":3566,"reported_epoch":509,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:49.859058+0000","last_change":"2023-10-09T10:24:42.005778+0000","last_active":"2023-10-09T10:24:49.859058+0000","last_peered":"2023-10-09T10:24:49.859058+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:24:41.997801+0000","last_became_peered":"2023-10-09T10:24:41.997801+0000","last_unstale":"2023-10-09T10:24:49.859058+0000","last_undegraded":"2023-10-09T10:24:41.678061+0000","last_fullsized":"2023-10-09T10:24:41.651535+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3523,"log_dups_size":0,"ondisk_log_size":3523,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168984576,"num_objects":2578,"num_object_clones":0,"num_object_copies":7734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2319,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2578,"num_write_kb":165024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2319},{"shards":"0,5","objects":259}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":200,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.679436+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:24:41.679436+0000","last_peered":"2023-10-09T10:24:41.679436+0000","last_clean":"2023-10-09T10:24:41.679436+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:24:41.679436+0000","last_undegraded":"2023-10-09T10:24:41.679436+0000","last_fullsized":"2023-10-09T10:24:41.679436+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.664974+0000","last_change":"2023-10-09T10:24:40.664182+0000","last_active":"2023-10-09T10:24:41.664974+0000","last_peered":"2023-10-09T10:24:41.664974+0000","last_clean":"2023-10-09T10:24:41.664974+0000","last_became_active":"2023-10-09T10:24:40.663881+0000","last_became_peered":"2023-10-09T10:24:40.663881+0000","last_unstale":"2023-10-09T10:24:41.664974+0000","last_undegraded":"2023-10-09T10:24:41.664974+0000","last_fullsized":"2023-10-09T10:24:41.664974+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"508'3396","reported_seq":4291,"reported_epoch":508,"state":"active+recovering+remapped","last_fresh":"2023-10-09T10:24:47.662398+0000","last_change":"2023-10-09T10:24:41.866871+0000","last_active":"2023-10-09T10:24:47.662398+0000","last_peered":"2023-10-09T10:24:47.662398+0000","last_clean":"2023-10-09T10:24:39.570134+0000","last_became_active":"2023-10-09T10:24:41.848692+0000","last_became_peered":"2023-10-09T10:24:41.848692+0000","last_unstale":"2023-10-09T10:24:47.662398+0000","last_undegraded":"2023-10-09T10:24:47.662398+0000","last_fullsized":"2023-10-09T10:24:47.662398+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":1965,"log_size":3396,"log_dups_size":0,"ondisk_log_size":3396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:31.581241+0000","scrub_duration":0.53049767999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160366592,"num_objects":2447,"num_object_clones":0,"num_object_copies":7341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1861,"num_objects_unfound":0,"num_objects_dirty":2447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":482,"num_write_kb":30848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26411008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":586}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":184,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.612488+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:24:41.612488+0000","last_peered":"2023-10-09T10:24:41.612488+0000","last_clean":"2023-10-09T10:24:41.612488+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:24:41.612488+0000","last_undegraded":"2023-10-09T10:24:41.612488+0000","last_fullsized":"2023-10-09T10:24:41.612488+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":407,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.608267+0000","last_change":"2023-10-09T10:22:13.366463+0000","last_active":"2023-10-09T10:24:40.608267+0000","last_peered":"2023-10-09T10:24:40.608267+0000","last_clean":"2023-10-09T10:24:40.608267+0000","last_became_active":"2023-10-09T10:22:13.366125+0000","last_became_peered":"2023-10-09T10:22:13.366125+0000","last_unstale":"2023-10-09T10:24:40.608267+0000","last_undegraded":"2023-10-09T10:24:40.608267+0000","last_fullsized":"2023-10-09T10:24:40.608267+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:41.113925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"509'3427","reported_seq":3468,"reported_epoch":509,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:49.889441+0000","last_change":"2023-10-09T10:24:41.975717+0000","last_active":"2023-10-09T10:24:49.889441+0000","last_peered":"2023-10-09T10:24:49.889441+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:24:49.889441+0000","last_undegraded":"2023-10-09T10:24:41.665622+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3427,"log_dups_size":0,"ondisk_log_size":3427,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164265984,"num_objects":2506,"num_object_clones":0,"num_object_copies":7518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2240,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2506,"num_write_kb":160416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2240},{"shards":"2,5","objects":266}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":104,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.608178+0000","last_change":"2023-10-09T10:23:25.554673+0000","last_active":"2023-10-09T10:24:40.608178+0000","last_peered":"2023-10-09T10:24:40.608178+0000","last_clean":"2023-10-09T10:24:40.608178+0000","last_became_active":"2023-10-09T10:23:25.436984+0000","last_became_peered":"2023-10-09T10:23:25.436984+0000","last_unstale":"2023-10-09T10:24:40.608178+0000","last_undegraded":"2023-10-09T10:24:40.608178+0000","last_fullsized":"2023-10-09T10:24:40.608178+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:00.646924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":119,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.664110+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:24:40.664110+0000","last_peered":"2023-10-09T10:24:40.664110+0000","last_clean":"2023-10-09T10:24:40.664110+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:24:40.664110+0000","last_undegraded":"2023-10-09T10:24:40.664110+0000","last_fullsized":"2023-10-09T10:24:40.664110+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"508'3387","reported_seq":3474,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:47.553808+0000","last_change":"2023-10-09T10:24:32.360877+0000","last_active":"2023-10-09T10:24:47.553808+0000","last_peered":"2023-10-09T10:24:47.553808+0000","last_clean":"2023-10-09T10:24:47.553808+0000","last_became_active":"2023-10-09T10:24:13.548575+0000","last_became_peered":"2023-10-09T10:24:13.548575+0000","last_unstale":"2023-10-09T10:24:47.553808+0000","last_undegraded":"2023-10-09T10:24:47.553808+0000","last_fullsized":"2023-10-09T10:24:47.553808+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":3387,"log_dups_size":0,"ondisk_log_size":3387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:06.832855+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160169984,"num_objects":2444,"num_object_clones":0,"num_object_copies":7332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":499,"num_write_kb":31936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":119,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.663877+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:24:40.663877+0000","last_peered":"2023-10-09T10:24:40.663877+0000","last_clean":"2023-10-09T10:24:40.663877+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:24:40.663877+0000","last_undegraded":"2023-10-09T10:24:40.663877+0000","last_fullsized":"2023-10-09T10:24:40.663877+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.659689+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:24:40.659689+0000","last_peered":"2023-10-09T10:24:40.659689+0000","last_clean":"2023-10-09T10:24:40.659689+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:24:40.659689+0000","last_undegraded":"2023-10-09T10:24:40.659689+0000","last_fullsized":"2023-10-09T10:24:40.659689+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"508'3393","reported_seq":3434,"reported_epoch":508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:47.604271+0000","last_change":"2023-10-09T10:24:41.952925+0000","last_active":"2023-10-09T10:24:47.604271+0000","last_peered":"2023-10-09T10:24:47.604271+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:24:41.949382+0000","last_became_peered":"2023-10-09T10:24:41.949382+0000","last_unstale":"2023-10-09T10:24:47.604271+0000","last_undegraded":"2023-10-09T10:24:41.626648+0000","last_fullsized":"2023-10-09T10:24:41.606845+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3393,"log_dups_size":0,"ondisk_log_size":3393,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160432128,"num_objects":2448,"num_object_clones":0,"num_object_copies":7344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2448,"num_write_kb":156672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,2],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2","objects":194},{"shards":"0,2,4","objects":2254}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":101,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.633134+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:24:40.633134+0000","last_peered":"2023-10-09T10:24:40.633134+0000","last_clean":"2023-10-09T10:24:40.633134+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:24:40.633134+0000","last_undegraded":"2023-10-09T10:24:40.633134+0000","last_fullsized":"2023-10-09T10:24:40.633134+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":128,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.660818+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:24:40.660818+0000","last_peered":"2023-10-09T10:24:40.660818+0000","last_clean":"2023-10-09T10:24:40.660818+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:24:40.660818+0000","last_undegraded":"2023-10-09T10:24:40.660818+0000","last_fullsized":"2023-10-09T10:24:40.660818+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"508'3378","reported_seq":3479,"reported_epoch":508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:47.585910+0000","last_change":"2023-10-09T10:24:41.972698+0000","last_active":"2023-10-09T10:24:47.585910+0000","last_peered":"2023-10-09T10:24:47.585910+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:24:47.585910+0000","last_undegraded":"2023-10-09T10:24:41.669040+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":3378,"log_dups_size":0,"ondisk_log_size":3378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159776768,"num_objects":2438,"num_object_clones":0,"num_object_copies":7314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":271,"num_write_kb":17344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2233},{"shards":"0,3","objects":205}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":452,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.687625+0000","last_change":"2023-10-09T10:24:40.669558+0000","last_active":"2023-10-09T10:24:41.687625+0000","last_peered":"2023-10-09T10:24:41.687625+0000","last_clean":"2023-10-09T10:24:41.687625+0000","last_became_active":"2023-10-09T10:24:40.669249+0000","last_became_peered":"2023-10-09T10:24:40.669249+0000","last_unstale":"2023-10-09T10:24:41.687625+0000","last_undegraded":"2023-10-09T10:24:41.687625+0000","last_fullsized":"2023-10-09T10:24:41.687625+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":522,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.634876+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:24:40.634876+0000","last_peered":"2023-10-09T10:24:40.634876+0000","last_clean":"2023-10-09T10:24:40.634876+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:24:40.634876+0000","last_undegraded":"2023-10-09T10:24:40.634876+0000","last_fullsized":"2023-10-09T10:24:40.634876+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"508'4964","reported_seq":4992,"reported_epoch":508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:48.091269+0000","last_change":"2023-10-09T10:24:41.999730+0000","last_active":"2023-10-09T10:24:48.091269+0000","last_peered":"2023-10-09T10:24:48.091269+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:24:41.995277+0000","last_became_peered":"2023-10-09T10:24:41.995277+0000","last_unstale":"2023-10-09T10:24:48.091269+0000","last_undegraded":"2023-10-09T10:24:41.635131+0000","last_fullsized":"2023-10-09T10:24:41.618705+0000","mapping_epoch":507,"log_start":"498'200","ondisk_log_start":"498'200","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4764,"log_dups_size":0,"ondisk_log_size":4764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325320704,"num_objects":4964,"num_object_clones":0,"num_object_copies":14892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4964,"num_write_kb":317696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":4964}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":219,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.617348+0000","last_change":"2023-10-09T10:23:25.462977+0000","last_active":"2023-10-09T10:24:40.617348+0000","last_peered":"2023-10-09T10:24:40.617348+0000","last_clean":"2023-10-09T10:24:40.617348+0000","last_became_active":"2023-10-09T10:23:25.462666+0000","last_became_peered":"2023-10-09T10:23:25.462666+0000","last_unstale":"2023-10-09T10:24:40.617348+0000","last_undegraded":"2023-10-09T10:24:40.617348+0000","last_fullsized":"2023-10-09T10:24:40.617348+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:25.535385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":184,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.684721+0000","last_change":"2023-10-09T10:24:40.666651+0000","last_active":"2023-10-09T10:24:41.684721+0000","last_peered":"2023-10-09T10:24:41.684721+0000","last_clean":"2023-10-09T10:24:41.684721+0000","last_became_active":"2023-10-09T10:24:40.666311+0000","last_became_peered":"2023-10-09T10:24:40.666311+0000","last_unstale":"2023-10-09T10:24:41.684721+0000","last_undegraded":"2023-10-09T10:24:41.684721+0000","last_fullsized":"2023-10-09T10:24:41.684721+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":473,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.665043+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:24:41.665043+0000","last_peered":"2023-10-09T10:24:41.665043+0000","last_clean":"2023-10-09T10:24:41.665043+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:24:41.665043+0000","last_undegraded":"2023-10-09T10:24:41.665043+0000","last_fullsized":"2023-10-09T10:24:41.665043+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"508'4993","reported_seq":5016,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:47.643628+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:24:47.643628+0000","last_peered":"2023-10-09T10:24:47.643628+0000","last_clean":"2023-10-09T10:24:47.643628+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:24:47.643628+0000","last_undegraded":"2023-10-09T10:24:47.643628+0000","last_fullsized":"2023-10-09T10:24:47.643628+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4993,"log_dups_size":0,"ondisk_log_size":4993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327221248,"num_objects":4993,"num_object_clones":0,"num_object_copies":14979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4993,"num_write_kb":319552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":193,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.664896+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:24:41.664896+0000","last_peered":"2023-10-09T10:24:41.664896+0000","last_clean":"2023-10-09T10:24:41.664896+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:24:41.664896+0000","last_undegraded":"2023-10-09T10:24:41.664896+0000","last_fullsized":"2023-10-09T10:24:41.664896+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":110,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.659539+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:24:40.659539+0000","last_peered":"2023-10-09T10:24:40.659539+0000","last_clean":"2023-10-09T10:24:40.659539+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:24:40.659539+0000","last_undegraded":"2023-10-09T10:24:40.659539+0000","last_fullsized":"2023-10-09T10:24:40.659539+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":70,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.617849+0000","last_change":"2023-10-09T10:24:40.649214+0000","last_active":"2023-10-09T10:24:41.617849+0000","last_peered":"2023-10-09T10:24:41.617849+0000","last_clean":"2023-10-09T10:24:41.617849+0000","last_became_active":"2023-10-09T10:24:40.648857+0000","last_became_peered":"2023-10-09T10:24:40.648857+0000","last_unstale":"2023-10-09T10:24:41.617849+0000","last_undegraded":"2023-10-09T10:24:41.617849+0000","last_fullsized":"2023-10-09T10:24:41.617849+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"509'3511","reported_seq":3552,"reported_epoch":509,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:49.882091+0000","last_change":"2023-10-09T10:24:41.985591+0000","last_active":"2023-10-09T10:24:49.882091+0000","last_peered":"2023-10-09T10:24:49.882091+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:24:41.979369+0000","last_became_peered":"2023-10-09T10:24:41.979369+0000","last_unstale":"2023-10-09T10:24:49.882091+0000","last_undegraded":"2023-10-09T10:24:41.685845+0000","last_fullsized":"2023-10-09T10:24:41.667241+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3511,"log_dups_size":0,"ondisk_log_size":3511,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166821888,"num_objects":2545,"num_object_clones":0,"num_object_copies":7635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2290,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2545,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":2290},{"shards":"2,5","objects":255}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":448,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.647153+0000","last_change":"2023-10-09T10:24:40.645960+0000","last_active":"2023-10-09T10:24:40.647153+0000","last_peered":"2023-10-09T10:24:40.647153+0000","last_clean":"2023-10-09T10:24:40.647153+0000","last_became_active":"2023-10-09T10:24:40.645619+0000","last_became_peered":"2023-10-09T10:24:40.645619+0000","last_unstale":"2023-10-09T10:24:40.647153+0000","last_undegraded":"2023-10-09T10:24:40.647153+0000","last_fullsized":"2023-10-09T10:24:40.647153+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:43.981027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.647168+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:24:40.647168+0000","last_peered":"2023-10-09T10:24:40.647168+0000","last_clean":"2023-10-09T10:24:40.647168+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:24:40.647168+0000","last_undegraded":"2023-10-09T10:24:40.647168+0000","last_fullsized":"2023-10-09T10:24:40.647168+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":504,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.619742+0000","last_change":"2023-10-09T10:23:25.521874+0000","last_active":"2023-10-09T10:24:41.619742+0000","last_peered":"2023-10-09T10:24:41.619742+0000","last_clean":"2023-10-09T10:24:41.619742+0000","last_became_active":"2023-10-09T10:23:25.521446+0000","last_became_peered":"2023-10-09T10:23:25.521446+0000","last_unstale":"2023-10-09T10:24:41.619742+0000","last_undegraded":"2023-10-09T10:24:41.619742+0000","last_fullsized":"2023-10-09T10:24:41.619742+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"509'5089","reported_seq":5133,"reported_epoch":509,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:24:49.885630+0000","last_change":"2023-10-09T10:24:41.858672+0000","last_active":"2023-10-09T10:24:49.885630+0000","last_peered":"2023-10-09T10:24:49.885630+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:24:49.885630+0000","last_undegraded":"2023-10-09T10:24:49.885630+0000","last_fullsized":"2023-10-09T10:24:49.885630+0000","mapping_epoch":507,"log_start":"498'500","ondisk_log_start":"498'500","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4589,"log_dups_size":0,"ondisk_log_size":4589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333512704,"num_objects":5089,"num_object_clones":0,"num_object_copies":15267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5088,"num_objects_unfound":0,"num_objects_dirty":5089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5089,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":5089}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":163,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.667864+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:24:41.667864+0000","last_peered":"2023-10-09T10:24:41.667864+0000","last_clean":"2023-10-09T10:24:41.667864+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:24:41.667864+0000","last_undegraded":"2023-10-09T10:24:41.667864+0000","last_fullsized":"2023-10-09T10:24:41.667864+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":700,"reported_epoch":508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:41.977919+0000","last_change":"2023-10-09T10:24:41.660008+0000","last_active":"2023-10-09T10:24:41.977919+0000","last_peered":"2023-10-09T10:24:41.977919+0000","last_clean":"2023-10-09T10:24:37.578053+0000","last_became_active":"2023-10-09T10:24:41.622788+0000","last_became_peered":"2023-10-09T10:24:41.622788+0000","last_unstale":"2023-10-09T10:24:41.977919+0000","last_undegraded":"2023-10-09T10:24:41.659804+0000","last_fullsized":"2023-10-09T10:24:41.605128+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:18:51.498413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":1},{"shards":"2,4","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":202,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.664678+0000","last_change":"2023-10-09T10:24:40.671489+0000","last_active":"2023-10-09T10:24:41.664678+0000","last_peered":"2023-10-09T10:24:41.664678+0000","last_clean":"2023-10-09T10:24:41.664678+0000","last_became_active":"2023-10-09T10:24:40.671115+0000","last_became_peered":"2023-10-09T10:24:40.671115+0000","last_unstale":"2023-10-09T10:24:41.664678+0000","last_undegraded":"2023-10-09T10:24:41.664678+0000","last_fullsized":"2023-10-09T10:24:41.664678+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":119,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.659733+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:24:40.659733+0000","last_peered":"2023-10-09T10:24:40.659733+0000","last_clean":"2023-10-09T10:24:40.659733+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:24:40.659733+0000","last_undegraded":"2023-10-09T10:24:40.659733+0000","last_fullsized":"2023-10-09T10:24:40.659733+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"509'3491","reported_seq":3526,"reported_epoch":509,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:49.796402+0000","last_change":"2023-10-09T10:24:41.957653+0000","last_active":"2023-10-09T10:24:49.796402+0000","last_peered":"2023-10-09T10:24:49.796402+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:24:41.949300+0000","last_became_peered":"2023-10-09T10:24:41.949300+0000","last_unstale":"2023-10-09T10:24:49.796402+0000","last_undegraded":"2023-10-09T10:24:41.660576+0000","last_fullsized":"2023-10-09T10:24:41.611975+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3491,"log_dups_size":0,"ondisk_log_size":3491,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168263680,"num_objects":2567,"num_object_clones":0,"num_object_copies":7701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2306,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2567,"num_write_kb":164320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":261}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":192,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.609831+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:24:40.609831+0000","last_peered":"2023-10-09T10:24:40.609831+0000","last_clean":"2023-10-09T10:24:40.609831+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:24:40.609831+0000","last_undegraded":"2023-10-09T10:24:40.609831+0000","last_fullsized":"2023-10-09T10:24:40.609831+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":442,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.615458+0000","last_change":"2023-10-09T10:19:43.292390+0000","last_active":"2023-10-09T10:24:40.615458+0000","last_peered":"2023-10-09T10:24:40.615458+0000","last_clean":"2023-10-09T10:24:40.615458+0000","last_became_active":"2023-10-09T10:19:43.290733+0000","last_became_peered":"2023-10-09T10:19:43.290733+0000","last_unstale":"2023-10-09T10:24:40.615458+0000","last_undegraded":"2023-10-09T10:24:40.615458+0000","last_fullsized":"2023-10-09T10:24:40.615458+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:53.407598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":65,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.614292+0000","last_change":"2023-10-09T10:23:25.531957+0000","last_active":"2023-10-09T10:24:41.614292+0000","last_peered":"2023-10-09T10:24:41.614292+0000","last_clean":"2023-10-09T10:24:41.614292+0000","last_became_active":"2023-10-09T10:23:25.531614+0000","last_became_peered":"2023-10-09T10:23:25.531614+0000","last_unstale":"2023-10-09T10:24:41.614292+0000","last_undegraded":"2023-10-09T10:24:41.614292+0000","last_fullsized":"2023-10-09T10:24:41.614292+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.c","version":"508'5033","reported_seq":5055,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:48.082107+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:24:48.082107+0000","last_peered":"2023-10-09T10:24:48.082107+0000","last_clean":"2023-10-09T10:24:48.082107+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:24:48.082107+0000","last_undegraded":"2023-10-09T10:24:48.082107+0000","last_fullsized":"2023-10-09T10:24:48.082107+0000","mapping_epoch":494,"log_start":"498'300","ondisk_log_start":"498'300","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4733,"log_dups_size":0,"ondisk_log_size":4733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329842688,"num_objects":5033,"num_object_clones":0,"num_object_copies":15099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5033,"num_write_kb":322112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":128,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.663783+0000","last_change":"2023-10-09T10:24:40.660988+0000","last_active":"2023-10-09T10:24:40.663783+0000","last_peered":"2023-10-09T10:24:40.663783+0000","last_clean":"2023-10-09T10:24:40.663783+0000","last_became_active":"2023-10-09T10:24:40.660470+0000","last_became_peered":"2023-10-09T10:24:40.660470+0000","last_unstale":"2023-10-09T10:24:40.663783+0000","last_undegraded":"2023-10-09T10:24:40.663783+0000","last_fullsized":"2023-10-09T10:24:40.663783+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":481,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.612952+0000","last_change":"2023-10-09T10:24:40.654902+0000","last_active":"2023-10-09T10:24:41.612952+0000","last_peered":"2023-10-09T10:24:41.612952+0000","last_clean":"2023-10-09T10:24:41.612952+0000","last_became_active":"2023-10-09T10:24:40.654383+0000","last_became_peered":"2023-10-09T10:24:40.654383+0000","last_unstale":"2023-10-09T10:24:41.612952+0000","last_undegraded":"2023-10-09T10:24:41.612952+0000","last_fullsized":"2023-10-09T10:24:41.612952+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":410,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.608465+0000","last_change":"2023-10-09T10:23:25.458876+0000","last_active":"2023-10-09T10:24:40.608465+0000","last_peered":"2023-10-09T10:24:40.608465+0000","last_clean":"2023-10-09T10:24:40.608465+0000","last_became_active":"2023-10-09T10:23:25.458037+0000","last_became_peered":"2023-10-09T10:23:25.458037+0000","last_unstale":"2023-10-09T10:24:40.608465+0000","last_undegraded":"2023-10-09T10:24:40.608465+0000","last_fullsized":"2023-10-09T10:24:40.608465+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:34.186003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"508'4814","reported_seq":4837,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:47.620800+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:24:47.620800+0000","last_peered":"2023-10-09T10:24:47.620800+0000","last_clean":"2023-10-09T10:24:47.620800+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:24:47.620800+0000","last_undegraded":"2023-10-09T10:24:47.620800+0000","last_fullsized":"2023-10-09T10:24:47.620800+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4814,"log_dups_size":0,"ondisk_log_size":4814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315490304,"num_objects":4814,"num_object_clones":0,"num_object_copies":14442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4814,"num_write_kb":308096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":307,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.609298+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:24:40.609298+0000","last_peered":"2023-10-09T10:24:40.609298+0000","last_clean":"2023-10-09T10:24:40.609298+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:24:40.609298+0000","last_undegraded":"2023-10-09T10:24:40.609298+0000","last_fullsized":"2023-10-09T10:24:40.609298+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":489,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.660770+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:24:40.660770+0000","last_peered":"2023-10-09T10:24:40.660770+0000","last_clean":"2023-10-09T10:24:40.660770+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:24:40.660770+0000","last_undegraded":"2023-10-09T10:24:40.660770+0000","last_fullsized":"2023-10-09T10:24:40.660770+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":385,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.991279+0000","last_change":"2023-10-09T10:24:41.978614+0000","last_active":"2023-10-09T10:24:41.991279+0000","last_peered":"2023-10-09T10:24:41.991279+0000","last_clean":"2023-10-09T10:24:41.991279+0000","last_became_active":"2023-10-09T10:24:41.978246+0000","last_became_peered":"2023-10-09T10:24:41.978246+0000","last_unstale":"2023-10-09T10:24:41.991279+0000","last_undegraded":"2023-10-09T10:24:41.991279+0000","last_fullsized":"2023-10-09T10:24:41.991279+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.e","version":"508'5051","reported_seq":5073,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:48.070848+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:24:48.070848+0000","last_peered":"2023-10-09T10:24:48.070848+0000","last_clean":"2023-10-09T10:24:48.070848+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:24:48.070848+0000","last_undegraded":"2023-10-09T10:24:48.070848+0000","last_fullsized":"2023-10-09T10:24:48.070848+0000","mapping_epoch":494,"log_start":"498'300","ondisk_log_start":"498'300","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4751,"log_dups_size":0,"ondisk_log_size":4751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331022336,"num_objects":5051,"num_object_clones":0,"num_object_copies":15153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5051,"num_write_kb":323264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.609032+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:24:41.609032+0000","last_peered":"2023-10-09T10:24:41.609032+0000","last_clean":"2023-10-09T10:24:41.609032+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:24:41.609032+0000","last_undegraded":"2023-10-09T10:24:41.609032+0000","last_fullsized":"2023-10-09T10:24:41.609032+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":195,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.737162+0000","last_change":"2023-10-09T10:24:40.736665+0000","last_active":"2023-10-09T10:24:40.737162+0000","last_peered":"2023-10-09T10:24:40.737162+0000","last_clean":"2023-10-09T10:24:40.737162+0000","last_became_active":"2023-10-09T10:24:40.671370+0000","last_became_peered":"2023-10-09T10:24:40.671370+0000","last_unstale":"2023-10-09T10:24:40.737162+0000","last_undegraded":"2023-10-09T10:24:40.737162+0000","last_fullsized":"2023-10-09T10:24:40.737162+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":189,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.706256+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:24:41.706256+0000","last_peered":"2023-10-09T10:24:41.706256+0000","last_clean":"2023-10-09T10:24:41.706256+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:24:41.706256+0000","last_undegraded":"2023-10-09T10:24:41.706256+0000","last_fullsized":"2023-10-09T10:24:41.706256+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"509'3409","reported_seq":3449,"reported_epoch":509,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:49.869660+0000","last_change":"2023-10-09T10:24:41.974012+0000","last_active":"2023-10-09T10:24:49.869660+0000","last_peered":"2023-10-09T10:24:49.869660+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:24:49.869660+0000","last_undegraded":"2023-10-09T10:24:41.649146+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3409,"log_dups_size":0,"ondisk_log_size":3409,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163086336,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2230,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2489,"num_write_kb":159264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2230},{"shards":"2,5","objects":259}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":128,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.979287+0000","last_change":"2023-10-09T10:24:41.975859+0000","last_active":"2023-10-09T10:24:41.979287+0000","last_peered":"2023-10-09T10:24:41.979287+0000","last_clean":"2023-10-09T10:24:41.979287+0000","last_became_active":"2023-10-09T10:24:41.973757+0000","last_became_peered":"2023-10-09T10:24:41.973757+0000","last_unstale":"2023-10-09T10:24:41.979287+0000","last_undegraded":"2023-10-09T10:24:41.979287+0000","last_fullsized":"2023-10-09T10:24:41.979287+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":494,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.629504+0000","last_change":"2023-10-09T10:24:41.629504+0000","last_active":"2023-10-09T10:24:41.629504+0000","last_peered":"2023-10-09T10:24:41.629504+0000","last_clean":"2023-10-09T10:24:41.629504+0000","last_became_active":"2023-10-09T10:24:41.629048+0000","last_became_peered":"2023-10-09T10:24:41.629048+0000","last_unstale":"2023-10-09T10:24:41.629504+0000","last_undegraded":"2023-10-09T10:24:41.629504+0000","last_fullsized":"2023-10-09T10:24:41.629504+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.14","version":"508'3406","reported_seq":3445,"reported_epoch":508,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:24:47.624854+0000","last_change":"2023-10-09T10:24:41.893439+0000","last_active":"2023-10-09T10:24:47.624854+0000","last_peered":"2023-10-09T10:24:47.624854+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:24:47.624854+0000","last_undegraded":"2023-10-09T10:24:47.624854+0000","last_fullsized":"2023-10-09T10:24:47.624854+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3406,"log_dups_size":0,"ondisk_log_size":3406,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162627584,"num_objects":2482,"num_object_clones":0,"num_object_copies":7446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2332,"num_objects_unfound":0,"num_objects_dirty":2482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2482,"num_write_kb":158816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":150}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":239,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.612605+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:24:41.612605+0000","last_peered":"2023-10-09T10:24:41.612605+0000","last_clean":"2023-10-09T10:24:41.612605+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:24:41.612605+0000","last_undegraded":"2023-10-09T10:24:41.612605+0000","last_fullsized":"2023-10-09T10:24:41.612605+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":101,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.634380+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:24:40.634380+0000","last_peered":"2023-10-09T10:24:40.634380+0000","last_clean":"2023-10-09T10:24:40.634380+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:24:40.634380+0000","last_undegraded":"2023-10-09T10:24:40.634380+0000","last_fullsized":"2023-10-09T10:24:40.634380+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"509'3465","reported_seq":3503,"reported_epoch":509,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:24:49.845930+0000","last_change":"2023-10-09T10:24:42.000716+0000","last_active":"2023-10-09T10:24:49.845930+0000","last_peered":"2023-10-09T10:24:49.845930+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:24:41.996323+0000","last_became_peered":"2023-10-09T10:24:41.996323+0000","last_unstale":"2023-10-09T10:24:49.845930+0000","last_undegraded":"2023-10-09T10:24:41.649962+0000","last_fullsized":"2023-10-09T10:24:41.612603+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":3465,"log_dups_size":0,"ondisk_log_size":3465,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165183488,"num_objects":2521,"num_object_clones":0,"num_object_copies":7563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2240,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2521,"num_write_kb":161312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2240},{"shards":"0,5","objects":281}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":222,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.680623+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:24:41.680623+0000","last_peered":"2023-10-09T10:24:41.680623+0000","last_clean":"2023-10-09T10:24:41.680623+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:24:41.680623+0000","last_undegraded":"2023-10-09T10:24:41.680623+0000","last_fullsized":"2023-10-09T10:24:41.680623+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":330,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.608402+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:24:40.608402+0000","last_peered":"2023-10-09T10:24:40.608402+0000","last_clean":"2023-10-09T10:24:40.608402+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:24:40.608402+0000","last_undegraded":"2023-10-09T10:24:40.608402+0000","last_fullsized":"2023-10-09T10:24:40.608402+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"508'3391","reported_seq":3427,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:48.016749+0000","last_change":"2023-10-09T10:24:34.732693+0000","last_active":"2023-10-09T10:24:48.016749+0000","last_peered":"2023-10-09T10:24:48.016749+0000","last_clean":"2023-10-09T10:24:48.016749+0000","last_became_active":"2023-10-09T10:24:13.564880+0000","last_became_peered":"2023-10-09T10:24:13.564880+0000","last_unstale":"2023-10-09T10:24:48.016749+0000","last_undegraded":"2023-10-09T10:24:48.016749+0000","last_fullsized":"2023-10-09T10:24:48.016749+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":3391,"log_dups_size":0,"ondisk_log_size":3391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164560896,"num_objects":2511,"num_object_clones":0,"num_object_copies":7533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":454,"num_write_kb":29056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":71,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.647259+0000","last_change":"2023-10-09T10:24:40.646030+0000","last_active":"2023-10-09T10:24:40.647259+0000","last_peered":"2023-10-09T10:24:40.647259+0000","last_clean":"2023-10-09T10:24:40.647259+0000","last_became_active":"2023-10-09T10:24:40.645673+0000","last_became_peered":"2023-10-09T10:24:40.645673+0000","last_unstale":"2023-10-09T10:24:40.647259+0000","last_undegraded":"2023-10-09T10:24:40.647259+0000","last_fullsized":"2023-10-09T10:24:40.647259+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:03.543808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":550,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.614935+0000","last_change":"2023-10-09T10:16:18.394108+0000","last_active":"2023-10-09T10:24:40.614935+0000","last_peered":"2023-10-09T10:24:40.614935+0000","last_clean":"2023-10-09T10:24:40.614935+0000","last_became_active":"2023-10-09T10:16:18.376228+0000","last_became_peered":"2023-10-09T10:16:18.376228+0000","last_unstale":"2023-10-09T10:24:40.614935+0000","last_undegraded":"2023-10-09T10:24:40.614935+0000","last_fullsized":"2023-10-09T10:24:40.614935+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:52.767306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"509'3504","reported_seq":3540,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-09T10:24:49.850596+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:24:49.850596+0000","last_peered":"2023-10-09T10:24:49.850596+0000","last_clean":"2023-10-09T10:24:49.850596+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:24:49.850596+0000","last_undegraded":"2023-10-09T10:24:49.850596+0000","last_fullsized":"2023-10-09T10:24:49.850596+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":3504,"log_dups_size":0,"ondisk_log_size":3504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165543936,"num_objects":2526,"num_object_clones":0,"num_object_copies":7578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":243,"num_write_kb":15552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":524,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-09T10:24:41.821644+0000","last_change":"2023-10-09T10:24:41.815914+0000","last_active":"2023-10-09T10:24:41.821644+0000","last_peered":"2023-10-09T10:24:41.821644+0000","last_clean":"2023-10-09T10:24:41.821644+0000","last_became_active":"2023-10-09T10:24:41.635750+0000","last_became_peered":"2023-10-09T10:24:41.635750+0000","last_unstale":"2023-10-09T10:24:41.821644+0000","last_undegraded":"2023-10-09T10:24:41.821644+0000","last_fullsized":"2023-10-09T10:24:41.821644+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":7756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":119,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-09T10:24:40.659157+0000","last_change":"2023-10-09T10:24:40.657199+0000","last_active":"2023-10-09T10:24:40.659157+0000","last_peered":"2023-10-09T10:24:40.659157+0000","last_clean":"2023-10-09T10:24:40.659157+0000","last_became_active":"2023-10-09T10:24:40.656592+0000","last_became_peered":"2023-10-09T10:24:40.656592+0000","last_unstale":"2023-10-09T10:24:40.659157+0000","last_undegraded":"2023-10-09T10:24:40.659157+0000","last_fullsized":"2023-10-09T10:24:40.659157+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":5237866496,"num_objects":79925,"num_object_clones":0,"num_object_copies":239775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32159,"num_objects_misplaced":9281,"num_objects_unfound":0,"num_objects_dirty":79925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":65349,"num_write_kb":4182240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":409,"num_bytes_recovered":26804224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15268708352,"data_stored":15268708352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97411,"ondisk_log_size":97411,"up":78,"acting":65,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":32346,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3829760,"data_stored":3804896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430662,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626890,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1161648,"kb_used_data":262376,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":92583504,"statfs":{"total":95995035648,"available":94805508096,"internally_reserved":0,"allocated":268673024,"data_stored":267835610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.96,"15min":1.71},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.4260000000000002,"5min":44.055,"15min":44.055},"last":0.216},{"interface":"front","average":{"1min":1.6839999999999999,"5min":2.319,"15min":1.9750000000000001},"min":{"1min":0.34200000000000003,"5min":0.251,"15min":0.22},"max":{"1min":9.2609999999999992,"5min":43.869999999999997,"15min":43.869999999999997},"last":1.4079999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":3.0289999999999999,"15min":2.4340000000000002},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":5.7919999999999998,"5min":155.01400000000001,"15min":155.01400000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.86199999999999999,"5min":3.165,"15min":2.5760000000000001},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":3.5310000000000001,"5min":154.71100000000001,"15min":154.71100000000001},"last":0.48499999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:24:11 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.647,"15min":1.798},"min":{"1min":0.39200000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":6.1920000000000002},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.5740000000000001,"15min":1.772},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.1470000000000002,"5min":7.5629999999999997,"15min":14.542999999999999},"last":1.4410000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:24:02 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.536,"15min":1.506},"min":{"1min":0.32700000000000001,"5min":0.308,"15min":0.308},"max":{"1min":4.4989999999999997,"5min":7.4889999999999999,"15min":8.0229999999999997},"last":1.3300000000000001},{"interface":"front","average":{"1min":1.875,"5min":1.998,"15min":1.6220000000000001},"min":{"1min":0.40799999999999997,"5min":0.316,"15min":0.31},"max":{"1min":8.0950000000000006,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":1.3149999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:24:02 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":3.335,"15min":1.764},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26000000000000001},"max":{"1min":8.6649999999999991,"5min":122.041,"15min":122.041},"last":4.6890000000000001},{"interface":"front","average":{"1min":1.7310000000000001,"5min":3.2469999999999999,"15min":1.663},"min":{"1min":0.313,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":8.4589999999999996,"5min":122.13200000000001,"15min":122.13200000000001},"last":2.726}]},{"osd":5,"last update":"Mon Oct 9 10:24:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5139999999999998,"5min":3.488,"15min":1.8700000000000001},"min":{"1min":0.32600000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.6769999999999996,"5min":121.98999999999999,"15min":121.98999999999999},"last":9.6890000000000001},{"interface":"front","average":{"1min":1.1779999999999999,"5min":2.7909999999999999,"15min":1.6000000000000001},"min":{"1min":0.34799999999999998,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":6.4989999999999997,"5min":122.346,"15min":122.346},"last":0.35799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":3.4710000000000001,"15min":1.9219999999999999},"min":{"1min":0.34899999999999998,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":1.8640000000000001,"5min":130.66999999999999,"15min":130.66999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.72299999999999998,"5min":3.5139999999999998,"15min":1.9650000000000001},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.34100000000000003},"max":{"1min":1.915,"5min":130.702,"15min":130.702},"last":0.497}]}]},{"osd":1,"up_from":0,"seq":360777253203,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416727,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3663192,"kb_used_data":2817680,"kb_used_omap":28,"kb_used_meta":845475,"kb_avail":90081960,"statfs":{"total":95995035648,"available":92243927040,"internally_reserved":0,"allocated":2885304320,"data_stored":2883418168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":865767248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:24:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.28,"15min":1.996},"min":{"1min":0.248,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.835,"5min":8.8699999999999992,"15min":54.518000000000001},"last":2.254},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.3360000000000001,"15min":2.6230000000000002},"min":{"1min":0.23899999999999999,"5min":0.192,"15min":0.192},"max":{"1min":3.5499999999999998,"5min":17.033000000000001,"15min":56.950000000000003},"last":1.1220000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:24:18 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.8129999999999999,"15min":2.1280000000000001},"min":{"1min":0.26100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.8369999999999997,"5min":47.215000000000003,"15min":53.988999999999997},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.962,"15min":2.0910000000000002},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":5.8899999999999997,"5min":47.000999999999998,"15min":54.119},"last":2.2839999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.3640000000000001,"15min":1.401},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":11.933,"5min":16.879999999999999,"15min":56.984999999999999},"last":1.8859999999999999},{"interface":"front","average":{"1min":1.581,"5min":1.3400000000000001,"15min":1.345},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.0020000000000007,"5min":8.0020000000000007,"15min":56.475999999999999},"last":1.875}]},{"osd":4,"last update":"Mon Oct 9 10:24:13 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.857,"15min":2.3399999999999999},"min":{"1min":0.38800000000000001,"5min":0.251,"15min":0.251},"max":{"1min":3.827,"5min":28.289000000000001,"15min":57.719000000000001},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.901,"15min":2.2349999999999999},"min":{"1min":0.36799999999999999,"5min":0.34000000000000002,"15min":0.32400000000000001},"max":{"1min":8.0640000000000001,"5min":28.331,"15min":56.378},"last":8.2479999999999993}]},{"osd":5,"last update":"Mon Oct 9 10:24:13 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.8340000000000001,"15min":3.137},"min":{"1min":0.312,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":8.0920000000000005,"5min":11.775,"15min":229.589},"last":2.8860000000000001},{"interface":"front","average":{"1min":1.446,"5min":1.5129999999999999,"15min":2.742},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":5.7709999999999999,"5min":8.7530000000000001,"15min":228.88900000000001},"last":3.2149999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:24:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6080000000000001,"5min":1.78,"15min":2.1709999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.33700000000000002},"max":{"1min":5.8090000000000002,"5min":8.7810000000000006,"15min":54.482999999999997},"last":4.8079999999999998},{"interface":"front","average":{"1min":1.4410000000000001,"5min":2.0899999999999999,"15min":2.6539999999999999},"min":{"1min":0.42199999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.8849999999999998,"5min":16.954999999999998,"15min":57.753},"last":1.847}]},{"osd":7,"last update":"Mon Oct 9 10:24:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":1.8360000000000001,"15min":2.4009999999999998},"min":{"1min":0.40500000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.0329999999999995,"5min":8.0329999999999995,"15min":56.411999999999999},"last":2.5910000000000002},{"interface":"front","average":{"1min":2.1000000000000001,"5min":1.944,"15min":2.1970000000000001},"min":{"1min":0.39900000000000002,"5min":0.38600000000000001,"15min":0.36499999999999999},"max":{"1min":10.215999999999999,"5min":11.175000000000001,"15min":54.451999999999998},"last":3.1749999999999998}]}]},{"osd":2,"up_from":217,"seq":932007903463,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3948132,"kb_used_data":3160348,"kb_used_omap":83,"kb_used_meta":787692,"kb_avail":89797020,"statfs":{"total":95995035648,"available":91952148480,"internally_reserved":0,"allocated":3236196352,"data_stored":3234483364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":806597103},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.8400000000000001,"15min":2.032},"min":{"1min":0.21299999999999999,"5min":0.19,"15min":0.185},"max":{"1min":4.2930000000000001,"5min":48.990000000000002,"15min":48.990000000000002},"last":0.754},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.8999999999999999,"15min":2.431},"min":{"1min":0.248,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.1139999999999999,"5min":49.412999999999997,"15min":49.412999999999997},"last":2.339}]},{"osd":1,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":1.353,"15min":2.1920000000000002},"min":{"1min":0.22900000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.8220000000000001,"5min":8.2010000000000005,"15min":34.893999999999998},"last":2.383},{"interface":"front","average":{"1min":1.109,"5min":1.357,"15min":2.1749999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":3.4489999999999998,"5min":6.9710000000000001,"15min":34.962000000000003},"last":2.0299999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.2310000000000001,"15min":1.121},"min":{"1min":0.27700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":5.3010000000000002,"5min":7.54,"15min":35.043999999999997},"last":3.677},{"interface":"front","average":{"1min":1.089,"5min":0.92000000000000004,"15min":1.2030000000000001},"min":{"1min":0.32700000000000001,"5min":0.246,"15min":0.246},"max":{"1min":5.181,"5min":6.617,"15min":50.406999999999996},"last":1.0980000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:24:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":1.6220000000000001,"15min":1.663},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.26600000000000001},"max":{"1min":47.503999999999998,"5min":47.503999999999998,"15min":47.503999999999998},"last":1.2749999999999999},{"interface":"front","average":{"1min":3.0169999999999999,"5min":1.7669999999999999,"15min":1.8080000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":47.704000000000001,"5min":47.704000000000001,"15min":47.704000000000001},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.5920000000000001,"15min":2.008},"min":{"1min":0.308,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":5.2809999999999997,"5min":10.448,"15min":57.298000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.7,"5min":1.764,"15min":2.0089999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.31},"max":{"1min":6.2469999999999999,"5min":8.1630000000000003,"15min":57.828000000000003},"last":0.85399999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":2.004,"15min":1.9510000000000001},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":4.9180000000000001,"5min":12.411,"15min":18.943999999999999},"last":1.238},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.7290000000000001,"15min":1.944},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.36599999999999999},"max":{"1min":5.3159999999999998,"5min":9.8399999999999999,"15min":24.379000000000001},"last":0.55700000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":2.2080000000000002,"15min":2.25},"min":{"1min":0.41799999999999998,"5min":0.41699999999999998,"15min":0.308},"max":{"1min":10.17,"5min":13.821,"15min":35.008000000000003},"last":1.3029999999999999},{"interface":"front","average":{"1min":1.5569999999999999,"5min":1.5109999999999999,"15min":2.1190000000000002},"min":{"1min":0.45300000000000001,"5min":0.41799999999999998,"15min":0.35899999999999999},"max":{"1min":10.507,"5min":10.507,"15min":24.323},"last":5.3040000000000003}]}]},{"osd":3,"up_from":352,"seq":1511828488303,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2596136,"kb_used_data":2259808,"kb_used_omap":15,"kb_used_meta":336304,"kb_avail":91149016,"statfs":{"total":95995035648,"available":93336592384,"internally_reserved":0,"allocated":2314043392,"data_stored":2311861735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":344375750},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.7909999999999999,"15min":1.3100000000000001},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":3.0209999999999999,"5min":65.674999999999997,"15min":65.674999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.81699999999999995,"5min":1.78,"15min":1.288},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.19900000000000001},"max":{"1min":5.7519999999999998,"5min":21.149999999999999,"15min":21.149999999999999},"last":0.28100000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":3.052,"15min":1.8169999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6779999999999999,"5min":105.401,"15min":105.401},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.80000000000000004,"5min":3.165,"15min":1.5880000000000001},"min":{"1min":0.32900000000000001,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":2.8010000000000002,"5min":104.914,"15min":104.914},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":1.8520000000000001,"15min":1.9610000000000001},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":9.4369999999999994,"5min":9.4369999999999994,"15min":9.4369999999999994},"last":0.253},{"interface":"front","average":{"1min":1.109,"5min":1.383,"15min":1.298},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":3.8359999999999999,"5min":25.338999999999999,"15min":25.338999999999999},"last":0.46300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":2.3069999999999999,"15min":1.2649999999999999},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":2.2629999999999999,"5min":21.831,"15min":21.831},"last":0.80900000000000005},{"interface":"front","average":{"1min":2.8490000000000002,"5min":2.8300000000000001,"15min":1.8280000000000001},"min":{"1min":0.379,"5min":0.35699999999999998,"15min":0.30199999999999999},"max":{"1min":9.2599999999999998,"5min":22.097999999999999,"15min":22.097999999999999},"last":2.4399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.782,"15min":1.157},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":2.629,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.617,"5min":2.4129999999999998,"15min":1.361},"min":{"1min":0.45400000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8099999999999996,"5min":13.394,"15min":13.394},"last":2.831}]},{"osd":6,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":2.1669999999999998,"15min":1.6419999999999999},"min":{"1min":0.41299999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.3719999999999999,"5min":19.114000000000001,"15min":19.114000000000001},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.262,"5min":1.966,"15min":1.3300000000000001},"min":{"1min":0.434,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":3.4209999999999998,"5min":25.306999999999999,"15min":25.306999999999999},"last":1.2869999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":2.2799999999999998,"15min":1.371},"min":{"1min":0.40999999999999998,"5min":0.39900000000000002,"15min":0.38400000000000001},"max":{"1min":4.6790000000000003,"5min":9.7880000000000003,"15min":10.459},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.744,"5min":2.1760000000000002,"15min":1.7130000000000001},"min":{"1min":0.443,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":11.413,"5min":19.280000000000001,"15min":19.280000000000001},"last":0.82999999999999996}]}]},{"osd":4,"up_from":266,"seq":1142461300932,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4049764,"kb_used_data":3108444,"kb_used_omap":35,"kb_used_meta":941276,"kb_avail":89695388,"statfs":{"total":95995035648,"available":91848077312,"internally_reserved":0,"allocated":3183046656,"data_stored":3182212078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":963866862},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.3039999999999998,"15min":2.0939999999999999},"min":{"1min":0.224,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.548,"5min":97.721999999999994,"15min":97.721999999999994},"last":0.249},{"interface":"front","average":{"1min":2.1400000000000001,"5min":4.1319999999999997,"15min":2.75},"min":{"1min":0.32400000000000001,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":10.003,"5min":97.915999999999997,"15min":97.915999999999997},"last":2.649}]},{"osd":1,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":2.569,"15min":2.335},"min":{"1min":0.379,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.250999999999999,"5min":11.32,"15min":71.457999999999998},"last":1.1850000000000001},{"interface":"front","average":{"1min":4.5140000000000002,"5min":2.577,"15min":2.1709999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":49.781999999999996,"5min":49.781999999999996,"15min":71.658000000000001},"last":2.101}]},{"osd":2,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":2.085,"15min":1.752},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.944000000000001,"5min":15.944000000000001,"15min":15.944000000000001},"last":1.2010000000000001},{"interface":"front","average":{"1min":2.0409999999999999,"5min":1.927,"15min":1.74},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":14.473000000000001,"5min":14.473000000000001,"15min":14.473000000000001},"last":0.67000000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:23:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":2.5299999999999998,"15min":1.885},"min":{"1min":0.32300000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.093999999999999,"5min":12.074,"15min":12.074},"last":1.2669999999999999},{"interface":"front","average":{"1min":2.4489999999999998,"5min":2.488,"15min":1.53},"min":{"1min":0.33800000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.067,"5min":16.067,"15min":16.067},"last":1.5600000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":2.633,"15min":1.9239999999999999},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":16.094000000000001,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":4.9089999999999998,"5min":2.597,"15min":1.768},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":49.915999999999997,"5min":49.915999999999997,"15min":99.384},"last":0.39200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.056,"15min":1.3220000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":14.499000000000001,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.617,"5min":2.657,"15min":1.599},"min":{"1min":0.312,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":11.096,"5min":13.016,"15min":13.016},"last":1.2430000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:24:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8889999999999998,"5min":2.7360000000000002,"15min":1.71},"min":{"1min":0.40300000000000002,"5min":0.315,"15min":0.307},"max":{"1min":10.343999999999999,"5min":17.369,"15min":17.369},"last":1.216},{"interface":"front","average":{"1min":5.0679999999999996,"5min":2.8029999999999999,"15min":1.694},"min":{"1min":0.38400000000000001,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":51.026000000000003,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.437}]}]},{"osd":5,"up_from":36,"seq":154618823117,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4151072,"kb_used_data":3233368,"kb_used_omap":0,"kb_used_meta":917696,"kb_avail":89594080,"statfs":{"total":95995035648,"available":91744337920,"internally_reserved":0,"allocated":3310968832,"data_stored":3310141412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":939720704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8359999999999999,"5min":2.2890000000000001,"15min":1.8480000000000001},"min":{"1min":0.35899999999999999,"5min":0.23899999999999999,"15min":0.21099999999999999},"max":{"1min":10.254,"5min":22.469999999999999,"15min":22.469999999999999},"last":1.613},{"interface":"front","average":{"1min":1.5669999999999999,"5min":2.0619999999999998,"15min":1.6739999999999999},"min":{"1min":0.309,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":8.7929999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.78100000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":2.6429999999999998,"15min":2.1480000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":4.306,"5min":29.370000000000001,"15min":29.370000000000001},"last":1.71},{"interface":"front","average":{"1min":1.1879999999999999,"5min":2.2229999999999999,"15min":1.865},"min":{"1min":0.28499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.4790000000000001,"5min":29.065000000000001,"15min":29.065000000000001},"last":0.86299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":2.3050000000000002,"15min":2.0369999999999999},"min":{"1min":0.32800000000000001,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":7.5339999999999998,"5min":13.965999999999999,"15min":15.973000000000001},"last":1.7430000000000001},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.996,"15min":1.6539999999999999},"min":{"1min":0.35099999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.6929999999999996,"5min":13.095000000000001,"15min":15.929},"last":0.81000000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":2.3479999999999999,"15min":1.7370000000000001},"min":{"1min":0.32300000000000001,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":7.04,"5min":10.561999999999999,"15min":10.561999999999999},"last":2.6989999999999998},{"interface":"front","average":{"1min":2.1200000000000001,"5min":2.0870000000000002,"15min":1.7010000000000001},"min":{"1min":0.48599999999999999,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9169999999999998,"5min":13.624000000000001,"15min":13.624000000000001},"last":4.5019999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":2.355,"15min":1.4830000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":12.023,"5min":12.023,"15min":12.023},"last":2.7759999999999998},{"interface":"front","average":{"1min":2.5739999999999998,"5min":2.2650000000000001,"15min":1.333},"min":{"1min":0.28899999999999998,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":8.8350000000000009,"5min":11.956,"15min":11.956},"last":0.83999999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":2.653,"15min":1.6459999999999999},"min":{"1min":0.38900000000000001,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":7.069,"5min":12.772,"15min":12.772},"last":5.9080000000000004},{"interface":"front","average":{"1min":1.887,"5min":2.1200000000000001,"15min":1.597},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.26300000000000001},"max":{"1min":7.3609999999999998,"5min":10.403,"15min":15.997},"last":1.657}]},{"osd":7,"last update":"Mon Oct 9 10:24:30 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.4700000000000002,"15min":1.7230000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":7.5750000000000002,"5min":21.109000000000002,"15min":21.109000000000002},"last":1.6850000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.397,"15min":1.1279999999999999},"min":{"1min":0.371,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":2.1709999999999998,"5min":11.994,"15min":11.994},"last":0.88600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2912944128,"data_stored":2912944128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3254517760,"data_stored":3254517760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2315780096,"data_stored":2315780096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3178102784,"data_stored":3178102784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3332833280,"data_stored":3332833280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":274530304,"data_stored":274530304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:24:53.313 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 31 2023-10-09T10:24:53.352 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:53 smithi110 ceph-mon[136032]: pgmap v1423: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 12 MiB/s wr, 192 op/s; 32160/239847 objects degraded (13.409%); 9281/239847 objects misplaced (3.870%); 2.6 MiB/s, 0 keys/s, 42 objects/s recovering 2023-10-09T10:24:53.405 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:53.464 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:53.583 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:53.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:53.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:53.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:24:53.747 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:53.850 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:24:53.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:53.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:24:53.954 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:54.035 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:54.100 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:54 smithi110 ceph-mon[136032]: from='client.36948 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:54.100 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:54 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 32160/239847 objects degraded (13.409%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:54.108 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:54.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:54.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:54.139 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:54.142 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 54 16 85536 85520 98.9447 61.3125 0.009057 0.0101013 2023-10-09T10:24:54.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:24:54.191 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:54.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:54.301 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:54 smithi012 ceph-mon[147926]: from='client.36948 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:54.301 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:54 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 32160/239847 objects degraded (13.409%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:54.301 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:54 smithi012 ceph-mon[152263]: from='client.36948 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:24:54.301 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:54 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 32160/239847 objects degraded (13.409%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:54.309 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:54.606 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:24:54.606 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:24:54.606 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:24:54.607 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:24:54.607 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:24:55.143 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 55 16 86606 86590 98.3617 66.875 0.0121177 0.0101616 2023-10-09T10:24:55.205 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:55.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:55.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:55.312 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:24:55.379 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:55.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:55 smithi012 ceph-mon[147926]: pgmap v1424: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 32 MiB/s wr, 513 op/s; 32487/249402 objects degraded (13.026%); 8939/249402 objects misplaced (3.584%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:24:55.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:55 smithi012 ceph-mon[152263]: pgmap v1424: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 32 MiB/s wr, 513 op/s; 32487/249402 objects degraded (13.026%); 8939/249402 objects misplaced (3.584%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:24:55.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:55 smithi110 ceph-mon[136032]: pgmap v1424: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 32 MiB/s wr, 513 op/s; 32487/249402 objects degraded (13.026%); 8939/249402 objects misplaced (3.584%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:24:56.143 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 56 16 87633 87617 97.7515 64.1875 0.0131845 0.0102251 2023-10-09T10:24:56.425 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:24:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:56.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:24:56.449 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:24:56.536 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:24:56.601 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:57.076 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:24:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:57.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:24:57.143 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 57 16 88632 88616 97.1321 62.4375 0.00257142 0.010289 2023-10-09T10:24:57.182 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:24:57.191 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:24:57.256 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:57.332 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:24:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:57.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:24:57.361 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:24:57.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:57.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:57.517 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:57 smithi012 ceph-mon[147926]: pgmap v1425: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 32487/254952 objects degraded (12.742%); 9253/254952 objects misplaced (3.629%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:24:57.517 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:57 smithi012 ceph-mon[152263]: pgmap v1425: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 32487/254952 objects degraded (12.742%); 9253/254952 objects misplaced (3.629%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:24:57.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:57 smithi110 ceph-mon[136032]: pgmap v1425: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 32487/254952 objects degraded (12.742%); 9253/254952 objects misplaced (3.629%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:24:58.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 58 16 89619 89603 96.5195 61.6875 0.0299961 0.0103558 2023-10-09T10:24:58.166 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:24:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:24:57] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.33.4" 2023-10-09T10:24:58.216 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T10:24:58.351 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:58.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:58.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:58.458 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:24:58.518 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1770003479' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T10:24:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T10:24:58.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 32487/254952 objects degraded (12.742%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T10:24:58.699 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1770003479' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T10:24:58.699 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T10:24:58.699 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 32487/254952 objects degraded (12.742%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:58.699 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1770003479' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T10:24:58.699 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T10:24:58.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 32487/254952 objects degraded (12.742%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:24:59.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 59 16 90675 90659 96.0022 66 0.0165528 0.0104123 2023-10-09T10:24:59.205 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:24:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:59.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:24:59.310 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:24:59.371 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:59.443 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:24:59.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:59.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:24:59.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:59 smithi110 ceph-mon[136032]: pgmap v1426: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 32487/260868 objects degraded (12.453%); 8869/260868 objects misplaced (3.400%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:24:59.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3314221314' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:59.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:59.471 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:59.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:59.472 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:24:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1971654591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:59.472 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:24:59.473 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:24:59.507 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:24:59.552 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:24:59.555 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:24:59.597 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:24:59.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:24:59.638 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:24:59.682 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:24:59.723 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:24:59.767 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:24:59.774 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:24:59.774 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:24:59.774 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:24:59.774 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:24:59.774 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:24:59.774 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:24:59.774 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:24:59.774 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:24:59.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[147926]: pgmap v1426: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 32487/260868 objects degraded (12.453%); 8869/260868 objects misplaced (3.400%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:24:59.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3314221314' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:59.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:59.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:59.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:59.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1971654591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:59.806 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[152263]: pgmap v1426: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s; 32487/260868 objects degraded (12.453%); 8869/260868 objects misplaced (3.400%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:24:59.807 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3314221314' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:59.807 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:24:59.807 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:24:59.807 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:24:59.807 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:24:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1971654591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:24:59.869 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:24:59.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:24:59.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:24:59.977 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:00.035 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:00.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:25:00.144564+0000 min lat: 0.00117322 max lat: 2.44365 avg lat: 0.0104729 2023-10-09T10:25:00.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:25:00.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 60 16 91668 91652 95.4366 62.0625 0.0196384 0.0104729 2023-10-09T10:25:00.215 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:00.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:00.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:00.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:00.364 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:00.414 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:00.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:00.477 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:00.477 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:00.477 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: osdmap e510: 8 total, 8 up, 6 in 2023-10-09T10:25:00.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.478 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[147926]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: osdmap e510: 8 total, 8 up, 6 in 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.479 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:00 smithi012 ceph-mon[152263]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:00.601 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:00.666 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: osdmap e510: 8 total, 8 up, 6 in 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.671 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.671 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:00 smithi110 ceph-mon[136032]: failed to encode map e510 with expected crc 2023-10-09T10:25:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:00.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:00.772 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:00.869 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:00.893 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:25:00.893 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:25:00.940 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:00.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:00.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:01.048 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:01.098 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:01.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 61 16 92678 92662 94.9069 63.125 0.00647893 0.0105327 2023-10-09T10:25:01.152 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:01.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:01.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:01.257 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:01.323 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:01.347 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:25:01.398 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:01.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:01.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:01.503 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:01.600 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:01 smithi110 ceph-mon[136032]: pgmap v1428: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 32795/264483 objects degraded (12.400%); 8869/264483 objects misplaced (3.353%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:25:01.601 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:01.677 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:01.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:01.786 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:01.840 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:01 smithi012 ceph-mon[147926]: pgmap v1428: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 32795/264483 objects degraded (12.400%); 8869/264483 objects misplaced (3.353%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:25:01.840 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:01 smithi012 ceph-mon[152263]: pgmap v1428: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 32795/264483 objects degraded (12.400%); 8869/264483 objects misplaced (3.353%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:25:01.843 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:01.899 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:01.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:01.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:01.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:25:02.008 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:02.087 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:02.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 62 16 93657 93641 94.3631 61.1875 0.014345 0.0105921 2023-10-09T10:25:02.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:02.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:02.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:02.192 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:25:02.267 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:02.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:25:02.321 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:02.444 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:02.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:02.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:02.630 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:02.646 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:02.932 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:02.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:02.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:03.038 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:03.099 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:03.131 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:03.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 63 16 94618 94602 93.8187 60.0625 0.0148691 0.0106534 2023-10-09T10:25:03.435 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:03.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:03.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:03.604 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:03.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:03 smithi110 ceph-mon[136032]: pgmap v1429: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 32795/270231 objects degraded (12.136%); 9205/270231 objects misplaced (3.406%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:25:03.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:03 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 32795/270231 objects degraded (12.136%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:03.701 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4, 6] out_osds: [1, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:25:03.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T10:25:03.707 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:03.710 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:25:03.710 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:25:03.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:03.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:03.761 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:03 smithi012 ceph-mon[147926]: pgmap v1429: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 32795/270231 objects degraded (12.136%); 9205/270231 objects misplaced (3.406%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:25:03.761 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:03 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 32795/270231 objects degraded (12.136%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:03.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:03 smithi012 ceph-mon[152263]: pgmap v1429: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 32795/270231 objects degraded (12.136%); 9205/270231 objects misplaced (3.406%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:25:03.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:03 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 32795/270231 objects degraded (12.136%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:03.813 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:03.908 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:04.018 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:04.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:04.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:04.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:04.136 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:25:04.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 64 16 95571 95555 93.2835 59.5625 0.012823 0.0107159 2023-10-09T10:25:04.187 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:04.264 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:04.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:04.377 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:04.450 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:04.536 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:04.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:04.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:04.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:04.721 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:04.850 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:04.854 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:04.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:04.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:04.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:04.960 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:25:05.013 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:05.128 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:05.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:05.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:05.144 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 65 16 96509 96493 92.7503 58.625 0.0207447 0.0107769 2023-10-09T10:25:05.234 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:05.314 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:05.388 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:05.391 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:25:05.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:05.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:05.424 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:25:05.458 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:25:05.492 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:25:05.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:05.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:05 smithi110 ceph-mon[136032]: pgmap v1430: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 33097/279717 objects degraded (11.832%); 8783/279717 objects misplaced (3.140%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:25:05.522 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2522929092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:25:05.523 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3148552889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:05.532 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:25:05.557 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:25:05.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:05 smithi012 ceph-mon[147926]: pgmap v1430: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 33097/279717 objects degraded (11.832%); 8783/279717 objects misplaced (3.140%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:25:05.565 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2522929092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:25:05.565 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3148552889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:05.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:05 smithi012 ceph-mon[152263]: pgmap v1430: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 33097/279717 objects degraded (11.832%); 8783/279717 objects misplaced (3.140%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:25:05.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2522929092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:25:05.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3148552889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:05.578 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:25:05.580 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:05.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:25:05.670 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:25:05.671 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:25:05.671 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:25:05.671 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:25:05.671 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:25:05.671 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:25:05.671 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:25:05.671 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:25:05.672 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:25:05.700 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:05.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:05.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:05.808 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:05.860 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:06.098 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:06.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:06.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:06.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 66 16 97445 97429 92.2314 58.5 0.0138132 0.0108369 2023-10-09T10:25:06.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:06.270 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:06.370 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:06.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:06.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:06.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:06.478 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:25:06.478 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:25:06.478 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:25:06.478 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:25:06.478 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:25:06.574 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:06.675 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[147926]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2522929092' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:25:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[147926]: osdmap e511: 8 total, 8 up, 6 in 2023-10-09T10:25:06.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[147926]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[147926]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:06.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[152263]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2522929092' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:25:06.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[152263]: osdmap e511: 8 total, 8 up, 6 in 2023-10-09T10:25:06.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[152263]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:06 smithi012 ceph-mon[152263]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:06.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:06.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:06.864 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:06.920 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:06 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:06.920 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:06 smithi110 ceph-mon[136032]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.920 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2522929092' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:25:06.920 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:06 smithi110 ceph-mon[136032]: osdmap e511: 8 total, 8 up, 6 in 2023-10-09T10:25:06.920 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:06 smithi110 ceph-mon[136032]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.920 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:06 smithi110 ceph-mon[136032]: failed to encode map e511 with expected crc 2023-10-09T10:25:06.926 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:25:06.926 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:06.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:06.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:07.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:07.084 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:07.141 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:07.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:07.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:07.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 67 16 98398 98382 91.7439 59.5625 0.00978087 0.0108944 2023-10-09T10:25:07.246 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:07.334 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:07.393 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:07.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:07.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:07.503 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:07.561 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:07.567 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T10:25:07.617 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:07.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:07 smithi110 ceph-mon[136032]: pgmap v1432: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 33097/285180 objects degraded (11.606%); 9100/285180 objects misplaced (3.191%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T10:25:07.620 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1185916382' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:25:07.621 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:25:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:07.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:07.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:07.751 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:07.811 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:25:07.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:07 smithi012 ceph-mon[147926]: pgmap v1432: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 33097/285180 objects degraded (11.606%); 9100/285180 objects misplaced (3.191%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T10:25:07.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1185916382' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:25:07.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:25:07.916 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:07 smithi012 ceph-mon[152263]: pgmap v1432: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 33097/285180 objects degraded (11.606%); 9100/285180 objects misplaced (3.191%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T10:25:07.916 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1185916382' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:25:07.916 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:25:07.922 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:07.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:07.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:07.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T10:25:08.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:08.055 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:08.092 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:08.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:08.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:08.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 68 16 98832 98816 90.7937 27.125 0.00391494 0.0109218 2023-10-09T10:25:08.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:25:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:25:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:25:07] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.33.4" 2023-10-09T10:25:08.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:08.226 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:08.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:08.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:08.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:08.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:25:08.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:08.510 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:08.595 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:08.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:08.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:08.611 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: osdmap e512: 8 total, 8 up, 6 in 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 33097/285180 objects degraded (11.606%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:08.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:08 smithi110 ceph-mon[136032]: osdmap e513: 8 total, 8 up, 6 in 2023-10-09T10:25:08.699 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: osdmap e512: 8 total, 8 up, 6 in 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 33097/285180 objects degraded (11.606%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[147926]: osdmap e513: 8 total, 8 up, 6 in 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: osdmap e512: 8 total, 8 up, 6 in 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: failed to encode map e512 with expected crc 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 33097/285180 objects degraded (11.606%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:08 smithi012 ceph-mon[152263]: osdmap e513: 8 total, 8 up, 6 in 2023-10-09T10:25:08.726 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:08.763 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:08.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:08.867 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:08.896 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:08.934 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:08.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:08.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:09.027 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:09.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:09.093 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114942 9.11 9:8d57c7a6:::benchmark_data_smithi012_313365_object2467:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 594.89788928300004, 2023-10-09T10:25:09.115 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0078972580000001, 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114942 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:25:09.116 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213943+0000", 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999967 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213940+0000", 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.791e-05 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.117 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213958+0000", 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5698999999999998e-05 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.213994+0000", 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.436e-05 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.118 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214008+0000", 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122045 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214130+0000", 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3575999999999999e-05 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214154+0000", 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99465683800000004 2023-10-09T10:25:09.119 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.208810+0000", 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042772410000000002 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213088+0000", 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031397499999999997 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.120 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.213402+0000", 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0061402180000000002 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.219542+0000", 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021434869999999999 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221685+0000", 2023-10-09T10:25:09.121 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7170999999999999e-05 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221743+0000", 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8197999999999997e-05 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221841+0000", 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0078972580000001 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.122 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114943 9.7 9:e38f5304:::benchmark_data_smithi012_313365_object2468:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 594.89779984100005, 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1890109880000002, 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.123 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114943 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214033+0000", 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.124 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214032+0000", 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9030000000000004e-06 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214040+0000", 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.774e-05 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214057+0000", 2023-10-09T10:25:09.125 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6160000000000008e-06 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214067+0000", 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013272499999999999 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214200+0000", 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2798000000000004e-05 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.126 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214262+0000", 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98051140199999998 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194774+0000", 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8810000000000008e-06 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.127 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194784+0000", 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1642592760000001 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359043+0000", 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000316631 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359360+0000", 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026861052 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.128 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386221+0000", 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8226999999999994e-05 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386309+0000", 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030679600000000002 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:25:09.129 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386616+0000", 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060189639999999999 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392635+0000", 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1379999999999999e-05 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392646+0000", 2023-10-09T10:25:09.130 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095982680000000001 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402244+0000", 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8916999999999999e-05 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402303+0000", 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00074067200000000005 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.131 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.403044+0000", 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1890109880000002 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114945 9.7 9:e1cd6305:::benchmark_data_smithi012_313365_object2470:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 594.897738839, 2023-10-09T10:25:09.132 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.171928715, 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114945 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:25:09.133 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214094+0000", 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214093+0000", 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3230000000000003e-06 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.134 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214099+0000", 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6458999999999999e-05 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214116+0000", 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1797000000000001e-05 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.135 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214128+0000", 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015993 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214287+0000", 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.214295+0000", 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98052440100000005 2023-10-09T10:25:09.136 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194820+0000", 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0089999999999998e-06 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194825+0000", 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1645754820000001 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.137 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359400+0000", 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035076999999999998 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359751+0000", 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015525427 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375276+0000", 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0332000000000004e-05 2023-10-09T10:25:09.138 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375347+0000", 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032258599999999998 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.375669+0000", 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0095005880000000008 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.139 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385170+0000", 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0859999999999997e-05 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385231+0000", 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5485999999999999e-05 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385266+0000", 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9339999999999999e-05 2023-10-09T10:25:09.140 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385306+0000", 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00071704500000000003 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386023+0000", 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.171928715 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.141 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114953 9.1 9:85379d0d:::benchmark_data_smithi012_313365_object2478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 594.87729456800002, 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98747086900000003, 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.142 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114953 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234538+0000", 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:09.143 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234537+0000", 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234543+0000", 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3334e-05 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.144 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234557+0000", 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1240000000000004e-06 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234562+0000", 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0877000000000002e-05 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234603+0000", 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5950000000000001e-06 2023-10-09T10:25:09.145 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.234610+0000", 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98059611300000005 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215206+0000", 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012311700000000001 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.146 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215329+0000", 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038846200000000001 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 6", 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.215718+0000", 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0038298339999999998 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.219548+0000", 2023-10-09T10:25:09.147 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023326010000000001 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221880+0000", 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4744999999999999e-05 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.221925+0000", 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3997000000000002e-05 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.148 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.222009+0000", 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.98747086900000003 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114959 9.13 9:cf007434:::benchmark_data_smithi012_313365_object2484:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 594.83612670900004, 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96965389000000002, 2023-10-09T10:25:09.149 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114959 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.150 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275706+0000", 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275705+0000", 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1540000000000002e-06 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.151 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275713+0000", 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4321000000000001e-05 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275727+0000", 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4679999999999997e-06 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275733+0000", 2023-10-09T10:25:09.152 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8029999999999999e-05 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275782+0000", 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0797e-05 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.275792+0000", 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.93510872899999997 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.153 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.210901+0000", 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013108099999999999 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.211032+0000", 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035745900000000002 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.154 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1", 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.211390+0000", 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052776840000000004 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.216667+0000", 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.028541474000000001 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245209+0000", 2023-10-09T10:25:09.155 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6895000000000003e-05 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245266+0000", 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4310999999999999e-05 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.245360+0000", 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.96965389000000002 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.156 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114960 9.17 9:ee9c2079:::benchmark_data_smithi012_313365_object2485:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 594.833634598, 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1241943320000001, 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.157 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114960 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278198+0000", 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.158 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278197+0000", 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6739999999999996e-06 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278202+0000", 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1857e-05 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278214+0000", 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4840000000000003e-06 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.159 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.160 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.160 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278219+0000", 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1851000000000002e-05 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278271+0000", 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6990000000000003e-06 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:14.278279+0000", 2023-10-09T10:25:09.161 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.91653466100000003 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194814+0000", 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7139999999999996e-06 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.194820+0000", 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1641976199999999 2023-10-09T10:25:09.162 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359018+0000", 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000340787 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359359+0000", 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.026806301000000001 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.163 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386165+0000", 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1144999999999996e-05 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386246+0000", 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000216963 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386463+0000", 2023-10-09T10:25:09.164 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025360859999999999 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.388999+0000", 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003603794 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.392603+0000", 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0077717259999999996 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.165 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400375+0000", 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0828999999999999e-05 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400406+0000", 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019869430000000001 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.166 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.402393+0000", 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1241943320000001 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:114968 9.7 9:e6edcf82:::benchmark_data_smithi012_313365_object2493:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 593.86027596300005, 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.148768977, 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.167 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 114968 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.168 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251557+0000", 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251556+0000", 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1169999999999998e-06 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251563+0000", 2023-10-09T10:25:09.169 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1269e-05 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251574+0000", 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5419999999999996e-06 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.251579+0000", 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0049893740000000004 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.170 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.256569+0000", 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0169e-05 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:15.256579+0000", 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.103208593 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.171 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.359788+0000", 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028333999999999998 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for degraded object", 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.360071+0000", 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.018386280000000001 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378457+0000", 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0075999999999998e-05 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.172 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378527+0000", 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023643399999999999 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.378764+0000", 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0064134869999999998 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.385177+0000", 2023-10-09T10:25:09.173 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00090589800000000003 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.386083+0000", 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013396399 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.399480+0000", 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0691999999999993e-05 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.174 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.399550+0000", 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00077554999999999996 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:15:16.400326+0000", 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.148768977 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.175 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128018 9.17 9:e84c3436:::benchmark_data_smithi012_313365_object15543:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 541.45854425499999, 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0015004270000001, 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128018 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.176 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653289+0000", 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.177 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653287+0000", 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8730000000000004e-06 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653293+0000", 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6543999999999999e-05 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653310+0000", 2023-10-09T10:25:09.178 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3600000000000001e-06 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653316+0000", 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0634999999999999e-05 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653367+0000", 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000173043 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.179 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653540+0000", 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036610800000000003 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.653906+0000", 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023299039999999998 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.656236+0000", 2023-10-09T10:25:09.180 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042009109999999999 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.660437+0000", 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1856e-05 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.660448+0000", 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97237176999999997 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.181 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.632820+0000", 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013645000000000001 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.632957+0000", 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.021710798 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.182 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.654667+0000", 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121478 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.654789+0000", 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0015004270000001 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.183 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128022 9.13 9:cae3607a:::benchmark_data_smithi012_313365_object15547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 541.31724853399999, 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.71981764000000004, 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128022 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.184 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794584+0000", 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999971 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.185 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794581+0000", 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7827999999999999e-05 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794599+0000", 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0599000000000003e-05 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794630+0000", 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2262000000000001e-05 2023-10-09T10:25:09.186 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794642+0000", 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7308e-05 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794709+0000", 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1711e-05 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.187 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.794721+0000", 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.712568863 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507290+0000", 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013420899999999999 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507424+0000", 2023-10-09T10:25:09.188 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036085199999999999 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.507785+0000", 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003524667 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.511310+0000", 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00080993999999999999 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.189 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512119+0000", 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00066952899999999998 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512789+0000", 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0370999999999997e-05 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.512829+0000", 2023-10-09T10:25:09.190 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015724840000000001 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.514402+0000", 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.71981764000000004 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:128023 9.17 9:e92c6749:::benchmark_data_smithi012_313365_object15548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-10-09T10:25:09.191 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 541.30690001899995, 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86638305000000004, 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 128023 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.192 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804933+0000", 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804931+0000", 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3567000000000001e-05 2023-10-09T10:25:09.193 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804945+0000", 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9168e-05 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804964+0000", 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2880000000000004e-06 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.194 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.804971+0000", 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1679000000000004e-05 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.805033+0000", 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3217999999999999e-05 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:07.805046+0000", 2023-10-09T10:25:09.195 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.85093918000000002 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.655985+0000", 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000130085 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.656115+0000", 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031436700000000002 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.196 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.656430+0000", 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065475230000000004 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.662977+0000", 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4881000000000004e-05 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.197 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.663052+0000", 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058596330000000004 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.668912+0000", 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9499999999999998e-05 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.668951+0000", 2023-10-09T10:25:09.198 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023647130000000001 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:16:08.671316+0000", 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86638305000000004 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156571 9.11 e377/365 9:88722a35:::benchmark_data_smithi012_313365_object44096:head v 377'8328, mlcod=376'8326)", 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.412584+0000", 2023-10-09T10:25:09.199 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 480.69924829000001, 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20419113999999999, 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412584+0000", 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412584+0000", 2023-10-09T10:25:09.200 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412584+0000", 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3465999999999999e-05 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412598+0000", 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9460999999999999e-05 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.201 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412617+0000", 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0070000000000003e-06 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.412626+0000", 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0066720659999999999 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.419298+0000", 2023-10-09T10:25:09.202 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8828999999999999e-05 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.419317+0000", 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.197365453 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.616682+0000", 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3162e-05 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.203 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.616776+0000", 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20419113999999999 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.45908.0:156718 9.17 9:ea7220c0:::benchmark_data_smithi012_313365_object44243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:18.139525+0000", 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 470.97230819800001, 2023-10-09T10:25:09.204 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.206501929, 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.45908", 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 156718 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139525+0000", 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.205 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139525+0000", 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139523+0000", 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5899999999999996e-06 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.206 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139530+0000", 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2329e-05 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139542+0000", 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8539999999999996e-06 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139547+0000", 2023-10-09T10:25:09.207 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1578000000000001e-05 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139588+0000", 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102212 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.139691+0000", 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037007000000000001 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.208 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.140061+0000", 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017259549999999999 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.141787+0000", 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00049306999999999997 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.142280+0000", 2023-10-09T10:25:09.209 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.20358306800000001 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.345863+0000", 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9493000000000003e-05 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.210 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.345902+0000", 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012420000000000001 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:18.346027+0000", 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.206501929 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.211 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "MOSDPGPull(3.13 e485/484 cost 1049576)", 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:25.682507+0000", 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 103.429325762, 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.40614320599999998, 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.682507+0000", 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.212 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.682507+0000", 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.682506+0000", 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4029999999999998e-05 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.213 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.682540+0000", 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.40587983799999999 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088420+0000", 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4996e-05 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088435+0000", 2023-10-09T10:25:09.214 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2153e-05 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088487+0000", 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000163413 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088650+0000", 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.40614320599999998 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.215 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:210964 10.d e485/484 10:b494e22a:::benchmark_data_smithi012_366312_object99635:head v 485'12552, mlcod=0'0)", 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:26.088617+0000", 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 103.023215533, 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178878856, 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.216 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088617+0000", 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088617+0000", 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088617+0000", 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5679999999999999e-06 2023-10-09T10:25:09.217 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088622+0000", 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8169999999999999e-06 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088630+0000", 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6439999999999997e-06 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.218 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088636+0000", 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.066188245000000007 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.154824+0000", 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6020000000000002e-05 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.154850+0000", 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.112612428 2023-10-09T10:25:09.219 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.267463+0000", 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3461999999999999e-05 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.267496+0000", 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178878856 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.220 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:210978 10.d e485/484 10:b0ccb7ce:::benchmark_data_smithi012_366312_object99649:head v 485'12553, mlcod=0'0)", 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:26.088640+0000", 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 103.023192349, 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17894791600000001, 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088640+0000", 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.221 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088640+0000", 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088640+0000", 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6199999999999994e-06 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.222 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088650+0000", 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2280000000000003e-06 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088656+0000", 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6670000000000004e-06 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.088661+0000", 2023-10-09T10:25:09.223 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.128415154 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.217076+0000", 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4825999999999999e-05 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.217101+0000", 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.050460613000000001 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.224 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.267561+0000", 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7118999999999999e-05 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.267588+0000", 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.17894791600000001 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.225 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:210967 10.8 e486/485 10:17c91626:::benchmark_data_smithi012_366312_object99638:head v 486'10664, mlcod=0'0)", 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:26.616785+0000", 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 102.49504811200001, 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.120056223, 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.226 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.616785+0000", 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.616785+0000", 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.616784+0000", 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2110000000000001e-06 2023-10-09T10:25:09.227 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.616791+0000", 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0275000000000001e-05 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.616801+0000", 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6010000000000002e-06 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.228 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.616807+0000", 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0071286639999999998 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.623936+0000", 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3268000000000002e-05 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.623959+0000", 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.11284654099999999 2023-10-09T10:25:09.229 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.736805+0000", 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5410000000000001e-05 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.736841+0000", 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.120056223 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.230 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:210983 10.8 e486/485 10:145c4c7d:::benchmark_data_smithi012_366312_object99654:head v 486'10666, mlcod=0'0)", 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:26.622187+0000", 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 102.48964554299999, 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.117981661, 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.231 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.622187+0000", 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.622187+0000", 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.622186+0000", 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7730000000000005e-06 2023-10-09T10:25:09.232 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.622195+0000", 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1443e-05 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.622207+0000", 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0550000000000002e-06 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.233 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.622214+0000", 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.097732839000000002 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.719946+0000", 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4040999999999999e-05 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.234 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.719970+0000", 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020139520000000001 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.740110+0000", 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8924e-05 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.740169+0000", 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.117981661 2023-10-09T10:25:09.235 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:210966 10.18 e486/485 10:186c943a:::benchmark_data_smithi012_366312_object99637:head v 486'10702, mlcod=0'0)", 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:26.624757+0000", 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 102.487075721, 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.123302254, 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.236 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.624757+0000", 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.624757+0000", 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.237 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.624756+0000", 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4530000000000008e-06 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.624765+0000", 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2024e-05 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.624777+0000", 2023-10-09T10:25:09.238 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.263e-06 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.624784+0000", 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.093246165000000006 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.718030+0000", 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6913000000000001e-05 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.239 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.718057+0000", 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.029956663000000001 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.748013+0000", 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5834999999999997e-05 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.748059+0000", 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.123302254 2023-10-09T10:25:09.240 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:211003 10.19 e486/485 10:995f601f:::benchmark_data_smithi012_366312_object99674:head v 486'10706, mlcod=0'0)", 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:26.626679+0000", 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 102.485153599, 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.122011247, 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.626679+0000", 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.626679+0000", 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.626679+0000", 2023-10-09T10:25:09.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8269999999999997e-06 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.626684+0000", 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9160000000000004e-06 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.626691+0000", 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2749999999999997e-06 2023-10-09T10:25:09.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.626695+0000", 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021721079999999999 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.628867+0000", 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1415000000000001e-05 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.628888+0000", 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.119762909 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.748651+0000", 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9174999999999997e-05 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.748690+0000", 2023-10-09T10:25:09.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.122011247 2023-10-09T10:25:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:25:09.246 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:25:09.246 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:25:09.247 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 69 16 98832 98816 89.4782 0 - 0.0109218 2023-10-09T10:25:09.280 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:09.302 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:09.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:09.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:09.406 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:09.446 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:09.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[152263]: pgmap v1434: 99 pgs: 2 remapped+peering, 11 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 28579/290292 objects degraded (9.845%); 8593/290292 objects misplaced (2.960%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T10:25:09.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[152263]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[152263]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[152263]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.569 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[152263]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[152263]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[152263]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.570 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[152263]: osdmap e514: 8 total, 8 up, 6 in 2023-10-09T10:25:09.571 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:09.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:09.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:09.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:25:09.685 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:09 smithi110 ceph-mon[136032]: pgmap v1434: 99 pgs: 2 remapped+peering, 11 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 28579/290292 objects degraded (9.845%); 8593/290292 objects misplaced (2.960%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T10:25:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:09 smithi110 ceph-mon[136032]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:09 smithi110 ceph-mon[136032]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:09 smithi110 ceph-mon[136032]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:09 smithi110 ceph-mon[136032]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:09 smithi110 ceph-mon[136032]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:09 smithi110 ceph-mon[136032]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:09 smithi110 ceph-mon[136032]: osdmap e514: 8 total, 8 up, 6 in 2023-10-09T10:25:09.716 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:09.757 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:09.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:09.865 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:09.920 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[147926]: pgmap v1434: 99 pgs: 2 remapped+peering, 11 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 28579/290292 objects degraded (9.845%); 8593/290292 objects misplaced (2.960%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T10:25:09.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[147926]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[147926]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[147926]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[147926]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[147926]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[147926]: failed to encode map e513 with expected crc 2023-10-09T10:25:09.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:09 smithi012 ceph-mon[147926]: osdmap e514: 8 total, 8 up, 6 in 2023-10-09T10:25:10.001 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:10.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 70 16 99164 99148 88.4966 10.375 0.0161309 0.0112952 2023-10-09T10:25:10.296 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:10.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:10.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:10.410 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:10.457 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:10.524 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:10.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:10.585 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[147926]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.585 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[147926]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.586 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[147926]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.632 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:10.691 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:10.749 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:10.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:10.756 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:10 smithi110 ceph-mon[136032]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.756 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:10 smithi110 ceph-mon[136032]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.756 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:10 smithi110 ceph-mon[136032]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.756 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:10 smithi110 ceph-mon[136032]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.756 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:10 smithi110 ceph-mon[136032]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:10.859 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:10.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[152263]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[152263]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[152263]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.889 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[152263]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.889 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[152263]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[147926]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.896 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:10 smithi012 ceph-mon[147926]: failed to encode map e514 with expected crc 2023-10-09T10:25:10.906 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:10.974 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:10.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:11.082 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:11.114 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:11.134 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:11.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 71 15 100038 100023 88.0205 54.6875 0.0197043 0.0113545 2023-10-09T10:25:11.187 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:11.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:11.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:11.295 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:11.350 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:11.411 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:11.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:11.519 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:11.555 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:25:11.575 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:11.588 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:25:11.621 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T10:25:11.628 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:25:11.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:11.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:11.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:11.670 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:25:11.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:11 smithi110 ceph-mon[136032]: pgmap v1437: 99 pgs: 2 remapped+peering, 15 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 41 MiB/s wr, 652 op/s; 28856/293463 objects degraded (9.833%); 8593/293463 objects misplaced (2.928%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T10:25:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:11 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:25:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/850821708' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:25:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:11 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:25:11.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/12580807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:11.696 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:25:11.732 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:25:11.747 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[147926]: pgmap v1437: 99 pgs: 2 remapped+peering, 15 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 41 MiB/s wr, 652 op/s; 28856/293463 objects degraded (9.833%); 8593/293463 objects misplaced (2.928%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T10:25:11.747 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:25:11.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/850821708' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:25:11.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:25:11.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/12580807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:11.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[152263]: pgmap v1437: 99 pgs: 2 remapped+peering, 15 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 41 MiB/s wr, 652 op/s; 28856/293463 objects degraded (9.833%); 8593/293463 objects misplaced (2.928%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T10:25:11.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:25:11.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/850821708' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:25:11.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:25:11.749 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/12580807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:11.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:11.765 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:25:11.784 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:11.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:25:11.798 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:25:11.798 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:25:11.798 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:25:11.798 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:25:11.799 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:25:11.799 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:25:11.799 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:25:11.799 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:25:11.821 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:11.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:11.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:11.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:11.953 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:11.987 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:11.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:12.092 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:12.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 72 16 100532 100516 87.226 30.8125 0.003761 0.0113875 2023-10-09T10:25:12.177 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:12.620 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:25:12.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/983215264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:12.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:25:12.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: osdmap e515: 8 total, 8 up, 6 in 2023-10-09T10:25:12.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.647 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/983215264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[147926]: osdmap e516: 8 total, 8 up, 6 in 2023-10-09T10:25:12.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/983215264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: osdmap e515: 8 total, 8 up, 6 in 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.649 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:25:12.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/983215264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:12.650 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:12 smithi012 ceph-mon[152263]: osdmap e516: 8 total, 8 up, 6 in 2023-10-09T10:25:12.775 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:12.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:12.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:12.880 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/983215264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: osdmap e515: 8 total, 8 up, 6 in 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: failed to encode map e515 with expected crc 2023-10-09T10:25:12.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:25:12.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/983215264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:12.906 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:12 smithi110 ceph-mon[136032]: osdmap e516: 8 total, 8 up, 6 in 2023-10-09T10:25:12.907 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:12.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:12.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:12.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:13.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:13.086 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:13.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 73 16 100596 100580 86.0861 4 0.00356117 0.0114289 2023-10-09T10:25:13.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T10:25:13.255 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:13.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:13.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:13.387 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:13.424 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:13.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:13.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:13.528 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:13.555 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:13.587 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:13.588 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:25:13.588 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:25:13.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:13.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:13.691 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:13.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:13 smithi110 ceph-mon[136032]: pgmap v1439: 99 pgs: 2 remapped+peering, 15 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 57 MiB/s wr, 908 op/s; 40952/296679 objects degraded (13.803%); 8561/296679 objects misplaced (2.886%); 15 MiB/s, 235 objects/s recovering 2023-10-09T10:25:13.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:13 smithi110 ceph-mon[136032]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:13 smithi110 ceph-mon[136032]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:13 smithi110 ceph-mon[136032]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:13 smithi110 ceph-mon[136032]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:13 smithi110 ceph-mon[136032]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 40952/296679 objects degraded (13.803%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:13.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:13 smithi110 ceph-mon[136032]: osdmap e517: 8 total, 8 up, 6 in 2023-10-09T10:25:13.719 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:13.754 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:13.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:13.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:13.859 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:13.924 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[147926]: pgmap v1439: 99 pgs: 2 remapped+peering, 15 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 57 MiB/s wr, 908 op/s; 40952/296679 objects degraded (13.803%); 8561/296679 objects misplaced (2.886%); 15 MiB/s, 235 objects/s recovering 2023-10-09T10:25:13.925 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[147926]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.925 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[147926]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.925 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[147926]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.925 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[147926]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.925 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[147926]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.925 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 40952/296679 objects degraded (13.803%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:13.925 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[147926]: osdmap e517: 8 total, 8 up, 6 in 2023-10-09T10:25:13.925 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[152263]: pgmap v1439: 99 pgs: 2 remapped+peering, 15 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 6.0 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 57 MiB/s wr, 908 op/s; 40952/296679 objects degraded (13.803%); 8561/296679 objects misplaced (2.886%); 15 MiB/s, 235 objects/s recovering 2023-10-09T10:25:13.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[152263]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[152263]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[152263]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[152263]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[152263]: failed to encode map e516 with expected crc 2023-10-09T10:25:13.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 40952/296679 objects degraded (13.803%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:13.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:13 smithi012 ceph-mon[152263]: osdmap e517: 8 total, 8 up, 6 in 2023-10-09T10:25:13.932 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:14.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 74 16 100966 100950 85.2355 23.125 0.0101639 0.0117277 2023-10-09T10:25:14.146 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:14.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:14.169 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:14.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:25:14.251 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:14.310 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:14.366 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:14.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:14.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:14.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:14.593 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:14.649 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:14 smithi110 ceph-mon[136032]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.650 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:14 smithi110 ceph-mon[136032]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.650 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:14 smithi110 ceph-mon[136032]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.650 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:14 smithi110 ceph-mon[136032]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.650 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:14 smithi110 ceph-mon[136032]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.650 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:14.650 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:14.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:25:14.756 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:14.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:14.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[147926]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[147926]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[147926]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[147926]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[147926]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:14.842 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:14.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[152263]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.842 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[152263]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[152263]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[152263]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[152263]: failed to encode map e517 with expected crc 2023-10-09T10:25:14.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:14.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:14.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:14.931 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:15.146 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 75 16 101801 101785 84.7949 52.1875 0.00985308 0.0117875 2023-10-09T10:25:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:15.180 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:15.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:15.281 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:15.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:15.373 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:25:15.422 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:15.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:15.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:15.444 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:15.532 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:15.568 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:15.625 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:15.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:15.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:15 smithi012 ceph-mon[147926]: pgmap v1442: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 28 MiB/s wr, 455 op/s; 57769/299394 objects degraded (19.295%); 8560/299394 objects misplaced (2.859%); 9.3 MiB/s, 146 objects/s recovering 2023-10-09T10:25:15.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:15 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:25:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:15 smithi012 ceph-mon[152263]: pgmap v1442: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 28 MiB/s wr, 455 op/s; 57769/299394 objects degraded (19.295%); 8560/299394 objects misplaced (2.859%); 9.3 MiB/s, 146 objects/s recovering 2023-10-09T10:25:15.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:15 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:25:15.737 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:15.785 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:15.833 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:15.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:15.939 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:15 smithi110 ceph-mon[136032]: pgmap v1442: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 28 MiB/s wr, 455 op/s; 57769/299394 objects degraded (19.295%); 8560/299394 objects misplaced (2.859%); 9.3 MiB/s, 146 objects/s recovering 2023-10-09T10:25:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:15 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:25:15.954 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:15.980 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:16.037 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:16.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:16.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:16.142 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:16.145 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 76 16 102564 102548 84.3065 47.6875 0.0106877 0.0118562 2023-10-09T10:25:16.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:16.240 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:16.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:16.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:16.453 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:16.665 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:16 smithi012 ceph-mon[147926]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 33 MiB/s wr, 535 op/s; 61529/303105 objects degraded (20.300%); 17063/303105 objects misplaced (5.629%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:25:16.666 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:16 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3157354320' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:25:16.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:16 smithi110 ceph-mon[136032]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 33 MiB/s wr, 535 op/s; 61529/303105 objects degraded (20.300%); 17063/303105 objects misplaced (5.629%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:25:16.666 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:16 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3157354320' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:25:16.672 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:16.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:16.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:16.694 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T10:25:16.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:16.825 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:16.885 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:16.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:16.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:16 smithi012 ceph-mon[152263]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 33 MiB/s wr, 535 op/s; 61529/303105 objects degraded (20.300%); 17063/303105 objects misplaced (5.629%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:25:16.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:16 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3157354320' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:25:16.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:17.024 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:17.146 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 77 16 103261 103245 83.7774 43.5625 0.00361097 0.0118923 2023-10-09T10:25:17.267 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:17.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:17.338 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:17.370 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T10:25:17.372 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:17.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:17.432 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:17.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:17.536 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:17.562 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:17.596 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:17.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:17.700 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:17.734 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:17.875 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[147926]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3157354320' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:25:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[147926]: osdmap e518: 8 total, 8 up, 6 in 2023-10-09T10:25:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[147926]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[147926]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[147926]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[147926]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[147926]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:17.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:17.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[152263]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3157354320' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:25:17.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[152263]: osdmap e518: 8 total, 8 up, 6 in 2023-10-09T10:25:17.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[152263]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[152263]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[152263]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[152263]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-mon[152263]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.891 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:25:17.892 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:25:17.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:17 smithi110 ceph-mon[136032]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:17 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3157354320' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:25:17.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:17 smithi110 ceph-mon[136032]: osdmap e518: 8 total, 8 up, 6 in 2023-10-09T10:25:17.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:17 smithi110 ceph-mon[136032]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:17 smithi110 ceph-mon[136032]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:17 smithi110 ceph-mon[136032]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:17 smithi110 ceph-mon[136032]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:17 smithi110 ceph-mon[136032]: failed to encode map e518 with expected crc 2023-10-09T10:25:17.981 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:17.991 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:25:18.041 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:25:18.077 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:25:18.077 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:18.118 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:25:18.146 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 78 16 103405 103389 82.8187 9 0.028036 0.0120697 2023-10-09T10:25:18.151 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:25:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:25:17] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.33.4" 2023-10-09T10:25:18.162 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:18.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:18.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:18.174 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:25:18.230 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:25:18.271 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:25:18.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:18.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:25:18.333 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:25:18.333 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:25:18.333 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:25:18.333 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:25:18.333 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:25:18.334 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:25:18.334 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:25:18.334 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:25:18.345 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:18.582 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:18.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:18.630 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:25:18.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:18.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: osdmap e519: 8 total, 8 up, 6 in 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: 11.d starting backfill to osd.2 from (0'0,0'0] MAX to 517'6456 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: 11.d continuing backfill to osd.6 from (498'1100,517'6456] 11:b0015e16:::benchmark_data_smithi012_404402_object88153:head to 517'6456 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: pgmap v1446: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 38 MiB/s wr, 600 op/s; 61524/306000 objects degraded (20.106%); 17063/306000 objects misplaced (5.576%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1488936145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:18.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 61529/303105 objects degraded (20.300%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:18.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: osdmap e519: 8 total, 8 up, 6 in 2023-10-09T10:25:18.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: 11.d starting backfill to osd.2 from (0'0,0'0] MAX to 517'6456 2023-10-09T10:25:18.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.777 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: 11.d continuing backfill to osd.6 from (498'1100,517'6456] 11:b0015e16:::benchmark_data_smithi012_404402_object88153:head to 517'6456 2023-10-09T10:25:18.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: pgmap v1446: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 38 MiB/s wr, 600 op/s; 61524/306000 objects degraded (20.106%); 17063/306000 objects misplaced (5.576%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T10:25:18.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1488936145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:18.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 61529/303105 objects degraded (20.300%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:18.782 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: osdmap e519: 8 total, 8 up, 6 in 2023-10-09T10:25:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: 11.d starting backfill to osd.2 from (0'0,0'0] MAX to 517'6456 2023-10-09T10:25:18.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: 11.d continuing backfill to osd.6 from (498'1100,517'6456] 11:b0015e16:::benchmark_data_smithi012_404402_object88153:head to 517'6456 2023-10-09T10:25:18.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: pgmap v1446: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.2 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 38 MiB/s wr, 600 op/s; 61524/306000 objects degraded (20.106%); 17063/306000 objects misplaced (5.576%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T10:25:18.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: failed to encode map e519 with expected crc 2023-10-09T10:25:18.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1488936145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:18.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 61529/303105 objects degraded (20.300%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:19.100 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:19.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:19.146 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 79 16 104183 104167 82.3859 48.625 0.0250299 0.0121316 2023-10-09T10:25:19.211 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:19.285 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:19.320 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:19.354 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:19.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:19.463 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:19.521 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:19.585 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:19.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:19.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:19.694 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:19.790 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:19.876 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:19.926 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:19.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:20.038 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:20.090 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:20.136 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:20.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:20.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:20.146 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:2023-10-09T10:25:20.147007+0000 min lat: 0.00117322 max lat: 2.44365 avg lat: 0.0121987 2023-10-09T10:25:20.146 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:25:20.146 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 80 16 104936 104920 81.9442 47.0625 0.0163318 0.0121987 2023-10-09T10:25:20.244 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:20.351 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:20.836 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:20 smithi012 ceph-mon[147926]: pgmap v1447: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 30 MiB/s wr, 473 op/s; 61725/308151 objects degraded (20.031%); 23543/308151 objects misplaced (7.640%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:25:20.836 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:20 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3492243679' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:20.837 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:20 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:20.893 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:25:20.934 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:20.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:21.045 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:21.091 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:20 smithi012 ceph-mon[152263]: pgmap v1447: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 30 MiB/s wr, 473 op/s; 61725/308151 objects degraded (20.031%); 23543/308151 objects misplaced (7.640%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:25:21.092 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:20 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3492243679' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:21.092 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:20 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:21.147 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 81 16 105724 105708 81.5405 49.25 0.0123513 0.0122591 2023-10-09T10:25:21.166 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:20 smithi110 ceph-mon[136032]: pgmap v1447: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 30 MiB/s wr, 473 op/s; 61725/308151 objects degraded (20.031%); 23543/308151 objects misplaced (7.640%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:25:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:20 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3492243679' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:20 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:21.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:25:21.371 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:21.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:21.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:21.482 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:21.582 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:21.649 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:21.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:21.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:21.762 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:21.796 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:21.845 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:21.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:21.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:25:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[152263]: osdmap e520: 8 total, 8 up, 6 in 2023-10-09T10:25:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[152263]: failed to encode map e520 with expected crc 2023-10-09T10:25:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[152263]: failed to encode map e520 with expected crc 2023-10-09T10:25:21.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[152263]: failed to encode map e520 with expected crc 2023-10-09T10:25:21.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:21.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:25:21.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[147926]: osdmap e520: 8 total, 8 up, 6 in 2023-10-09T10:25:21.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[147926]: failed to encode map e520 with expected crc 2023-10-09T10:25:21.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[147926]: failed to encode map e520 with expected crc 2023-10-09T10:25:21.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:21 smithi012 ceph-mon[147926]: failed to encode map e520 with expected crc 2023-10-09T10:25:21.949 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:22.007 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:156579 9.7 e377/361 9:e10cf41e:::benchmark_data_smithi012_313365_object44104:head v 377'8119, mlcod=327'3621)", 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.633803+0000", 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 493.39269891200001, 2023-10-09T10:25:22.037 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10126631799999999, 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633803+0000", 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.038 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633803+0000", 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633803+0000", 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633810+0000", 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5270000000000001e-06 2023-10-09T10:25:22.039 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633818+0000", 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5259999999999996e-06 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.633826+0000", 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7139999999999999e-06 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.040 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.634044+0000", 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00021737500000000001 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.634073+0000", 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9030000000000002e-05 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.734986+0000", 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.100913221 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.041 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.735070+0000", 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3572000000000006e-05 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.1f 398/377 [PushOp(9:ff99a8f0:::benchmark_data_smithi012_313365_object80195:head, version: 333'5053, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:ff99a8f0:::benchmark_data_smithi012_313365_object80195:head@333'5053, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 412.976080873, 2023-10-09T10:25:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071748132000000006, 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.043 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050420+0000", 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050484+0000", 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3660999999999999e-05 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.044 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050499+0000", 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4959e-05 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050510+0000", 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1165999999999999e-05 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050573+0000", 2023-10-09T10:25:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2452000000000004e-05 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050587+0000", 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4083000000000001e-05 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.122169+0000", 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071582693000000003 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.046 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:192723 9.16 e398/361 9:6c284d1c:::benchmark_data_smithi012_313365_object80248:head v 398'9382, mlcod=340'6834)", 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:29.054442+0000", 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 412.97206055300001, 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070307176999999998, 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.047 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.054442+0000", 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.054442+0000", 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.054441+0000", 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.048 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.054450+0000", 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9479999999999997e-06 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120912+0000", 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066462038000000001 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.049 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120929+0000", 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7533e-05 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120982+0000", 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3090999999999997e-05 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.121010+0000", 2023-10-09T10:25:22.050 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7526999999999999e-05 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.124712+0000", 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0037020870000000002 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.124749+0000", 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7020999999999997e-05 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.051 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.45908.0:192666 9.f e398/378 9:f5dbcf8b:::benchmark_data_smithi012_313365_object80191:head v 398'11046, mlcod=0'0)", 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:29.060753+0000", 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 412.96574938399999, 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.065306681000000005, 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.052 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.060753+0000", 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.060753+0000", 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.060752+0000", 2023-10-09T10:25:22.053 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.060760+0000", 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4149999999999999e-06 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120924+0000", 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060163258999999997 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.054 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120933+0000", 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9760000000000001e-06 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120975+0000", 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2363000000000001e-05 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.055 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.120999+0000", 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4000000000000001e-05 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.126037+0000", 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050383399999999997 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.126059+0000", 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2204e-05 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.056 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53350 10.5 e433/424 10:a3334060:::benchmark_data_smithi012_366312_object53349:head v 433'3357, mlcod=433'3355)", 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.338586+0000", 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 305.68791579399999, 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.104754867, 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.057 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338586+0000", 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338586+0000", 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338585+0000", 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338614+0000", 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8943999999999999e-05 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338629+0000", 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4309e-05 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.059 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338638+0000", 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4679999999999998e-06 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338684+0000", 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6298000000000003e-05 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338704+0000", 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9813000000000002e-05 2023-10-09T10:25:22.060 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.342706+0000", 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040014919999999997 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443341+0000", 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10063549400000001 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.061 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53352 10.e e433/424 10:732e479f:::benchmark_data_smithi012_366312_object53351:head v 433'3315, mlcod=433'3312)", 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.340487+0000", 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 305.68601509199999, 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10286616599999999, 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.062 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340487+0000", 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340487+0000", 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340486+0000", 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.063 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340529+0000", 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3291000000000003e-05 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340540+0000", 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0995999999999999e-05 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.064 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340548+0000", 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4379999999999998e-06 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340591+0000", 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2740000000000001e-05 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340612+0000", 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0996e-05 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.065 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.342863+0000", 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022512690000000002 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443353+0000", 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.100490447 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.066 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53356 10.5 e433/424 10:aeaac023:::benchmark_data_smithi012_366312_object53355:head v 433'3358, mlcod=433'3355)", 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.340839+0000", 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 305.68566351700002, 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.107630108, 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340839+0000", 2023-10-09T10:25:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340839+0000", 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340838+0000", 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.068 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340862+0000", 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3927e-05 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340872+0000", 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0509999999999999e-05 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340879+0000", 2023-10-09T10:25:22.069 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4110000000000001e-06 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340939+0000", 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0615000000000002e-05 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340958+0000", 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8502999999999999e-05 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.070 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.448424+0000", 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.107465777 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.448469+0000", 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5042e-05 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.071 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.072 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.072 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53363 10.5 e433/424 10:a71fae04:::benchmark_data_smithi012_366312_object53362:head v 433'3359, mlcod=433'3355)", 2023-10-09T10:25:22.072 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.343610+0000", 2023-10-09T10:25:22.072 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 305.68289203500001, 2023-10-09T10:25:22.072 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.104896063, 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.343610+0000", 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.074 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.343610+0000", 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.343609+0000", 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.343634+0000", 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5088000000000001e-05 2023-10-09T10:25:22.075 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443297+0000", 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.099662744999999997 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443322+0000", 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5011000000000002e-05 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.076 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443361+0000", 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9066999999999997e-05 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443378+0000", 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6393e-05 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.448478+0000", 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051004739999999998 2023-10-09T10:25:22.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.448506+0000", 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8220000000000001e-05 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53361 10.e e433/424 10:7ba25ff8:::benchmark_data_smithi012_366312_object53360:head v 433'3316, mlcod=433'3313)", 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.344867+0000", 2023-10-09T10:25:22.078 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 305.68163552300001, 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.102124111, 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.344867+0000", 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.079 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.344867+0000", 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.344866+0000", 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.344918+0000", 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2333000000000002e-05 2023-10-09T10:25:22.080 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443328+0000", 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.098409900999999994 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443338+0000", 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0236e-05 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.081 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443375+0000", 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6925000000000003e-05 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443390+0000", 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4415e-05 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.446946+0000", 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035561339999999999 2023-10-09T10:25:22.082 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.446991+0000", 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4795000000000001e-05 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36474.0:169260 10.f 10:f45f3781:::benchmark_data_smithi012_366312_object57931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e468)", 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:22:28.766840+0000", 2023-10-09T10:25:22.083 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 173.25966227699999, 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.05782698, 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36474", 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 169260 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.084 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.766840+0000", 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.766840+0000", 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.766838+0000", 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:22.085 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.766845+0000", 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1630000000000001e-06 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.766858+0000", 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3831e-05 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.086 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.766868+0000", 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4800000000000007e-06 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.766940+0000", 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2205000000000004e-05 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.767044+0000", 2023-10-09T10:25:22.087 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0001036 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4,6", 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.767157+0000", 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000113622 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.769547+0000", 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023893220000000001 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.769730+0000", 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018337099999999999 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.775219+0000", 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0054893110000000002 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.824495+0000", 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.049275775000000001 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.824541+0000", 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5407000000000001e-05 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:28.824667+0000", 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012634500000000001 2023-10-09T10:25:22.090 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36474.0:175656 10.f 10:f3e3488b:::benchmark_data_smithi012_366312_object64327:head [delete] snapc 0=[] ondisk+write+known_if_redirected e471)", 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:22:37.793176+0000", 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 164.23332648900001, 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062675038000000002, 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:22.091 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36474", 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 175656 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793176+0000", 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.092 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793176+0000", 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793174+0000", 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.093 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793183+0000", 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5189999999999998e-06 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793199+0000", 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5753e-05 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793209+0000", 2023-10-09T10:25:22.094 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8320000000000001e-06 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793264+0000", 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4676999999999998e-05 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793397+0000", 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013299699999999999 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.095 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,4,6", 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.793507+0000", 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0001108 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.796801+0000", 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032935680000000002 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.796843+0000", 2023-10-09T10:25:22.096 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2354999999999997e-05 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.813756+0000", 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016912178 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.855679+0000", 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041923167999999997 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.097 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.855741+0000", 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2397999999999994e-05 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:37.855851+0000", 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109645 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.098 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:182365 10.14 e474/463 10:29b20442:::benchmark_data_smithi012_366312_object71036:head v 474'9809, mlcod=474'9807)", 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:22:47.945625+0000", 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 154.080877098, 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.084676611999999998, 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.099 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945625+0000", 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945625+0000", 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945625+0000", 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:25:22.100 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945629+0000", 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8299999999999998e-06 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945635+0000", 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8930000000000002e-06 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945639+0000", 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2610000000000004e-06 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945677+0000", 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8506e-05 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945692+0000", 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4191e-05 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.030267+0000", 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.084575633999999997 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.030302+0000", 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4527999999999999e-05 2023-10-09T10:25:22.103 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:182367 10.a e474/467 10:5ed41c5c:::benchmark_data_smithi012_366312_object71038:head v 474'11310, mlcod=474'11307)", 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:22:47.946616+0000", 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 154.079886434, 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.091614227000000006, 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946616+0000", 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946616+0000", 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946615+0000", 2023-10-09T10:25:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946621+0000", 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4580000000000001e-06 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946629+0000", 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4889999999999995e-06 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946635+0000", 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4609999999999999e-06 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.032469+0000", 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.085834405000000003 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.032515+0000", 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6402000000000001e-05 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.038199+0000", 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056834420000000004 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.038230+0000", 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1109000000000003e-05 2023-10-09T10:25:22.108 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:182369 10.e e474/467 10:7c2770ab:::benchmark_data_smithi012_366312_object71040:head v 474'11217, mlcod=474'11214)", 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:22:47.949829+0000", 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 154.07667318, 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.089488281000000003, 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.949829+0000", 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.949829+0000", 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.949828+0000", 2023-10-09T10:25:22.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.949834+0000", 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0109999999999997e-06 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.029649+0000", 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.079814211999999995 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.029665+0000", 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6007000000000001e-05 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.032286+0000", 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026212729999999999 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.032318+0000", 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2382000000000003e-05 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.039276+0000", 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0069573680000000002 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.039317+0000", 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1610000000000003e-05 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.113 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:53232 11.c e503/494 11:3e78d13f:::benchmark_data_smithi012_404402_object53231:head v 503'3403, mlcod=503'3401)", 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:25.984523+0000", 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 56.041979294999997, 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063839707999999995, 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.984523+0000", 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.984523+0000", 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.984522+0000", 2023-10-09T10:25:22.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.984565+0000", 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2979999999999998e-05 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.984579+0000", 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3993e-05 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.984587+0000", 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6930000000000007e-06 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.984634+0000", 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6927e-05 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.984654+0000", 2023-10-09T10:25:22.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0692000000000001e-05 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986772+0000", 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021174380000000001 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048363+0000", 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061590903000000002 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.118 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:53226 11.11 e503/501 11:8ad5e31a:::benchmark_data_smithi012_404402_object53225:head v 503'2589, mlcod=503'2587)", 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:25.986205+0000", 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 56.040297569000003, 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062162191999999998, 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986205+0000", 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986205+0000", 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986204+0000", 2023-10-09T10:25:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986234+0000", 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0446e-05 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986246+0000", 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1508999999999999e-05 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986253+0000", 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4030000000000001e-06 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986297+0000", 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4076000000000003e-05 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986336+0000", 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9320999999999998e-05 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048264+0000", 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.06192719 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048367+0000", 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103272 2023-10-09T10:25:22.123 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:53228 11.1 e503/501 11:823952c2:::benchmark_data_smithi012_404402_object53227:head v 503'2624, mlcod=503'2622)", 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:25.986274+0000", 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 56.040227829999999, 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062164725999999997, 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986274+0000", 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986274+0000", 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986274+0000", 2023-10-09T10:25:22.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986294+0000", 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0265e-05 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986301+0000", 2023-10-09T10:25:22.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8120000000000002e-06 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986306+0000", 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1279999999999999e-06 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986369+0000", 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2766000000000002e-05 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986390+0000", 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0594999999999998e-05 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048377+0000", 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061987140000000003 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048439+0000", 2023-10-09T10:25:22.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2325999999999999e-05 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:53235 11.2 e503/501 11:46eac87f:::benchmark_data_smithi012_404402_object53234:head v 503'2606, mlcod=503'2604)", 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:25.986443+0000", 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 56.040059601000003, 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066656706999999996, 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.129 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986443+0000", 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986443+0000", 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986442+0000", 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986463+0000", 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0982e-05 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986473+0000", 2023-10-09T10:25:22.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0000000000000001e-05 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986479+0000", 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6270000000000002e-06 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986543+0000", 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3631000000000001e-05 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.986561+0000", 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7881e-05 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.053062+0000", 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066500661000000003 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.053099+0000", 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7672000000000001e-05 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:53234 11.18 e503/501 11:1d0b5055:::benchmark_data_smithi012_404402_object53233:head v 503'2682, mlcod=503'2680)", 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:25.987139+0000", 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 56.039363113999997, 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064055716999999998, 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.134 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.987139+0000", 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.987139+0000", 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.987138+0000", 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.987186+0000", 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7614999999999997e-05 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048270+0000", 2023-10-09T10:25:22.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061083542999999997 2023-10-09T10:25:22.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048297+0000", 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7932999999999998e-05 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048459+0000", 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000161173 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048523+0000", 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4125e-05 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.051154+0000", 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026316880000000001 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.051195+0000", 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0349999999999998e-05 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:53241 11.0 e503/501 11:03b2e66b:::benchmark_data_smithi012_404402_object53240:head v 503'2573, mlcod=503'2571)", 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:24:25.987861+0000", 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 56.038641445000003, 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063370917999999998, 2023-10-09T10:25:22.140 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.987861+0000", 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.987861+0000", 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:22.141 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.987860+0000", 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:25.987887+0000", 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7268999999999999e-05 2023-10-09T10:25:22.142 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048223+0000", 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.060335633999999999 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048280+0000", 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7524e-05 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048409+0000", 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128932 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.048474+0000", 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4263000000000005e-05 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:22.144 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.051198+0000", 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027247059999999999 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:24:26.051232+0000", 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3420999999999997e-05 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:22.145 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:25:22.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:22.147 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:22.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:22.147 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:21 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:22.147 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:21 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:25:22.147 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:21 smithi110 ceph-mon[136032]: osdmap e520: 8 total, 8 up, 6 in 2023-10-09T10:25:22.147 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:21 smithi110 ceph-mon[136032]: failed to encode map e520 with expected crc 2023-10-09T10:25:22.148 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:21 smithi110 ceph-mon[136032]: failed to encode map e520 with expected crc 2023-10-09T10:25:22.148 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:21 smithi110 ceph-mon[136032]: failed to encode map e520 with expected crc 2023-10-09T10:25:22.148 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 82 16 106480 106464 81.1223 47.25 0.0284087 0.0123212 2023-10-09T10:25:22.248 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:22.292 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:22.349 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:22.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:22.454 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:22.496 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:25:22.525 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:22.531 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:22.602 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:22.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:22.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:22.781 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:22.882 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:22.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:22.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:22.904 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T10:25:22.993 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:23.030 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:22 smithi012 ceph-mon[147926]: pgmap v1449: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 30 MiB/s wr, 484 op/s; 63354/311841 objects degraded (20.316%); 21740/311841 objects misplaced (6.972%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T10:25:23.030 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/363711458' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:25:23.030 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:25:23.030 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:22 smithi012 ceph-mon[152263]: pgmap v1449: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 30 MiB/s wr, 484 op/s; 63354/311841 objects degraded (20.316%); 21740/311841 objects misplaced (6.972%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T10:25:23.030 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/363711458' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:25:23.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:25:23.071 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:23.147 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 83 16 107233 107217 80.7119 47.0625 0.0235684 0.0123835 2023-10-09T10:25:23.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:22 smithi110 ceph-mon[136032]: pgmap v1449: 99 pgs: 1 active+recovery_wait+remapped, 2 activating, 1 activating+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 30 MiB/s wr, 484 op/s; 63354/311841 objects degraded (20.316%); 21740/311841 objects misplaced (6.972%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T10:25:23.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/363711458' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:25:23.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:25:23.274 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:25:23.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:25:23.439 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:23.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:23.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:23.632 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:24.141 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:25:24.142 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[147926]: osdmap e521: 8 total, 8 up, 6 in 2023-10-09T10:25:24.142 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[147926]: failed to encode map e521 with expected crc 2023-10-09T10:25:24.142 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[147926]: failed to encode map e521 with expected crc 2023-10-09T10:25:24.142 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 63354/311841 objects degraded (20.316%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:24.142 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:25:24.143 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[152263]: osdmap e521: 8 total, 8 up, 6 in 2023-10-09T10:25:24.143 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[152263]: failed to encode map e521 with expected crc 2023-10-09T10:25:24.143 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[152263]: failed to encode map e521 with expected crc 2023-10-09T10:25:24.143 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 63354/311841 objects degraded (20.316%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:24.147 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 84 16 108066 108050 80.3709 52.0625 0.0279407 0.0124364 2023-10-09T10:25:24.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:23 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:25:24.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:23 smithi110 ceph-mon[136032]: osdmap e521: 8 total, 8 up, 6 in 2023-10-09T10:25:24.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:23 smithi110 ceph-mon[136032]: failed to encode map e521 with expected crc 2023-10-09T10:25:24.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:23 smithi110 ceph-mon[136032]: failed to encode map e521 with expected crc 2023-10-09T10:25:24.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 63354/311841 objects degraded (20.316%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:24.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T10:25:24.273 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:24.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:24.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:24.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:24.536 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:24.563 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:24.641 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:24.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:24.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:24.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:24.878 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:24.954 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:25:24.974 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:24.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:24.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:25.004 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:25:25.037 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:25:25.073 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:25:25.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:25.119 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:25:25.149 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 85 16 108778 108762 79.9466 44.5 0.00945247 0.0125026 2023-10-09T10:25:25.152 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:25:25.170 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:25.194 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:25:25.230 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:25:25.242 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:25.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:25.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:25.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:25:25.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:25:25.255 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:25:25.256 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:25:25.256 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:25:25.256 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:25:25.256 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:25:25.256 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:25:25.351 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:25.403 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:25.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:25 smithi110 ceph-mon[136032]: pgmap v1451: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 61355/317787 objects degraded (19.307%); 23912/317787 objects misplaced (7.525%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T10:25:25.432 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:25 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4212825682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:25.813 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:25.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:25.837 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:25 smithi012 ceph-mon[152263]: pgmap v1451: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 61355/317787 objects degraded (19.307%); 23912/317787 objects misplaced (7.525%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T10:25:25.837 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:25 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4212825682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:25.838 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:25 smithi012 ceph-mon[147926]: pgmap v1451: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.5 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 61355/317787 objects degraded (19.307%); 23912/317787 objects misplaced (7.525%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T10:25:25.838 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:25 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4212825682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:25.919 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:26.028 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:26.043 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:26.150 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 86 16 109602 109586 79.6159 51.5 0.0222279 0.012556 2023-10-09T10:25:26.179 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:26.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:26.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:26.373 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:26.430 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:25:26.430 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:25:26.452 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:26.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:26.559 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:26.659 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:26.738 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:26.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:26.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:26.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:26.879 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:25:26.928 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:27.059 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:27.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:27.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:27.151 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 87 16 110370 110354 79.2525 48 0.0155361 0.0126124 2023-10-09T10:25:27.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:27.249 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:27.319 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:27.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:27.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:27.502 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:25:27.536 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:27.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:27 smithi012 ceph-mon[147926]: pgmap v1452: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 45 MiB/s wr, 718 op/s; 61355/323259 objects degraded (18.980%); 24161/323259 objects misplaced (7.474%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T10:25:27.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:27 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/931581191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:27.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:27.651 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:27 smithi012 ceph-mon[152263]: pgmap v1452: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 45 MiB/s wr, 718 op/s; 61355/323259 objects degraded (18.980%); 24161/323259 objects misplaced (7.474%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T10:25:27.651 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:27 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/931581191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:27.651 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:27.654 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:25:27.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:25:27.931 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:25:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:25:27] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.33.4" 2023-10-09T10:25:27.939 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:27 smithi110 ceph-mon[136032]: pgmap v1452: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 45 MiB/s wr, 718 op/s; 61355/323259 objects degraded (18.980%); 24161/323259 objects misplaced (7.474%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T10:25:27.940 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:27 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/931581191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:27.940 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:27.940 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:27.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:27.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:28.053 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:28.151 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 88 16 111155 111139 78.9086 49.0625 0.0144297 0.0126676 2023-10-09T10:25:28.171 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:28.275 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:28.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:28.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:28.399 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:28.469 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:28.603 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:28.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:28.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:28 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:28.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:28 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:28.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:28 smithi110 ceph-mon[136032]: osdmap e522: 8 total, 8 up, 6 in 2023-10-09T10:25:28.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:28 smithi110 ceph-mon[136032]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:28 smithi110 ceph-mon[136032]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.615 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:28 smithi110 ceph-mon[136032]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.616 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:28 smithi110 ceph-mon[136032]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.616 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 61355/323259 objects degraded (18.980%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:28.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:28.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:28.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[152263]: osdmap e522: 8 total, 8 up, 6 in 2023-10-09T10:25:28.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[152263]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[152263]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[152263]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[152263]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 61355/323259 objects degraded (18.980%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:28.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:28.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:28.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[147926]: osdmap e522: 8 total, 8 up, 6 in 2023-10-09T10:25:28.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[147926]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[147926]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[147926]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[147926]: failed to encode map e522 with expected crc 2023-10-09T10:25:28.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 61355/323259 objects degraded (18.980%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:28.714 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:28.784 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:28.816 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:28.918 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:28.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:28.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:29.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:29.030 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:25:29.030 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:25:29.091 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:29.151 INFO:tasks.radosbench.radosbench.0.smithi012.stdout: 89 16 111934 111918 78.5689 48.6875 0.0123455 0.0127223 2023-10-09T10:25:29.457 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:25:29.516 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T10:25:29.757 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:29.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:29.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:29.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[152263]: pgmap v1454: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 61355/327369 objects degraded (18.742%); 23859/327369 objects misplaced (7.288%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:25:29.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2235536427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:25:29.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:25:29.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:29.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:29.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[147926]: pgmap v1454: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 61355/327369 objects degraded (18.742%); 23859/327369 objects misplaced (7.288%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:25:29.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2235536427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:25:29.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:25:29.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:29.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:29.867 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:29.928 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:29.931 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:29 smithi110 ceph-mon[136032]: pgmap v1454: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 61355/327369 objects degraded (18.742%); 23859/327369 objects misplaced (7.288%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:25:29.931 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2235536427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:25:29.931 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:25:29.931 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:29.931 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:29.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:25:29.997 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:30.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:30.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:30.151 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total time run: 90.0176 2023-10-09T10:25:30.151 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Total writes made: 112780 2023-10-09T10:25:30.151 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Write size: 65536 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Object size: 65536 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Bandwidth (MB/sec): 78.3041 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Bandwidth: 42.9173 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max bandwidth (MB/sec): 227.5 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min bandwidth (MB/sec): 0 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average IOPS: 1252 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev IOPS: 686.677 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max IOPS: 3640 2023-10-09T10:25:30.152 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min IOPS: 0 2023-10-09T10:25:30.153 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Average Latency(s): 0.0127661 2023-10-09T10:25:30.153 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Stddev Latency(s): 0.0482268 2023-10-09T10:25:30.153 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Max latency(s): 2.44365 2023-10-09T10:25:30.153 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Min latency(s): 0.00117322 2023-10-09T10:25:30.155 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:25:30.182 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:30.243 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:25:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:25:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:30 smithi012 ceph-mon[147926]: osdmap e523: 8 total, 8 up, 6 in 2023-10-09T10:25:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:30 smithi012 ceph-mon[147926]: failed to encode map e523 with expected crc 2023-10-09T10:25:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:30 smithi012 ceph-mon[147926]: failed to encode map e523 with expected crc 2023-10-09T10:25:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:25:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:30 smithi012 ceph-mon[152263]: osdmap e523: 8 total, 8 up, 6 in 2023-10-09T10:25:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:30 smithi012 ceph-mon[152263]: failed to encode map e523 with expected crc 2023-10-09T10:25:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:30 smithi012 ceph-mon[152263]: failed to encode map e523 with expected crc 2023-10-09T10:25:30.793 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:25:30.794 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:30 smithi110 ceph-mon[136032]: osdmap e523: 8 total, 8 up, 6 in 2023-10-09T10:25:30.794 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:30 smithi110 ceph-mon[136032]: failed to encode map e523 with expected crc 2023-10-09T10:25:30.794 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:30 smithi110 ceph-mon[136032]: failed to encode map e523 with expected crc 2023-10-09T10:25:31.276 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:31.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:31.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:31.385 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:31.446 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:31.520 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T10:25:31.560 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:31.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:31.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:31 smithi012 ceph-mon[152263]: pgmap v1456: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 44 MiB/s wr, 703 op/s; 61585/330168 objects degraded (18.653%); 24089/330168 objects misplaced (7.296%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T10:25:31.592 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:31 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:25:31.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:31 smithi012 ceph-mon[147926]: pgmap v1456: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 44 MiB/s wr, 703 op/s; 61585/330168 objects degraded (18.653%); 24089/330168 objects misplaced (7.296%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T10:25:31.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:31 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:25:31.669 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:31.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:31 smithi110 ceph-mon[136032]: pgmap v1456: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 44 MiB/s wr, 703 op/s; 61585/330168 objects degraded (18.653%); 24089/330168 objects misplaced (7.296%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T10:25:31.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:31 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:25:31.742 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:31.899 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:31.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:31.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:32.005 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:32.073 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:25:32.073 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:25:32.109 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:32.200 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:32.228 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:32.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:32.339 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:32.395 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:32.629 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:32.741 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:25:32.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:25:32.811 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:32.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:32.824 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:25:32.862 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:25:32.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:25:32.907 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:25:32.915 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:32.951 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:25:32.970 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:33.005 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:25:33.011 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:33.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:33.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:33.039 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:25:33.065 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:25:33.066 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:25:33.066 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:25:33.066 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:25:33.066 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:25:33.066 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:25:33.066 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:25:33.066 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:25:33.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:33.274 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:33.593 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:33.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:33.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:33.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:33.757 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:33.824 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:33 smithi012 ceph-mon[147926]: pgmap v1457: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 45 MiB/s wr, 721 op/s; 61585/335106 objects degraded (18.378%); 24314/335106 objects misplaced (7.256%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:25:33.824 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/962657317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:33.824 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 61585/335106 objects degraded (18.378%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:33.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:33 smithi012 ceph-mon[152263]: pgmap v1457: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 45 MiB/s wr, 721 op/s; 61585/335106 objects degraded (18.378%); 24314/335106 objects misplaced (7.256%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:25:33.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/962657317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:33.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 61585/335106 objects degraded (18.378%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:33.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:33 smithi110 ceph-mon[136032]: pgmap v1457: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 45 MiB/s wr, 721 op/s; 61585/335106 objects degraded (18.378%); 24314/335106 objects misplaced (7.256%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T10:25:33.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/962657317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:33.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 61585/335106 objects degraded (18.378%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:34.063 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:34.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:34.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:34.170 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:34.260 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:34.397 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:34.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:34.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:34.591 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:34.649 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:34.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:34.759 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:34.832 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:34.909 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:34.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:34.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:35.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:35.132 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:35.137 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:35.208 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:35.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:35.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:35.421 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:35.554 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:35.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:35.608 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:25:35.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:35 smithi012 ceph-mon[147926]: pgmap v1458: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 38 MiB/s wr, 728 op/s; 61367/335292 objects degraded (18.303%); 23980/335292 objects misplaced (7.152%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:25:35.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2362303914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:25:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:35 smithi012 ceph-mon[152263]: pgmap v1458: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 38 MiB/s wr, 728 op/s; 61367/335292 objects degraded (18.303%); 23980/335292 objects misplaced (7.152%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:25:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2362303914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:25:35.762 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:35.838 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:35.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:35.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:35 smithi110 ceph-mon[136032]: pgmap v1458: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 38 MiB/s wr, 728 op/s; 61367/335292 objects degraded (18.303%); 23980/335292 objects misplaced (7.152%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:25:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2362303914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:25:35.955 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:36.017 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:36.175 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:36.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:36.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:36.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:25:36.288 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:36.381 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:36.486 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:36.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:36.565 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:36.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2362303914' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:25:36.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: osdmap e524: 8 total, 8 up, 6 in 2023-10-09T10:25:36.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4005739363' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:25:36.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:25:36.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:36 smithi110 ceph-mon[136032]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.591 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:36.650 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:36.650 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T10:25:36.721 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:36.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:36.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:36.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:36.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:36.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2362303914' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:25:36.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: osdmap e524: 8 total, 8 up, 6 in 2023-10-09T10:25:36.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4005739363' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:25:36.865 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:25:36.865 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[152263]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:36.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2362303914' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:25:36.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: osdmap e524: 8 total, 8 up, 6 in 2023-10-09T10:25:36.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4005739363' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:25:36.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:25:36.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:36 smithi012 ceph-mon[147926]: failed to encode map e524 with expected crc 2023-10-09T10:25:36.997 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:37.133 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:37.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:37.168 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T10:25:37.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:25:37.234 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:37.336 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:37.419 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:37.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:37.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:37.527 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:37.561 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:37.610 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(11.8 525/513 [PushOp(11:11dd857b:::benchmark_data_smithi012_404402_object4581:head, version: 498'310, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:11dd857b:::benchmark_data_smithi012_404402_object4581:head@498'310, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:37.605908+0000", 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0019058059999999999, 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001941522, 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.605908+0000", 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.605908+0000", 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.605907+0000", 2023-10-09T10:25:37.612 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.605963+0000", 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6023999999999998e-05 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.605993+0000", 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0175000000000002e-05 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.613 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606012+0000", 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9267e-05 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606095+0000", 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3038e-05 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.614 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606111+0000", 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5834000000000001e-05 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(11.5 525/513 [PushOp(11:a3b71f65:::benchmark_data_smithi012_404402_object4583:head, version: 498'288, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:a3b71f65:::benchmark_data_smithi012_404402_object4583:head@498'288, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:37.606033+0000", 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.001779997, 2023-10-09T10:25:37.615 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001868918, 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606033+0000", 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:37.616 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606033+0000", 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606033+0000", 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606079+0000", 2023-10-09T10:25:37.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6140999999999998e-05 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606093+0000", 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4007e-05 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606107+0000", 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4141000000000001e-05 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606344+0000", 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023654000000000001 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606358+0000", 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3893000000000001e-05 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:117361 11.2 e525/513 11:47ffb878:::benchmark_data_smithi012_404402_object4578:head v 525'4594, mlcod=0'0)", 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:37.606159+0000", 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0016541119999999999, 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00179108, 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606159+0000", 2023-10-09T10:25:37.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606159+0000", 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606159+0000", 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606164+0000", 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5010000000000004e-06 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606174+0000", 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6760000000000002e-06 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:37.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606180+0000", 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4019999999999999e-06 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.607925+0000", 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00174453 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.607950+0000", 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4771999999999999e-05 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.623 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:117368 11.0 e525/507 11:0188e910:::benchmark_data_smithi012_404402_object4585:head v 525'4544, mlcod=498'332)", 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:37.606194+0000", 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0016198320000000001, 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017924970000000001, 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606194+0000", 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606194+0000", 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606193+0000", 2023-10-09T10:25:37.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606198+0000", 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4129999999999999e-06 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606204+0000", 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5100000000000004e-06 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.626 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.606209+0000", 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9420000000000003e-06 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.607465+0000", 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0012563710000000001 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.607496+0000", 2023-10-09T10:25:37.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0731000000000001e-05 2023-10-09T10:25:37.628 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.628 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:37.628 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.628 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:37.628 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:25:37.628 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 4 2023-10-09T10:25:37.628 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:25:37.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:37.629 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:37.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[147926]: pgmap v1460: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 29 MiB/s wr, 746 op/s; 60551/331428 objects degraded (18.270%); 23729/331428 objects misplaced (7.160%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:25:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:25:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[147926]: osdmap e525: 8 total, 8 up, 6 in 2023-10-09T10:25:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[147926]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[147926]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[147926]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:37.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:37.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[152263]: pgmap v1460: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 29 MiB/s wr, 746 op/s; 60551/331428 objects degraded (18.270%); 23729/331428 objects misplaced (7.160%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:25:37.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:25:37.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[152263]: osdmap e525: 8 total, 8 up, 6 in 2023-10-09T10:25:37.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[152263]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[152263]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[152263]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:37.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:37.719 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:37.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:37.830 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:37 smithi110 ceph-mon[136032]: pgmap v1460: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 29 MiB/s wr, 746 op/s; 60551/331428 objects degraded (18.270%); 23729/331428 objects misplaced (7.160%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:25:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:25:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:37 smithi110 ceph-mon[136032]: osdmap e525: 8 total, 8 up, 6 in 2023-10-09T10:25:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:37 smithi110 ceph-mon[136032]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:37 smithi110 ceph-mon[136032]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:37 smithi110 ceph-mon[136032]: failed to encode map e525 with expected crc 2023-10-09T10:25:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:37 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:37 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:38.085 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:38.089 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:25:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:25:37] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.33.4" 2023-10-09T10:25:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:38.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:38.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:38.197 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:25:38.264 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:38.445 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:38.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:38.553 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:38.578 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:38.593 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:25:38.598 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:38.647 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:38.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:38.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:38.753 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:38.939 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:39.047 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:39.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:39.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:39.213 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:39.363 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:39 smithi012 ceph-mon[147926]: pgmap v1462: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 21 MiB/s wr, 629 op/s; 60551/331428 objects degraded (18.270%); 23729/331428 objects misplaced (7.160%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:25:39.363 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:39 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:39.363 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:39 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:39.363 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:39 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 60551/331428 objects degraded (18.270%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:39.364 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:39 smithi012 ceph-mon[152263]: pgmap v1462: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 21 MiB/s wr, 629 op/s; 60551/331428 objects degraded (18.270%); 23729/331428 objects misplaced (7.160%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:25:39.364 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:39 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:39.364 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:39 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:39.364 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:39 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 60551/331428 objects degraded (18.270%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:39.409 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:39.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:39.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:39.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:39 smithi110 ceph-mon[136032]: pgmap v1462: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 127 B/s rd, 21 MiB/s wr, 629 op/s; 60551/331428 objects degraded (18.270%); 23729/331428 objects misplaced (7.160%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:25:39.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:39 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:39.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:39 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:39.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:39 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 60551/331428 objects degraded (18.270%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T10:25:39.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:39.615 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:39.676 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:39.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:39.788 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:39.879 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:39.968 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:39.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:40.050 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:40.077 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:40.115 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:40.222 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:40.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:40.243 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:25:40.299 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:25:40.317 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:40.336 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:40.343 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:25:40.385 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:25:40.394 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:40.426 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:25:40.459 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:40.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:40.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:40.473 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:25:40.513 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:25:40.544 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:25:40.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:25:40.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:25:40.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:25:40.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:25:40.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:25:40.558 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:25:40.559 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:25:40.559 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:25:40.572 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:40.650 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:40.691 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:40.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:40.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:40.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:40.867 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:41.020 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:41.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:41.141 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:41.153 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:41 smithi012 ceph-mon[147926]: pgmap v1463: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 127 B/s rd, 8.4 MiB/s wr, 640 op/s; 60002/326214 objects degraded (18.393%); 23194/326214 objects misplaced (7.110%); 11 MiB/s, 175 objects/s recovering 2023-10-09T10:25:41.153 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2847119472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:41.153 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:41 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:41.153 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:41 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:41.201 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:41.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:41 smithi012 ceph-mon[152263]: pgmap v1463: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 127 B/s rd, 8.4 MiB/s wr, 640 op/s; 60002/326214 objects degraded (18.393%); 23194/326214 objects misplaced (7.110%); 11 MiB/s, 175 objects/s recovering 2023-10-09T10:25:41.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2847119472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:41.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:41 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:41.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:41 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:41.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:41 smithi110 ceph-mon[136032]: pgmap v1463: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 127 B/s rd, 8.4 MiB/s wr, 640 op/s; 60002/326214 objects degraded (18.393%); 23194/326214 objects misplaced (7.110%); 11 MiB/s, 175 objects/s recovering 2023-10-09T10:25:41.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2847119472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:41.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:41 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:41.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:41 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:41.594 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:25:41.610 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:25:42.071 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:42.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:42.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:42.179 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:42.238 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:42.334 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:42.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[152263]: from='client.47033 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:25:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:25:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:25:42.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[147926]: from='client.47033 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:25:42.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:25:42.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:25:42.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:42 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:42.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:42 smithi110 ceph-mon[136032]: from='client.47033 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:25:42.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:42 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:42 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:25:42.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:25:42.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:42 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:25:42.504 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:42.561 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:42.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:42.666 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:42.714 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:42.784 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:42.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:42.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:42.892 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:42.984 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:43.036 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:43.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:43.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:43.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:43.228 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:43.272 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1463,"stamp":"2023-10-09T10:25:39.782773+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7125112760,"num_objects":108738,"num_object_clones":0,"num_object_copies":326214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":60002,"num_objects_misplaced":23194,"num_objects_unfound":0,"num_objects_dirty":108738,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":102492,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5630,"num_bytes_recovered":361006300,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124206,"ondisk_log_size":124206,"up":297,"acting":280,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":336,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":25974308,"kb_used_data":20814772,"kb_used_omap":163,"kb_used_meta":5159004,"kb_avail":536496604,"statfs":{"total":575970213888,"available":549372522496,"internally_reserved":0,"allocated":21314326528,"data_stored":21305523732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":5282821107},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,0,2,2,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-194314216,"num_objects":-2964,"num_object_clones":0,"num_object_copies":-8892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1583,"num_objects_misplaced":-1120,"num_objects_unfound":0,"num_objects_dirty":-2964,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5130,"num_write_kb":69249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1409,"num_bytes_recovered":92340224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.009260"},"pg_stats":[{"pgid":"11.12","version":"525'4692","reported_seq":4750,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:37.742505+0000","last_change":"2023-10-09T10:25:09.709209+0000","last_active":"2023-10-09T10:25:37.742505+0000","last_peered":"2023-10-09T10:25:37.742505+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:25:37.742505+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4692,"log_dups_size":0,"ondisk_log_size":4692,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224100352,"num_objects":3420,"num_object_clones":0,"num_object_copies":10260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3119,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3744,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":301},{"shards":"0,2,5","objects":2957}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":118,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602429+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:25:35.602429+0000","last_peered":"2023-10-09T10:25:35.602429+0000","last_clean":"2023-10-09T10:25:35.602429+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:25:35.602429+0000","last_undegraded":"2023-10-09T10:25:35.602429+0000","last_fullsized":"2023-10-09T10:25:35.602429+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":479,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605281+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:25:35.605281+0000","last_peered":"2023-10-09T10:25:35.605281+0000","last_clean":"2023-10-09T10:25:35.605281+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:25:35.605281+0000","last_undegraded":"2023-10-09T10:25:35.605281+0000","last_fullsized":"2023-10-09T10:25:35.605281+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"525'4626","reported_seq":5144,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:37.741169+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:25:37.741169+0000","last_peered":"2023-10-09T10:25:37.741169+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:25:37.741169+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4626,"log_dups_size":0,"ondisk_log_size":4626,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221577216,"num_objects":3381,"num_object_clones":0,"num_object_copies":10143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5073,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3681,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":9830400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":2114},{"shards":"0,2,6","objects":845},{"shards":"0,6","objects":422}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":556,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602073+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:25:35.602073+0000","last_peered":"2023-10-09T10:25:35.602073+0000","last_clean":"2023-10-09T10:25:35.602073+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:25:35.602073+0000","last_undegraded":"2023-10-09T10:25:35.602073+0000","last_fullsized":"2023-10-09T10:25:35.602073+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":449,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587774+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:25:35.587774+0000","last_peered":"2023-10-09T10:25:35.587774+0000","last_clean":"2023-10-09T10:25:35.587774+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:25:35.587774+0000","last_undegraded":"2023-10-09T10:25:35.587774+0000","last_fullsized":"2023-10-09T10:25:35.587774+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"525'4567","reported_seq":7182,"reported_epoch":525,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:25:37.764189+0000","last_change":"2023-10-09T10:25:17.878455+0000","last_active":"2023-10-09T10:25:37.764189+0000","last_peered":"2023-10-09T10:25:37.764189+0000","last_clean":"2023-10-09T10:24:39.547747+0000","last_became_active":"2023-10-09T10:24:41.974279+0000","last_became_peered":"2023-10-09T10:24:41.974279+0000","last_unstale":"2023-10-09T10:25:37.764189+0000","last_undegraded":"2023-10-09T10:25:37.764189+0000","last_fullsized":"2023-10-09T10:24:41.670066+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4567,"log_dups_size":0,"ondisk_log_size":4567,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219643904,"num_objects":3352,"num_object_clones":0,"num_object_copies":10056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":955,"num_objects_unfound":0,"num_objects_dirty":3352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3630,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1278,"num_bytes_recovered":83755008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":955},{"shards":"0,3","objects":2397}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583268+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:25:35.583268+0000","last_peered":"2023-10-09T10:25:35.583268+0000","last_clean":"2023-10-09T10:25:35.583268+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:25:35.583268+0000","last_undegraded":"2023-10-09T10:25:35.583268+0000","last_fullsized":"2023-10-09T10:25:35.583268+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.585008+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:25:35.585008+0000","last_peered":"2023-10-09T10:25:35.585008+0000","last_clean":"2023-10-09T10:25:35.585008+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:25:35.585008+0000","last_undegraded":"2023-10-09T10:25:35.585008+0000","last_fullsized":"2023-10-09T10:25:35.585008+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":483,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.585436+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:25:35.585436+0000","last_peered":"2023-10-09T10:25:35.585436+0000","last_clean":"2023-10-09T10:25:35.585436+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:25:35.585436+0000","last_undegraded":"2023-10-09T10:25:35.585436+0000","last_fullsized":"2023-10-09T10:25:35.585436+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":431,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T10:25:29.510340+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:25:29.510340+0000","last_peered":"2023-10-09T10:25:29.510340+0000","last_clean":"2023-10-09T10:25:29.510340+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:25:29.510340+0000","last_undegraded":"2023-10-09T10:25:29.510340+0000","last_fullsized":"2023-10-09T10:25:29.510340+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":198,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.474769+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:25:27.474769+0000","last_peered":"2023-10-09T10:25:27.474769+0000","last_clean":"2023-10-09T10:25:27.474769+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:25:27.474769+0000","last_undegraded":"2023-10-09T10:25:27.474769+0000","last_fullsized":"2023-10-09T10:25:27.474769+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":431,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T10:25:29.510966+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:25:29.510966+0000","last_peered":"2023-10-09T10:25:29.510966+0000","last_clean":"2023-10-09T10:25:29.510966+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:25:29.510966+0000","last_undegraded":"2023-10-09T10:25:29.510966+0000","last_fullsized":"2023-10-09T10:25:29.510966+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":130,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605423+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:25:35.605423+0000","last_peered":"2023-10-09T10:25:35.605423+0000","last_clean":"2023-10-09T10:25:35.605423+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:25:35.605423+0000","last_undegraded":"2023-10-09T10:25:35.605423+0000","last_fullsized":"2023-10-09T10:25:35.605423+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587179+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:25:35.587179+0000","last_peered":"2023-10-09T10:25:35.587179+0000","last_clean":"2023-10-09T10:25:35.587179+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:25:35.587179+0000","last_undegraded":"2023-10-09T10:25:35.587179+0000","last_fullsized":"2023-10-09T10:25:35.587179+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":176,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602706+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:25:35.602706+0000","last_peered":"2023-10-09T10:25:35.602706+0000","last_clean":"2023-10-09T10:25:35.602706+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:25:35.602706+0000","last_undegraded":"2023-10-09T10:25:35.602706+0000","last_fullsized":"2023-10-09T10:25:35.602706+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602608+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:25:35.602608+0000","last_peered":"2023-10-09T10:25:35.602608+0000","last_clean":"2023-10-09T10:25:35.602608+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:25:35.602608+0000","last_undegraded":"2023-10-09T10:25:35.602608+0000","last_fullsized":"2023-10-09T10:25:35.602608+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":96,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602530+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:25:35.602530+0000","last_peered":"2023-10-09T10:25:35.602530+0000","last_clean":"2023-10-09T10:25:35.602530+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:25:35.602530+0000","last_undegraded":"2023-10-09T10:25:35.602530+0000","last_fullsized":"2023-10-09T10:25:35.602530+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":48,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.476007+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:25:27.476007+0000","last_peered":"2023-10-09T10:25:27.476007+0000","last_clean":"2023-10-09T10:25:27.476007+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:25:27.476007+0000","last_undegraded":"2023-10-09T10:25:27.476007+0000","last_fullsized":"2023-10-09T10:25:27.476007+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":447,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591435+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:25:35.591435+0000","last_peered":"2023-10-09T10:25:35.591435+0000","last_clean":"2023-10-09T10:25:35.591435+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:25:35.591435+0000","last_undegraded":"2023-10-09T10:25:35.591435+0000","last_fullsized":"2023-10-09T10:25:35.591435+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":189,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.480438+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:25:27.480438+0000","last_peered":"2023-10-09T10:25:27.480438+0000","last_clean":"2023-10-09T10:25:27.480438+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:25:27.480438+0000","last_undegraded":"2023-10-09T10:25:27.480438+0000","last_fullsized":"2023-10-09T10:25:27.480438+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"523'4564","reported_seq":5354,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:34.860052+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:25:34.860052+0000","last_peered":"2023-10-09T10:25:34.860052+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:25:34.860052+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4564,"log_dups_size":0,"ondisk_log_size":4564,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227049472,"num_objects":3465,"num_object_clones":0,"num_object_copies":10395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":3465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3641,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":314},{"shards":"3,5,6","objects":1116}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602189+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:25:35.602189+0000","last_peered":"2023-10-09T10:25:35.602189+0000","last_clean":"2023-10-09T10:25:35.602189+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:25:35.602189+0000","last_undegraded":"2023-10-09T10:25:35.602189+0000","last_fullsized":"2023-10-09T10:25:35.602189+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587450+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:25:35.587450+0000","last_peered":"2023-10-09T10:25:35.587450+0000","last_clean":"2023-10-09T10:25:35.587450+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:25:35.587450+0000","last_undegraded":"2023-10-09T10:25:35.587450+0000","last_fullsized":"2023-10-09T10:25:35.587450+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"523'4604","reported_seq":4940,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:34.860029+0000","last_change":"2023-10-09T10:25:09.702978+0000","last_active":"2023-10-09T10:25:34.860029+0000","last_peered":"2023-10-09T10:25:34.860029+0000","last_clean":"2023-10-09T10:24:39.576387+0000","last_became_active":"2023-10-09T10:25:09.696470+0000","last_became_peered":"2023-10-09T10:25:09.696470+0000","last_unstale":"2023-10-09T10:25:34.860029+0000","last_undegraded":"2023-10-09T10:25:09.546049+0000","last_fullsized":"2023-10-09T10:25:09.436538+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4604,"log_dups_size":0,"ondisk_log_size":4604,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226525184,"num_objects":3457,"num_object_clones":0,"num_object_copies":10371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5168,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3639,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":5963776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":2187},{"shards":"2,5,6","objects":794},{"shards":"5,6","objects":476}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":30,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591447+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:25:35.591447+0000","last_peered":"2023-10-09T10:25:35.591447+0000","last_clean":"2023-10-09T10:25:35.591447+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:25:35.591447+0000","last_undegraded":"2023-10-09T10:25:35.591447+0000","last_fullsized":"2023-10-09T10:25:35.591447+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T10:25:29.511261+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:25:29.511261+0000","last_peered":"2023-10-09T10:25:29.511261+0000","last_clean":"2023-10-09T10:25:29.511261+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:25:29.511261+0000","last_undegraded":"2023-10-09T10:25:29.511261+0000","last_fullsized":"2023-10-09T10:25:29.511261+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"523'4631","reported_seq":4682,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T10:25:34.889638+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:25:34.889638+0000","last_peered":"2023-10-09T10:25:34.889638+0000","last_clean":"2023-10-09T10:25:34.889638+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:25:34.889638+0000","last_undegraded":"2023-10-09T10:25:34.889638+0000","last_fullsized":"2023-10-09T10:25:34.889638+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":4631,"log_dups_size":0,"ondisk_log_size":4631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231014400,"num_objects":3525,"num_object_clones":0,"num_object_copies":10575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1343,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":26,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605288+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:25:35.605288+0000","last_peered":"2023-10-09T10:25:35.605288+0000","last_clean":"2023-10-09T10:25:35.605288+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:25:35.605288+0000","last_undegraded":"2023-10-09T10:25:35.605288+0000","last_fullsized":"2023-10-09T10:25:35.605288+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":198,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.474306+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:25:27.474306+0000","last_peered":"2023-10-09T10:25:27.474306+0000","last_clean":"2023-10-09T10:25:27.474306+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:25:27.474306+0000","last_undegraded":"2023-10-09T10:25:27.474306+0000","last_fullsized":"2023-10-09T10:25:27.474306+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"525'4478","reported_seq":4534,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:39.356111+0000","last_change":"2023-10-09T10:25:13.752880+0000","last_active":"2023-10-09T10:25:39.356111+0000","last_peered":"2023-10-09T10:25:39.356111+0000","last_clean":"2023-10-09T10:25:11.561932+0000","last_became_active":"2023-10-09T10:25:13.746279+0000","last_became_peered":"2023-10-09T10:25:13.746279+0000","last_unstale":"2023-10-09T10:25:39.356111+0000","last_undegraded":"2023-10-09T10:25:13.623380+0000","last_fullsized":"2023-10-09T10:25:13.604555+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":4478,"log_dups_size":0,"ondisk_log_size":4478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:39.267545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208601088,"num_objects":3183,"num_object_clones":0,"num_object_copies":9549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3012,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1718,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":171},{"shards":"2,4,5","objects":2840}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":152,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605270+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:25:35.605270+0000","last_peered":"2023-10-09T10:25:35.605270+0000","last_clean":"2023-10-09T10:25:35.605270+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:25:35.605270+0000","last_undegraded":"2023-10-09T10:25:35.605270+0000","last_fullsized":"2023-10-09T10:25:35.605270+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602235+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:25:35.602235+0000","last_peered":"2023-10-09T10:25:35.602235+0000","last_clean":"2023-10-09T10:25:35.602235+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:25:35.602235+0000","last_undegraded":"2023-10-09T10:25:35.602235+0000","last_fullsized":"2023-10-09T10:25:35.602235+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"523'4564","reported_seq":4891,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:34.847165+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:25:34.847165+0000","last_peered":"2023-10-09T10:25:34.847165+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:25:34.847165+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4564,"log_dups_size":0,"ondisk_log_size":4564,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225804288,"num_objects":3445,"num_object_clones":0,"num_object_copies":10335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5277,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3619,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":5701632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2232},{"shards":"0,5,6","objects":813},{"shards":"5,6","objects":400}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":214,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.476085+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:25:27.476085+0000","last_peered":"2023-10-09T10:25:27.476085+0000","last_clean":"2023-10-09T10:25:27.476085+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:25:27.476085+0000","last_undegraded":"2023-10-09T10:25:27.476085+0000","last_fullsized":"2023-10-09T10:25:27.476085+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":115,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.478043+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:25:27.478043+0000","last_peered":"2023-10-09T10:25:27.478043+0000","last_clean":"2023-10-09T10:25:27.478043+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:25:27.478043+0000","last_undegraded":"2023-10-09T10:25:27.478043+0000","last_fullsized":"2023-10-09T10:25:27.478043+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"523'4559","reported_seq":8735,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T10:25:34.835895+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:25:34.835895+0000","last_peered":"2023-10-09T10:25:34.835895+0000","last_clean":"2023-10-09T10:25:34.835895+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:25:34.835895+0000","last_undegraded":"2023-10-09T10:25:34.835895+0000","last_fullsized":"2023-10-09T10:25:34.835895+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":4559,"log_dups_size":0,"ondisk_log_size":4559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224002048,"num_objects":3418,"num_object_clones":0,"num_object_copies":10254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1645,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":198,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.475602+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:25:27.475602+0000","last_peered":"2023-10-09T10:25:27.475602+0000","last_clean":"2023-10-09T10:25:27.475602+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:25:27.475602+0000","last_undegraded":"2023-10-09T10:25:27.475602+0000","last_fullsized":"2023-10-09T10:25:27.475602+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":434,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583987+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:25:35.583987+0000","last_peered":"2023-10-09T10:25:35.583987+0000","last_clean":"2023-10-09T10:25:35.583987+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:25:35.583987+0000","last_undegraded":"2023-10-09T10:25:35.583987+0000","last_fullsized":"2023-10-09T10:25:35.583987+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"523'4522","reported_seq":5802,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:34.886565+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:25:34.886565+0000","last_peered":"2023-10-09T10:25:34.886565+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:25:34.886565+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4522,"log_dups_size":0,"ondisk_log_size":4522,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223969280,"num_objects":3417,"num_object_clones":0,"num_object_copies":10251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3601,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1629},{"shards":"2,5","objects":1788}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":135,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583915+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:25:35.583915+0000","last_peered":"2023-10-09T10:25:35.583915+0000","last_clean":"2023-10-09T10:25:35.583915+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:25:35.583915+0000","last_undegraded":"2023-10-09T10:25:35.583915+0000","last_fullsized":"2023-10-09T10:25:35.583915+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602627+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:25:35.602627+0000","last_peered":"2023-10-09T10:25:35.602627+0000","last_clean":"2023-10-09T10:25:35.602627+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:25:35.602627+0000","last_undegraded":"2023-10-09T10:25:35.602627+0000","last_fullsized":"2023-10-09T10:25:35.602627+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"525'4598","reported_seq":4709,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:37.754480+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:25:37.754480+0000","last_peered":"2023-10-09T10:25:37.754480+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:25:37.754480+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":4598,"log_dups_size":0,"ondisk_log_size":4598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220004352,"num_objects":3357,"num_object_clones":0,"num_object_copies":10071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1710,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":299},{"shards":"0,2,5","objects":2909}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602313+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:25:35.602313+0000","last_peered":"2023-10-09T10:25:35.602313+0000","last_clean":"2023-10-09T10:25:35.602313+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:25:35.602313+0000","last_undegraded":"2023-10-09T10:25:35.602313+0000","last_fullsized":"2023-10-09T10:25:35.602313+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":102,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605289+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:25:35.605289+0000","last_peered":"2023-10-09T10:25:35.605289+0000","last_clean":"2023-10-09T10:25:35.605289+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:25:35.605289+0000","last_undegraded":"2023-10-09T10:25:35.605289+0000","last_fullsized":"2023-10-09T10:25:35.605289+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"525'4616","reported_seq":5100,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:37.760518+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:25:37.760518+0000","last_peered":"2023-10-09T10:25:37.760518+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:25:37.760518+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4616,"log_dups_size":0,"ondisk_log_size":4616,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222363648,"num_objects":3393,"num_object_clones":0,"num_object_copies":10179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3671,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":2115},{"shards":"0,2,6","objects":806},{"shards":"0,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602313+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:25:35.602313+0000","last_peered":"2023-10-09T10:25:35.602313+0000","last_clean":"2023-10-09T10:25:35.602313+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:25:35.602313+0000","last_undegraded":"2023-10-09T10:25:35.602313+0000","last_fullsized":"2023-10-09T10:25:35.602313+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":145,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605172+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:25:35.605172+0000","last_peered":"2023-10-09T10:25:35.605172+0000","last_clean":"2023-10-09T10:25:35.605172+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:25:35.605172+0000","last_undegraded":"2023-10-09T10:25:35.605172+0000","last_fullsized":"2023-10-09T10:25:35.605172+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"525'4545","reported_seq":4970,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:37.617096+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:25:37.617096+0000","last_peered":"2023-10-09T10:25:37.617096+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:25:37.617096+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":4545,"log_dups_size":0,"ondisk_log_size":4545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220790784,"num_objects":3369,"num_object_clones":0,"num_object_copies":10107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1438,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2081},{"shards":"0,3","objects":1288}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":475,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.474229+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:25:27.474229+0000","last_peered":"2023-10-09T10:25:27.474229+0000","last_clean":"2023-10-09T10:25:27.474229+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:25:27.474229+0000","last_undegraded":"2023-10-09T10:25:27.474229+0000","last_fullsized":"2023-10-09T10:25:27.474229+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":539,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605508+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:25:35.605508+0000","last_peered":"2023-10-09T10:25:35.605508+0000","last_clean":"2023-10-09T10:25:35.605508+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:25:35.605508+0000","last_undegraded":"2023-10-09T10:25:35.605508+0000","last_fullsized":"2023-10-09T10:25:35.605508+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"525'7340","reported_seq":7414,"reported_epoch":525,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:25:38.104053+0000","last_change":"2023-10-09T10:25:17.929569+0000","last_active":"2023-10-09T10:25:38.104053+0000","last_peered":"2023-10-09T10:25:38.104053+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:25:38.104053+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"499'1500","ondisk_log_start":"499'1500","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442695704,"num_objects":6756,"num_object_clones":0,"num_object_copies":20268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6753,"num_objects_misplaced":6756,"num_objects_unfound":0,"num_objects_dirty":6756,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7340,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":6756}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":245,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.586501+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:25:35.586501+0000","last_peered":"2023-10-09T10:25:35.586501+0000","last_clean":"2023-10-09T10:25:35.586501+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:25:35.586501+0000","last_undegraded":"2023-10-09T10:25:35.586501+0000","last_fullsized":"2023-10-09T10:25:35.586501+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":207,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.480395+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:25:27.480395+0000","last_peered":"2023-10-09T10:25:27.480395+0000","last_clean":"2023-10-09T10:25:27.480395+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:25:27.480395+0000","last_undegraded":"2023-10-09T10:25:27.480395+0000","last_fullsized":"2023-10-09T10:25:27.480395+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":487,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.478076+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:25:27.478076+0000","last_peered":"2023-10-09T10:25:27.478076+0000","last_clean":"2023-10-09T10:25:27.478076+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:25:27.478076+0000","last_undegraded":"2023-10-09T10:25:27.478076+0000","last_fullsized":"2023-10-09T10:25:27.478076+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"525'7410","reported_seq":7450,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:37.769300+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:25:37.769300+0000","last_peered":"2023-10-09T10:25:37.769300+0000","last_clean":"2023-10-09T10:25:37.769300+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:25:37.769300+0000","last_undegraded":"2023-10-09T10:25:37.769300+0000","last_fullsized":"2023-10-09T10:25:37.769300+0000","mapping_epoch":494,"log_start":"503'2406","ondisk_log_start":"503'2406","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5004,"log_dups_size":0,"ondisk_log_size":5004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":450101248,"num_objects":6868,"num_object_clones":0,"num_object_copies":20604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7410,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":207,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.478001+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:25:27.478001+0000","last_peered":"2023-10-09T10:25:27.478001+0000","last_clean":"2023-10-09T10:25:27.478001+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:25:27.478001+0000","last_undegraded":"2023-10-09T10:25:27.478001+0000","last_fullsized":"2023-10-09T10:25:27.478001+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":127,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602593+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:25:35.602593+0000","last_peered":"2023-10-09T10:25:35.602593+0000","last_clean":"2023-10-09T10:25:35.602593+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:25:35.602593+0000","last_undegraded":"2023-10-09T10:25:35.602593+0000","last_fullsized":"2023-10-09T10:25:35.602593+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":437,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.578271+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:25:35.578271+0000","last_peered":"2023-10-09T10:25:35.578271+0000","last_clean":"2023-10-09T10:25:35.578271+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:25:35.578271+0000","last_undegraded":"2023-10-09T10:25:35.578271+0000","last_fullsized":"2023-10-09T10:25:35.578271+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"523'4573","reported_seq":4945,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:34.859632+0000","last_change":"2023-10-09T10:25:09.703037+0000","last_active":"2023-10-09T10:25:34.859632+0000","last_peered":"2023-10-09T10:25:34.859632+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:25:09.696864+0000","last_became_peered":"2023-10-09T10:25:09.696864+0000","last_unstale":"2023-10-09T10:25:34.859632+0000","last_undegraded":"2023-10-09T10:25:09.545735+0000","last_fullsized":"2023-10-09T10:25:09.430540+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4573,"log_dups_size":0,"ondisk_log_size":4573,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222920704,"num_objects":3401,"num_object_clones":0,"num_object_copies":10203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5178,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3607,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":103,"num_bytes_recovered":6750208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":2187},{"shards":"2,5,6","objects":804},{"shards":"5,6","objects":410}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":474,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.589498+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:25:35.589498+0000","last_peered":"2023-10-09T10:25:35.589498+0000","last_clean":"2023-10-09T10:25:35.589498+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:25:35.589498+0000","last_undegraded":"2023-10-09T10:25:35.589498+0000","last_fullsized":"2023-10-09T10:25:35.589498+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":92,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591364+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:25:35.591364+0000","last_peered":"2023-10-09T10:25:35.591364+0000","last_clean":"2023-10-09T10:25:35.591364+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:25:35.591364+0000","last_undegraded":"2023-10-09T10:25:35.591364+0000","last_fullsized":"2023-10-09T10:25:35.591364+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":536,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.475348+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:25:27.475348+0000","last_peered":"2023-10-09T10:25:27.475348+0000","last_clean":"2023-10-09T10:25:27.475348+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:25:27.475348+0000","last_undegraded":"2023-10-09T10:25:27.475348+0000","last_fullsized":"2023-10-09T10:25:27.475348+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"523'7251","reported_seq":7309,"reported_epoch":523,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:25:34.886832+0000","last_change":"2023-10-09T10:24:41.858672+0000","last_active":"2023-10-09T10:25:34.886832+0000","last_peered":"2023-10-09T10:25:34.886832+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:25:34.886832+0000","last_undegraded":"2023-10-09T10:25:34.886832+0000","last_fullsized":"2023-10-09T10:25:34.886832+0000","mapping_epoch":507,"log_start":"503'2400","ondisk_log_start":"503'2400","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4851,"log_dups_size":0,"ondisk_log_size":4851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":449642496,"num_objects":6861,"num_object_clones":0,"num_object_copies":20583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6860,"num_objects_unfound":0,"num_objects_dirty":6861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7251,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":6861}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":177,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.481961+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:25:27.481961+0000","last_peered":"2023-10-09T10:25:27.481961+0000","last_clean":"2023-10-09T10:25:27.481961+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:25:27.481961+0000","last_undegraded":"2023-10-09T10:25:27.481961+0000","last_fullsized":"2023-10-09T10:25:27.481961+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":724,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:35.576978+0000","last_change":"2023-10-09T10:25:13.642678+0000","last_active":"2023-10-09T10:25:35.576978+0000","last_peered":"2023-10-09T10:25:35.576978+0000","last_clean":"2023-10-09T10:24:37.578053+0000","last_became_active":"2023-10-09T10:25:13.642243+0000","last_became_peered":"2023-10-09T10:25:13.642243+0000","last_unstale":"2023-10-09T10:25:35.576978+0000","last_undegraded":"2023-10-09T10:25:13.604924+0000","last_fullsized":"2023-10-09T10:25:13.604107+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:47.058901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[2,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":225,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.477889+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:25:27.477889+0000","last_peered":"2023-10-09T10:25:27.477889+0000","last_clean":"2023-10-09T10:25:27.477889+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:25:27.477889+0000","last_undegraded":"2023-10-09T10:25:27.477889+0000","last_fullsized":"2023-10-09T10:25:27.477889+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605360+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:25:35.605360+0000","last_peered":"2023-10-09T10:25:35.605360+0000","last_clean":"2023-10-09T10:25:35.605360+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:25:35.605360+0000","last_undegraded":"2023-10-09T10:25:35.605360+0000","last_fullsized":"2023-10-09T10:25:35.605360+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"523'4659","reported_seq":5022,"reported_epoch":523,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:25:34.937017+0000","last_change":"2023-10-09T10:25:13.759251+0000","last_active":"2023-10-09T10:25:34.937017+0000","last_peered":"2023-10-09T10:25:34.937017+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:25:34.937017+0000","last_undegraded":"2023-10-09T10:25:13.707357+0000","last_fullsized":"2023-10-09T10:25:34.937017+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4659,"log_dups_size":0,"ondisk_log_size":4659,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229605376,"num_objects":3503,"num_object_clones":0,"num_object_copies":10509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2188,"num_objects_misplaced":3141,"num_objects_unfound":0,"num_objects_dirty":3503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3735,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":7602176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2188},{"shards":"3,5,6","objects":362}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":209,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591523+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:25:35.591523+0000","last_peered":"2023-10-09T10:25:35.591523+0000","last_clean":"2023-10-09T10:25:35.591523+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:25:35.591523+0000","last_undegraded":"2023-10-09T10:25:35.591523+0000","last_fullsized":"2023-10-09T10:25:35.591523+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":469,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.589638+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:25:35.589638+0000","last_peered":"2023-10-09T10:25:35.589638+0000","last_clean":"2023-10-09T10:25:35.589638+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:25:35.589638+0000","last_undegraded":"2023-10-09T10:25:35.589638+0000","last_fullsized":"2023-10-09T10:25:35.589638+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587256+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:25:35.587256+0000","last_peered":"2023-10-09T10:25:35.587256+0000","last_clean":"2023-10-09T10:25:35.587256+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:25:35.587256+0000","last_undegraded":"2023-10-09T10:25:35.587256+0000","last_fullsized":"2023-10-09T10:25:35.587256+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"525'7401","reported_seq":7440,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:38.093608+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:25:38.093608+0000","last_peered":"2023-10-09T10:25:38.093608+0000","last_clean":"2023-10-09T10:25:38.093608+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:25:38.093608+0000","last_undegraded":"2023-10-09T10:25:38.093608+0000","last_fullsized":"2023-10-09T10:25:38.093608+0000","mapping_epoch":494,"log_start":"499'1600","ondisk_log_start":"499'1600","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5801,"log_dups_size":0,"ondisk_log_size":5801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446365696,"num_objects":6811,"num_object_clones":0,"num_object_copies":20433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7401,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584506+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:25:35.584506+0000","last_peered":"2023-10-09T10:25:35.584506+0000","last_clean":"2023-10-09T10:25:35.584506+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:25:35.584506+0000","last_undegraded":"2023-10-09T10:25:35.584506+0000","last_fullsized":"2023-10-09T10:25:35.584506+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":504,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.475865+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:25:27.475865+0000","last_peered":"2023-10-09T10:25:27.475865+0000","last_clean":"2023-10-09T10:25:27.475865+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:25:27.475865+0000","last_undegraded":"2023-10-09T10:25:27.475865+0000","last_fullsized":"2023-10-09T10:25:27.475865+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":437,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584255+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:25:35.584255+0000","last_peered":"2023-10-09T10:25:35.584255+0000","last_clean":"2023-10-09T10:25:35.584255+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:25:35.584255+0000","last_undegraded":"2023-10-09T10:25:35.584255+0000","last_fullsized":"2023-10-09T10:25:35.584255+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"525'7168","reported_seq":7208,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:37.739558+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:25:37.739558+0000","last_peered":"2023-10-09T10:25:37.739558+0000","last_clean":"2023-10-09T10:25:37.739558+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:25:37.739558+0000","last_undegraded":"2023-10-09T10:25:37.739558+0000","last_fullsized":"2023-10-09T10:25:37.739558+0000","mapping_epoch":494,"log_start":"503'2076","ondisk_log_start":"503'2076","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5092,"log_dups_size":0,"ondisk_log_size":5092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430440448,"num_objects":6568,"num_object_clones":0,"num_object_copies":19704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7168,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":325,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.585297+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:25:35.585297+0000","last_peered":"2023-10-09T10:25:35.585297+0000","last_clean":"2023-10-09T10:25:35.585297+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:25:35.585297+0000","last_undegraded":"2023-10-09T10:25:35.585297+0000","last_fullsized":"2023-10-09T10:25:35.585297+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":506,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605136+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:25:35.605136+0000","last_peered":"2023-10-09T10:25:35.605136+0000","last_clean":"2023-10-09T10:25:35.605136+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:25:35.605136+0000","last_undegraded":"2023-10-09T10:25:35.605136+0000","last_fullsized":"2023-10-09T10:25:35.605136+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583173+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:25:35.583173+0000","last_peered":"2023-10-09T10:25:35.583173+0000","last_clean":"2023-10-09T10:25:35.583173+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:25:35.583173+0000","last_undegraded":"2023-10-09T10:25:35.583173+0000","last_fullsized":"2023-10-09T10:25:35.583173+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"525'7423","reported_seq":7462,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:38.084082+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:25:38.084082+0000","last_peered":"2023-10-09T10:25:38.084082+0000","last_clean":"2023-10-09T10:25:38.084082+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:25:38.084082+0000","last_undegraded":"2023-10-09T10:25:38.084082+0000","last_fullsized":"2023-10-09T10:25:38.084082+0000","mapping_epoch":494,"log_start":"499'1600","ondisk_log_start":"499'1600","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5823,"log_dups_size":0,"ondisk_log_size":5823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448331776,"num_objects":6841,"num_object_clones":0,"num_object_copies":20523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7423,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.473465+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:25:27.473465+0000","last_peered":"2023-10-09T10:25:27.473465+0000","last_clean":"2023-10-09T10:25:27.473465+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:25:27.473465+0000","last_undegraded":"2023-10-09T10:25:27.473465+0000","last_fullsized":"2023-10-09T10:25:27.473465+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":24,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587521+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:25:35.587521+0000","last_peered":"2023-10-09T10:25:35.587521+0000","last_clean":"2023-10-09T10:25:35.587521+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:25:35.587521+0000","last_undegraded":"2023-10-09T10:25:35.587521+0000","last_fullsized":"2023-10-09T10:25:35.587521+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":203,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.483121+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:25:27.483121+0000","last_peered":"2023-10-09T10:25:27.483121+0000","last_clean":"2023-10-09T10:25:27.483121+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:25:27.483121+0000","last_undegraded":"2023-10-09T10:25:27.483121+0000","last_fullsized":"2023-10-09T10:25:27.483121+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"523'4488","reported_seq":4573,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:34.859706+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:25:34.859706+0000","last_peered":"2023-10-09T10:25:34.859706+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:25:34.859706+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4488,"log_dups_size":0,"ondisk_log_size":4488,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220430336,"num_objects":3364,"num_object_clones":0,"num_object_copies":10092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3568,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2128},{"shards":"2,5","objects":1148}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584188+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:25:35.584188+0000","last_peered":"2023-10-09T10:25:35.584188+0000","last_clean":"2023-10-09T10:25:35.584188+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:25:35.584188+0000","last_undegraded":"2023-10-09T10:25:35.584188+0000","last_fullsized":"2023-10-09T10:25:35.584188+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":29,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583460+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:25:35.583460+0000","last_peered":"2023-10-09T10:25:35.583460+0000","last_clean":"2023-10-09T10:25:35.583460+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:25:35.583460+0000","last_undegraded":"2023-10-09T10:25:35.583460+0000","last_fullsized":"2023-10-09T10:25:35.583460+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"525'4603","reported_seq":4664,"reported_epoch":525,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:25:37.768866+0000","last_change":"2023-10-09T10:25:08.466144+0000","last_active":"2023-10-09T10:25:37.768866+0000","last_peered":"2023-10-09T10:25:37.768866+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:25:37.768866+0000","last_undegraded":"2023-10-09T10:25:37.768866+0000","last_fullsized":"2023-10-09T10:25:37.768866+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4603,"log_dups_size":0,"ondisk_log_size":4603,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221020160,"num_objects":3373,"num_object_clones":0,"num_object_copies":10119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2331,"num_objects_unfound":0,"num_objects_dirty":3373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3679,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1042}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":253,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.475682+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:25:27.475682+0000","last_peered":"2023-10-09T10:25:27.475682+0000","last_clean":"2023-10-09T10:25:27.475682+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:25:27.475682+0000","last_undegraded":"2023-10-09T10:25:27.475682+0000","last_fullsized":"2023-10-09T10:25:27.475682+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":118,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602675+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:25:35.602675+0000","last_peered":"2023-10-09T10:25:35.602675+0000","last_clean":"2023-10-09T10:25:35.602675+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:25:35.602675+0000","last_undegraded":"2023-10-09T10:25:35.602675+0000","last_fullsized":"2023-10-09T10:25:35.602675+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"523'4583","reported_seq":4921,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:34.933979+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:25:34.933979+0000","last_peered":"2023-10-09T10:25:34.933979+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:25:34.933979+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4583,"log_dups_size":0,"ondisk_log_size":4583,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226263040,"num_objects":3453,"num_object_clones":0,"num_object_copies":10359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3639,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":6094848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2147},{"shards":"0,5,6","objects":833},{"shards":"5,6","objects":472}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":236,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T10:25:27.480308+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:25:27.480308+0000","last_peered":"2023-10-09T10:25:27.480308+0000","last_clean":"2023-10-09T10:25:27.480308+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:25:27.480308+0000","last_undegraded":"2023-10-09T10:25:27.480308+0000","last_fullsized":"2023-10-09T10:25:27.480308+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":347,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591287+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:25:35.591287+0000","last_peered":"2023-10-09T10:25:35.591287+0000","last_clean":"2023-10-09T10:25:35.591287+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:25:35.591287+0000","last_undegraded":"2023-10-09T10:25:35.591287+0000","last_fullsized":"2023-10-09T10:25:35.591287+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"525'4570","reported_seq":4626,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:39.384240+0000","last_change":"2023-10-09T10:25:13.762065+0000","last_active":"2023-10-09T10:25:39.384240+0000","last_peered":"2023-10-09T10:25:39.384240+0000","last_clean":"2023-10-09T10:25:11.551777+0000","last_became_active":"2023-10-09T10:25:13.754797+0000","last_became_peered":"2023-10-09T10:25:13.754797+0000","last_unstale":"2023-10-09T10:25:39.384240+0000","last_undegraded":"2023-10-09T10:25:13.625003+0000","last_fullsized":"2023-10-09T10:25:13.604794+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":4570,"log_dups_size":0,"ondisk_log_size":4570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:58.037975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219283456,"num_objects":3346,"num_object_clones":0,"num_object_copies":10038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1633,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":209},{"shards":"2,4,5","objects":2965}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":22,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T10:25:29.510975+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:25:29.510975+0000","last_peered":"2023-10-09T10:25:29.510975+0000","last_clean":"2023-10-09T10:25:29.510975+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:25:29.510975+0000","last_undegraded":"2023-10-09T10:25:29.510975+0000","last_fullsized":"2023-10-09T10:25:29.510975+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":577,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.589427+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:25:35.589427+0000","last_peered":"2023-10-09T10:25:35.589427+0000","last_clean":"2023-10-09T10:25:35.589427+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:25:35.589427+0000","last_undegraded":"2023-10-09T10:25:35.589427+0000","last_fullsized":"2023-10-09T10:25:35.589427+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"523'4559","reported_seq":4609,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T10:25:34.698554+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:25:34.698554+0000","last_peered":"2023-10-09T10:25:34.698554+0000","last_clean":"2023-10-09T10:25:34.698554+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:25:34.698554+0000","last_undegraded":"2023-10-09T10:25:34.698554+0000","last_fullsized":"2023-10-09T10:25:34.698554+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":4559,"log_dups_size":0,"ondisk_log_size":4559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221970432,"num_objects":3387,"num_object_clones":0,"num_object_copies":10161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1298,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":41,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584872+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:25:35.584872+0000","last_peered":"2023-10-09T10:25:35.584872+0000","last_clean":"2023-10-09T10:25:35.584872+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:25:35.584872+0000","last_undegraded":"2023-10-09T10:25:35.584872+0000","last_fullsized":"2023-10-09T10:25:35.584872+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":145,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602036+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:25:35.602036+0000","last_peered":"2023-10-09T10:25:35.602036+0000","last_clean":"2023-10-09T10:25:35.602036+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:25:35.602036+0000","last_undegraded":"2023-10-09T10:25:35.602036+0000","last_fullsized":"2023-10-09T10:25:35.602036+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":7124516888,"num_objects":108714,"num_object_clones":0,"num_object_copies":326142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":60000,"num_objects_misplaced":23194,"num_objects_unfound":0,"num_objects_dirty":108714,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":102232,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5472,"num_bytes_recovered":358612992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21234065408,"data_stored":21234057264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124012,"ondisk_log_size":124012,"up":78,"acting":62,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430671,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626899,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2300820,"kb_used_data":1389260,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":91444332,"statfs":{"total":95995035648,"available":93638995968,"internally_reserved":0,"allocated":1422602240,"data_stored":1421714228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:15 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.3999999999999999,"15min":1.6739999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":4.8099999999999996,"5min":9.4260000000000002,"15min":44.055},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.756,"15min":1.919},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.22},"max":{"1min":3.6760000000000002,"5min":9.2609999999999992,"15min":43.869999999999997},"last":0.29599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":3.0289999999999999,"15min":2.4340000000000002},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":5.7919999999999998,"5min":155.01400000000001,"15min":155.01400000000001},"last":5.0419999999999998},{"interface":"front","average":{"1min":0.86199999999999999,"5min":3.165,"15min":2.5760000000000001},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":3.5310000000000001,"5min":154.71100000000001,"15min":154.71100000000001},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:25:15 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":1.7889999999999999,"15min":1.887},"min":{"1min":0.32800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.8460000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":1.4319999999999999},{"interface":"front","average":{"1min":1.6970000000000001,"5min":1.649,"15min":1.8029999999999999},"min":{"1min":0.36299999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.875,"5min":7.5629999999999997,"15min":14.542999999999999},"last":0.46100000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.619,"15min":1.514},"min":{"1min":0.498,"5min":0.308,"15min":0.308},"max":{"1min":7.681,"5min":7.681,"15min":8.0229999999999997},"last":0.434},{"interface":"front","average":{"1min":1.496,"5min":1.927,"15min":1.6220000000000001},"min":{"1min":0.36399999999999999,"5min":0.316,"15min":0.31},"max":{"1min":7.6920000000000002,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.443}]},{"osd":4,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.881,"5min":3.4460000000000002,"15min":1.8600000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":10.618,"5min":122.041,"15min":122.041},"last":1.395},{"interface":"front","average":{"1min":1.8069999999999999,"5min":3.3130000000000002,"15min":1.75},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.9470000000000001,"5min":122.13200000000001,"15min":122.13200000000001},"last":4.4379999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.4620000000000002,"15min":1.976},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.6890000000000001,"5min":121.98999999999999,"15min":121.98999999999999},"last":4.9989999999999997},{"interface":"front","average":{"1min":0.95299999999999996,"5min":2.6280000000000001,"15min":1.6200000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":5.1420000000000003,"5min":122.346,"15min":122.346},"last":0.36499999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":3.4710000000000001,"15min":1.9219999999999999},"min":{"1min":0.34899999999999998,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":1.8640000000000001,"5min":130.66999999999999,"15min":130.66999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.72299999999999998,"5min":3.5139999999999998,"15min":1.9650000000000001},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.34100000000000003},"max":{"1min":1.915,"5min":130.702,"15min":130.702},"last":0.47399999999999998}]}]},{"osd":1,"up_from":0,"seq":360777253213,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416737,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4860968,"kb_used_data":3957216,"kb_used_omap":28,"kb_used_meta":903715,"kb_avail":88884184,"statfs":{"total":95995035648,"available":91017404416,"internally_reserved":0,"allocated":4052189184,"data_stored":4050174655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":925405008},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,1,2,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:24:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.28,"15min":1.996},"min":{"1min":0.248,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.835,"5min":8.8699999999999992,"15min":54.518000000000001},"last":0.622},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.3360000000000001,"15min":2.6230000000000002},"min":{"1min":0.23899999999999999,"5min":0.192,"15min":0.192},"max":{"1min":3.5499999999999998,"5min":17.033000000000001,"15min":56.950000000000003},"last":2.552}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.9630000000000001,"15min":2.1419999999999999},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.7839999999999998,"5min":47.215000000000003,"15min":53.988999999999997},"last":6.6399999999999997},{"interface":"front","average":{"1min":1.99,"5min":2.254,"15min":2.1339999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.173,"5min":47.000999999999998,"15min":54.119},"last":1.3759999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.488,"15min":1.4119999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.702,"5min":16.879999999999999,"15min":56.984999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.5750000000000002,"5min":1.6830000000000001,"15min":1.4450000000000001},"min":{"1min":0.51700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1749999999999998,"5min":8.0020000000000007,"15min":56.475999999999999},"last":4.0919999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.869,"15min":2.3159999999999998},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":3.3399999999999999,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.1230000000000002,"15min":2.246},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.32400000000000001},"max":{"1min":8.2479999999999993,"5min":28.331,"15min":56.378},"last":0.90100000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.1259999999999999,"15min":3.129},"min":{"1min":0.46800000000000003,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":6.923,"5min":11.775,"15min":229.589},"last":4.0220000000000002},{"interface":"front","average":{"1min":2.238,"5min":1.8029999999999999,"15min":2.8279999999999998},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":7.032,"5min":8.7530000000000001,"15min":228.88900000000001},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.919,"15min":2.2320000000000002},"min":{"1min":0.435,"5min":0.379,"15min":0.33700000000000002},"max":{"1min":6.8230000000000004,"5min":8.7810000000000006,"15min":54.482999999999997},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.248,"5min":2.0859999999999999,"15min":2.6349999999999998},"min":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.7160000000000002,"5min":16.954999999999998,"15min":57.753},"last":0.53300000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.9930000000000001,"15min":2.391},"min":{"1min":0.54500000000000004,"5min":0.38600000000000001,"15min":0.34999999999999998},"max":{"1min":6.6079999999999997,"5min":8.0329999999999995,"15min":56.411999999999999},"last":4.0599999999999996},{"interface":"front","average":{"1min":2.0070000000000001,"5min":2.0640000000000001,"15min":2.258},"min":{"1min":0.47099999999999997,"5min":0.38600000000000001,"15min":0.36499999999999999},"max":{"1min":6.891,"5min":11.175000000000001,"15min":54.451999999999998},"last":1.0229999999999999}]}]},{"osd":2,"up_from":217,"seq":932007903473,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4895400,"kb_used_data":4072800,"kb_used_omap":83,"kb_used_meta":822316,"kb_avail":88849752,"statfs":{"total":95995035648,"available":90982146048,"internally_reserved":0,"allocated":4170547200,"data_stored":4168717465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":842052079},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.778,"15min":2.0600000000000001},"min":{"1min":0.216,"5min":0.20899999999999999,"15min":0.185},"max":{"1min":5.2480000000000002,"5min":48.990000000000002,"15min":48.990000000000002},"last":4.2240000000000002},{"interface":"front","average":{"1min":1.5089999999999999,"5min":2.0649999999999999,"15min":2.3849999999999998},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.590999999999999,"5min":49.412999999999997,"15min":49.412999999999997},"last":1.3089999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.3089999999999999,"15min":2.0899999999999999},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":5.008,"5min":8.2010000000000005,"15min":34.893999999999998},"last":4.1769999999999996},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.4910000000000001,"15min":2.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":7.8029999999999999,"5min":7.8029999999999999,"15min":34.962000000000003},"last":1.9419999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.4139999999999999,"15min":1.155},"min":{"1min":0.25700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":3.9860000000000002,"5min":7.54,"15min":35.043999999999997},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.90900000000000003,"15min":1.1819999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.1269999999999998,"5min":6.617,"15min":50.406999999999996},"last":1.4850000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:25:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.772,"15min":1.6990000000000001},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.26600000000000001},"max":{"1min":5.6749999999999998,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.8859999999999999,"15min":1.845},"min":{"1min":0.40699999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.2720000000000002,"5min":47.704000000000001,"15min":47.704000000000001},"last":6.5469999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.603,"15min":1.992},"min":{"1min":0.39100000000000001,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":3.0329999999999999,"5min":10.448,"15min":57.298000000000002},"last":1.339},{"interface":"front","average":{"1min":1.534,"5min":1.7050000000000001,"15min":2.0310000000000001},"min":{"1min":0.374,"5min":0.31,"15min":0.31},"max":{"1min":5.8810000000000002,"5min":8.1630000000000003,"15min":57.828000000000003},"last":4.2039999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.99,"5min":2.2360000000000002,"15min":2.0329999999999999},"min":{"1min":0.45000000000000001,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":5.5510000000000002,"5min":12.411,"15min":18.943999999999999},"last":3.4830000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.744,"15min":1.95},"min":{"1min":0.442,"5min":0.38300000000000001,"15min":0.36599999999999999},"max":{"1min":3.0950000000000002,"5min":9.8399999999999999,"15min":24.379000000000001},"last":1.2669999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.3370000000000002,"15min":2.2850000000000001},"min":{"1min":0.53500000000000003,"5min":0.41799999999999998,"15min":0.308},"max":{"1min":3.5760000000000001,"5min":13.821,"15min":35.008000000000003},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.655,"5min":1.6619999999999999,"15min":2.1160000000000001},"min":{"1min":0.439,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":6.2140000000000004,"5min":10.507,"15min":24.323},"last":2.452}]}]},{"osd":3,"up_from":352,"seq":1511828488312,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3567232,"kb_used_data":3172536,"kb_used_omap":15,"kb_used_meta":394544,"kb_avail":90177920,"statfs":{"total":95995035648,"available":92342190080,"internally_reserved":0,"allocated":3248676864,"data_stored":3246396872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":404013510},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.95,"15min":1.3160000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":3.2309999999999999,"5min":65.674999999999997,"15min":65.674999999999997},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.97,"15min":1.319},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.19900000000000001},"max":{"1min":4.758,"5min":21.149999999999999,"15min":21.149999999999999},"last":0.22600000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":3.1309999999999998,"15min":1.8069999999999999},"min":{"1min":0.246,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.3760000000000003,"5min":105.401,"15min":105.401},"last":0.375},{"interface":"front","average":{"1min":1.7889999999999999,"5min":3.4079999999999999,"15min":1.6619999999999999},"min":{"1min":0.311,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":6.7050000000000001,"5min":104.914,"15min":104.914},"last":0.30299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.954,"15min":1.883},"min":{"1min":0.253,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":2.2650000000000001,"5min":9.4369999999999994,"15min":9.4369999999999994},"last":1.3819999999999999},{"interface":"front","average":{"1min":1.248,"5min":1.508,"15min":1.3},"min":{"1min":0.313,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":4.4930000000000003,"5min":25.338999999999999,"15min":25.338999999999999},"last":0.55000000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":2.4220000000000002,"15min":1.319},"min":{"1min":0.39100000000000001,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":4.7770000000000001,"5min":21.831,"15min":21.831},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.105,"5min":2.8759999999999999,"15min":1.8169999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.4510000000000001,"5min":22.097999999999999,"15min":22.097999999999999},"last":7.952}]},{"osd":5,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":1.8819999999999999,"15min":1.1919999999999999},"min":{"1min":0.44,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":6.6710000000000003,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.6779999999999999,"5min":2.4039999999999999,"15min":1.4259999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":4.6600000000000001,"5min":13.394,"15min":13.394},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":2.1749999999999998,"15min":1.631},"min":{"1min":0.36199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.617,"5min":19.114000000000001,"15min":19.114000000000001},"last":3.7509999999999999},{"interface":"front","average":{"1min":1.538,"5min":1.958,"15min":1.373},"min":{"1min":0.47099999999999997,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":5.2519999999999998,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":2.2120000000000002,"15min":1.4239999999999999},"min":{"1min":0.40699999999999997,"5min":0.39900000000000002,"15min":0.38400000000000001},"max":{"1min":4.8109999999999999,"5min":9.7880000000000003,"15min":10.459},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.097,"5min":2.1829999999999998,"15min":1.6970000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.3860000000000001,"5min":19.280000000000001,"15min":19.280000000000001},"last":1.4750000000000001}]}]},{"osd":4,"up_from":266,"seq":1142461300942,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4832884,"kb_used_data":3903852,"kb_used_omap":35,"kb_used_meta":928988,"kb_avail":88912268,"statfs":{"total":95995035648,"available":91046162432,"internally_reserved":0,"allocated":3997544448,"data_stored":3996639552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":951283950},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":3.286,"15min":2.1200000000000001},"min":{"1min":0.23699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.4699999999999998,"5min":97.721999999999994,"15min":97.721999999999994},"last":1.409},{"interface":"front","average":{"1min":1.8540000000000001,"5min":4.1630000000000003,"15min":2.8100000000000001},"min":{"1min":0.30199999999999999,"5min":0.252,"15min":0.224},"max":{"1min":5.931,"5min":97.915999999999997,"15min":97.915999999999997},"last":3.4409999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":2.5379999999999998,"15min":2.3889999999999998},"min":{"1min":0.41599999999999998,"5min":0.32300000000000001,"15min":0.24299999999999999},"max":{"1min":5.6630000000000003,"5min":11.32,"15min":71.457999999999998},"last":0.434},{"interface":"front","average":{"1min":1.754,"5min":2.714,"15min":2.2029999999999998},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":5.8730000000000002,"5min":49.781999999999996,"15min":71.658000000000001},"last":2.4900000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":2.3279999999999998,"15min":1.8360000000000001},"min":{"1min":0.313,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":6.468,"5min":15.944000000000001,"15min":15.944000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.875,"15min":1.7450000000000001},"min":{"1min":0.28699999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.8420000000000001,"5min":14.473000000000001,"15min":14.473000000000001},"last":3.5680000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":2.5779999999999998,"15min":1.887},"min":{"1min":0.45100000000000001,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":5.7060000000000004,"5min":12.074,"15min":12.074},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.444,"5min":2.746,"15min":1.6220000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4240000000000004,"5min":16.067,"15min":16.067},"last":4.1900000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.7719999999999998,"15min":2.0579999999999998},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":10.023999999999999,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.5819999999999999,"15min":1.7889999999999999},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.335,"5min":49.915999999999997,"15min":99.384},"last":1.321}]},{"osd":6,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.9530000000000001,"15min":1.3280000000000001},"min":{"1min":0.371,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.7709999999999999,"5min":14.499000000000001,"15min":14.499000000000001},"last":1.373},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.9279999999999999,"15min":1.734},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":18.952999999999999,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.59399999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":2.9159999999999999,"15min":1.8049999999999999},"min":{"1min":0.33000000000000002,"5min":0.315,"15min":0.307},"max":{"1min":9.3789999999999996,"5min":17.369,"15min":17.369},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.554,"5min":2.9129999999999998,"15min":1.7430000000000001},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":5.9020000000000001,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.626}]}]},{"osd":5,"up_from":36,"seq":154618823126,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5517004,"kb_used_data":4319108,"kb_used_omap":0,"kb_used_meta":1197888,"kb_avail":88228148,"statfs":{"total":95995035648,"available":90345623552,"internally_reserved":0,"allocated":4422766592,"data_stored":4421880960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1226637312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.468,"15min":1.9159999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":7.6390000000000002,"5min":22.469999999999999,"15min":22.469999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.365,"5min":2.0390000000000001,"15min":1.6950000000000001},"min":{"1min":0.248,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":5.7119999999999997,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.63700000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":2.6429999999999998,"15min":2.1480000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":4.306,"5min":29.370000000000001,"15min":29.370000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.1879999999999999,"5min":2.2229999999999999,"15min":1.865},"min":{"1min":0.28499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.4790000000000001,"5min":29.065000000000001,"15min":29.065000000000001},"last":0.54900000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6899999999999999,"5min":2.5680000000000001,"15min":2.1269999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":16.629999999999999,"5min":16.629999999999999,"15min":16.629999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.403,"5min":2.2890000000000001,"15min":1.7270000000000001},"min":{"1min":0.30499999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.8930000000000007,"5min":13.095000000000001,"15min":15.929},"last":3.1389999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":2.1899999999999999,"15min":1.7430000000000001},"min":{"1min":0.33300000000000002,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":4.3220000000000001,"5min":10.561999999999999,"15min":10.561999999999999},"last":2.8380000000000001},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.976,"15min":1.702},"min":{"1min":0.376,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":4.5019999999999998,"5min":13.624000000000001,"15min":13.624000000000001},"last":2.8330000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":2.1920000000000002,"15min":1.524},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.24299999999999999},"max":{"1min":4.1159999999999997,"5min":12.023,"15min":12.023},"last":2.3439999999999999},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.081,"15min":1.341},"min":{"1min":0.29899999999999999,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":2.7170000000000001,"5min":11.956,"15min":11.956},"last":2.2120000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":2.4249999999999998,"15min":1.6890000000000001},"min":{"1min":0.309,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":5.9080000000000004,"5min":11.686,"15min":12.772},"last":2.3780000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":2.0750000000000002,"15min":1.6100000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":2.7330000000000001,"5min":10.403,"15min":15.997},"last":0.69599999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:25:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.6429999999999998,"15min":1.798},"min":{"1min":0.34000000000000002,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":16.686,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.397,"15min":1.1719999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":8.9719999999999995,"5min":11.994,"15min":11.994},"last":0.59799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4033937408,"data_stored":4033937408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4149149696,"data_stored":4149149696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3230466048,"data_stored":3230466048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3986362368,"data_stored":3986358296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4417130496,"data_stored":4417126424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1417019392,"data_stored":1417019392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:25:43.279 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:25:43.420 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:43 smithi012 ceph-mon[147926]: pgmap v1464: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 887 KiB/s wr, 562 op/s; 59148/322452 objects degraded (18.343%); 22919/322452 objects misplaced (7.108%); 11 MiB/s, 180 objects/s recovering 2023-10-09T10:25:43.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:43 smithi012 ceph-mon[152263]: pgmap v1464: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 887 KiB/s wr, 562 op/s; 59148/322452 objects degraded (18.343%); 22919/322452 objects misplaced (7.108%); 11 MiB/s, 180 objects/s recovering 2023-10-09T10:25:43.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:43 smithi110 ceph-mon[136032]: pgmap v1464: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.6 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 887 KiB/s wr, 562 op/s; 59148/322452 objects degraded (18.343%); 22919/322452 objects misplaced (7.108%); 11 MiB/s, 180 objects/s recovering 2023-10-09T10:25:43.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:25:43.546 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:43.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:43.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:43.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:43.738 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:43.831 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:43.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:43.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:43.938 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:44.062 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:44.145 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:44.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:44.232 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:25:44.232 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:25:44.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:44.264 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:44.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:44 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 59148/322452 objects degraded (18.343%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:44.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:44.338 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:44 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 59148/322452 objects degraded (18.343%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:44.338 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:44.363 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:44.649 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:44.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:44.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:44.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:44 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 59148/322452 objects degraded (18.343%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:44.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:44.734 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:25:44.763 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:44.901 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:44.968 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:44.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:44.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:45.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:45.172 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:45.174 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:45.254 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:45.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:45.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:45.284 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:25:45.364 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:45.416 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:45.474 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:45.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:45.484 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:45 smithi110 ceph-mon[136032]: pgmap v1465: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 58524/317496 objects degraded (18.433%); 22302/317496 objects misplaced (7.024%); 12 MiB/s, 198 objects/s recovering 2023-10-09T10:25:45.485 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2170982332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:45.485 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:45 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:45.583 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:45.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:45 smithi012 ceph-mon[147926]: pgmap v1465: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 58524/317496 objects degraded (18.433%); 22302/317496 objects misplaced (7.024%); 12 MiB/s, 198 objects/s recovering 2023-10-09T10:25:45.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2170982332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:45.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:45 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:45.607 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:45 smithi012 ceph-mon[152263]: pgmap v1465: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 58524/317496 objects degraded (18.433%); 22302/317496 objects misplaced (7.024%); 12 MiB/s, 198 objects/s recovering 2023-10-09T10:25:45.607 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2170982332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:45.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:45 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:25:45.654 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:45.713 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:45.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:45.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:45.879 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:45.958 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:45.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:46.064 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:46.114 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:46.356 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:46.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:46.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:46.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:46 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:25:46.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:46 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:46.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:46 smithi012 ceph-mon[147926]: osdmap e526: 8 total, 8 up, 6 in 2023-10-09T10:25:46.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:46 smithi012 ceph-mon[147926]: failed to encode map e526 with expected crc 2023-10-09T10:25:46.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:46 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:25:46.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:46 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:46.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:46 smithi012 ceph-mon[152263]: osdmap e526: 8 total, 8 up, 6 in 2023-10-09T10:25:46.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:46 smithi012 ceph-mon[152263]: failed to encode map e526 with expected crc 2023-10-09T10:25:46.465 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:46.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:46 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:25:46.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:46 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:25:46.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:46 smithi110 ceph-mon[136032]: osdmap e526: 8 total, 8 up, 6 in 2023-10-09T10:25:46.543 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:46 smithi110 ceph-mon[136032]: failed to encode map e526 with expected crc 2023-10-09T10:25:46.549 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:46.608 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:46.613 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:46.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:46.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:46.720 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:46.810 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:46.966 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:25:46.988 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:25:47.003 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:47.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:47.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:47.109 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:47.170 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:47.229 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-mon[152263]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.229 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-mon[152263]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.229 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-mon[152263]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.230 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-mon[152263]: pgmap v1467: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 670 op/s; 57637/312951 objects degraded (18.417%); 22013/312951 objects misplaced (7.034%); 14 MiB/s, 230 objects/s recovering 2023-10-09T10:25:47.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-mon[147926]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-mon[147926]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-mon[147926]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-mon[147926]: pgmap v1467: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 670 op/s; 57637/312951 objects degraded (18.417%); 22013/312951 objects misplaced (7.034%); 14 MiB/s, 230 objects/s recovering 2023-10-09T10:25:47.381 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:47.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:47.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:47.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:47.554 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:47.616 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:47.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:47.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:47.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:47 smithi110 ceph-mon[136032]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:47 smithi110 ceph-mon[136032]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:47 smithi110 ceph-mon[136032]: failed to encode map e526 with expected crc 2023-10-09T10:25:47.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:47 smithi110 ceph-mon[136032]: pgmap v1467: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 670 op/s; 57637/312951 objects degraded (18.417%); 22013/312951 objects misplaced (7.034%); 14 MiB/s, 230 objects/s recovering 2023-10-09T10:25:47.698 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1467,"stamp":"2023-10-09T10:25:45.785933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6835378104,"num_objects":104317,"num_object_clones":0,"num_object_copies":312951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":57637,"num_objects_misplaced":22013,"num_objects_unfound":0,"num_objects_dirty":104317,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":106913,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7211,"num_bytes_recovered":464618716,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127327,"ondisk_log_size":127327,"up":297,"acting":280,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":336,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":25712692,"kb_used_data":20206212,"kb_used_omap":163,"kb_used_meta":5506204,"kb_avail":536758220,"statfs":{"total":575970213888,"available":549640417280,"internally_reserved":0,"allocated":20691161088,"data_stored":20682327738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":5638353907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-403636224,"num_objects":-6159,"num_object_clones":0,"num_object_copies":-18477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2914,"num_objects_misplaced":-1716,"num_objects_unfound":0,"num_objects_dirty":-6159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6159,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2119,"num_bytes_recovered":138870784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.191095"},"pg_stats":[{"pgid":"11.12","version":"525'4782","reported_seq":4840,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:42.619450+0000","last_change":"2023-10-09T10:25:09.709209+0000","last_active":"2023-10-09T10:25:42.619450+0000","last_peered":"2023-10-09T10:25:42.619450+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:25:42.619450+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218202112,"num_objects":3330,"num_object_clones":0,"num_object_copies":9990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3119,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3834,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":211},{"shards":"0,2,5","objects":2867}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":118,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602429+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:25:35.602429+0000","last_peered":"2023-10-09T10:25:35.602429+0000","last_clean":"2023-10-09T10:25:35.602429+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:25:35.602429+0000","last_undegraded":"2023-10-09T10:25:35.602429+0000","last_fullsized":"2023-10-09T10:25:35.602429+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":479,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605281+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:25:35.605281+0000","last_peered":"2023-10-09T10:25:35.605281+0000","last_clean":"2023-10-09T10:25:35.605281+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:25:35.605281+0000","last_undegraded":"2023-10-09T10:25:35.605281+0000","last_fullsized":"2023-10-09T10:25:35.605281+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"525'4743","reported_seq":5611,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:42.661040+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:25:42.661040+0000","last_peered":"2023-10-09T10:25:42.661040+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:25:42.661040+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4743,"log_dups_size":0,"ondisk_log_size":4743,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213909504,"num_objects":3264,"num_object_clones":0,"num_object_copies":9792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4839,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3798,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":267,"num_bytes_recovered":17498112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1997},{"shards":"0,2,6","objects":845},{"shards":"0,6","objects":422}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":556,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602073+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:25:35.602073+0000","last_peered":"2023-10-09T10:25:35.602073+0000","last_clean":"2023-10-09T10:25:35.602073+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:25:35.602073+0000","last_undegraded":"2023-10-09T10:25:35.602073+0000","last_fullsized":"2023-10-09T10:25:35.602073+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":449,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587774+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:25:35.587774+0000","last_peered":"2023-10-09T10:25:35.587774+0000","last_clean":"2023-10-09T10:25:35.587774+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:25:35.587774+0000","last_undegraded":"2023-10-09T10:25:35.587774+0000","last_fullsized":"2023-10-09T10:25:35.587774+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"525'4668","reported_seq":8117,"reported_epoch":525,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:25:42.784697+0000","last_change":"2023-10-09T10:25:17.878455+0000","last_active":"2023-10-09T10:25:42.784697+0000","last_peered":"2023-10-09T10:25:42.784697+0000","last_clean":"2023-10-09T10:24:39.547747+0000","last_became_active":"2023-10-09T10:24:41.974279+0000","last_became_peered":"2023-10-09T10:24:41.974279+0000","last_unstale":"2023-10-09T10:25:42.784697+0000","last_undegraded":"2023-10-09T10:25:42.784697+0000","last_fullsized":"2023-10-09T10:24:41.670066+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4668,"log_dups_size":0,"ondisk_log_size":4668,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213024768,"num_objects":3251,"num_object_clones":0,"num_object_copies":9753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":538,"num_objects_unfound":0,"num_objects_dirty":3251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3731,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1695,"num_bytes_recovered":111083520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":538},{"shards":"0,3","objects":2713}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583268+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:25:35.583268+0000","last_peered":"2023-10-09T10:25:35.583268+0000","last_clean":"2023-10-09T10:25:35.583268+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:25:35.583268+0000","last_undegraded":"2023-10-09T10:25:35.583268+0000","last_fullsized":"2023-10-09T10:25:35.583268+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.585008+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:25:35.585008+0000","last_peered":"2023-10-09T10:25:35.585008+0000","last_clean":"2023-10-09T10:25:35.585008+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:25:35.585008+0000","last_undegraded":"2023-10-09T10:25:35.585008+0000","last_fullsized":"2023-10-09T10:25:35.585008+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":483,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.585436+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:25:35.585436+0000","last_peered":"2023-10-09T10:25:35.585436+0000","last_clean":"2023-10-09T10:25:35.585436+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:25:35.585436+0000","last_undegraded":"2023-10-09T10:25:35.585436+0000","last_fullsized":"2023-10-09T10:25:35.585436+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":433,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:36.585907+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:25:36.585907+0000","last_peered":"2023-10-09T10:25:36.585907+0000","last_clean":"2023-10-09T10:25:36.585907+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:25:36.585907+0000","last_undegraded":"2023-10-09T10:25:36.585907+0000","last_fullsized":"2023-10-09T10:25:36.585907+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":200,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.579412+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:25:35.579412+0000","last_peered":"2023-10-09T10:25:35.579412+0000","last_clean":"2023-10-09T10:25:35.579412+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:25:35.579412+0000","last_undegraded":"2023-10-09T10:25:35.579412+0000","last_fullsized":"2023-10-09T10:25:35.579412+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":433,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:36.589227+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:25:36.589227+0000","last_peered":"2023-10-09T10:25:36.589227+0000","last_clean":"2023-10-09T10:25:36.589227+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:25:36.589227+0000","last_undegraded":"2023-10-09T10:25:36.589227+0000","last_fullsized":"2023-10-09T10:25:36.589227+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":130,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605423+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:25:35.605423+0000","last_peered":"2023-10-09T10:25:35.605423+0000","last_clean":"2023-10-09T10:25:35.605423+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:25:35.605423+0000","last_undegraded":"2023-10-09T10:25:35.605423+0000","last_fullsized":"2023-10-09T10:25:35.605423+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587179+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:25:35.587179+0000","last_peered":"2023-10-09T10:25:35.587179+0000","last_clean":"2023-10-09T10:25:35.587179+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:25:35.587179+0000","last_undegraded":"2023-10-09T10:25:35.587179+0000","last_fullsized":"2023-10-09T10:25:35.587179+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":176,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602706+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:25:35.602706+0000","last_peered":"2023-10-09T10:25:35.602706+0000","last_clean":"2023-10-09T10:25:35.602706+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:25:35.602706+0000","last_undegraded":"2023-10-09T10:25:35.602706+0000","last_fullsized":"2023-10-09T10:25:35.602706+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602608+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:25:35.602608+0000","last_peered":"2023-10-09T10:25:35.602608+0000","last_clean":"2023-10-09T10:25:35.602608+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:25:35.602608+0000","last_undegraded":"2023-10-09T10:25:35.602608+0000","last_fullsized":"2023-10-09T10:25:35.602608+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":96,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602530+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:25:35.602530+0000","last_peered":"2023-10-09T10:25:35.602530+0000","last_clean":"2023-10-09T10:25:35.602530+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:25:35.602530+0000","last_undegraded":"2023-10-09T10:25:35.602530+0000","last_fullsized":"2023-10-09T10:25:35.602530+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":50,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.579991+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:25:35.579991+0000","last_peered":"2023-10-09T10:25:35.579991+0000","last_clean":"2023-10-09T10:25:35.579991+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:25:35.579991+0000","last_undegraded":"2023-10-09T10:25:35.579991+0000","last_fullsized":"2023-10-09T10:25:35.579991+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":447,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591435+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:25:35.591435+0000","last_peered":"2023-10-09T10:25:35.591435+0000","last_clean":"2023-10-09T10:25:35.591435+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:25:35.591435+0000","last_undegraded":"2023-10-09T10:25:35.591435+0000","last_fullsized":"2023-10-09T10:25:35.591435+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":191,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.585270+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:25:35.585270+0000","last_peered":"2023-10-09T10:25:35.585270+0000","last_clean":"2023-10-09T10:25:35.585270+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:25:35.585270+0000","last_undegraded":"2023-10-09T10:25:35.585270+0000","last_fullsized":"2023-10-09T10:25:35.585270+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"525'4766","reported_seq":5558,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.880007+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:25:44.880007+0000","last_peered":"2023-10-09T10:25:44.880007+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:25:44.880007+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4766,"log_dups_size":0,"ondisk_log_size":4766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213811200,"num_objects":3263,"num_object_clones":0,"num_object_copies":9789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":3263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3843,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":112},{"shards":"3,5,6","objects":914}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602189+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:25:35.602189+0000","last_peered":"2023-10-09T10:25:35.602189+0000","last_clean":"2023-10-09T10:25:35.602189+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:25:35.602189+0000","last_undegraded":"2023-10-09T10:25:35.602189+0000","last_fullsized":"2023-10-09T10:25:35.602189+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587450+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:25:35.587450+0000","last_peered":"2023-10-09T10:25:35.587450+0000","last_clean":"2023-10-09T10:25:35.587450+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:25:35.587450+0000","last_undegraded":"2023-10-09T10:25:35.587450+0000","last_fullsized":"2023-10-09T10:25:35.587450+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"525'4800","reported_seq":5726,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.915928+0000","last_change":"2023-10-09T10:25:09.702978+0000","last_active":"2023-10-09T10:25:44.915928+0000","last_peered":"2023-10-09T10:25:44.915928+0000","last_clean":"2023-10-09T10:24:39.576387+0000","last_became_active":"2023-10-09T10:25:09.696470+0000","last_became_peered":"2023-10-09T10:25:09.696470+0000","last_unstale":"2023-10-09T10:25:44.915928+0000","last_undegraded":"2023-10-09T10:25:09.546049+0000","last_fullsized":"2023-10-09T10:25:09.436538+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4800,"log_dups_size":0,"ondisk_log_size":4800,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213680128,"num_objects":3261,"num_object_clones":0,"num_object_copies":9783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4776,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3835,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":18808832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1991},{"shards":"2,5,6","objects":794},{"shards":"5,6","objects":476}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":30,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591447+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:25:35.591447+0000","last_peered":"2023-10-09T10:25:35.591447+0000","last_clean":"2023-10-09T10:25:35.591447+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:25:35.591447+0000","last_undegraded":"2023-10-09T10:25:35.591447+0000","last_fullsized":"2023-10-09T10:25:35.591447+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:36.586336+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:25:36.586336+0000","last_peered":"2023-10-09T10:25:36.586336+0000","last_clean":"2023-10-09T10:25:36.586336+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:25:36.586336+0000","last_undegraded":"2023-10-09T10:25:36.586336+0000","last_fullsized":"2023-10-09T10:25:36.586336+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"525'4854","reported_seq":4907,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:44.940764+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:25:44.940764+0000","last_peered":"2023-10-09T10:25:44.940764+0000","last_clean":"2023-10-09T10:25:44.940764+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:25:44.940764+0000","last_undegraded":"2023-10-09T10:25:44.940764+0000","last_fullsized":"2023-10-09T10:25:44.940764+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":4854,"log_dups_size":0,"ondisk_log_size":4854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216399872,"num_objects":3302,"num_object_clones":0,"num_object_copies":9906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1566,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":26,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605288+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:25:35.605288+0000","last_peered":"2023-10-09T10:25:35.605288+0000","last_clean":"2023-10-09T10:25:35.605288+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:25:35.605288+0000","last_undegraded":"2023-10-09T10:25:35.605288+0000","last_fullsized":"2023-10-09T10:25:35.605288+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":200,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583902+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:25:35.583902+0000","last_peered":"2023-10-09T10:25:35.583902+0000","last_clean":"2023-10-09T10:25:35.583902+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:25:35.583902+0000","last_undegraded":"2023-10-09T10:25:35.583902+0000","last_fullsized":"2023-10-09T10:25:35.583902+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"525'4585","reported_seq":4641,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.344949+0000","last_change":"2023-10-09T10:25:13.752880+0000","last_active":"2023-10-09T10:25:44.344949+0000","last_peered":"2023-10-09T10:25:44.344949+0000","last_clean":"2023-10-09T10:25:11.561932+0000","last_became_active":"2023-10-09T10:25:13.746279+0000","last_became_peered":"2023-10-09T10:25:13.746279+0000","last_unstale":"2023-10-09T10:25:44.344949+0000","last_undegraded":"2023-10-09T10:25:13.623380+0000","last_fullsized":"2023-10-09T10:25:13.604555+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":4585,"log_dups_size":0,"ondisk_log_size":4585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:39.267545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201588736,"num_objects":3076,"num_object_clones":0,"num_object_copies":9228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3012,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1825,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":64},{"shards":"2,4,5","objects":2733}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":152,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605270+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:25:35.605270+0000","last_peered":"2023-10-09T10:25:35.605270+0000","last_clean":"2023-10-09T10:25:35.605270+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:25:35.605270+0000","last_undegraded":"2023-10-09T10:25:35.605270+0000","last_fullsized":"2023-10-09T10:25:35.605270+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602235+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:25:35.602235+0000","last_peered":"2023-10-09T10:25:35.602235+0000","last_clean":"2023-10-09T10:25:35.602235+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:25:35.602235+0000","last_undegraded":"2023-10-09T10:25:35.602235+0000","last_fullsized":"2023-10-09T10:25:35.602235+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"525'4766","reported_seq":5701,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.908343+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:25:44.908343+0000","last_peered":"2023-10-09T10:25:44.908343+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:25:44.908343+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4766,"log_dups_size":0,"ondisk_log_size":4766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212566016,"num_objects":3243,"num_object_clones":0,"num_object_copies":9729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3821,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2030},{"shards":"0,5,6","objects":813},{"shards":"5,6","objects":400}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":216,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.580617+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:25:35.580617+0000","last_peered":"2023-10-09T10:25:35.580617+0000","last_clean":"2023-10-09T10:25:35.580617+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:25:35.580617+0000","last_undegraded":"2023-10-09T10:25:35.580617+0000","last_fullsized":"2023-10-09T10:25:35.580617+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.580989+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:25:35.580989+0000","last_peered":"2023-10-09T10:25:35.580989+0000","last_clean":"2023-10-09T10:25:35.580989+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:25:35.580989+0000","last_undegraded":"2023-10-09T10:25:35.580989+0000","last_fullsized":"2023-10-09T10:25:35.580989+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"525'4752","reported_seq":8930,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:44.797596+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:25:44.797596+0000","last_peered":"2023-10-09T10:25:44.797596+0000","last_clean":"2023-10-09T10:25:44.797596+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:25:44.797596+0000","last_undegraded":"2023-10-09T10:25:44.797596+0000","last_fullsized":"2023-10-09T10:25:44.797596+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":4752,"log_dups_size":0,"ondisk_log_size":4752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211353600,"num_objects":3225,"num_object_clones":0,"num_object_copies":9675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1838,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":200,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.579688+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:25:35.579688+0000","last_peered":"2023-10-09T10:25:35.579688+0000","last_clean":"2023-10-09T10:25:35.579688+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:25:35.579688+0000","last_undegraded":"2023-10-09T10:25:35.579688+0000","last_fullsized":"2023-10-09T10:25:35.579688+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":434,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583987+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:25:35.583987+0000","last_peered":"2023-10-09T10:25:35.583987+0000","last_clean":"2023-10-09T10:25:35.583987+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:25:35.583987+0000","last_undegraded":"2023-10-09T10:25:35.583987+0000","last_fullsized":"2023-10-09T10:25:35.583987+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"525'4731","reported_seq":6013,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.946551+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:25:44.946551+0000","last_peered":"2023-10-09T10:25:44.946551+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:25:44.946551+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4731,"log_dups_size":0,"ondisk_log_size":4731,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210272256,"num_objects":3208,"num_object_clones":0,"num_object_copies":9624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3810,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1629},{"shards":"2,5","objects":1579}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":135,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583915+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:25:35.583915+0000","last_peered":"2023-10-09T10:25:35.583915+0000","last_clean":"2023-10-09T10:25:35.583915+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:25:35.583915+0000","last_undegraded":"2023-10-09T10:25:35.583915+0000","last_fullsized":"2023-10-09T10:25:35.583915+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602627+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:25:35.602627+0000","last_peered":"2023-10-09T10:25:35.602627+0000","last_clean":"2023-10-09T10:25:35.602627+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:25:35.602627+0000","last_undegraded":"2023-10-09T10:25:35.602627+0000","last_fullsized":"2023-10-09T10:25:35.602627+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"525'4693","reported_seq":4804,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:42.688618+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:25:42.688618+0000","last_peered":"2023-10-09T10:25:42.688618+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:25:42.688618+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213778432,"num_objects":3262,"num_object_clones":0,"num_object_copies":9786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1805,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":204},{"shards":"0,2,5","objects":2814}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602313+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:25:35.602313+0000","last_peered":"2023-10-09T10:25:35.602313+0000","last_clean":"2023-10-09T10:25:35.602313+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:25:35.602313+0000","last_undegraded":"2023-10-09T10:25:35.602313+0000","last_fullsized":"2023-10-09T10:25:35.602313+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":102,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605289+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:25:35.605289+0000","last_peered":"2023-10-09T10:25:35.605289+0000","last_clean":"2023-10-09T10:25:35.605289+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:25:35.605289+0000","last_undegraded":"2023-10-09T10:25:35.605289+0000","last_fullsized":"2023-10-09T10:25:35.605289+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"525'4711","reported_seq":5480,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:42.788136+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:25:42.788136+0000","last_peered":"2023-10-09T10:25:42.788136+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:25:42.788136+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4711,"log_dups_size":0,"ondisk_log_size":4711,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216137728,"num_objects":3298,"num_object_clones":0,"num_object_copies":9894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4846,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3766,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":234,"num_bytes_recovered":15335424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":2020},{"shards":"0,2,6","objects":806},{"shards":"0,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602313+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:25:35.602313+0000","last_peered":"2023-10-09T10:25:35.602313+0000","last_clean":"2023-10-09T10:25:35.602313+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:25:35.602313+0000","last_undegraded":"2023-10-09T10:25:35.602313+0000","last_fullsized":"2023-10-09T10:25:35.602313+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":145,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605172+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:25:35.605172+0000","last_peered":"2023-10-09T10:25:35.605172+0000","last_clean":"2023-10-09T10:25:35.605172+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:25:35.605172+0000","last_undegraded":"2023-10-09T10:25:35.605172+0000","last_fullsized":"2023-10-09T10:25:35.605172+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"525'4641","reported_seq":5066,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:42.760163+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:25:42.760163+0000","last_peered":"2023-10-09T10:25:42.760163+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:25:42.760163+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":4641,"log_dups_size":0,"ondisk_log_size":4641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214499328,"num_objects":3273,"num_object_clones":0,"num_object_copies":9819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1534,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2019},{"shards":"0,3","objects":1192}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":477,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583827+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:25:35.583827+0000","last_peered":"2023-10-09T10:25:35.583827+0000","last_clean":"2023-10-09T10:25:35.583827+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:25:35.583827+0000","last_undegraded":"2023-10-09T10:25:35.583827+0000","last_fullsized":"2023-10-09T10:25:35.583827+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":539,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605508+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:25:35.605508+0000","last_peered":"2023-10-09T10:25:35.605508+0000","last_clean":"2023-10-09T10:25:35.605508+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:25:35.605508+0000","last_undegraded":"2023-10-09T10:25:35.605508+0000","last_fullsized":"2023-10-09T10:25:35.605508+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"525'7540","reported_seq":7614,"reported_epoch":525,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:25:43.098569+0000","last_change":"2023-10-09T10:25:17.929569+0000","last_active":"2023-10-09T10:25:43.098569+0000","last_peered":"2023-10-09T10:25:43.098569+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:25:43.098569+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"500'1700","ondisk_log_start":"500'1700","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429588504,"num_objects":6556,"num_object_clones":0,"num_object_copies":19668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6553,"num_objects_misplaced":6556,"num_objects_unfound":0,"num_objects_dirty":6556,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7540,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":6556}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":245,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.586501+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:25:35.586501+0000","last_peered":"2023-10-09T10:25:35.586501+0000","last_clean":"2023-10-09T10:25:35.586501+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:25:35.586501+0000","last_undegraded":"2023-10-09T10:25:35.586501+0000","last_fullsized":"2023-10-09T10:25:35.586501+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":209,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.585190+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:25:35.585190+0000","last_peered":"2023-10-09T10:25:35.585190+0000","last_clean":"2023-10-09T10:25:35.585190+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:25:35.585190+0000","last_undegraded":"2023-10-09T10:25:35.585190+0000","last_fullsized":"2023-10-09T10:25:35.585190+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":489,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.581053+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:25:35.581053+0000","last_peered":"2023-10-09T10:25:35.581053+0000","last_clean":"2023-10-09T10:25:35.581053+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:25:35.581053+0000","last_undegraded":"2023-10-09T10:25:35.581053+0000","last_fullsized":"2023-10-09T10:25:35.581053+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"525'7602","reported_seq":7642,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:42.733889+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:25:42.733889+0000","last_peered":"2023-10-09T10:25:42.733889+0000","last_clean":"2023-10-09T10:25:42.733889+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:25:42.733889+0000","last_undegraded":"2023-10-09T10:25:42.733889+0000","last_fullsized":"2023-10-09T10:25:42.733889+0000","mapping_epoch":494,"log_start":"503'2506","ondisk_log_start":"503'2506","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437518336,"num_objects":6676,"num_object_clones":0,"num_object_copies":20028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7602,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":209,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.580922+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:25:35.580922+0000","last_peered":"2023-10-09T10:25:35.580922+0000","last_clean":"2023-10-09T10:25:35.580922+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:25:35.580922+0000","last_undegraded":"2023-10-09T10:25:35.580922+0000","last_fullsized":"2023-10-09T10:25:35.580922+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":127,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602593+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:25:35.602593+0000","last_peered":"2023-10-09T10:25:35.602593+0000","last_clean":"2023-10-09T10:25:35.602593+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:25:35.602593+0000","last_undegraded":"2023-10-09T10:25:35.602593+0000","last_fullsized":"2023-10-09T10:25:35.602593+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":437,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.578271+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:25:35.578271+0000","last_peered":"2023-10-09T10:25:35.578271+0000","last_clean":"2023-10-09T10:25:35.578271+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:25:35.578271+0000","last_undegraded":"2023-10-09T10:25:35.578271+0000","last_fullsized":"2023-10-09T10:25:35.578271+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"525'4767","reported_seq":5723,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.877972+0000","last_change":"2023-10-09T10:25:09.703037+0000","last_active":"2023-10-09T10:25:44.877972+0000","last_peered":"2023-10-09T10:25:44.877972+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:25:09.696864+0000","last_became_peered":"2023-10-09T10:25:09.696864+0000","last_unstale":"2023-10-09T10:25:44.877972+0000","last_undegraded":"2023-10-09T10:25:09.545735+0000","last_fullsized":"2023-10-09T10:25:09.430540+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4767,"log_dups_size":0,"ondisk_log_size":4767,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210206720,"num_objects":3207,"num_object_clones":0,"num_object_copies":9621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4790,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3801,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":19464192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1993},{"shards":"2,5,6","objects":804},{"shards":"5,6","objects":410}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":475,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:36.620454+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:25:36.620454+0000","last_peered":"2023-10-09T10:25:36.620454+0000","last_clean":"2023-10-09T10:25:36.620454+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:25:36.620454+0000","last_undegraded":"2023-10-09T10:25:36.620454+0000","last_fullsized":"2023-10-09T10:25:36.620454+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":92,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591364+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:25:35.591364+0000","last_peered":"2023-10-09T10:25:35.591364+0000","last_clean":"2023-10-09T10:25:35.591364+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:25:35.591364+0000","last_undegraded":"2023-10-09T10:25:35.591364+0000","last_fullsized":"2023-10-09T10:25:35.591364+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":538,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.579998+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:25:35.579998+0000","last_peered":"2023-10-09T10:25:35.579998+0000","last_clean":"2023-10-09T10:25:35.579998+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:25:35.579998+0000","last_undegraded":"2023-10-09T10:25:35.579998+0000","last_fullsized":"2023-10-09T10:25:35.579998+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"525'7655","reported_seq":7715,"reported_epoch":525,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:25:44.921621+0000","last_change":"2023-10-09T10:24:41.858672+0000","last_active":"2023-10-09T10:25:44.921621+0000","last_peered":"2023-10-09T10:25:44.921621+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:25:44.921621+0000","last_undegraded":"2023-10-09T10:25:44.921621+0000","last_fullsized":"2023-10-09T10:25:44.921621+0000","mapping_epoch":507,"log_start":"503'2800","ondisk_log_start":"503'2800","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4855,"log_dups_size":0,"ondisk_log_size":4855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423165952,"num_objects":6457,"num_object_clones":0,"num_object_copies":19371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6457,"num_objects_unfound":0,"num_objects_dirty":6457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7655,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":6457}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":179,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.582827+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:25:35.582827+0000","last_peered":"2023-10-09T10:25:35.582827+0000","last_clean":"2023-10-09T10:25:35.582827+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:25:35.582827+0000","last_undegraded":"2023-10-09T10:25:35.582827+0000","last_fullsized":"2023-10-09T10:25:35.582827+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":724,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:35.576978+0000","last_change":"2023-10-09T10:25:13.642678+0000","last_active":"2023-10-09T10:25:35.576978+0000","last_peered":"2023-10-09T10:25:35.576978+0000","last_clean":"2023-10-09T10:24:37.578053+0000","last_became_active":"2023-10-09T10:25:13.642243+0000","last_became_peered":"2023-10-09T10:25:13.642243+0000","last_unstale":"2023-10-09T10:25:35.576978+0000","last_undegraded":"2023-10-09T10:25:13.604924+0000","last_fullsized":"2023-10-09T10:25:13.604107+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:47.058901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[2,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":227,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.580739+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:25:35.580739+0000","last_peered":"2023-10-09T10:25:35.580739+0000","last_clean":"2023-10-09T10:25:35.580739+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:25:35.580739+0000","last_undegraded":"2023-10-09T10:25:35.580739+0000","last_fullsized":"2023-10-09T10:25:35.580739+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":136,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605360+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:25:35.605360+0000","last_peered":"2023-10-09T10:25:35.605360+0000","last_clean":"2023-10-09T10:25:35.605360+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:25:35.605360+0000","last_undegraded":"2023-10-09T10:25:35.605360+0000","last_fullsized":"2023-10-09T10:25:35.605360+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"525'4820","reported_seq":5666,"reported_epoch":525,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:25:44.942338+0000","last_change":"2023-10-09T10:25:13.759251+0000","last_active":"2023-10-09T10:25:44.942338+0000","last_peered":"2023-10-09T10:25:44.942338+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:25:44.942338+0000","last_undegraded":"2023-10-09T10:25:13.707357+0000","last_fullsized":"2023-10-09T10:25:44.942338+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4820,"log_dups_size":0,"ondisk_log_size":4820,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219054080,"num_objects":3342,"num_object_clones":0,"num_object_copies":10026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2029,"num_objects_misplaced":2980,"num_objects_unfound":0,"num_objects_dirty":3342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3896,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":277,"num_bytes_recovered":18153472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2029},{"shards":"3,5,6","objects":362}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":209,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591523+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:25:35.591523+0000","last_peered":"2023-10-09T10:25:35.591523+0000","last_clean":"2023-10-09T10:25:35.591523+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:25:35.591523+0000","last_undegraded":"2023-10-09T10:25:35.591523+0000","last_fullsized":"2023-10-09T10:25:35.591523+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":469,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.589638+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:25:35.589638+0000","last_peered":"2023-10-09T10:25:35.589638+0000","last_clean":"2023-10-09T10:25:35.589638+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:25:35.589638+0000","last_undegraded":"2023-10-09T10:25:35.589638+0000","last_fullsized":"2023-10-09T10:25:35.589638+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587256+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:25:35.587256+0000","last_peered":"2023-10-09T10:25:35.587256+0000","last_clean":"2023-10-09T10:25:35.587256+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:25:35.587256+0000","last_undegraded":"2023-10-09T10:25:35.587256+0000","last_fullsized":"2023-10-09T10:25:35.587256+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"525'7602","reported_seq":7641,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:43.118700+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:25:43.118700+0000","last_peered":"2023-10-09T10:25:43.118700+0000","last_clean":"2023-10-09T10:25:43.118700+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:25:43.118700+0000","last_undegraded":"2023-10-09T10:25:43.118700+0000","last_fullsized":"2023-10-09T10:25:43.118700+0000","mapping_epoch":494,"log_start":"500'1800","ondisk_log_start":"500'1800","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433192960,"num_objects":6610,"num_object_clones":0,"num_object_copies":19830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7602,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584506+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:25:35.584506+0000","last_peered":"2023-10-09T10:25:35.584506+0000","last_clean":"2023-10-09T10:25:35.584506+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:25:35.584506+0000","last_undegraded":"2023-10-09T10:25:35.584506+0000","last_fullsized":"2023-10-09T10:25:35.584506+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":506,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.579898+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:25:35.579898+0000","last_peered":"2023-10-09T10:25:35.579898+0000","last_clean":"2023-10-09T10:25:35.579898+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:25:35.579898+0000","last_undegraded":"2023-10-09T10:25:35.579898+0000","last_fullsized":"2023-10-09T10:25:35.579898+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":437,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584255+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:25:35.584255+0000","last_peered":"2023-10-09T10:25:35.584255+0000","last_clean":"2023-10-09T10:25:35.584255+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:25:35.584255+0000","last_undegraded":"2023-10-09T10:25:35.584255+0000","last_fullsized":"2023-10-09T10:25:35.584255+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"525'7351","reported_seq":7391,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:42.724759+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:25:42.724759+0000","last_peered":"2023-10-09T10:25:42.724759+0000","last_clean":"2023-10-09T10:25:42.724759+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:25:42.724759+0000","last_undegraded":"2023-10-09T10:25:42.724759+0000","last_fullsized":"2023-10-09T10:25:42.724759+0000","mapping_epoch":494,"log_start":"503'2276","ondisk_log_start":"503'2276","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5075,"log_dups_size":0,"ondisk_log_size":5075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418447360,"num_objects":6385,"num_object_clones":0,"num_object_copies":19155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7351,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":325,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.585297+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:25:35.585297+0000","last_peered":"2023-10-09T10:25:35.585297+0000","last_clean":"2023-10-09T10:25:35.585297+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:25:35.585297+0000","last_undegraded":"2023-10-09T10:25:35.585297+0000","last_fullsized":"2023-10-09T10:25:35.585297+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":506,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.605136+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:25:35.605136+0000","last_peered":"2023-10-09T10:25:35.605136+0000","last_clean":"2023-10-09T10:25:35.605136+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:25:35.605136+0000","last_undegraded":"2023-10-09T10:25:35.605136+0000","last_fullsized":"2023-10-09T10:25:35.605136+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583173+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:25:35.583173+0000","last_peered":"2023-10-09T10:25:35.583173+0000","last_clean":"2023-10-09T10:25:35.583173+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:25:35.583173+0000","last_undegraded":"2023-10-09T10:25:35.583173+0000","last_fullsized":"2023-10-09T10:25:35.583173+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"525'7611","reported_seq":7650,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:43.025864+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:25:43.025864+0000","last_peered":"2023-10-09T10:25:43.025864+0000","last_clean":"2023-10-09T10:25:43.025864+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:25:43.025864+0000","last_undegraded":"2023-10-09T10:25:43.025864+0000","last_fullsized":"2023-10-09T10:25:43.025864+0000","mapping_epoch":494,"log_start":"500'1800","ondisk_log_start":"500'1800","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5811,"log_dups_size":0,"ondisk_log_size":5811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436011008,"num_objects":6653,"num_object_clones":0,"num_object_copies":19959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7611,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.579150+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:25:35.579150+0000","last_peered":"2023-10-09T10:25:35.579150+0000","last_clean":"2023-10-09T10:25:35.579150+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:25:35.579150+0000","last_undegraded":"2023-10-09T10:25:35.579150+0000","last_fullsized":"2023-10-09T10:25:35.579150+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":24,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.587521+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:25:35.587521+0000","last_peered":"2023-10-09T10:25:35.587521+0000","last_clean":"2023-10-09T10:25:35.587521+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:25:35.587521+0000","last_undegraded":"2023-10-09T10:25:35.587521+0000","last_fullsized":"2023-10-09T10:25:35.587521+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":205,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583619+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:25:35.583619+0000","last_peered":"2023-10-09T10:25:35.583619+0000","last_clean":"2023-10-09T10:25:35.583619+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:25:35.583619+0000","last_undegraded":"2023-10-09T10:25:35.583619+0000","last_fullsized":"2023-10-09T10:25:35.583619+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"525'4677","reported_seq":4764,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.911245+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:25:44.911245+0000","last_peered":"2023-10-09T10:25:44.911245+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:25:44.911245+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4677,"log_dups_size":0,"ondisk_log_size":4677,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208044032,"num_objects":3175,"num_object_clones":0,"num_object_copies":9525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3757,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1939},{"shards":"2,5","objects":959}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":20,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584188+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:25:35.584188+0000","last_peered":"2023-10-09T10:25:35.584188+0000","last_clean":"2023-10-09T10:25:35.584188+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:25:35.584188+0000","last_undegraded":"2023-10-09T10:25:35.584188+0000","last_fullsized":"2023-10-09T10:25:35.584188+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":29,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.583460+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:25:35.583460+0000","last_peered":"2023-10-09T10:25:35.583460+0000","last_clean":"2023-10-09T10:25:35.583460+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:25:35.583460+0000","last_undegraded":"2023-10-09T10:25:35.583460+0000","last_fullsized":"2023-10-09T10:25:35.583460+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"525'4697","reported_seq":4758,"reported_epoch":525,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:25:42.742999+0000","last_change":"2023-10-09T10:25:08.466144+0000","last_active":"2023-10-09T10:25:42.742999+0000","last_peered":"2023-10-09T10:25:42.742999+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:25:42.742999+0000","last_undegraded":"2023-10-09T10:25:42.742999+0000","last_fullsized":"2023-10-09T10:25:42.742999+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4697,"log_dups_size":0,"ondisk_log_size":4697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214859776,"num_objects":3279,"num_object_clones":0,"num_object_copies":9837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2331,"num_objects_unfound":0,"num_objects_dirty":3279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3773,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":948}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":255,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.579761+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:25:35.579761+0000","last_peered":"2023-10-09T10:25:35.579761+0000","last_clean":"2023-10-09T10:25:35.579761+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:25:35.579761+0000","last_undegraded":"2023-10-09T10:25:35.579761+0000","last_fullsized":"2023-10-09T10:25:35.579761+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":118,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602675+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:25:35.602675+0000","last_peered":"2023-10-09T10:25:35.602675+0000","last_clean":"2023-10-09T10:25:35.602675+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:25:35.602675+0000","last_undegraded":"2023-10-09T10:25:35.602675+0000","last_fullsized":"2023-10-09T10:25:35.602675+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"525'4782","reported_seq":5720,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.894402+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:25:44.894402+0000","last_peered":"2023-10-09T10:25:44.894402+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:25:44.894402+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213221376,"num_objects":3254,"num_object_clones":0,"num_object_copies":9762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4730,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3838,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":292,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1948},{"shards":"0,5,6","objects":833},{"shards":"5,6","objects":472}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":238,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584506+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:25:35.584506+0000","last_peered":"2023-10-09T10:25:35.584506+0000","last_clean":"2023-10-09T10:25:35.584506+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:25:35.584506+0000","last_undegraded":"2023-10-09T10:25:35.584506+0000","last_fullsized":"2023-10-09T10:25:35.584506+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":347,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.591287+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:25:35.591287+0000","last_peered":"2023-10-09T10:25:35.591287+0000","last_clean":"2023-10-09T10:25:35.591287+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:25:35.591287+0000","last_undegraded":"2023-10-09T10:25:35.591287+0000","last_fullsized":"2023-10-09T10:25:35.591287+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"525'4667","reported_seq":4723,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:44.398505+0000","last_change":"2023-10-09T10:25:13.762065+0000","last_active":"2023-10-09T10:25:44.398505+0000","last_peered":"2023-10-09T10:25:44.398505+0000","last_clean":"2023-10-09T10:25:11.551777+0000","last_became_active":"2023-10-09T10:25:13.754797+0000","last_became_peered":"2023-10-09T10:25:13.754797+0000","last_unstale":"2023-10-09T10:25:44.398505+0000","last_undegraded":"2023-10-09T10:25:13.625003+0000","last_fullsized":"2023-10-09T10:25:13.604794+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":4667,"log_dups_size":0,"ondisk_log_size":4667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:58.037975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212926464,"num_objects":3249,"num_object_clones":0,"num_object_copies":9747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1730,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":112},{"shards":"2,4,5","objects":2868}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":24,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:36.586779+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:25:36.586779+0000","last_peered":"2023-10-09T10:25:36.586779+0000","last_clean":"2023-10-09T10:25:36.586779+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:25:36.586779+0000","last_undegraded":"2023-10-09T10:25:36.586779+0000","last_fullsized":"2023-10-09T10:25:36.586779+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":577,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.589427+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:25:35.589427+0000","last_peered":"2023-10-09T10:25:35.589427+0000","last_clean":"2023-10-09T10:25:35.589427+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:25:35.589427+0000","last_undegraded":"2023-10-09T10:25:35.589427+0000","last_fullsized":"2023-10-09T10:25:35.589427+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"525'4752","reported_seq":4804,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T10:25:44.913652+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:25:44.913652+0000","last_peered":"2023-10-09T10:25:44.913652+0000","last_clean":"2023-10-09T10:25:44.913652+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:25:44.913652+0000","last_undegraded":"2023-10-09T10:25:44.913652+0000","last_fullsized":"2023-10-09T10:25:44.913652+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":4752,"log_dups_size":0,"ondisk_log_size":4752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209321984,"num_objects":3194,"num_object_clones":0,"num_object_copies":9582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1491,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":41,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.584872+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:25:35.584872+0000","last_peered":"2023-10-09T10:25:35.584872+0000","last_clean":"2023-10-09T10:25:35.584872+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:25:35.584872+0000","last_undegraded":"2023-10-09T10:25:35.584872+0000","last_fullsized":"2023-10-09T10:25:35.584872+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":145,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T10:25:35.602036+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:25:35.602036+0000","last_peered":"2023-10-09T10:25:35.602036+0000","last_clean":"2023-10-09T10:25:35.602036+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:25:35.602036+0000","last_undegraded":"2023-10-09T10:25:35.602036+0000","last_fullsized":"2023-10-09T10:25:35.602036+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":6834782232,"num_objects":104293,"num_object_clones":0,"num_object_copies":312879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":57635,"num_objects_misplaced":22013,"num_objects_unfound":0,"num_objects_dirty":104293,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":106653,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7053,"num_bytes_recovered":462225408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20600201216,"data_stored":20600193072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127133,"ondisk_log_size":127133,"up":78,"acting":62,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430673,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626901,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2285228,"kb_used_data":1385956,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91459924,"statfs":{"total":95995035648,"available":93654962176,"internally_reserved":0,"allocated":1419218944,"data_stored":1418323274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:15 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.3999999999999999,"15min":1.6739999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":4.8099999999999996,"5min":9.4260000000000002,"15min":44.055},"last":3.6789999999999998},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.756,"15min":1.919},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.22},"max":{"1min":3.6760000000000002,"5min":9.2609999999999992,"15min":43.869999999999997},"last":0.38100000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":2.911,"15min":2.2400000000000002},"min":{"1min":0.29399999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.0419999999999998,"5min":155.01400000000001,"15min":155.01400000000001},"last":3.7690000000000001},{"interface":"front","average":{"1min":1.383,"5min":2.9460000000000002,"15min":2.4060000000000001},"min":{"1min":0.33200000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.5709999999999997,"5min":154.71100000000001,"15min":154.71100000000001},"last":1.1459999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:25:15 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":1.7889999999999999,"15min":1.887},"min":{"1min":0.32800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.8460000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":5.9870000000000001},{"interface":"front","average":{"1min":1.6970000000000001,"5min":1.649,"15min":1.8029999999999999},"min":{"1min":0.36299999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.875,"5min":7.5629999999999997,"15min":14.542999999999999},"last":3.7480000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.619,"15min":1.514},"min":{"1min":0.498,"5min":0.308,"15min":0.308},"max":{"1min":7.681,"5min":7.681,"15min":8.0229999999999997},"last":3.8140000000000001},{"interface":"front","average":{"1min":1.496,"5min":1.927,"15min":1.6220000000000001},"min":{"1min":0.36399999999999999,"5min":0.316,"15min":0.31},"max":{"1min":7.6920000000000002,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":1.1160000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.881,"5min":3.4460000000000002,"15min":1.8600000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":10.618,"5min":122.041,"15min":122.041},"last":5.9429999999999996},{"interface":"front","average":{"1min":1.8069999999999999,"5min":3.3130000000000002,"15min":1.75},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.9470000000000001,"5min":122.13200000000001,"15min":122.13200000000001},"last":3.7269999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.4620000000000002,"15min":1.976},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.6890000000000001,"5min":121.98999999999999,"15min":121.98999999999999},"last":6.1890000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":2.6280000000000001,"15min":1.6200000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":5.1420000000000003,"5min":122.346,"15min":122.346},"last":7.0060000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":3.532,"15min":1.9730000000000001},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":4.9169999999999998,"5min":130.66999999999999,"15min":130.66999999999999},"last":3.7930000000000001},{"interface":"front","average":{"1min":1.3779999999999999,"5min":3.573,"15min":2.016},"min":{"1min":0.374,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":4.9589999999999996,"5min":130.702,"15min":130.702},"last":3.8370000000000002}]}]},{"osd":1,"up_from":0,"seq":360777253214,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416738,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4780328,"kb_used_data":3868384,"kb_used_omap":28,"kb_used_meta":911907,"kb_avail":88964824,"statfs":{"total":95995035648,"available":91099979776,"internally_reserved":0,"allocated":3961225216,"data_stored":3959210687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":933793616},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:24:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.28,"15min":1.996},"min":{"1min":0.248,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.835,"5min":8.8699999999999992,"15min":54.518000000000001},"last":0.31},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.3360000000000001,"15min":2.6230000000000002},"min":{"1min":0.23899999999999999,"5min":0.192,"15min":0.192},"max":{"1min":3.5499999999999998,"5min":17.033000000000001,"15min":56.950000000000003},"last":0.26000000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.9630000000000001,"15min":2.1419999999999999},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.7839999999999998,"5min":47.215000000000003,"15min":53.988999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.99,"5min":2.254,"15min":2.1339999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.173,"5min":47.000999999999998,"15min":54.119},"last":1.2769999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.488,"15min":1.4119999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.702,"5min":16.879999999999999,"15min":56.984999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.5750000000000002,"5min":1.6830000000000001,"15min":1.4450000000000001},"min":{"1min":0.51700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1749999999999998,"5min":8.0020000000000007,"15min":56.475999999999999},"last":1.444}]},{"osd":4,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.869,"15min":2.3159999999999998},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":3.3399999999999999,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.1230000000000002,"15min":2.246},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.32400000000000001},"max":{"1min":8.2479999999999993,"5min":28.331,"15min":56.378},"last":1.486}]},{"osd":5,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.1259999999999999,"15min":3.129},"min":{"1min":0.46800000000000003,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":6.923,"5min":11.775,"15min":229.589},"last":1.5229999999999999},{"interface":"front","average":{"1min":2.238,"5min":1.8029999999999999,"15min":2.8279999999999998},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":7.032,"5min":8.7530000000000001,"15min":228.88900000000001},"last":5.7320000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.919,"15min":2.2320000000000002},"min":{"1min":0.435,"5min":0.379,"15min":0.33700000000000002},"max":{"1min":6.8230000000000004,"5min":8.7810000000000006,"15min":54.482999999999997},"last":1.663},{"interface":"front","average":{"1min":1.248,"5min":2.0859999999999999,"15min":2.6349999999999998},"min":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.7160000000000002,"5min":16.954999999999998,"15min":57.753},"last":0.67300000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.9930000000000001,"15min":2.391},"min":{"1min":0.54500000000000004,"5min":0.38600000000000001,"15min":0.34999999999999998},"max":{"1min":6.6079999999999997,"5min":8.0329999999999995,"15min":56.411999999999999},"last":3.1339999999999999},{"interface":"front","average":{"1min":2.0070000000000001,"5min":2.0640000000000001,"15min":2.258},"min":{"1min":0.47099999999999997,"5min":0.38600000000000001,"15min":0.36499999999999999},"max":{"1min":6.891,"5min":11.175000000000001,"15min":54.451999999999998},"last":0.70399999999999996}]}]},{"osd":2,"up_from":217,"seq":932007903474,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4812392,"kb_used_data":3985888,"kb_used_omap":83,"kb_used_meta":826412,"kb_avail":88932760,"statfs":{"total":95995035648,"available":91067146240,"internally_reserved":0,"allocated":4081549312,"data_stored":4079719577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":846246383},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.778,"15min":2.0600000000000001},"min":{"1min":0.216,"5min":0.20899999999999999,"15min":0.185},"max":{"1min":5.2480000000000002,"5min":48.990000000000002,"15min":48.990000000000002},"last":11.093999999999999},{"interface":"front","average":{"1min":1.5089999999999999,"5min":2.0649999999999999,"15min":2.3849999999999998},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.590999999999999,"5min":49.412999999999997,"15min":49.412999999999997},"last":6.1779999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.3089999999999999,"15min":2.0899999999999999},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":5.008,"5min":8.2010000000000005,"15min":34.893999999999998},"last":11.148},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.4910000000000001,"15min":2.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":7.8029999999999999,"5min":7.8029999999999999,"15min":34.962000000000003},"last":2.0910000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.4139999999999999,"15min":1.155},"min":{"1min":0.25700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":3.9860000000000002,"5min":7.54,"15min":35.043999999999997},"last":2.4079999999999999},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.90900000000000003,"15min":1.1819999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.1269999999999998,"5min":6.617,"15min":50.406999999999996},"last":2.335}]},{"osd":4,"last update":"Mon Oct 9 10:25:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.772,"15min":1.6990000000000001},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.26600000000000001},"max":{"1min":5.6749999999999998,"5min":47.503999999999998,"15min":47.503999999999998},"last":2.1200000000000001},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.8859999999999999,"15min":1.845},"min":{"1min":0.40699999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.2720000000000002,"5min":47.704000000000001,"15min":47.704000000000001},"last":1.9910000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.603,"15min":1.992},"min":{"1min":0.39100000000000001,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":3.0329999999999999,"5min":10.448,"15min":57.298000000000002},"last":1.9490000000000001},{"interface":"front","average":{"1min":1.534,"5min":1.7050000000000001,"15min":2.0310000000000001},"min":{"1min":0.374,"5min":0.31,"15min":0.31},"max":{"1min":5.8810000000000002,"5min":8.1630000000000003,"15min":57.828000000000003},"last":11.195}]},{"osd":6,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.99,"5min":2.2360000000000002,"15min":2.0329999999999999},"min":{"1min":0.45000000000000001,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":5.5510000000000002,"5min":12.411,"15min":18.943999999999999},"last":2.8900000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.744,"15min":1.95},"min":{"1min":0.442,"5min":0.38300000000000001,"15min":0.36599999999999999},"max":{"1min":3.0950000000000002,"5min":9.8399999999999999,"15min":24.379000000000001},"last":2.0270000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.3370000000000002,"15min":2.2850000000000001},"min":{"1min":0.53500000000000003,"5min":0.41799999999999998,"15min":0.308},"max":{"1min":3.5760000000000001,"5min":13.821,"15min":35.008000000000003},"last":2.161},{"interface":"front","average":{"1min":1.655,"5min":1.6619999999999999,"15min":2.1160000000000001},"min":{"1min":0.439,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":6.2140000000000004,"5min":10.507,"15min":24.323},"last":2.0550000000000002}]}]},{"osd":3,"up_from":352,"seq":1511828488314,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3459040,"kb_used_data":3028312,"kb_used_omap":15,"kb_used_meta":430640,"kb_avail":90286112,"statfs":{"total":95995035648,"available":92452978688,"internally_reserved":0,"allocated":3100991488,"data_stored":3098696258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":440975814},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.95,"15min":1.3160000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":3.2309999999999999,"5min":65.674999999999997,"15min":65.674999999999997},"last":7.4580000000000002},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.97,"15min":1.319},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.19900000000000001},"max":{"1min":4.758,"5min":21.149999999999999,"15min":21.149999999999999},"last":1.96}]},{"osd":1,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":3.1309999999999998,"15min":1.8069999999999999},"min":{"1min":0.246,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.3760000000000003,"5min":105.401,"15min":105.401},"last":7.1189999999999998},{"interface":"front","average":{"1min":1.7889999999999999,"5min":3.4079999999999999,"15min":1.6619999999999999},"min":{"1min":0.311,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":6.7050000000000001,"5min":104.914,"15min":104.914},"last":3.7810000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.954,"15min":1.883},"min":{"1min":0.253,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":2.2650000000000001,"5min":9.4369999999999994,"15min":9.4369999999999994},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.248,"5min":1.508,"15min":1.3},"min":{"1min":0.313,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":4.4930000000000003,"5min":25.338999999999999,"15min":25.338999999999999},"last":9.2050000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":2.4220000000000002,"15min":1.319},"min":{"1min":0.39100000000000001,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":4.7770000000000001,"5min":21.831,"15min":21.831},"last":6.726},{"interface":"front","average":{"1min":1.105,"5min":2.8759999999999999,"15min":1.8169999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.4510000000000001,"5min":22.097999999999999,"15min":22.097999999999999},"last":8.9060000000000006}]},{"osd":5,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":1.8819999999999999,"15min":1.1919999999999999},"min":{"1min":0.44,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":6.6710000000000003,"5min":25.251999999999999,"15min":25.251999999999999},"last":7.0800000000000001},{"interface":"front","average":{"1min":1.6779999999999999,"5min":2.4039999999999999,"15min":1.4259999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":4.6600000000000001,"5min":13.394,"15min":13.394},"last":3.8370000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":2.1749999999999998,"15min":1.631},"min":{"1min":0.36199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.617,"5min":19.114000000000001,"15min":19.114000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.538,"5min":1.958,"15min":1.373},"min":{"1min":0.47099999999999997,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":5.2519999999999998,"5min":25.306999999999999,"15min":25.306999999999999},"last":7.7610000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":2.2120000000000002,"15min":1.4239999999999999},"min":{"1min":0.40699999999999997,"5min":0.39900000000000002,"15min":0.38400000000000001},"max":{"1min":4.8109999999999999,"5min":9.7880000000000003,"15min":10.459},"last":3.7050000000000001},{"interface":"front","average":{"1min":1.097,"5min":2.1829999999999998,"15min":1.6970000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.3860000000000001,"5min":19.280000000000001,"15min":19.280000000000001},"last":7.056}]}]},{"osd":4,"up_from":266,"seq":1142461300943,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4751092,"kb_used_data":3817964,"kb_used_omap":35,"kb_used_meta":933084,"kb_avail":88994060,"statfs":{"total":95995035648,"available":91129917440,"internally_reserved":0,"allocated":3909595136,"data_stored":3908690240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":955478254},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":3.286,"15min":2.1200000000000001},"min":{"1min":0.23699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.4699999999999998,"5min":97.721999999999994,"15min":97.721999999999994},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.8540000000000001,"5min":4.1630000000000003,"15min":2.8100000000000001},"min":{"1min":0.30199999999999999,"5min":0.252,"15min":0.224},"max":{"1min":5.931,"5min":97.915999999999997,"15min":97.915999999999997},"last":0.34300000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":2.5379999999999998,"15min":2.3889999999999998},"min":{"1min":0.41599999999999998,"5min":0.32300000000000001,"15min":0.24299999999999999},"max":{"1min":5.6630000000000003,"5min":11.32,"15min":71.457999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.754,"5min":2.714,"15min":2.2029999999999998},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":5.8730000000000002,"5min":49.781999999999996,"15min":71.658000000000001},"last":1.95}]},{"osd":2,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":2.3279999999999998,"15min":1.8360000000000001},"min":{"1min":0.313,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":6.468,"5min":15.944000000000001,"15min":15.944000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.875,"15min":1.7450000000000001},"min":{"1min":0.28699999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.8420000000000001,"5min":14.473000000000001,"15min":14.473000000000001},"last":2.6299999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":2.5779999999999998,"15min":1.887},"min":{"1min":0.45100000000000001,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":5.7060000000000004,"5min":12.074,"15min":12.074},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.444,"5min":2.746,"15min":1.6220000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4240000000000004,"5min":16.067,"15min":16.067},"last":0.41499999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.7719999999999998,"15min":2.0579999999999998},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":10.023999999999999,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.5819999999999999,"15min":1.7889999999999999},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.335,"5min":49.915999999999997,"15min":99.384},"last":0.36499999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.9530000000000001,"15min":1.3280000000000001},"min":{"1min":0.371,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.7709999999999999,"5min":14.499000000000001,"15min":14.499000000000001},"last":9.5389999999999997},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.9279999999999999,"15min":1.734},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":18.952999999999999,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.84399999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":2.9159999999999999,"15min":1.8049999999999999},"min":{"1min":0.33000000000000002,"5min":0.315,"15min":0.307},"max":{"1min":9.3789999999999996,"5min":17.369,"15min":17.369},"last":7.8179999999999996},{"interface":"front","average":{"1min":1.554,"5min":2.9129999999999998,"15min":1.7430000000000001},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":5.9020000000000001,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.622}]}]},{"osd":5,"up_from":36,"seq":154618823128,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5624612,"kb_used_data":4119708,"kb_used_omap":0,"kb_used_meta":1504896,"kb_avail":88120540,"statfs":{"total":95995035648,"available":90235432960,"internally_reserved":0,"allocated":4218580992,"data_stored":4217687702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541013504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.468,"15min":1.9159999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":7.6390000000000002,"5min":22.469999999999999,"15min":22.469999999999999},"last":0.318},{"interface":"front","average":{"1min":1.365,"5min":2.0390000000000001,"15min":1.6950000000000001},"min":{"1min":0.248,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":5.7119999999999997,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.55000000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:25:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.6339999999999999,"15min":2.2090000000000001},"min":{"1min":0.29599999999999999,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":16.562999999999999,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.8,"5min":1.8759999999999999,"15min":1.8580000000000001},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":8.9529999999999994,"5min":13.143000000000001,"15min":29.065000000000001},"last":0.34499999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6899999999999999,"5min":2.5680000000000001,"15min":2.1269999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":16.629999999999999,"5min":16.629999999999999,"15min":16.629999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.403,"5min":2.2890000000000001,"15min":1.7270000000000001},"min":{"1min":0.30499999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.8930000000000007,"5min":13.095000000000001,"15min":15.929},"last":1.5209999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":2.1899999999999999,"15min":1.7430000000000001},"min":{"1min":0.33300000000000002,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":4.3220000000000001,"5min":10.561999999999999,"15min":10.561999999999999},"last":3.2280000000000002},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.976,"15min":1.702},"min":{"1min":0.376,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":4.5019999999999998,"5min":13.624000000000001,"15min":13.624000000000001},"last":0.47899999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":2.1920000000000002,"15min":1.524},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.24299999999999999},"max":{"1min":4.1159999999999997,"5min":12.023,"15min":12.023},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.081,"15min":1.341},"min":{"1min":0.29899999999999999,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":2.7170000000000001,"5min":11.956,"15min":11.956},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":2.4249999999999998,"15min":1.6890000000000001},"min":{"1min":0.309,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":5.9080000000000004,"5min":11.686,"15min":12.772},"last":3.3359999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":2.0750000000000002,"15min":1.6100000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":2.7330000000000001,"5min":10.403,"15min":15.997},"last":0.69499999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:25:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.6429999999999998,"15min":1.798},"min":{"1min":0.34000000000000002,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":16.686,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.397,"15min":1.1719999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":8.9719999999999995,"5min":11.994,"15min":11.994},"last":0.57199999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3937992704,"data_stored":3937992704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4060217344,"data_stored":4060217344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3080847360,"data_stored":3080847360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3896184832,"data_stored":3896180760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4211019776,"data_stored":4211015704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1413939200,"data_stored":1413939200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:25:47.706 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:25:47.721 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:47.826 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:47.975 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:25:48.083 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:25:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:25:47] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-10-09T10:25:48.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:48 smithi012 ceph-mon[147926]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:25:48.432 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:48 smithi012 ceph-mon[152263]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:25:48.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:48 smithi110 ceph-mon[136032]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:25:49.557 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:25:49.557 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:25:49.584 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:25:49.638 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:25:49.676 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:25:49.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:49 smithi012 ceph-mon[152263]: pgmap v1468: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 57637/312951 objects degraded (18.417%); 22013/312951 objects misplaced (7.034%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:25:49.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:49 smithi012 ceph-mon[152263]: failed to encode map e527 with expected crc 2023-10-09T10:25:49.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:49 smithi012 ceph-mon[152263]: osdmap e527: 8 total, 8 up, 6 in 2023-10-09T10:25:49.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:49 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 57637/312951 objects degraded (18.417%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:49.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:49 smithi012 ceph-mon[147926]: pgmap v1468: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 57637/312951 objects degraded (18.417%); 22013/312951 objects misplaced (7.034%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:25:49.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:49 smithi012 ceph-mon[147926]: failed to encode map e527 with expected crc 2023-10-09T10:25:49.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:49 smithi012 ceph-mon[147926]: osdmap e527: 8 total, 8 up, 6 in 2023-10-09T10:25:49.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:49 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 57637/312951 objects degraded (18.417%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:49 smithi110 ceph-mon[136032]: pgmap v1468: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 615 op/s; 57637/312951 objects degraded (18.417%); 22013/312951 objects misplaced (7.034%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:25:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:49 smithi110 ceph-mon[136032]: failed to encode map e527 with expected crc 2023-10-09T10:25:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:49 smithi110 ceph-mon[136032]: osdmap e527: 8 total, 8 up, 6 in 2023-10-09T10:25:49.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:49 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 57637/312951 objects degraded (18.417%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:49.722 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:25:49.765 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:49.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:49.778 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:25:49.807 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:25:49.816 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:25:49.817 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:25:49.817 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:25:49.817 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:25:49.817 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:25:49.817 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:25:49.817 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:25:49.817 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:25:49.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:49.988 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:50.167 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:50.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[147926]: osdmap e528: 8 total, 8 up, 6 in 2023-10-09T10:25:50.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[147926]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[147926]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[147926]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1355899832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:50.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[152263]: osdmap e528: 8 total, 8 up, 6 in 2023-10-09T10:25:50.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[152263]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[152263]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[152263]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1355899832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:50 smithi110 ceph-mon[136032]: osdmap e528: 8 total, 8 up, 6 in 2023-10-09T10:25:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:50 smithi110 ceph-mon[136032]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:50 smithi110 ceph-mon[136032]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:50 smithi110 ceph-mon[136032]: failed to encode map e528 with expected crc 2023-10-09T10:25:50.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1355899832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:51.004 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:51.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:51.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:51.115 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:51.177 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:51.232 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:51.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:51.348 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:51 smithi110 ceph-mon[136032]: pgmap v1471: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 54390/307326 objects degraded (17.698%); 23884/307326 objects misplaced (7.772%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:25:51.529 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:51 smithi012 ceph-mon[152263]: pgmap v1471: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 54390/307326 objects degraded (17.698%); 23884/307326 objects misplaced (7.772%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:25:51.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:51 smithi012 ceph-mon[147926]: pgmap v1471: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 54390/307326 objects degraded (17.698%); 23884/307326 objects misplaced (7.772%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:25:51.536 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:51.674 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:25:51.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:51.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:25:51.784 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:51.845 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:52.190 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:52.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:52.226 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:52.277 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:25:52.277 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:25:52.277 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:25:52.277 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:25:52.277 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.277 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:128066 9.1 9:8603ff6a:::benchmark_data_smithi012_313365_object15591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 582.58948843999997, 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55978837800000003, 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:52.278 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 128066 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682158+0000", 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.279 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682155+0000", 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682168+0000", 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2682e-05 2023-10-09T10:25:52.280 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682184+0000", 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6339000000000001e-05 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682194+0000", 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4420000000000004e-06 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.281 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682258+0000", 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4463999999999998e-05 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:09.682271+0000", 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3166000000000001e-05 2023-10-09T10:25:52.282 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237630+0000", 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.55535865299999998 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237768+0000", 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013818000000000001 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.283 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.237891+0000", 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012287299999999999 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.240596+0000", 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002704559 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:52.284 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241710+0000", 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00111455 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241856+0000", 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014592000000000001 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241905+0000", 2023-10-09T10:25:52.285 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9042000000000002e-05 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:16:10.241946+0000", 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0655999999999998e-05 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156571 9.11 9:88722a35:::benchmark_data_smithi012_313365_object44096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-10-09T10:25:52.286 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 523.871222192, 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.21718912700000001, 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156571 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.287 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:25:52.288 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400424+0000", 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400422+0000", 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.290 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400431+0000", 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2720000000000003e-06 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400450+0000", 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8981e-05 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.400462+0000", 2023-10-09T10:25:52.291 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2333e-05 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.402769+0000", 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023067069999999999 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.402909+0000", 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013978499999999999 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.292 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.403031+0000", 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012171300000000001 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.446104+0000", 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.043073587000000003 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.293 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575808+0000", 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12970353000000001 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617506+0000", 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.041698357999999998 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.294 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617561+0000", 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5191000000000001e-05 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.617613+0000", 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1649e-05 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.295 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156574 9.21 9:87a74460:::benchmark_data_smithi012_313365_object44099:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:25:52.296 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:25:52.296 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 523.86620954, 2023-10-09T10:25:52.296 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0291358530000001, 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156574 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.297 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405436+0000", 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405435+0000", 2023-10-09T10:25:52.298 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405441+0000", 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7290000000000003e-06 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405454+0000", 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2811e-05 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.299 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.405465+0000", 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0448000000000001e-05 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575963+0000", 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.17049810900000001 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575975+0000", 2023-10-09T10:25:52.300 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.221e-05 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.430808+0000", 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.85483296499999994 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.430955+0000", 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014732499999999999 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.301 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.431079+0000", 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012408499999999999 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.432894+0000", 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001814593 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:52.302 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434249+0000", 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001355537 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434463+0000", 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000213165 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434525+0000", 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2205000000000005e-05 2023-10-09T10:25:52.303 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:09.434572+0000", 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7500000000000003e-05 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:156578 9.11 9:8e383e1c:::benchmark_data_smithi012_313365_object44103:head [delete] snapc 0=[] ondisk+write+known_if_redirected e377)", 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:25:52.304 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 523.76002427000003, 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.19283726200000001, 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 156578 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.305 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511622+0000", 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511620+0000", 2023-10-09T10:25:52.306 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511628+0000", 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7139999999999999e-06 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511644+0000", 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6018000000000001e-05 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.307 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511654+0000", 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9119999999999993e-06 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511718+0000", 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3587e-05 2023-10-09T10:25:52.308 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.511874+0000", 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000156126 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.512003+0000", 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012904400000000001 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.309 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.575815+0000", 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063811965999999998 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.656235+0000", 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.080420144999999998 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704358+0000", 2023-10-09T10:25:52.310 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.048122282000000002 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704415+0000", 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7090999999999999e-05 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:17:08.704459+0000", 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4347000000000001e-05 2023-10-09T10:25:52.311 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27047 10.4 e428/424 10:2e4baf45:::benchmark_data_smithi012_366312_object27046:head v 428'1731, mlcod=428'1729)", 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 353.591395116, 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16898123100000001, 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.312 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.313 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680249+0000", 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680286+0000", 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6359000000000002e-05 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.314 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680298+0000", 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2072999999999999e-05 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680305+0000", 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2400000000000001e-06 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680357+0000", 2023-10-09T10:25:52.315 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1610000000000002e-05 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680379+0000", 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1732000000000001e-05 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849191+0000", 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.168812924 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.316 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849232+0000", 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0663e-05 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27057 10.5 e428/424 10:ac997561:::benchmark_data_smithi012_366312_object27056:head v 428'1679, mlcod=428'1677)", 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 353.590810154, 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.170571852, 2023-10-09T10:25:52.317 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:25:52.318 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680835+0000", 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680874+0000", 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9499000000000002e-05 2023-10-09T10:25:52.319 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680887+0000", 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.278e-05 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680895+0000", 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.644e-06 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.320 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680981+0000", 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6104000000000001e-05 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.681001+0000", 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0417000000000001e-05 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851371+0000", 2023-10-09T10:25:52.321 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.170369565 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851408+0000", 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6757000000000001e-05 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.322 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27067 10.5 e428/424 10:a3c37c8c:::benchmark_data_smithi012_366312_object27066:head v 428'1680, mlcod=428'1677)", 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 353.587348679, 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16713378700000001, 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.323 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684296+0000", 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.324 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684335+0000", 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9345999999999999e-05 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848078+0000", 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16374229300000001 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848092+0000", 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4222e-05 2023-10-09T10:25:52.325 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849076+0000", 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00098447500000000006 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849097+0000", 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0258000000000002e-05 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.326 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851412+0000", 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002315517 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851431+0000", 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8884000000000001e-05 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.327 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27069 10.4 e428/424 10:2f6e2957:::benchmark_data_smithi012_366312_object27068:head v 428'1732, mlcod=428'1729)", 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.685774+0000", 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 353.58587240399999, 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16706146599999999, 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.685774+0000", 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.328 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.685774+0000", 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.685773+0000", 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.329 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.685820+0000", 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6873999999999999e-05 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848088+0000", 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.162268312 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848099+0000", 2023-10-09T10:25:52.330 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1205e-05 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848146+0000", 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6663999999999999e-05 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848168+0000", 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1919000000000001e-05 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.331 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.852794+0000", 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0046263420000000003 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.852835+0000", 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1016000000000002e-05 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.332 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27078 10.f e428/424 10:f852e76d:::benchmark_data_smithi012_366312_object27077:head v 428'1695, mlcod=428'1693)", 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.690195+0000", 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 353.58145140099998, 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.160981017, 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.690195+0000", 2023-10-09T10:25:52.333 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.690195+0000", 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.690183+0000", 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999881 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.334 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.690236+0000", 2023-10-09T10:25:52.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2896999999999999e-05 2023-10-09T10:25:52.335 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.335 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.335 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.335 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848098+0000", 2023-10-09T10:25:52.335 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15786182700000001 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848105+0000", 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4800000000000004e-06 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.336 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848324+0000", 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000219018 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848345+0000", 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1382999999999998e-05 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851127+0000", 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027814850000000002 2023-10-09T10:25:52.337 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851176+0000", 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8639e-05 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:100348 10.d e446/443 10:b23cf803:::benchmark_data_smithi012_366312_object100347:head v 446'6294, mlcod=428'83)", 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:03.790773+0000", 2023-10-09T10:25:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 288.48087289, 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.066822567999999999, 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790773+0000", 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790773+0000", 2023-10-09T10:25:52.339 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790773+0000", 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790814+0000", 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1415999999999998e-05 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.340 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790822+0000", 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6120000000000001e-06 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.790827+0000", 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5489999999999997e-06 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.798748+0000", 2023-10-09T10:25:52.341 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0079209510000000007 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.798781+0000", 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2410000000000003e-05 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.857546+0000", 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058765036 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.342 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:03.857596+0000", 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9935999999999998e-05 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(10.d 447/443 [PushOp(10:b1f42904:::benchmark_data_smithi012_366312_object2884:head, version: 428'173, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:b1f42904:::benchmark_data_smithi012_366312_object2884:head@428'173, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:08.006676+0000", 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 284.26496969300001, 2023-10-09T10:25:52.343 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067742063000000005, 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006676+0000", 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006676+0000", 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.344 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006675+0000", 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006729+0000", 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4329000000000001e-05 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.345 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006750+0000", 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0428000000000002e-05 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.006767+0000", 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6912999999999999e-05 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.346 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.071299+0000", 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.064532701999999997 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.071318+0000", 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8717000000000001e-05 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.347 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:08.074418+0000", 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031001650000000002 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135021 10.d e457/443 10:b65cfb4e:::benchmark_data_smithi012_366312_object23692:head v 457'8492, mlcod=428'433)", 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.049439+0000", 2023-10-09T10:25:52.348 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.22220674499999, 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.075575955, 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049439+0000", 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049439+0000", 2023-10-09T10:25:52.349 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049438+0000", 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049449+0000", 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.163e-05 2023-10-09T10:25:52.350 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049466+0000", 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6501999999999999e-05 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.049475+0000", 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.9390000000000003e-06 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.351 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.057081+0000", 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0076062170000000002 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.057107+0000", 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5797000000000001e-05 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.352 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.124962+0000", 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.067854822999999995 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.125015+0000", 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3579999999999999e-05 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.353 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135039 10.d e457/443 10:b7499b5c:::benchmark_data_smithi012_366312_object23710:head v 457'8493, mlcod=428'433)", 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.060346+0000", 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.211300021, 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070359356999999997, 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060346+0000", 2023-10-09T10:25:52.354 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060346+0000", 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060345+0000", 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:52.355 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060355+0000", 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.062e-05 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060374+0000", 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8312e-05 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.356 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.060384+0000", 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0268000000000001e-05 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067438+0000", 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00705435 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067464+0000", 2023-10-09T10:25:52.357 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5636e-05 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130662+0000", 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063198174999999995 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130705+0000", 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3129e-05 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.358 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135046 10.5 e457/456 10:a6a4524b:::benchmark_data_smithi012_366312_object23717:head v 457'8418, mlcod=457'8416)", 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.061370+0000", 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.21027583, 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063689329000000003, 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.359 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061370+0000", 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061370+0000", 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061369+0000", 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:52.360 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061378+0000", 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7499999999999992e-06 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061396+0000", 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.7628000000000001e-05 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.361 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.061414+0000", 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8592000000000001e-05 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.065364+0000", 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003949508 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.362 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.065384+0000", 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0625e-05 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.125023+0000", 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059638459999999997 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.125059+0000", 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6664e-05 2023-10-09T10:25:52.363 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135042 10.d e457/443 10:b1be933a:::benchmark_data_smithi012_366312_object23713:head v 457'8494, mlcod=428'433)", 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.067349+0000", 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.20429725899999, 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063424356000000001, 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.364 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067349+0000", 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067349+0000", 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.365 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067348+0000", 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067357+0000", 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8330000000000006e-06 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067372+0000", 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4978e-05 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.366 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.067381+0000", 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7360000000000007e-06 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.078524+0000", 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011143125 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.367 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.078551+0000", 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6531999999999999e-05 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130748+0000", 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052196996000000002 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130773+0000", 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5272000000000001e-05 2023-10-09T10:25:52.368 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:135049 10.5 e457/456 10:a35ad3f9:::benchmark_data_smithi012_366312_object23720:head v 457'8419, mlcod=457'8416)", 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:49.068476+0000", 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 243.20317000899999, 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.062269128999999999, 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.369 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068476+0000", 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068476+0000", 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.370 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068475+0000", 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068484+0000", 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8040000000000007e-06 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068497+0000", 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.295e-05 2023-10-09T10:25:52.371 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.068504+0000", 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3409999999999997e-06 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.076436+0000", 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0079316730000000002 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.372 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.076460+0000", 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4379000000000002e-05 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130711+0000", 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054251313000000002 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:49.130745+0000", 2023-10-09T10:25:52.373 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3639000000000001e-05 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:136118 10.d e458/443 10:bce59347:::benchmark_data_smithi012_366312_object24789:head v 458'8570, mlcod=428'433)", 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:21:50.181296+0000", 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 242.090350293, 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.063169832999999995, 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.374 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181296+0000", 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181296+0000", 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181295+0000", 2023-10-09T10:25:52.375 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181301+0000", 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5779999999999998e-06 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181325+0000", 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4624999999999999e-05 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.376 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.181332+0000", 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5420000000000002e-06 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.188034+0000", 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0067019790000000003 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.188065+0000", 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0951999999999998e-05 2023-10-09T10:25:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.244405+0000", 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.056340135999999999 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:21:50.244466+0000", 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0458000000000003e-05 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.378 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:125160 11.8 e528/513 11:11046420:::benchmark_data_smithi012_404402_object12377:head v 528'4879, mlcod=498'797)", 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:50.028046+0000", 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.2435995470000001, 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028496178, 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.379 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.028046+0000", 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.028046+0000", 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.028045+0000", 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:52.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.028059+0000", 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3166999999999999e-05 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.028077+0000", 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8365000000000001e-05 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.381 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.028087+0000", 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7690000000000008e-06 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.028199+0000", 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000112434 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.382 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.028235+0000", 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5488999999999999e-05 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.056493+0000", 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028257886999999999 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.056543+0000", 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0009000000000002e-05 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.383 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:125172 11.18 e528/513 11:1ded7cec:::benchmark_data_smithi012_404402_object12389:head v 528'4905, mlcod=498'935)", 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:50.032631+0000", 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 2.239015363, 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025405785, 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.384 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.032631+0000", 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.032631+0000", 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.032630+0000", 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.032634+0000", 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0790000000000002e-06 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.032640+0000", 2023-10-09T10:25:52.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6450000000000002e-06 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.032644+0000", 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0439999999999997e-06 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.035350+0000", 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027057079999999998 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.035388+0000", 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7919000000000001e-05 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.057994+0000", 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022606281999999998 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.058036+0000", 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2379000000000002e-05 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:126664 11.b e528/494 11:df9598b9:::benchmark_data_smithi012_404402_object13881:head v 528'7987, mlcod=528'7985)", 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:52.181179+0000", 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.090467248, 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025123479000000001, 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:52.389 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.181179+0000", 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.181179+0000", 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:52.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.181178+0000", 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.181184+0000", 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2429999999999996e-06 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.181190+0000", 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6329999999999999e-06 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.181195+0000", 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0919999999999997e-06 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:52.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.184275+0000", 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030794260000000001 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.184295+0000", 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0406000000000001e-05 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.206253+0000", 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021958136 2023-10-09T10:25:52.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:52.206302+0000", 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8936e-05 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:52.394 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:25:52.395 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:52.436 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:52.509 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:52.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:52.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:52.692 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:52.830 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:52.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:52.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:52.937 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:25:52.946 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:53.009 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:53.066 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:53.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:53.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:53.170 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:53.219 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:53.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:53.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:53.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:53.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:25:53.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:53 smithi012 ceph-mon[147926]: pgmap v1472: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.2 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 48858/303300 objects degraded (16.109%); 28045/303300 objects misplaced (9.247%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:25:53.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:53 smithi012 ceph-mon[152263]: pgmap v1472: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.2 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 48858/303300 objects degraded (16.109%); 28045/303300 objects misplaced (9.247%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:25:53.481 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:53.592 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:25:53.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:53.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:25:53.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:53 smithi110 ceph-mon[136032]: pgmap v1472: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 6.2 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 48858/303300 objects degraded (16.109%); 28045/303300 objects misplaced (9.247%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:25:53.701 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:53.741 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:53.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:53.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:53.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:53.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:53.993 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:54.056 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:54.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:54.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:54.150 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:54.163 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:54.206 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:54.237 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:25:54.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:54.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:54.360 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:54.413 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:54.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:54 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 48858/303300 objects degraded (16.109%), 16 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:54.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:54 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 48858/303300 objects degraded (16.109%), 16 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:54.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:54.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:54.570 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:54.626 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:54 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 48858/303300 objects degraded (16.109%), 16 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:54.627 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:54.675 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:54.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:54.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:54.786 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:54.859 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:54.913 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:54.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:55.018 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:55.074 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:55.132 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:55.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:55.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:55.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:55.356 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:25:55.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:55.579 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:25:55.579 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 48282/298137 objects degraded (16.195%), 15 pgs degraded, 3 pgs undersized","count":18},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1872,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":529,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846541,"num_in_osds":6,"osd_in_since":1696847079,"num_remapped_pgs":18},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":12},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":99379,"data_bytes":6511761336,"bytes_used":25718013952,"bytes_avail":550252199936,"bytes_total":575970213888,"degraded_objects":48282,"degraded_total":298137,"degraded_ratio":0.16194568268950177,"misplaced_objects":27824,"misplaced_total":298137,"misplaced_ratio":0.093326222508444107,"recovering_objects_per_sec":201,"recovering_bytes_per_sec":13248838,"recovering_keys_per_sec":0,"num_objects_recovered":1611,"num_bytes_recovered":106037792,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":616},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":32,"modified":"2023-10-09T10:25:43.785386+0000","services":{}},"progress_events":{"4aaa0d12-13e6-4ea3-ae4b-c27ebe383851":{"message":"Global Recovery Event (4m)\n [======================......] (remaining: 72s)","progress":0.79797977209091187,"add_to_ceph_s":true},"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T10:25:55.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:55 smithi012 ceph-mon[147926]: pgmap v1473: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.1 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 48282/298137 objects degraded (16.195%); 27824/298137 objects misplaced (9.333%); 13 MiB/s, 201 objects/s recovering 2023-10-09T10:25:55.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3916875999' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:55.624 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:55 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:55.624 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:55 smithi012 ceph-mon[152263]: pgmap v1473: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.1 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 48282/298137 objects degraded (16.195%); 27824/298137 objects misplaced (9.333%); 13 MiB/s, 201 objects/s recovering 2023-10-09T10:25:55.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3916875999' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:55.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:55 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:55.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:55 smithi110 ceph-mon[136032]: pgmap v1473: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.1 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 48282/298137 objects degraded (16.195%); 27824/298137 objects misplaced (9.333%); 13 MiB/s, 201 objects/s recovering 2023-10-09T10:25:55.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3916875999' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:55.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:55 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:25:55.869 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:55.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:55.978 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:56.041 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:56.101 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:56.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:56.208 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:56.301 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:56.302 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:25:56.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:25:56.366 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:56.412 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:56.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:56.422 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:56.422 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:25:56.422 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[147926]: osdmap e529: 8 total, 8 up, 6 in 2023-10-09T10:25:56.423 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[147926]: failed to encode map e529 with expected crc 2023-10-09T10:25:56.423 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[147926]: failed to encode map e529 with expected crc 2023-10-09T10:25:56.423 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4191250922' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:25:56.423 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:56.423 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:25:56.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[152263]: osdmap e529: 8 total, 8 up, 6 in 2023-10-09T10:25:56.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[152263]: failed to encode map e529 with expected crc 2023-10-09T10:25:56.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[152263]: failed to encode map e529 with expected crc 2023-10-09T10:25:56.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4191250922' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:25:56.522 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:25:56.565 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:56 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:25:56.565 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:56 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:25:56.565 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:56 smithi110 ceph-mon[136032]: osdmap e529: 8 total, 8 up, 6 in 2023-10-09T10:25:56.565 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:56 smithi110 ceph-mon[136032]: failed to encode map e529 with expected crc 2023-10-09T10:25:56.565 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:56 smithi110 ceph-mon[136032]: failed to encode map e529 with expected crc 2023-10-09T10:25:56.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4191250922' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:25:56.568 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:56.609 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:25:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:56.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:25:56.713 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:25:56.753 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:129502 11.e e529/494 11:7d8482f5:::benchmark_data_smithi012_404402_object16719:head v 529'8159, mlcod=529'8156)", 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:56.769987+0000", 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.007712216, 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0077519149999999998, 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:56.782 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.769987+0000", 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.769987+0000", 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.783 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.769986+0000", 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.769998+0000", 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1484999999999999e-05 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:56.784 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.770017+0000", 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9592000000000001e-05 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.770025+0000", 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2279999999999998e-06 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.773497+0000", 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034714849999999998 2023-10-09T10:25:56.785 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.773518+0000", 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1178e-05 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:129506 11.17 e529/501 11:ef5c2794:::benchmark_data_smithi012_404402_object16723:head v 529'4990, mlcod=529'4987)", 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:56.772450+0000", 2023-10-09T10:25:56.786 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0052488980000000001, 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053713249999999997, 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772450+0000", 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.787 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772450+0000", 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772449+0000", 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:56.788 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772457+0000", 2023-10-09T10:25:56.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3520000000000007e-06 2023-10-09T10:25:56.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.789 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:56.789 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772469+0000", 2023-10-09T10:25:56.789 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1868999999999999e-05 2023-10-09T10:25:56.789 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.789 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:56.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772478+0000", 2023-10-09T10:25:56.790 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5869999999999993e-06 2023-10-09T10:25:56.790 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.790 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.790 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.790 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772522+0000", 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4453000000000003e-05 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772547+0000", 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4357000000000001e-05 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:129503 11.2 11:46c70a23:::benchmark_data_smithi012_404402_object16720:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-09T10:25:56.791 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:56.772691+0000", 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.005007933, 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052051470000000002, 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 129503 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.792 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772691+0000", 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772691+0000", 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772690+0000", 2023-10-09T10:25:56.793 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772696+0000", 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4849999999999997e-06 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772710+0000", 2023-10-09T10:25:56.794 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.342e-05 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772719+0000", 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0140000000000004e-06 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772765+0000", 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6693999999999997e-05 2023-10-09T10:25:56.795 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772915+0000", 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000149195 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.773059+0000", 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014424300000000001 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:56.796 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777603+0000", 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0045443519999999998 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777734+0000", 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013132199999999999 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.797 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:129505 11.6 e529/516 11:65759366:::benchmark_data_smithi012_404402_object16722:head v 529'4801, mlcod=500'1682)", 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:56.774742+0000", 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0029572069999999999, 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032471420000000002, 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.774742+0000", 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.798 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.774742+0000", 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.774741+0000", 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.774749+0000", 2023-10-09T10:25:56.799 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7149999999999998e-06 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.774759+0000", 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.006e-05 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.774767+0000", 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6960000000000005e-06 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.800 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.775840+0000", 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010734780000000001 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.775873+0000", 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3370000000000001e-05 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.801 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:129510 11.11 11:8e9c316f:::benchmark_data_smithi012_404402_object16727:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:56.777620+0000", 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 7.8881999999999995e-05, 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00040672000000000001, 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 129510 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:56.802 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777620+0000", 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777620+0000", 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:56.803 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777619+0000", 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777626+0000", 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9870000000000002e-06 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777636+0000", 2023-10-09T10:25:56.804 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0679000000000001e-05 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777646+0000", 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1200000000000008e-06 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.777699+0000", 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3492999999999998e-05 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:56.805 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:129490 11.11 11:8e5d863b:::benchmark_data_smithi012_404402_object16707:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:56.743466+0000", 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.034233263, 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.034623133, 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:25:56.806 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 129490 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.743466+0000", 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.743466+0000", 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.807 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.743464+0000", 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.743471+0000", 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9890000000000003e-06 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.808 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.743483+0000", 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2004000000000001e-05 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.743491+0000", 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6359999999999994e-06 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.744297+0000", 2023-10-09T10:25:56.809 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00080657500000000002 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.745059+0000", 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00076208500000000002 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.810 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.766265+0000", 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021205946 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.766343+0000", 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7667e-05 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.766435+0000", 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.1440000000000005e-05 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.811 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772041+0000", 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056062810000000003 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772962+0000", 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00092099399999999996 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.812 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:129495 11.11 11:8de0738e:::benchmark_data_smithi012_404402_object16712:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:56.752945+0000", 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.024754368999999998, 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025228922000000001, 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "delayed", 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 129495 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:56.813 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.752945+0000", 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.752945+0000", 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:56.814 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.752943+0000", 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.752951+0000", 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9300000000000003e-06 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:56.815 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.752971+0000", 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0108000000000001e-05 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.752979+0000", 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7069999999999992e-06 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.757127+0000", 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0041480859999999996 2023-10-09T10:25:56.816 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.757951+0000", 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082440200000000001 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:129475 11.1 11:83452ba5:::benchmark_data_smithi012_404402_object16692:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:56.725731+0000", 2023-10-09T10:25:56.817 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.051968281999999998, 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.052514193000000001, 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 129475 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:25:56.818 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.725731+0000", 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.725731+0000", 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.725729+0000", 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.819 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.725740+0000", 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1472e-05 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.725757+0000", 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6966e-05 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.725769+0000", 2023-10-09T10:25:56.820 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.165e-05 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.739191+0000", 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01342199 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.741664+0000", 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024736469999999998 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.821 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.766832+0000", 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025167513999999998 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.766932+0000", 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9759000000000005e-05 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.767091+0000", 2023-10-09T10:25:56.822 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015960799999999999 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.772978+0000", 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058866819999999999 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:56.776588+0000", 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0036095060000000002 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:25:56.823 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.824 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:25:56.824 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:25:56.824 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 8 2023-10-09T10:25:56.824 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:25:56.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:56.825 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:25:56.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:25:56.926 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:57.012 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:57.129 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:57.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:57.241 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:57.290 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:57.349 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:57.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:57.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:57.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:57 smithi012 ceph-mon[147926]: pgmap v1475: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 47577/293562 objects degraded (16.207%); 26851/293562 objects misplaced (9.147%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:25:57.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:57 smithi012 ceph-mon[152263]: pgmap v1475: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 47577/293562 objects degraded (16.207%); 26851/293562 objects misplaced (9.147%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:25:57.457 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:57.532 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:57.578 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:57.581 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:57 smithi110 ceph-mon[136032]: pgmap v1475: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 47577/293562 objects degraded (16.207%); 26851/293562 objects misplaced (9.147%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:25:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:57.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:57.683 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:25:57.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:25:57.732 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:57.803 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:25:57.805 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:57.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:57.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:25:57.908 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:25:57.916 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:25:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:25:57] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-10-09T10:25:57.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:25:57.962 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:58.007 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:25:58.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:58.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:25:58.114 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:25:58.247 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:58.317 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:25:58.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:58.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:25:58.428 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:58.433 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:25:58.466 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:25:58.502 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:25:58.541 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:25:58.549 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:58.588 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:25:58.629 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:25:58.666 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:25:58.699 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:25:58.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:25:58.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:25:58.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:25:58.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:25:58.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:25:58.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:25:58.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:25:58.708 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:25:59.179 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:59.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:59.289 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:25:59.322 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:59.367 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:25:59.513 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:25:59.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:25:59.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:25:59.521 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:59 smithi012 ceph-mon[147926]: pgmap v1476: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 556 op/s; 47577/293562 objects degraded (16.207%); 26851/293562 objects misplaced (9.147%); 13 MiB/s, 205 objects/s recovering 2023-10-09T10:25:59.521 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1078139661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:59.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:59 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 47577/293562 objects degraded (16.207%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:59.522 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:25:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:59.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:59 smithi012 ceph-mon[152263]: pgmap v1476: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 556 op/s; 47577/293562 objects degraded (16.207%); 26851/293562 objects misplaced (9.147%); 13 MiB/s, 205 objects/s recovering 2023-10-09T10:25:59.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1078139661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:59.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:59 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 47577/293562 objects degraded (16.207%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:59.522 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:25:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:59.568 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:25:59.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:25:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:59 smithi110 ceph-mon[136032]: pgmap v1476: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 556 op/s; 47577/293562 objects degraded (16.207%); 26851/293562 objects misplaced (9.147%); 13 MiB/s, 205 objects/s recovering 2023-10-09T10:25:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1078139661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:25:59.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:59 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 47577/293562 objects degraded (16.207%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:25:59.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:25:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:25:59.704 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:00.130 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:00.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:00.236 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:00.311 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:00.518 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:00.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:00.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:00.625 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:00.689 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:00.753 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:00.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:00.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:00.858 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:00.893 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:26:00.902 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:00.907 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:26:00.979 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:00.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:01.086 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:01.173 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:01.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:26:01.374 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:01 smithi110 ceph-mon[136032]: pgmap v1477: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 639 op/s; 47009/288144 objects degraded (16.314%); 26397/288144 objects misplaced (9.161%); 13 MiB/s, 210 objects/s recovering 2023-10-09T10:26:01.375 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:01 smithi012 ceph-mon[147926]: pgmap v1477: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 639 op/s; 47009/288144 objects degraded (16.314%); 26397/288144 objects misplaced (9.161%); 13 MiB/s, 210 objects/s recovering 2023-10-09T10:26:01.375 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:01 smithi012 ceph-mon[152263]: pgmap v1477: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 639 op/s; 47009/288144 objects degraded (16.314%); 26397/288144 objects misplaced (9.161%); 13 MiB/s, 210 objects/s recovering 2023-10-09T10:26:01.447 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:01.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:01.516 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1477,"stamp":"2023-10-09T10:25:59.791534+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6293460920,"num_objects":96048,"num_object_clones":0,"num_object_copies":288144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47009,"num_objects_misplaced":26397,"num_objects_unfound":0,"num_objects_dirty":96048,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":115182,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10114,"num_bytes_recovered":655329020,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132396,"ondisk_log_size":132396,"up":297,"acting":282,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":24615528,"kb_used_data":19018744,"kb_used_omap":163,"kb_used_meta":5596444,"kb_avail":537855384,"statfs":{"total":575970213888,"available":550763913216,"internally_reserved":0,"allocated":19475193856,"data_stored":19466233804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":5730759667},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,3,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-419037184,"num_objects":-6394,"num_object_clones":0,"num_object_copies":-19182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-7381,"num_objects_misplaced":2513,"num_objects_unfound":0,"num_objects_dirty":-6394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6394,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2104,"num_bytes_recovered":138347040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004292"},"pg_stats":[{"pgid":"11.12","version":"529'5071","reported_seq":5133,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:57.757441+0000","last_change":"2023-10-09T10:25:09.709209+0000","last_active":"2023-10-09T10:25:57.757441+0000","last_peered":"2023-10-09T10:25:57.757441+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:25:57.757441+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5071,"log_dups_size":0,"ondisk_log_size":5071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199262208,"num_objects":3041,"num_object_clones":0,"num_object_copies":9123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3119,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4123,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-78},{"shards":"0,2,5","objects":2578}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":122,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.283855+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:25:49.283855+0000","last_peered":"2023-10-09T10:25:49.283855+0000","last_clean":"2023-10-09T10:25:49.283855+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:25:49.283855+0000","last_undegraded":"2023-10-09T10:25:49.283855+0000","last_fullsized":"2023-10-09T10:25:49.283855+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":483,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284282+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:25:49.284282+0000","last_peered":"2023-10-09T10:25:49.284282+0000","last_clean":"2023-10-09T10:25:49.284282+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:25:49.284282+0000","last_undegraded":"2023-10-09T10:25:49.284282+0000","last_fullsized":"2023-10-09T10:25:49.284282+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"529'5010","reported_seq":6683,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:57.759956+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:25:57.759956+0000","last_peered":"2023-10-09T10:25:57.759956+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:25:57.759956+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196411392,"num_objects":2997,"num_object_clones":0,"num_object_copies":8991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4305,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4065,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":534,"num_bytes_recovered":34996224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1730},{"shards":"0,2,6","objects":845},{"shards":"0,6","objects":422}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":560,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.283902+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:25:49.283902+0000","last_peered":"2023-10-09T10:25:49.283902+0000","last_clean":"2023-10-09T10:25:49.283902+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:25:49.283902+0000","last_undegraded":"2023-10-09T10:25:49.283902+0000","last_fullsized":"2023-10-09T10:25:49.283902+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":453,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.248515+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:25:49.248515+0000","last_peered":"2023-10-09T10:25:49.248515+0000","last_clean":"2023-10-09T10:25:49.248515+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:25:49.248515+0000","last_undegraded":"2023-10-09T10:25:49.248515+0000","last_fullsized":"2023-10-09T10:25:49.248515+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"529'4945","reported_seq":9484,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:57.554484+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:25:57.554484+0000","last_peered":"2023-10-09T10:25:57.554484+0000","last_clean":"2023-10-09T10:25:57.554484+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:25:57.554484+0000","last_undegraded":"2023-10-09T10:25:57.554484+0000","last_fullsized":"2023-10-09T10:25:57.554484+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4945,"log_dups_size":0,"ondisk_log_size":4945,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194871296,"num_objects":2974,"num_object_clones":0,"num_object_copies":8922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4008,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":24,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.249409+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:25:49.249409+0000","last_peered":"2023-10-09T10:25:49.249409+0000","last_clean":"2023-10-09T10:25:49.249409+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:25:49.249409+0000","last_undegraded":"2023-10-09T10:25:49.249409+0000","last_fullsized":"2023-10-09T10:25:49.249409+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":105,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.247342+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:25:49.247342+0000","last_peered":"2023-10-09T10:25:49.247342+0000","last_clean":"2023-10-09T10:25:49.247342+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:25:49.247342+0000","last_undegraded":"2023-10-09T10:25:49.247342+0000","last_fullsized":"2023-10-09T10:25:49.247342+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":487,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.247712+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:25:49.247712+0000","last_peered":"2023-10-09T10:25:49.247712+0000","last_clean":"2023-10-09T10:25:49.247712+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:25:49.247712+0000","last_undegraded":"2023-10-09T10:25:49.247712+0000","last_fullsized":"2023-10-09T10:25:49.247712+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":436,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.243407+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:25:49.243407+0000","last_peered":"2023-10-09T10:25:49.243407+0000","last_clean":"2023-10-09T10:25:49.243407+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:25:49.243407+0000","last_undegraded":"2023-10-09T10:25:49.243407+0000","last_fullsized":"2023-10-09T10:25:49.243407+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":203,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.255777+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:25:48.255777+0000","last_peered":"2023-10-09T10:25:48.255777+0000","last_clean":"2023-10-09T10:25:48.255777+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:25:48.255777+0000","last_undegraded":"2023-10-09T10:25:48.255777+0000","last_fullsized":"2023-10-09T10:25:48.255777+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":436,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.244525+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:25:49.244525+0000","last_peered":"2023-10-09T10:25:49.244525+0000","last_clean":"2023-10-09T10:25:49.244525+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:25:49.244525+0000","last_undegraded":"2023-10-09T10:25:49.244525+0000","last_fullsized":"2023-10-09T10:25:49.244525+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":134,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284408+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:25:49.284408+0000","last_peered":"2023-10-09T10:25:49.284408+0000","last_clean":"2023-10-09T10:25:49.284408+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:25:49.284408+0000","last_undegraded":"2023-10-09T10:25:49.284408+0000","last_fullsized":"2023-10-09T10:25:49.284408+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":24,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.246382+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:25:49.246382+0000","last_peered":"2023-10-09T10:25:49.246382+0000","last_clean":"2023-10-09T10:25:49.246382+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:25:49.246382+0000","last_undegraded":"2023-10-09T10:25:49.246382+0000","last_fullsized":"2023-10-09T10:25:49.246382+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":180,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284572+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:25:49.284572+0000","last_peered":"2023-10-09T10:25:49.284572+0000","last_clean":"2023-10-09T10:25:49.284572+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:25:49.284572+0000","last_undegraded":"2023-10-09T10:25:49.284572+0000","last_fullsized":"2023-10-09T10:25:49.284572+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":140,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284404+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:25:49.284404+0000","last_peered":"2023-10-09T10:25:49.284404+0000","last_clean":"2023-10-09T10:25:49.284404+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:25:49.284404+0000","last_undegraded":"2023-10-09T10:25:49.284404+0000","last_fullsized":"2023-10-09T10:25:49.284404+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":100,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284090+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:25:49.284090+0000","last_peered":"2023-10-09T10:25:49.284090+0000","last_clean":"2023-10-09T10:25:49.284090+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:25:49.284090+0000","last_undegraded":"2023-10-09T10:25:49.284090+0000","last_fullsized":"2023-10-09T10:25:49.284090+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":53,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.256574+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:25:48.256574+0000","last_peered":"2023-10-09T10:25:48.256574+0000","last_clean":"2023-10-09T10:25:48.256574+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:25:48.256574+0000","last_undegraded":"2023-10-09T10:25:48.256574+0000","last_fullsized":"2023-10-09T10:25:48.256574+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":452,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:52.797208+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:25:52.797208+0000","last_peered":"2023-10-09T10:25:52.797208+0000","last_clean":"2023-10-09T10:25:52.797208+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:25:52.797208+0000","last_undegraded":"2023-10-09T10:25:52.797208+0000","last_fullsized":"2023-10-09T10:25:52.797208+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":194,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.262644+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:25:48.262644+0000","last_peered":"2023-10-09T10:25:48.262644+0000","last_clean":"2023-10-09T10:25:48.262644+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:25:48.262644+0000","last_undegraded":"2023-10-09T10:25:48.262644+0000","last_fullsized":"2023-10-09T10:25:48.262644+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"528'4984","reported_seq":5779,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:54.963270+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:25:54.963270+0000","last_peered":"2023-10-09T10:25:54.963270+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:25:54.963270+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4984,"log_dups_size":0,"ondisk_log_size":4984,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199524352,"num_objects":3045,"num_object_clones":0,"num_object_copies":9135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":3045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4061,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-106},{"shards":"3,5,6","objects":843}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":140,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284030+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:25:49.284030+0000","last_peered":"2023-10-09T10:25:49.284030+0000","last_clean":"2023-10-09T10:25:49.284030+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:25:49.284030+0000","last_undegraded":"2023-10-09T10:25:49.284030+0000","last_fullsized":"2023-10-09T10:25:49.284030+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":24,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.248200+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:25:49.248200+0000","last_peered":"2023-10-09T10:25:49.248200+0000","last_clean":"2023-10-09T10:25:49.248200+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:25:49.248200+0000","last_undegraded":"2023-10-09T10:25:49.248200+0000","last_fullsized":"2023-10-09T10:25:49.248200+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"528'5001","reported_seq":7020,"reported_epoch":528,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:25:54.969916+0000","last_change":"2023-10-09T10:25:48.137669+0000","last_active":"2023-10-09T10:25:54.969916+0000","last_peered":"2023-10-09T10:25:54.969916+0000","last_clean":"2023-10-09T10:24:39.576387+0000","last_became_active":"2023-10-09T10:25:09.696470+0000","last_became_peered":"2023-10-09T10:25:09.696470+0000","last_unstale":"2023-10-09T10:25:54.969916+0000","last_undegraded":"2023-10-09T10:25:54.969916+0000","last_fullsized":"2023-10-09T10:25:09.436538+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5001,"log_dups_size":0,"ondisk_log_size":5001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200507392,"num_objects":3060,"num_object_clones":0,"num_object_copies":9180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4050,"num_objects_unfound":0,"num_objects_dirty":3060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4036,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":650,"num_bytes_recovered":42598400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1628},{"shards":"2,5,6","objects":794},{"shards":"5,6","objects":638}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":34,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.251496+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:25:49.251496+0000","last_peered":"2023-10-09T10:25:49.251496+0000","last_clean":"2023-10-09T10:25:49.251496+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:25:49.251496+0000","last_undegraded":"2023-10-09T10:25:49.251496+0000","last_fullsized":"2023-10-09T10:25:49.251496+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":27,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.244376+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:25:49.244376+0000","last_peered":"2023-10-09T10:25:49.244376+0000","last_clean":"2023-10-09T10:25:49.244376+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:25:49.244376+0000","last_undegraded":"2023-10-09T10:25:49.244376+0000","last_fullsized":"2023-10-09T10:25:49.244376+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"528'5057","reported_seq":5113,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:54.896657+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:25:54.896657+0000","last_peered":"2023-10-09T10:25:54.896657+0000","last_clean":"2023-10-09T10:25:54.896657+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:25:54.896657+0000","last_undegraded":"2023-10-09T10:25:54.896657+0000","last_fullsized":"2023-10-09T10:25:54.896657+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":5057,"log_dups_size":0,"ondisk_log_size":5057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203096064,"num_objects":3099,"num_object_clones":0,"num_object_copies":9297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1769,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":30,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284682+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:25:49.284682+0000","last_peered":"2023-10-09T10:25:49.284682+0000","last_clean":"2023-10-09T10:25:49.284682+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:25:49.284682+0000","last_undegraded":"2023-10-09T10:25:49.284682+0000","last_fullsized":"2023-10-09T10:25:49.284682+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":203,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.261320+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:25:48.261320+0000","last_peered":"2023-10-09T10:25:48.261320+0000","last_clean":"2023-10-09T10:25:48.261320+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:25:48.261320+0000","last_undegraded":"2023-10-09T10:25:48.261320+0000","last_fullsized":"2023-10-09T10:25:48.261320+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"529'4839","reported_seq":6110,"reported_epoch":529,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:59.423012+0000","last_change":"2023-10-09T10:25:48.142610+0000","last_active":"2023-10-09T10:25:59.423012+0000","last_peered":"2023-10-09T10:25:59.423012+0000","last_clean":"2023-10-09T10:25:11.561932+0000","last_became_active":"2023-10-09T10:25:13.746279+0000","last_became_peered":"2023-10-09T10:25:13.746279+0000","last_unstale":"2023-10-09T10:25:59.423012+0000","last_undegraded":"2023-10-09T10:25:13.623380+0000","last_fullsized":"2023-10-09T10:25:13.604555+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":4839,"log_dups_size":0,"ondisk_log_size":4839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:39.267545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184942592,"num_objects":2822,"num_object_clones":0,"num_object_copies":8466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":343,"num_objects_misplaced":2064,"num_objects_unfound":0,"num_objects_dirty":2822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2079,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":605,"num_bytes_recovered":39649280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":415},{"shards":"2,4,5","objects":2064}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":156,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284899+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:25:49.284899+0000","last_peered":"2023-10-09T10:25:49.284899+0000","last_clean":"2023-10-09T10:25:49.284899+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:25:49.284899+0000","last_undegraded":"2023-10-09T10:25:49.284899+0000","last_fullsized":"2023-10-09T10:25:49.284899+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":143,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284110+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:25:49.284110+0000","last_peered":"2023-10-09T10:25:49.284110+0000","last_clean":"2023-10-09T10:25:49.284110+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:25:49.284110+0000","last_undegraded":"2023-10-09T10:25:49.284110+0000","last_fullsized":"2023-10-09T10:25:49.284110+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"528'4972","reported_seq":6528,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:54.870070+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:25:54.870070+0000","last_peered":"2023-10-09T10:25:54.870070+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:25:54.870070+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4972,"log_dups_size":0,"ondisk_log_size":4972,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199065600,"num_objects":3037,"num_object_clones":0,"num_object_copies":9111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4027,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":495,"num_bytes_recovered":32440320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1824},{"shards":"0,5,6","objects":813},{"shards":"5,6","objects":400}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":219,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.261624+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:25:48.261624+0000","last_peered":"2023-10-09T10:25:48.261624+0000","last_clean":"2023-10-09T10:25:48.261624+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:25:48.261624+0000","last_undegraded":"2023-10-09T10:25:48.261624+0000","last_fullsized":"2023-10-09T10:25:48.261624+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.256559+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:25:48.256559+0000","last_peered":"2023-10-09T10:25:48.256559+0000","last_clean":"2023-10-09T10:25:48.256559+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:25:48.256559+0000","last_undegraded":"2023-10-09T10:25:48.256559+0000","last_fullsized":"2023-10-09T10:25:48.256559+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"528'4934","reported_seq":9115,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:54.888594+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:25:54.888594+0000","last_peered":"2023-10-09T10:25:54.888594+0000","last_clean":"2023-10-09T10:25:54.888594+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:25:54.888594+0000","last_undegraded":"2023-10-09T10:25:54.888594+0000","last_fullsized":"2023-10-09T10:25:54.888594+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199426048,"num_objects":3043,"num_object_clones":0,"num_object_copies":9129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2020,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":203,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.255162+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:25:48.255162+0000","last_peered":"2023-10-09T10:25:48.255162+0000","last_clean":"2023-10-09T10:25:48.255162+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:25:48.255162+0000","last_undegraded":"2023-10-09T10:25:48.255162+0000","last_fullsized":"2023-10-09T10:25:48.255162+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":438,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.246347+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:25:49.246347+0000","last_peered":"2023-10-09T10:25:49.246347+0000","last_clean":"2023-10-09T10:25:49.246347+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:25:49.246347+0000","last_undegraded":"2023-10-09T10:25:49.246347+0000","last_fullsized":"2023-10-09T10:25:49.246347+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"528'4933","reported_seq":6218,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:54.837320+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:25:54.837320+0000","last_peered":"2023-10-09T10:25:54.837320+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:25:54.837320+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4933,"log_dups_size":0,"ondisk_log_size":4933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197033984,"num_objects":3006,"num_object_clones":0,"num_object_copies":9018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4012,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1629},{"shards":"2,5","objects":1377}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":139,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.246342+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:25:49.246342+0000","last_peered":"2023-10-09T10:25:49.246342+0000","last_clean":"2023-10-09T10:25:49.246342+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:25:49.246342+0000","last_undegraded":"2023-10-09T10:25:49.246342+0000","last_fullsized":"2023-10-09T10:25:49.246342+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":140,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284475+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:25:49.284475+0000","last_peered":"2023-10-09T10:25:49.284475+0000","last_clean":"2023-10-09T10:25:49.284475+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:25:49.284475+0000","last_undegraded":"2023-10-09T10:25:49.284475+0000","last_fullsized":"2023-10-09T10:25:49.284475+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"529'4999","reported_seq":5114,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:57.734977+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:25:57.734977+0000","last_peered":"2023-10-09T10:25:57.734977+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:25:57.734977+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":4999,"log_dups_size":0,"ondisk_log_size":4999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193724416,"num_objects":2956,"num_object_clones":0,"num_object_copies":8868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2111,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-102},{"shards":"0,2,5","objects":2508}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":140,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.283801+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:25:49.283801+0000","last_peered":"2023-10-09T10:25:49.283801+0000","last_clean":"2023-10-09T10:25:49.283801+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:25:49.283801+0000","last_undegraded":"2023-10-09T10:25:49.283801+0000","last_fullsized":"2023-10-09T10:25:49.283801+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":106,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284314+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:25:49.284314+0000","last_peered":"2023-10-09T10:25:49.284314+0000","last_clean":"2023-10-09T10:25:49.284314+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:25:49.284314+0000","last_undegraded":"2023-10-09T10:25:49.284314+0000","last_fullsized":"2023-10-09T10:25:49.284314+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"529'5013","reported_seq":6693,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:57.798721+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:25:57.798721+0000","last_peered":"2023-10-09T10:25:57.798721+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:25:57.798721+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5013,"log_dups_size":0,"ondisk_log_size":5013,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196345856,"num_objects":2996,"num_object_clones":0,"num_object_copies":8988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4068,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":536,"num_bytes_recovered":35127296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1717},{"shards":"0,2,4,6","objects":1},{"shards":"0,2,6","objects":806},{"shards":"0,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":122,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284264+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:25:49.284264+0000","last_peered":"2023-10-09T10:25:49.284264+0000","last_clean":"2023-10-09T10:25:49.284264+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:25:49.284264+0000","last_undegraded":"2023-10-09T10:25:49.284264+0000","last_fullsized":"2023-10-09T10:25:49.284264+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":149,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284718+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:25:49.284718+0000","last_peered":"2023-10-09T10:25:49.284718+0000","last_clean":"2023-10-09T10:25:49.284718+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:25:49.284718+0000","last_undegraded":"2023-10-09T10:25:49.284718+0000","last_fullsized":"2023-10-09T10:25:49.284718+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"529'4971","reported_seq":5401,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:57.683931+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:25:57.683931+0000","last_peered":"2023-10-09T10:25:57.683931+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:25:57.683931+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":4971,"log_dups_size":0,"ondisk_log_size":4971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192872448,"num_objects":2943,"num_object_clones":0,"num_object_copies":8829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1864,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1689},{"shards":"0,3","objects":862}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":480,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.261267+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:25:48.261267+0000","last_peered":"2023-10-09T10:25:48.261267+0000","last_clean":"2023-10-09T10:25:48.261267+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:25:48.261267+0000","last_undegraded":"2023-10-09T10:25:48.261267+0000","last_fullsized":"2023-10-09T10:25:48.261267+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":543,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284827+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:25:49.284827+0000","last_peered":"2023-10-09T10:25:49.284827+0000","last_clean":"2023-10-09T10:25:49.284827+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:25:49.284827+0000","last_undegraded":"2023-10-09T10:25:49.284827+0000","last_fullsized":"2023-10-09T10:25:49.284827+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"529'8121","reported_seq":8199,"reported_epoch":529,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:25:58.119581+0000","last_change":"2023-10-09T10:25:17.929569+0000","last_active":"2023-10-09T10:25:58.119581+0000","last_peered":"2023-10-09T10:25:58.119581+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:25:58.119581+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"503'2200","ondisk_log_start":"503'2200","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5921,"log_dups_size":0,"ondisk_log_size":5921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391512088,"num_objects":5975,"num_object_clones":0,"num_object_copies":17925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5972,"num_objects_misplaced":5975,"num_objects_unfound":0,"num_objects_dirty":5975,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8121,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":5975}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":249,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.247881+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:25:49.247881+0000","last_peered":"2023-10-09T10:25:49.247881+0000","last_clean":"2023-10-09T10:25:49.247881+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:25:49.247881+0000","last_undegraded":"2023-10-09T10:25:49.247881+0000","last_fullsized":"2023-10-09T10:25:49.247881+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":212,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.262561+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:25:48.262561+0000","last_peered":"2023-10-09T10:25:48.262561+0000","last_clean":"2023-10-09T10:25:48.262561+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:25:48.262561+0000","last_undegraded":"2023-10-09T10:25:48.262561+0000","last_fullsized":"2023-10-09T10:25:48.262561+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":492,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.256635+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:25:48.256635+0000","last_peered":"2023-10-09T10:25:48.256635+0000","last_clean":"2023-10-09T10:25:48.256635+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:25:48.256635+0000","last_undegraded":"2023-10-09T10:25:48.256635+0000","last_fullsized":"2023-10-09T10:25:48.256635+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"529'8211","reported_seq":8255,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:57.746354+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:25:57.746354+0000","last_peered":"2023-10-09T10:25:57.746354+0000","last_clean":"2023-10-09T10:25:57.746354+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:25:57.746354+0000","last_undegraded":"2023-10-09T10:25:57.746354+0000","last_fullsized":"2023-10-09T10:25:57.746354+0000","mapping_epoch":494,"log_start":"503'3206","ondisk_log_start":"503'3206","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397606912,"num_objects":6067,"num_object_clones":0,"num_object_copies":18201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8211,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":212,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.256471+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:25:48.256471+0000","last_peered":"2023-10-09T10:25:48.256471+0000","last_clean":"2023-10-09T10:25:48.256471+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:25:48.256471+0000","last_undegraded":"2023-10-09T10:25:48.256471+0000","last_fullsized":"2023-10-09T10:25:48.256471+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":131,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284325+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:25:49.284325+0000","last_peered":"2023-10-09T10:25:49.284325+0000","last_clean":"2023-10-09T10:25:49.284325+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:25:49.284325+0000","last_undegraded":"2023-10-09T10:25:49.284325+0000","last_fullsized":"2023-10-09T10:25:49.284325+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":441,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.243652+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:25:49.243652+0000","last_peered":"2023-10-09T10:25:49.243652+0000","last_clean":"2023-10-09T10:25:49.243652+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:25:49.243652+0000","last_undegraded":"2023-10-09T10:25:49.243652+0000","last_fullsized":"2023-10-09T10:25:49.243652+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"528'4994","reported_seq":6637,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:54.974406+0000","last_change":"2023-10-09T10:25:09.703037+0000","last_active":"2023-10-09T10:25:54.974406+0000","last_peered":"2023-10-09T10:25:54.974406+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:25:09.696864+0000","last_became_peered":"2023-10-09T10:25:09.696864+0000","last_unstale":"2023-10-09T10:25:54.974406+0000","last_undegraded":"2023-10-09T10:25:09.545735+0000","last_fullsized":"2023-10-09T10:25:09.430540+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4994,"log_dups_size":0,"ondisk_log_size":4994,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195330048,"num_objects":2980,"num_object_clones":0,"num_object_copies":8940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4334,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4028,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":525,"num_bytes_recovered":34406400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1765},{"shards":"2,5,6","objects":804},{"shards":"5,6","objects":411}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":479,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.250973+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:25:49.250973+0000","last_peered":"2023-10-09T10:25:49.250973+0000","last_clean":"2023-10-09T10:25:49.250973+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:25:49.250973+0000","last_undegraded":"2023-10-09T10:25:49.250973+0000","last_fullsized":"2023-10-09T10:25:49.250973+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":96,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.251406+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:25:49.251406+0000","last_peered":"2023-10-09T10:25:49.251406+0000","last_clean":"2023-10-09T10:25:49.251406+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:25:49.251406+0000","last_undegraded":"2023-10-09T10:25:49.251406+0000","last_fullsized":"2023-10-09T10:25:49.251406+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":541,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.256333+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:25:48.256333+0000","last_peered":"2023-10-09T10:25:48.256333+0000","last_clean":"2023-10-09T10:25:48.256333+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:25:48.256333+0000","last_undegraded":"2023-10-09T10:25:48.256333+0000","last_fullsized":"2023-10-09T10:25:48.256333+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"528'8075","reported_seq":8138,"reported_epoch":528,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:25:54.944923+0000","last_change":"2023-10-09T10:24:41.858672+0000","last_active":"2023-10-09T10:25:54.944923+0000","last_peered":"2023-10-09T10:25:54.944923+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:25:54.944923+0000","last_undegraded":"2023-10-09T10:25:54.944923+0000","last_fullsized":"2023-10-09T10:25:54.944923+0000","mapping_epoch":507,"log_start":"503'3200","ondisk_log_start":"503'3200","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4875,"log_dups_size":0,"ondisk_log_size":4875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395640832,"num_objects":6037,"num_object_clones":0,"num_object_copies":18111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6037,"num_objects_unfound":0,"num_objects_dirty":6037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8075,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":6037}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":182,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.257696+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:25:48.257696+0000","last_peered":"2023-10-09T10:25:48.257696+0000","last_clean":"2023-10-09T10:25:48.257696+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:25:48.257696+0000","last_undegraded":"2023-10-09T10:25:48.257696+0000","last_fullsized":"2023-10-09T10:25:48.257696+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":722,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:52.797046+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:25:52.797046+0000","last_peered":"2023-10-09T10:25:52.797046+0000","last_clean":"2023-10-09T10:25:52.797046+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:25:52.797046+0000","last_undegraded":"2023-10-09T10:25:52.797046+0000","last_fullsized":"2023-10-09T10:25:52.797046+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":230,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.256251+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:25:48.256251+0000","last_peered":"2023-10-09T10:25:48.256251+0000","last_clean":"2023-10-09T10:25:48.256251+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:25:48.256251+0000","last_undegraded":"2023-10-09T10:25:48.256251+0000","last_fullsized":"2023-10-09T10:25:48.256251+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":140,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284365+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:25:49.284365+0000","last_peered":"2023-10-09T10:25:49.284365+0000","last_clean":"2023-10-09T10:25:49.284365+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:25:49.284365+0000","last_undegraded":"2023-10-09T10:25:49.284365+0000","last_fullsized":"2023-10-09T10:25:49.284365+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"528'5010","reported_seq":6430,"reported_epoch":528,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:25:54.977412+0000","last_change":"2023-10-09T10:25:13.759251+0000","last_active":"2023-10-09T10:25:54.977412+0000","last_peered":"2023-10-09T10:25:54.977412+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:25:54.977412+0000","last_undegraded":"2023-10-09T10:25:13.707357+0000","last_fullsized":"2023-10-09T10:25:54.977412+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206602240,"num_objects":3152,"num_object_clones":0,"num_object_copies":9456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1838,"num_objects_misplaced":2789,"num_objects_unfound":0,"num_objects_dirty":3152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4086,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":468,"num_bytes_recovered":30670848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1838},{"shards":"3,5,6","objects":363}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":213,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.251571+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:25:49.251571+0000","last_peered":"2023-10-09T10:25:49.251571+0000","last_clean":"2023-10-09T10:25:49.251571+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:25:49.251571+0000","last_undegraded":"2023-10-09T10:25:49.251571+0000","last_fullsized":"2023-10-09T10:25:49.251571+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":473,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.251135+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:25:49.251135+0000","last_peered":"2023-10-09T10:25:49.251135+0000","last_clean":"2023-10-09T10:25:49.251135+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:25:49.251135+0000","last_undegraded":"2023-10-09T10:25:49.251135+0000","last_fullsized":"2023-10-09T10:25:49.251135+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":24,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.247982+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:25:49.247982+0000","last_peered":"2023-10-09T10:25:49.247982+0000","last_clean":"2023-10-09T10:25:49.247982+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:25:49.247982+0000","last_undegraded":"2023-10-09T10:25:49.247982+0000","last_fullsized":"2023-10-09T10:25:49.247982+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"529'8202","reported_seq":8245,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:58.115820+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:25:58.115820+0000","last_peered":"2023-10-09T10:25:58.115820+0000","last_clean":"2023-10-09T10:25:58.115820+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:25:58.115820+0000","last_undegraded":"2023-10-09T10:25:58.115820+0000","last_fullsized":"2023-10-09T10:25:58.115820+0000","mapping_epoch":494,"log_start":"503'2300","ondisk_log_start":"503'2300","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393871360,"num_objects":6010,"num_object_clones":0,"num_object_copies":18030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8202,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":24,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.246901+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:25:49.246901+0000","last_peered":"2023-10-09T10:25:49.246901+0000","last_clean":"2023-10-09T10:25:49.246901+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:25:49.246901+0000","last_undegraded":"2023-10-09T10:25:49.246901+0000","last_fullsized":"2023-10-09T10:25:49.246901+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":509,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.256432+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:25:48.256432+0000","last_peered":"2023-10-09T10:25:48.256432+0000","last_clean":"2023-10-09T10:25:48.256432+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:25:48.256432+0000","last_undegraded":"2023-10-09T10:25:48.256432+0000","last_fullsized":"2023-10-09T10:25:48.256432+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":441,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.246638+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:25:49.246638+0000","last_peered":"2023-10-09T10:25:49.246638+0000","last_clean":"2023-10-09T10:25:49.246638+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:25:49.246638+0000","last_undegraded":"2023-10-09T10:25:49.246638+0000","last_fullsized":"2023-10-09T10:25:49.246638+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"529'7937","reported_seq":7981,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:57.793714+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:25:57.793714+0000","last_peered":"2023-10-09T10:25:57.793714+0000","last_clean":"2023-10-09T10:25:57.793714+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:25:57.793714+0000","last_undegraded":"2023-10-09T10:25:57.793714+0000","last_fullsized":"2023-10-09T10:25:57.793714+0000","mapping_epoch":494,"log_start":"503'2876","ondisk_log_start":"503'2876","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5061,"log_dups_size":0,"ondisk_log_size":5061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380043264,"num_objects":5799,"num_object_clones":0,"num_object_copies":17397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7937,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":329,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.247595+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:25:49.247595+0000","last_peered":"2023-10-09T10:25:49.247595+0000","last_clean":"2023-10-09T10:25:49.247595+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:25:49.247595+0000","last_undegraded":"2023-10-09T10:25:49.247595+0000","last_fullsized":"2023-10-09T10:25:49.247595+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":510,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284622+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:25:49.284622+0000","last_peered":"2023-10-09T10:25:49.284622+0000","last_clean":"2023-10-09T10:25:49.284622+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:25:49.284622+0000","last_undegraded":"2023-10-09T10:25:49.284622+0000","last_fullsized":"2023-10-09T10:25:49.284622+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":24,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.249333+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:25:49.249333+0000","last_peered":"2023-10-09T10:25:49.249333+0000","last_clean":"2023-10-09T10:25:49.249333+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:25:49.249333+0000","last_undegraded":"2023-10-09T10:25:49.249333+0000","last_fullsized":"2023-10-09T10:25:49.249333+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"529'8201","reported_seq":8244,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:57.923412+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:25:57.923412+0000","last_peered":"2023-10-09T10:25:57.923412+0000","last_clean":"2023-10-09T10:25:57.923412+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:25:57.923412+0000","last_undegraded":"2023-10-09T10:25:57.923412+0000","last_fullsized":"2023-10-09T10:25:57.923412+0000","mapping_epoch":494,"log_start":"503'2300","ondisk_log_start":"503'2300","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5901,"log_dups_size":0,"ondisk_log_size":5901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397344768,"num_objects":6063,"num_object_clones":0,"num_object_copies":18189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8201,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":31,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.250705+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:25:48.250705+0000","last_peered":"2023-10-09T10:25:48.250705+0000","last_clean":"2023-10-09T10:25:48.250705+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:25:48.250705+0000","last_undegraded":"2023-10-09T10:25:48.250705+0000","last_fullsized":"2023-10-09T10:25:48.250705+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":28,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.248274+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:25:49.248274+0000","last_peered":"2023-10-09T10:25:49.248274+0000","last_clean":"2023-10-09T10:25:49.248274+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:25:49.248274+0000","last_undegraded":"2023-10-09T10:25:49.248274+0000","last_fullsized":"2023-10-09T10:25:49.248274+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":208,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.254982+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:25:48.254982+0000","last_peered":"2023-10-09T10:25:48.254982+0000","last_clean":"2023-10-09T10:25:48.254982+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:25:48.254982+0000","last_undegraded":"2023-10-09T10:25:48.254982+0000","last_fullsized":"2023-10-09T10:25:48.254982+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"528'4883","reported_seq":4973,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:54.954622+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:25:54.954622+0000","last_peered":"2023-10-09T10:25:54.954622+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:25:54.954622+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4883,"log_dups_size":0,"ondisk_log_size":4883,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194543616,"num_objects":2969,"num_object_clones":0,"num_object_copies":8907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3963,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1733},{"shards":"2,5","objects":753}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":24,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.246537+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:25:49.246537+0000","last_peered":"2023-10-09T10:25:49.246537+0000","last_clean":"2023-10-09T10:25:49.246537+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:25:49.246537+0000","last_undegraded":"2023-10-09T10:25:49.246537+0000","last_fullsized":"2023-10-09T10:25:49.246537+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":33,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.246749+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:25:49.246749+0000","last_peered":"2023-10-09T10:25:49.246749+0000","last_clean":"2023-10-09T10:25:49.246749+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:25:49.246749+0000","last_undegraded":"2023-10-09T10:25:49.246749+0000","last_fullsized":"2023-10-09T10:25:49.246749+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"529'5006","reported_seq":5071,"reported_epoch":529,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:25:57.782374+0000","last_change":"2023-10-09T10:25:08.466144+0000","last_active":"2023-10-09T10:25:57.782374+0000","last_peered":"2023-10-09T10:25:57.782374+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:25:57.782374+0000","last_undegraded":"2023-10-09T10:25:57.782374+0000","last_fullsized":"2023-10-09T10:25:57.782374+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194609152,"num_objects":2970,"num_object_clones":0,"num_object_copies":8910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2331,"num_objects_unfound":0,"num_objects_dirty":2970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4082,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":639}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":258,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.255238+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:25:48.255238+0000","last_peered":"2023-10-09T10:25:48.255238+0000","last_clean":"2023-10-09T10:25:48.255238+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:25:48.255238+0000","last_undegraded":"2023-10-09T10:25:48.255238+0000","last_fullsized":"2023-10-09T10:25:48.255238+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":122,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.284481+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:25:49.284481+0000","last_peered":"2023-10-09T10:25:49.284481+0000","last_clean":"2023-10-09T10:25:49.284481+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:25:49.284481+0000","last_undegraded":"2023-10-09T10:25:49.284481+0000","last_fullsized":"2023-10-09T10:25:49.284481+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"528'4983","reported_seq":6527,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:54.965054+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:25:54.965054+0000","last_peered":"2023-10-09T10:25:54.965054+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:25:54.965054+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4983,"log_dups_size":0,"ondisk_log_size":4983,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200048640,"num_objects":3053,"num_object_clones":0,"num_object_copies":9159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4328,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4039,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":493,"num_bytes_recovered":32309248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1747},{"shards":"0,5,6","objects":833},{"shards":"5,6","objects":472}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":241,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T10:25:48.262396+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:25:48.262396+0000","last_peered":"2023-10-09T10:25:48.262396+0000","last_clean":"2023-10-09T10:25:48.262396+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:25:48.262396+0000","last_undegraded":"2023-10-09T10:25:48.262396+0000","last_fullsized":"2023-10-09T10:25:48.262396+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":351,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.250248+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:25:49.250248+0000","last_peered":"2023-10-09T10:25:49.250248+0000","last_clean":"2023-10-09T10:25:49.250248+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:25:49.250248+0000","last_undegraded":"2023-10-09T10:25:49.250248+0000","last_fullsized":"2023-10-09T10:25:49.250248+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"529'4978","reported_seq":5038,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:25:59.420141+0000","last_change":"2023-10-09T10:25:13.762065+0000","last_active":"2023-10-09T10:25:59.420141+0000","last_peered":"2023-10-09T10:25:59.420141+0000","last_clean":"2023-10-09T10:25:11.551777+0000","last_became_active":"2023-10-09T10:25:13.754797+0000","last_became_peered":"2023-10-09T10:25:13.754797+0000","last_unstale":"2023-10-09T10:25:59.420141+0000","last_undegraded":"2023-10-09T10:25:13.625003+0000","last_fullsized":"2023-10-09T10:25:13.604794+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":4978,"log_dups_size":0,"ondisk_log_size":4978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:58.037975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192544768,"num_objects":2938,"num_object_clones":0,"num_object_copies":8814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2041,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":-199},{"shards":"2,4,5","objects":2557}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":27,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.244394+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:25:49.244394+0000","last_peered":"2023-10-09T10:25:49.244394+0000","last_clean":"2023-10-09T10:25:49.244394+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:25:49.244394+0000","last_undegraded":"2023-10-09T10:25:49.244394+0000","last_fullsized":"2023-10-09T10:25:49.244394+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":581,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.250896+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:25:49.250896+0000","last_peered":"2023-10-09T10:25:49.250896+0000","last_clean":"2023-10-09T10:25:49.250896+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:25:49.250896+0000","last_undegraded":"2023-10-09T10:25:49.250896+0000","last_fullsized":"2023-10-09T10:25:49.250896+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"528'4954","reported_seq":5009,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:54.738808+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:25:54.738808+0000","last_peered":"2023-10-09T10:25:54.738808+0000","last_clean":"2023-10-09T10:25:54.738808+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:25:54.738808+0000","last_undegraded":"2023-10-09T10:25:54.738808+0000","last_fullsized":"2023-10-09T10:25:54.738808+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":4954,"log_dups_size":0,"ondisk_log_size":4954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196083712,"num_objects":2992,"num_object_clones":0,"num_object_copies":8976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1693,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":45,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.247210+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:25:49.247210+0000","last_peered":"2023-10-09T10:25:49.247210+0000","last_clean":"2023-10-09T10:25:49.247210+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:25:49.247210+0000","last_undegraded":"2023-10-09T10:25:49.247210+0000","last_fullsized":"2023-10-09T10:25:49.247210+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":149,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T10:25:49.283960+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:25:49.283960+0000","last_peered":"2023-10-09T10:25:49.283960+0000","last_clean":"2023-10-09T10:25:49.283960+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:25:49.283960+0000","last_undegraded":"2023-10-09T10:25:49.283960+0000","last_fullsized":"2023-10-09T10:25:49.283960+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":6292865048,"num_objects":96024,"num_object_clones":0,"num_object_copies":288072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47009,"num_objects_misplaced":26397,"num_objects_unfound":0,"num_objects_dirty":96024,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114922,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9954,"num_bytes_recovered":652345344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19413147648,"data_stored":19413139504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132202,"ondisk_log_size":132202,"up":78,"acting":63,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430675,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626903,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2278480,"kb_used_data":1379208,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91466672,"statfs":{"total":95995035648,"available":93661872128,"internally_reserved":0,"allocated":1412308992,"data_stored":1411401719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:15 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.3999999999999999,"15min":1.6739999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":4.8099999999999996,"5min":9.4260000000000002,"15min":44.055},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.756,"15min":1.919},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.22},"max":{"1min":3.6760000000000002,"5min":9.2609999999999992,"15min":43.869999999999997},"last":0.44400000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":2.911,"15min":2.2400000000000002},"min":{"1min":0.29399999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.0419999999999998,"5min":155.01400000000001,"15min":155.01400000000001},"last":1.8400000000000001},{"interface":"front","average":{"1min":1.383,"5min":2.9460000000000002,"15min":2.4060000000000001},"min":{"1min":0.33200000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.5709999999999997,"5min":154.71100000000001,"15min":154.71100000000001},"last":0.29999999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:25:15 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":1.7889999999999999,"15min":1.887},"min":{"1min":0.32800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.8460000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":0.309},{"interface":"front","average":{"1min":1.6970000000000001,"5min":1.649,"15min":1.8029999999999999},"min":{"1min":0.36299999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.875,"5min":7.5629999999999997,"15min":14.542999999999999},"last":0.66100000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.619,"15min":1.514},"min":{"1min":0.498,"5min":0.308,"15min":0.308},"max":{"1min":7.681,"5min":7.681,"15min":8.0229999999999997},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.496,"5min":1.927,"15min":1.6220000000000001},"min":{"1min":0.36399999999999999,"5min":0.316,"15min":0.31},"max":{"1min":7.6920000000000002,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.82499999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.881,"5min":3.4460000000000002,"15min":1.8600000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":10.618,"5min":122.041,"15min":122.041},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.8069999999999999,"5min":3.3130000000000002,"15min":1.75},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.9470000000000001,"5min":122.13200000000001,"15min":122.13200000000001},"last":1.7969999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.4620000000000002,"15min":1.976},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.6890000000000001,"5min":121.98999999999999,"15min":121.98999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.95299999999999996,"5min":2.6280000000000001,"15min":1.6200000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":5.1420000000000003,"5min":122.346,"15min":122.346},"last":2.2000000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":3.532,"15min":1.9730000000000001},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":4.9169999999999998,"5min":130.66999999999999,"15min":130.66999999999999},"last":0.373},{"interface":"front","average":{"1min":1.3779999999999999,"5min":3.573,"15min":2.016},"min":{"1min":0.374,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":4.9589999999999996,"5min":130.702,"15min":130.702},"last":0.40400000000000003}]}]},{"osd":1,"up_from":0,"seq":360777253217,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416741,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4553580,"kb_used_data":3601316,"kb_used_omap":28,"kb_used_meta":952099,"kb_avail":89191572,"statfs":{"total":95995035648,"available":91332169728,"internally_reserved":0,"allocated":3687747584,"data_stored":3685698903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":974950224},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:25:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.6200000000000001,"15min":2.0059999999999998},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":17.161999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.585,"5min":1.506,"15min":2.4689999999999999},"min":{"1min":0.218,"5min":0.214,"15min":0.192},"max":{"1min":10.786,"5min":17.033000000000001,"15min":56.950000000000003},"last":0.29099999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.9630000000000001,"15min":2.1419999999999999},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.7839999999999998,"5min":47.215000000000003,"15min":53.988999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.99,"5min":2.254,"15min":2.1339999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.173,"5min":47.000999999999998,"15min":54.119},"last":0.60099999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.488,"15min":1.4119999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.702,"5min":16.879999999999999,"15min":56.984999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.5750000000000002,"5min":1.6830000000000001,"15min":1.4450000000000001},"min":{"1min":0.51700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1749999999999998,"5min":8.0020000000000007,"15min":56.475999999999999},"last":1.712}]},{"osd":4,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.869,"15min":2.3159999999999998},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":3.3399999999999999,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.1230000000000002,"15min":2.246},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.32400000000000001},"max":{"1min":8.2479999999999993,"5min":28.331,"15min":56.378},"last":1.7709999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.1259999999999999,"15min":3.129},"min":{"1min":0.46800000000000003,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":6.923,"5min":11.775,"15min":229.589},"last":90.412000000000006},{"interface":"front","average":{"1min":2.238,"5min":1.8029999999999999,"15min":2.8279999999999998},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":7.032,"5min":8.7530000000000001,"15min":228.88900000000001},"last":90.393000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.919,"15min":2.2320000000000002},"min":{"1min":0.435,"5min":0.379,"15min":0.33700000000000002},"max":{"1min":6.8230000000000004,"5min":8.7810000000000006,"15min":54.482999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.248,"5min":2.0859999999999999,"15min":2.6349999999999998},"min":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.7160000000000002,"5min":16.954999999999998,"15min":57.753},"last":0.502}]},{"osd":7,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.9930000000000001,"15min":2.391},"min":{"1min":0.54500000000000004,"5min":0.38600000000000001,"15min":0.34999999999999998},"max":{"1min":6.6079999999999997,"5min":8.0329999999999995,"15min":56.411999999999999},"last":1.7969999999999999},{"interface":"front","average":{"1min":2.0070000000000001,"5min":2.0640000000000001,"15min":2.258},"min":{"1min":0.47099999999999997,"5min":0.38600000000000001,"15min":0.36499999999999999},"max":{"1min":6.891,"5min":11.175000000000001,"15min":54.451999999999998},"last":0.54200000000000004}]}]},{"osd":2,"up_from":217,"seq":932007903477,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4532004,"kb_used_data":3705500,"kb_used_omap":83,"kb_used_meta":826412,"kb_avail":89213148,"statfs":{"total":95995035648,"available":91354263552,"internally_reserved":0,"allocated":3794432000,"data_stored":3792575459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":846246383},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.778,"15min":2.0600000000000001},"min":{"1min":0.216,"5min":0.20899999999999999,"15min":0.185},"max":{"1min":5.2480000000000002,"5min":48.990000000000002,"15min":48.990000000000002},"last":3.7930000000000001},{"interface":"front","average":{"1min":1.5089999999999999,"5min":2.0649999999999999,"15min":2.3849999999999998},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.590999999999999,"5min":49.412999999999997,"15min":49.412999999999997},"last":0.45600000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.3089999999999999,"15min":2.0899999999999999},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":5.008,"5min":8.2010000000000005,"15min":34.893999999999998},"last":9.7230000000000008},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.4910000000000001,"15min":2.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":7.8029999999999999,"5min":7.8029999999999999,"15min":34.962000000000003},"last":1.655}]},{"osd":3,"last update":"Mon Oct 9 10:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.4139999999999999,"15min":1.202},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.248},"max":{"1min":9.3119999999999994,"5min":9.3119999999999994,"15min":35.043999999999997},"last":7.202},{"interface":"front","average":{"1min":2.7909999999999999,"5min":1.2829999999999999,"15min":1.3},"min":{"1min":0.35699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.206,"5min":11.206,"15min":50.406999999999996},"last":1.821}]},{"osd":4,"last update":"Mon Oct 9 10:25:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.772,"15min":1.6990000000000001},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.26600000000000001},"max":{"1min":5.6749999999999998,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.8859999999999999,"15min":1.845},"min":{"1min":0.40699999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.2720000000000002,"5min":47.704000000000001,"15min":47.704000000000001},"last":1.871}]},{"osd":5,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.603,"15min":1.992},"min":{"1min":0.39100000000000001,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":3.0329999999999999,"5min":10.448,"15min":57.298000000000002},"last":1.911},{"interface":"front","average":{"1min":1.534,"5min":1.7050000000000001,"15min":2.0310000000000001},"min":{"1min":0.374,"5min":0.31,"15min":0.31},"max":{"1min":5.8810000000000002,"5min":8.1630000000000003,"15min":57.828000000000003},"last":3.8340000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.99,"5min":2.2360000000000002,"15min":2.0329999999999999},"min":{"1min":0.45000000000000001,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":5.5510000000000002,"5min":12.411,"15min":18.943999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.744,"15min":1.95},"min":{"1min":0.442,"5min":0.38300000000000001,"15min":0.36599999999999999},"max":{"1min":3.0950000000000002,"5min":9.8399999999999999,"15min":24.379000000000001},"last":1.9430000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.3370000000000002,"15min":2.2850000000000001},"min":{"1min":0.53500000000000003,"5min":0.41799999999999998,"15min":0.308},"max":{"1min":3.5760000000000001,"5min":13.821,"15min":35.008000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.655,"5min":1.6619999999999999,"15min":2.1160000000000001},"min":{"1min":0.439,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":6.2140000000000004,"5min":10.507,"15min":24.323},"last":1.9950000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488316,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3357792,"kb_used_data":2872984,"kb_used_omap":15,"kb_used_meta":484784,"kb_avail":90387360,"statfs":{"total":95995035648,"available":92556656640,"internally_reserved":0,"allocated":2941935616,"data_stored":2939609786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":496419270},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.95,"15min":1.3160000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":3.2309999999999999,"5min":65.674999999999997,"15min":65.674999999999997},"last":1.6619999999999999},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.97,"15min":1.319},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.19900000000000001},"max":{"1min":4.758,"5min":21.149999999999999,"15min":21.149999999999999},"last":1.546}]},{"osd":1,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":3.1309999999999998,"15min":1.8069999999999999},"min":{"1min":0.246,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.3760000000000003,"5min":105.401,"15min":105.401},"last":0.5},{"interface":"front","average":{"1min":1.7889999999999999,"5min":3.4079999999999999,"15min":1.6619999999999999},"min":{"1min":0.311,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":6.7050000000000001,"5min":104.914,"15min":104.914},"last":9.2140000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.954,"15min":1.883},"min":{"1min":0.253,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":2.2650000000000001,"5min":9.4369999999999994,"15min":9.4369999999999994},"last":9.7780000000000005},{"interface":"front","average":{"1min":1.248,"5min":1.508,"15min":1.3},"min":{"1min":0.313,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":4.4930000000000003,"5min":25.338999999999999,"15min":25.338999999999999},"last":0.46999999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":2.4220000000000002,"15min":1.319},"min":{"1min":0.39100000000000001,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":4.7770000000000001,"5min":21.831,"15min":21.831},"last":1.6399999999999999},{"interface":"front","average":{"1min":1.105,"5min":2.8759999999999999,"15min":1.8169999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.4510000000000001,"5min":22.097999999999999,"15min":22.097999999999999},"last":0.67000000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":1.8819999999999999,"15min":1.1919999999999999},"min":{"1min":0.44,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":6.6710000000000003,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.6779999999999999,"5min":2.4039999999999999,"15min":1.4259999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":4.6600000000000001,"5min":13.394,"15min":13.394},"last":1.5960000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":2.1749999999999998,"15min":1.631},"min":{"1min":0.36199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.617,"5min":19.114000000000001,"15min":19.114000000000001},"last":1.3979999999999999},{"interface":"front","average":{"1min":1.538,"5min":1.958,"15min":1.373},"min":{"1min":0.47099999999999997,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":5.2519999999999998,"5min":25.306999999999999,"15min":25.306999999999999},"last":2.25}]},{"osd":7,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":2.2120000000000002,"15min":1.4239999999999999},"min":{"1min":0.40699999999999997,"5min":0.39900000000000002,"15min":0.38400000000000001},"max":{"1min":4.8109999999999999,"5min":9.7880000000000003,"15min":10.459},"last":1.7689999999999999},{"interface":"front","average":{"1min":1.097,"5min":2.1829999999999998,"15min":1.6970000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.3860000000000001,"5min":19.280000000000001,"15min":19.280000000000001},"last":0.45500000000000002}]}]},{"osd":4,"up_from":266,"seq":1142461300946,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4483040,"kb_used_data":3549912,"kb_used_omap":35,"kb_used_meta":933084,"kb_avail":89262112,"statfs":{"total":95995035648,"available":91404402688,"internally_reserved":0,"allocated":3635109888,"data_stored":3634193054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":955478254},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":3.286,"15min":2.1200000000000001},"min":{"1min":0.23699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.4699999999999998,"5min":97.721999999999994,"15min":97.721999999999994},"last":2.1040000000000001},{"interface":"front","average":{"1min":1.8540000000000001,"5min":4.1630000000000003,"15min":2.8100000000000001},"min":{"1min":0.30199999999999999,"5min":0.252,"15min":0.224},"max":{"1min":5.931,"5min":97.915999999999997,"15min":97.915999999999997},"last":2.1899999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":2.5379999999999998,"15min":2.3889999999999998},"min":{"1min":0.41599999999999998,"5min":0.32300000000000001,"15min":0.24299999999999999},"max":{"1min":5.6630000000000003,"5min":11.32,"15min":71.457999999999998},"last":1.5289999999999999},{"interface":"front","average":{"1min":1.754,"5min":2.714,"15min":2.2029999999999998},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":5.8730000000000002,"5min":49.781999999999996,"15min":71.658000000000001},"last":7.0129999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":2.3279999999999998,"15min":1.8360000000000001},"min":{"1min":0.313,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":6.468,"5min":15.944000000000001,"15min":15.944000000000001},"last":7.0540000000000003},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.875,"15min":1.7450000000000001},"min":{"1min":0.28699999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.8420000000000001,"5min":14.473000000000001,"15min":14.473000000000001},"last":7.1580000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":2.5779999999999998,"15min":1.887},"min":{"1min":0.45100000000000001,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":5.7060000000000004,"5min":12.074,"15min":12.074},"last":7.8259999999999996},{"interface":"front","average":{"1min":2.444,"5min":2.746,"15min":1.6220000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4240000000000004,"5min":16.067,"15min":16.067},"last":1.5620000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.7719999999999998,"15min":2.0579999999999998},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":10.023999999999999,"5min":16.094000000000001,"15min":99.585999999999999},"last":2.0209999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.5819999999999999,"15min":1.7889999999999999},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.335,"5min":49.915999999999997,"15min":99.384},"last":1.0009999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.9530000000000001,"15min":1.3280000000000001},"min":{"1min":0.371,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.7709999999999999,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.92700000000000005},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.9279999999999999,"15min":1.734},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":18.952999999999999,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.39100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":2.9159999999999999,"15min":1.8049999999999999},"min":{"1min":0.33000000000000002,"5min":0.315,"15min":0.307},"max":{"1min":9.3789999999999996,"5min":17.369,"15min":17.369},"last":1.8580000000000001},{"interface":"front","average":{"1min":1.554,"5min":2.9129999999999998,"15min":1.7430000000000001},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":5.9020000000000001,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.40200000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823130,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5410632,"kb_used_data":3909824,"kb_used_omap":0,"kb_used_meta":1500800,"kb_avail":88334520,"statfs":{"total":95995035648,"available":90454548480,"internally_reserved":0,"allocated":4003659776,"data_stored":4002754883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536819200},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.754,"15min":1.9370000000000001},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":16.536999999999999,"5min":22.469999999999999,"15min":22.469999999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.1309999999999998,"15min":1.7270000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":8.9339999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.63200000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:25:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.6339999999999999,"15min":2.2090000000000001},"min":{"1min":0.29599999999999999,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":16.562999999999999,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.8,"5min":1.8759999999999999,"15min":1.8580000000000001},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":8.9529999999999994,"5min":13.143000000000001,"15min":29.065000000000001},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6899999999999999,"5min":2.5680000000000001,"15min":2.1269999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":16.629999999999999,"5min":16.629999999999999,"15min":16.629999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.403,"5min":2.2890000000000001,"15min":1.7270000000000001},"min":{"1min":0.30499999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.8930000000000007,"5min":13.095000000000001,"15min":15.929},"last":1.675}]},{"osd":3,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.2170000000000001,"15min":1.7889999999999999},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":10.362,"5min":10.561999999999999,"15min":10.561999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.0830000000000002,"15min":1.7270000000000001},"min":{"1min":0.41599999999999998,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9130000000000003,"5min":13.624000000000001,"15min":13.624000000000001},"last":0.38400000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:24:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":2.1920000000000002,"15min":1.524},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.24299999999999999},"max":{"1min":4.1159999999999997,"5min":12.023,"15min":12.023},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.081,"15min":1.341},"min":{"1min":0.29899999999999999,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":2.7170000000000001,"5min":11.956,"15min":11.956},"last":0.30599999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.621,"15min":1.794},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":10.664999999999999,"5min":11.686,"15min":12.772},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.4500000000000002,"5min":2.1640000000000001,"15min":1.665},"min":{"1min":0.29699999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.42299999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:25:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.6429999999999998,"15min":1.798},"min":{"1min":0.34000000000000002,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":16.686,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.397,"15min":1.1719999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":8.9719999999999995,"5min":11.994,"15min":11.994},"last":0.66200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3667525632,"data_stored":3667525632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3785752576,"data_stored":3785752576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2934308864,"data_stored":2934308864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3623555072,"data_stored":3623551000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3996782592,"data_stored":3996778520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1405222912,"data_stored":1405222912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:26:01.556 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:01.631 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:01.805 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:01.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:01.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:01.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:01.984 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:02.074 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:02.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:02.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:02.217 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:26:02.217 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:26:02.328 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:02.356 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:02.425 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:02.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:02.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:02.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:02 smithi012 ceph-mon[147926]: from='client.37071 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:02.595 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:02 smithi012 ceph-mon[152263]: from='client.37071 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:02.595 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:02.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:02 smithi110 ceph-mon[136032]: from='client.37071 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:02.768 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:02.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:02.874 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:02.946 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:03.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:26:03.126 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:03.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:03.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:03.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:03.287 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:03.354 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:03.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:03.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:03.430 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:26:03.465 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:03 smithi012 ceph-mon[147926]: pgmap v1478: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.8 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 46317/284346 objects degraded (16.289%); 25682/284346 objects misplaced (9.032%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:26:03.465 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3008717143' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:03.465 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:03 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:03.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:03.472 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:03 smithi012 ceph-mon[152263]: pgmap v1478: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.8 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 46317/284346 objects degraded (16.289%); 25682/284346 objects misplaced (9.032%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:26:03.472 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3008717143' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:03.472 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:03 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:03.498 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:03.547 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:03.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:03.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:03.551 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:03 smithi110 ceph-mon[136032]: pgmap v1478: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.8 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 46317/284346 objects degraded (16.289%); 25682/284346 objects misplaced (9.032%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:26:03.551 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3008717143' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:03.551 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:03 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:03.652 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:03.703 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:03.769 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:03.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:03.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:03.876 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:03.937 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:04.005 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:04.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:04.108 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:04.184 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:04.244 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:04.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:04.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:04.354 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:04.379 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:04.401 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:04.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:04 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:04.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:04 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:04.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:04 smithi110 ceph-mon[136032]: osdmap e530: 8 total, 8 up, 6 in 2023-10-09T10:26:04.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:04 smithi110 ceph-mon[136032]: failed to encode map e530 with expected crc 2023-10-09T10:26:04.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:04 smithi110 ceph-mon[136032]: failed to encode map e530 with expected crc 2023-10-09T10:26:04.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:04 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 46317/284346 objects degraded (16.289%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:04.452 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:04.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:04.487 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:04.487 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[147926]: osdmap e530: 8 total, 8 up, 6 in 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[147926]: failed to encode map e530 with expected crc 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[147926]: failed to encode map e530 with expected crc 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 46317/284346 objects degraded (16.289%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[152263]: osdmap e530: 8 total, 8 up, 6 in 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[152263]: failed to encode map e530 with expected crc 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[152263]: failed to encode map e530 with expected crc 2023-10-09T10:26:04.488 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:04 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 46317/284346 objects degraded (16.289%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:04.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:26:04.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:04.681 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:04.780 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:04.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:04.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:04.842 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:26:04.886 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:26:04.893 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:04.929 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:26:04.937 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:04.980 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:26:05.001 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:05.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:05.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:05.009 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:26:05.043 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:26:05.080 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:26:05.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:05.120 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:26:05.149 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:26:05.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:26:05.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:26:05.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:26:05.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:26:05.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:26:05.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:26:05.150 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:26:05.172 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:05.239 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:05.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:05.347 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:05.414 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:05.470 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:05.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:05.577 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:05.629 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:05 smithi012 ceph-mon[147926]: failed to encode map e530 with expected crc 2023-10-09T10:26:05.629 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:05 smithi012 ceph-mon[147926]: pgmap v1480: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.7 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 45742/279150 objects degraded (16.386%); 25485/279150 objects misplaced (9.130%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T10:26:05.630 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2021867143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:05.630 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:05 smithi012 ceph-mon[152263]: failed to encode map e530 with expected crc 2023-10-09T10:26:05.630 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:05 smithi012 ceph-mon[152263]: pgmap v1480: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.7 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 45742/279150 objects degraded (16.386%); 25485/279150 objects misplaced (9.130%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T10:26:05.630 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2021867143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:05.680 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:05.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:05 smithi110 ceph-mon[136032]: failed to encode map e530 with expected crc 2023-10-09T10:26:05.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:05 smithi110 ceph-mon[136032]: pgmap v1480: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.7 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 45742/279150 objects degraded (16.386%); 25485/279150 objects misplaced (9.130%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T10:26:05.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2021867143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:05.765 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:05.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:05.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:05.874 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:05.927 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:05.973 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:05.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:05.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:06.082 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:06.141 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:06.192 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:06.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:06.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:06.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:26:06.275 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:06.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:06.383 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:26:06.383 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:26:06.383 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:26:06.383 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36474.0:210945 10.11 10:8de901af:::benchmark_data_smithi012_366312_object99616:head [delete] snapc 0=[] ondisk+write+known_if_redirected e483)", 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:23:24.397149+0000", 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 161.97883458699999, 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.2122347649999998, 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "reached pg", 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36474", 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 210945 2023-10-09T10:26:06.384 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.397149+0000", 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.397149+0000", 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.385 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.397147+0000", 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.397154+0000", 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2320000000000004e-06 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.397170+0000", 2023-10-09T10:26:06.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5965e-05 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.397180+0000", 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.414e-06 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.398516+0000", 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001335997 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.398616+0000", 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000100345 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 4,7", 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.399141+0000", 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00052529400000000004 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.402175+0000", 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0030335620000000001 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.415919+0000", 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013743887 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.415930+0000", 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1331999999999999e-05 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:25.404474+0000", 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98854342100000003 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:25.404487+0000", 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3733e-05 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.609264+0000", 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.204776721 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.609383+0000", 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119264 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36474.0:210956 10.c 10:37d7d511:::benchmark_data_smithi012_366312_object99627:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-10-09T10:26:06.391 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:23:24.407691+0000", 2023-10-09T10:26:06.392 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 161.96829227000001, 2023-10-09T10:26:06.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1947900640000002, 2023-10-09T10:26:06.393 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.393 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.393 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.393 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36474", 2023-10-09T10:26:06.393 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:06.393 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 210956 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.407691+0000", 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.407691+0000", 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.407689+0000", 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.407696+0000", 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7789999999999998e-06 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.408866+0000", 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001169855 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.408879+0000", 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3006e-05 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.414115+0000", 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0052359900000000003 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.414129+0000", 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4239e-05 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:25.402797+0000", 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.98866802099999995 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:26:06.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:25.402808+0000", 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1099e-05 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.592795+0000", 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1899865169999999 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.592947+0000", 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015287800000000001 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.593257+0000", 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000309625 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.598307+0000", 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0050499919999999997 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.599375+0000", 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0010674669999999999 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.599786+0000", 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00041109099999999998 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.602305+0000", 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025188749999999998 2023-10-09T10:26:06.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.602361+0000", 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6419000000000003e-05 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.602481+0000", 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011996 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.401 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36474.0:210963 10.21 10:866789cf:::benchmark_data_smithi012_366312_object99634:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:23:24.434662+0000", 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 161.94132089300001, 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1769924540000001, 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36474", 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:06.402 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 210963 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.434662+0000", 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.434662+0000", 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.404 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.404 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.434661+0000", 2023-10-09T10:26:06.404 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.404 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.404 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.434666+0000", 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3170000000000001e-06 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.434678+0000", 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1844999999999999e-05 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.434696+0000", 2023-10-09T10:26:06.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8527999999999999e-05 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.434734+0000", 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7880999999999999e-05 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:24.434742+0000", 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7419999999999997e-06 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:25.404518+0000", 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.96977592499999998 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for peered", 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:25.404528+0000", 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8970000000000003e-06 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.580126+0000", 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1755976889999999 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.580275+0000", 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014946 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.580577+0000", 2023-10-09T10:26:06.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00030221799999999999 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.609710+0000", 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029132568000000001 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.609746+0000", 2023-10-09T10:26:06.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6528e-05 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.609780+0000", 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3943999999999999e-05 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.611508+0000", 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001728165 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.611556+0000", 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7426000000000003e-05 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:23:26.611655+0000", 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8846999999999997e-05 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:115463 11.1 11:873fca37:::benchmark_data_smithi012_404402_object2680:head [delete] snapc 0=[] ondisk+write+known_if_redirected e523)", 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:34.364745+0000", 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 32.011238106, 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.112439951, 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 115463 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.364745+0000", 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.364745+0000", 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.364745+0000", 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.364747+0000", 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8100000000000002e-06 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.364753+0000", 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5969999999999999e-06 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.364759+0000", 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0719999999999996e-06 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.371920+0000", 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0071612150000000003 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.374329+0000", 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024091170000000001 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.455607+0000", 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.081277689 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.455710+0000", 2023-10-09T10:26:06.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000103349 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.455842+0000", 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013204499999999999 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.460835+0000", 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0049926550000000004 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.465202+0000", 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0043670139999999998 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.474595+0000", 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0093933200000000001 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.474662+0000", 2023-10-09T10:26:06.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6645999999999999e-05 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:34.477185+0000", 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002522832 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:117257 11.11 11:8c098e7a:::benchmark_data_smithi012_404402_object4474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:37.360650+0000", 2023-10-09T10:26:06.420 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 29.015332728000001, 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15309879200000001, 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117257 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.360650+0000", 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.360650+0000", 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.360649+0000", 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.360658+0000", 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3009999999999995e-06 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.360680+0000", 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1529e-05 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.360690+0000", 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0147e-05 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.380809+0000", 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020119442000000001 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.381910+0000", 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011012929999999999 2023-10-09T10:26:06.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.478494+0000", 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.096583345000000001 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.478613+0000", 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000119561 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.478768+0000", 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015446599999999999 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.501239+0000", 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022471259 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.511286+0000", 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010046681999999999 2023-10-09T10:26:06.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.511296+0000", 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.8910000000000007e-06 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.511363+0000", 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7273000000000005e-05 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.513749+0000", 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023863059999999999 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:117262 11.11 11:8a1fc22a:::benchmark_data_smithi012_404402_object4479:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:37.365799+0000", 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 29.010184218999999, 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.14095766400000001, 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.428 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117262 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.365799+0000", 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.365799+0000", 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.365797+0000", 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.365804+0000", 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1760000000000004e-06 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.365821+0000", 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6813e-05 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.365830+0000", 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0319999999999992e-06 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.381945+0000", 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016114995999999999 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.382841+0000", 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00089541600000000001 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.432 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.475927+0000", 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.093086800999999997 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.476037+0000", 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000109305 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.476183+0000", 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000145964 2023-10-09T10:26:06.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.486858+0000", 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.010675232999999999 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.498181+0000", 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01132295 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.501285+0000", 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031037410000000001 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.501369+0000", 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4689000000000002e-05 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:37.506757+0000", 2023-10-09T10:26:06.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0053872690000000001 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:117960 11.11 11:8935036e:::benchmark_data_smithi012_404402_object5177:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.562459+0000", 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.813524202, 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.108838817, 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.436 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117960 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.562459+0000", 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.562459+0000", 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.562457+0000", 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.562465+0000", 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.537e-06 2023-10-09T10:26:06.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.562476+0000", 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1039999999999999e-05 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.562484+0000", 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1219999999999995e-06 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.571558+0000", 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090740879999999993 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.572330+0000", 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00077153900000000001 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.633473+0000", 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061143445999999997 2023-10-09T10:26:06.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.633547+0000", 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3471000000000002e-05 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.633639+0000", 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2886999999999993e-05 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.644866+0000", 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011226277999999999 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.644872+0000", 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4439999999999996e-06 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.667686+0000", 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022813982 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.667770+0000", 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3436000000000005e-05 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.671298+0000", 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003528172 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:117969 11.1 11:873187d2:::benchmark_data_smithi012_404402_object5186:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.580789+0000", 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.795193926, 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.123260226, 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117969 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.444 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.580789+0000", 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.580789+0000", 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.580788+0000", 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.580795+0000", 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9530000000000002e-06 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.580806+0000", 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1191e-05 2023-10-09T10:26:06.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.580814+0000", 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9500000000000001e-06 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.601151+0000", 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020337158000000001 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.602589+0000", 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014376009999999999 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.644877+0000", 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.042288490999999997 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.644985+0000", 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000108118 2023-10-09T10:26:06.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.645121+0000", 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135931 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.654154+0000", 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090324020000000001 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.671891+0000", 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017737478000000001 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.701256+0000", 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029364773 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.701317+0000", 2023-10-09T10:26:06.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1319e-05 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.704049+0000", 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027320819999999998 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:117977 11.1 11:864a2f6c:::benchmark_data_smithi012_404402_object5194:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.611614+0000", 2023-10-09T10:26:06.451 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.764369574, 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11952452500000001, 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 117977 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.611614+0000", 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.611614+0000", 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.611612+0000", 2023-10-09T10:26:06.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.611618+0000", 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.178e-06 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.611630+0000", 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2228999999999999e-05 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.454 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.611639+0000", 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.259e-06 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.648680+0000", 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.037040923000000003 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.455 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.651833+0000", 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0031536239999999998 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.699009+0000", 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.047175949000000002 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.699119+0000", 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00011011899999999999 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.699271+0000", 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015177800000000001 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.705128+0000", 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0058572950000000002 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.708591+0000", 2023-10-09T10:26:06.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0034627529999999998 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.728593+0000", 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020002236 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.728667+0000", 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3510000000000006e-05 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.731138+0000", 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0024711989999999999 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118115 11.1 11:877a5d48:::benchmark_data_smithi012_404402_object5332:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.888687+0000", 2023-10-09T10:26:06.459 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.487296533999999, 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.115581832, 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118115 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.888687+0000", 2023-10-09T10:26:06.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.888687+0000", 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.888685+0000", 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.888690+0000", 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3299999999999998e-06 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.888700+0000", 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5089999999999999e-06 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.888706+0000", 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3640000000000004e-06 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.921614+0000", 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032907912999999997 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.924336+0000", 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027218020000000002 2023-10-09T10:26:06.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.947826+0000", 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023489677 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.947900+0000", 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3905999999999993e-05 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.947987+0000", 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.7146999999999998e-05 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.970929+0000", 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.022941884999999999 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.971005+0000", 2023-10-09T10:26:06.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6575999999999993e-05 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.001219+0000", 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030213331999999999 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.001292+0000", 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3875000000000007e-05 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.004268+0000", 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002975994 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118120 11.11 11:88ad4326:::benchmark_data_smithi012_404402_object5337:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.903849+0000", 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.472133862, 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.111701145, 2023-10-09T10:26:06.467 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118120 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903849+0000", 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903849+0000", 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903847+0000", 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903858+0000", 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0549e-05 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903881+0000", 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3640999999999999e-05 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903894+0000", 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2259000000000001e-05 2023-10-09T10:26:06.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.932591+0000", 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.028697038000000001 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.933511+0000", 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000920212 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.976684+0000", 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.043173204 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.976866+0000", 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018175400000000001 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.977127+0000", 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000261716 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.991921+0000", 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014793669000000001 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.012898+0000", 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.020976430000000001 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.012999+0000", 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000101597 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.013050+0000", 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1143999999999997e-05 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.015550+0000", 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025000859999999999 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.474 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118123 11.11 11:899c6df7:::benchmark_data_smithi012_404402_object5340:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.903959+0000", 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.472023878999998, 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11427617599999999, 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.475 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118123 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903959+0000", 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903959+0000", 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903959+0000", 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903963+0000", 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3769999999999998e-06 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903973+0000", 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.6350000000000001e-06 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.903980+0000", 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7379999999999994e-06 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.933554+0000", 2023-10-09T10:26:06.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.029573469000000002 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.934597+0000", 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001042814 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.989234+0000", 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.054637603999999999 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.989388+0000", 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015361500000000001 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.989563+0000", 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000175398 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.012943+0000", 2023-10-09T10:26:06.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0233796 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.012953+0000", 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7990000000000005e-06 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.015636+0000", 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026828860000000002 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.015690+0000", 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4901000000000002e-05 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.018235+0000", 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002544915 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118133 11.11 11:8989f753:::benchmark_data_smithi012_404402_object5350:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.926143+0000", 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.449840501000001, 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.113132574, 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118133 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.483 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926143+0000", 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926143+0000", 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926141+0000", 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926148+0000", 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3720000000000001e-06 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926160+0000", 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2414e-05 2023-10-09T10:26:06.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926168+0000", 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.943e-06 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.934640+0000", 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0084719689999999993 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.940367+0000", 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0057270309999999996 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.997884+0000", 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057517281000000003 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.998011+0000", 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012719200000000001 2023-10-09T10:26:06.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.998199+0000", 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000187794 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.012960+0000", 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.01476094 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.015596+0000", 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026361549999999998 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.036679+0000", 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.021082242000000001 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.036954+0000", 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00027553799999999998 2023-10-09T10:26:06.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.039275+0000", 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023210729999999999 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118135 11.11 11:8bf9a2d4:::benchmark_data_smithi012_404402_object5352:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.926197+0000", 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.449786433, 2023-10-09T10:26:06.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13142410900000001, 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118135 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926197+0000", 2023-10-09T10:26:06.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926197+0000", 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926196+0000", 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926199+0000", 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5110000000000002e-06 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926205+0000", 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2299999999999996e-06 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.926210+0000", 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3669999999999996e-06 2023-10-09T10:26:06.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.949826+0000", 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023615265 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.952375+0000", 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025496450000000001 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.010493+0000", 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.058118177 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.010605+0000", 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111808 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.010756+0000", 2023-10-09T10:26:06.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015065099999999999 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.015605+0000", 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048489020000000004 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.028269+0000", 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012664249000000001 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.055019+0000", 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.026750025 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.055095+0000", 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6383000000000004e-05 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.057621+0000", 2023-10-09T10:26:06.497 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025252959999999998 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118144 11.1 11:814ab67a:::benchmark_data_smithi012_404402_object5361:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.949563+0000", 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.426420265000001, 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.137833813, 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.498 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118144 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.949563+0000", 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.499 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.949563+0000", 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.949562+0000", 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.949565+0000", 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0110000000000001e-06 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.500 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.949572+0000", 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3450000000000002e-06 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.949577+0000", 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2769999999999996e-06 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.501 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.950765+0000", 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0011882939999999999 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.962973+0000", 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.012207391 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.031893+0000", 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068920079999999995 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.502 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.032007+0000", 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000114147 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.032142+0000", 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000135055 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.503 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.040668+0000", 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0085259340000000006 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.054000+0000", 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.013332111000000001 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.084811+0000", 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.030811158000000002 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.504 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.084869+0000", 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8179e-05 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.087397+0000", 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025272340000000002 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.505 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118154 11.11 11:8edb217a:::benchmark_data_smithi012_404402_object5371:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.990524+0000", 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.385458697000001, 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.11662120400000001, 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118154 2023-10-09T10:26:06.506 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990524+0000", 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990524+0000", 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.507 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990523+0000", 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990529+0000", 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6350000000000003e-06 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990538+0000", 2023-10-09T10:26:06.508 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3530000000000001e-06 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990545+0000", 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6320000000000002e-06 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.027270+0000", 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.036725089000000002 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.509 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.510 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.028219+0000", 2023-10-09T10:26:06.510 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00094944799999999998 2023-10-09T10:26:06.510 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.510 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.062179+0000", 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033959369000000003 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.062290+0000", 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000111672 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.511 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.070590+0000", 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0082992610000000005 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.101613+0000", 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.031023116999999999 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.512 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.101655+0000", 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2648000000000002e-05 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.104469+0000", 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0028135310000000002 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.104517+0000", 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8569000000000002e-05 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.513 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.107146+0000", 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002628226 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118160 11.11 11:8dc1bbe3:::benchmark_data_smithi012_404402_object5377:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.990551+0000", 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.38543202, 2023-10-09T10:26:06.514 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.126493156, 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118160 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990551+0000", 2023-10-09T10:26:06.515 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990551+0000", 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990551+0000", 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.516 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990553+0000", 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6460000000000002e-06 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990561+0000", 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.5220000000000001e-06 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.990566+0000", 2023-10-09T10:26:06.517 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.417e-06 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.028293+0000", 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.037727101999999998 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.029123+0000", 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00082943600000000002 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.518 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.077667+0000", 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.048543942999999999 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.077734+0000", 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6898999999999998e-05 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.077827+0000", 2023-10-09T10:26:06.519 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3775000000000003e-05 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.101693+0000", 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.023865685000000001 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.108355+0000", 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0066614990000000004 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.520 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.114810+0000", 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0064555849999999998 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.114865+0000", 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4444000000000003e-05 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.117044+0000", 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021796570000000002 2023-10-09T10:26:06.521 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:118164 11.11 11:8f147b94:::benchmark_data_smithi012_404402_object5381:head [delete] snapc 0=[] ondisk+write+known_if_redirected e525)", 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:38.996163+0000", 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.379819629, 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.12646100499999999, 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.522 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 118164 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.996163+0000", 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.996163+0000", 2023-10-09T10:26:06.523 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.996162+0000", 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.996176+0000", 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4634000000000001e-05 2023-10-09T10:26:06.524 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.996200+0000", 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3147e-05 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:38.996211+0000", 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1691e-05 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.525 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.029167+0000", 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.032955972 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.030144+0000", 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00097638800000000002 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.087443+0000", 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.057299431999999997 2023-10-09T10:26:06.526 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.087553+0000", 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010966700000000001 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.087699+0000", 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014601300000000001 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.104455+0000", 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016755902999999999 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.108363+0000", 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0039079179999999998 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.120192+0000", 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.011829209 2023-10-09T10:26:06.528 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.120248+0000", 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.6589000000000003e-05 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:39.122624+0000", 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0023761070000000001 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.529 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:125686 11.2 11:429f7a16:::benchmark_data_smithi012_404402_object12903:head [delete] snapc 0=[] ondisk+write+known_if_redirected e528)", 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:50.788827+0000", 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 15.587155804, 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.107898191, 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 125686 2023-10-09T10:26:06.530 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.788827+0000", 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.788827+0000", 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.531 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.788826+0000", 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.788833+0000", 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.3730000000000004e-06 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.788846+0000", 2023-10-09T10:26:06.532 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.29e-05 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.788855+0000", 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.0489999999999993e-06 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.789766+0000", 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000910365 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.533 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.789899+0000", 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00013365299999999999 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.790023+0000", 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012395599999999999 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.792587+0000", 2023-10-09T10:26:06.534 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025634939999999999 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.808644+0000", 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016057620000000002 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.817686+0000", 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0090415340000000004 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.535 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.817746+0000", 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0099000000000002e-05 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.896725+0000", 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.078979604999999994 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.536 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:130452 11.1 11:80a17b14:::benchmark_data_smithi012_404402_object17669:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:58.256966+0000", 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 8.1190167029999998, 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15199114, 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 130452 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:06.537 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.256966+0000", 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.256966+0000", 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:06.538 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.256965+0000", 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.256970+0000", 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9520000000000002e-06 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.256980+0000", 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7620000000000007e-06 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.539 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.256986+0000", 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0410000000000003e-06 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.271936+0000", 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.014949708000000001 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.275478+0000", 2023-10-09T10:26:06.540 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035420479999999999 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.347189+0000", 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071711393999999998 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.347311+0000", 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121654 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.541 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.347492+0000", 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018081999999999999 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.351750+0000", 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042580099999999996 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.358183+0000", 2023-10-09T10:26:06.542 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.006433439 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.406786+0000", 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.048602737999999999 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.406841+0000", 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5346999999999997e-05 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:06.543 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:58.408958+0000", 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021165089999999999 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:26:06.544 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:06.567 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:06.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:06.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:06.678 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:06.741 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:06.796 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:06.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:06.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:06.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:06.964 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:07.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:26:07.177 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:07.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:07.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:07.287 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:07.355 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:07.532 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:07.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:07.590 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:07 smithi012 ceph-mon[147926]: pgmap v1481: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.6 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 44993/274602 objects degraded (16.385%); 24477/274602 objects misplaced (8.914%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:26:07.590 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:07 smithi012 ceph-mon[152263]: pgmap v1481: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.6 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 44993/274602 objects degraded (16.385%); 24477/274602 objects misplaced (8.914%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:26:07.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:07.694 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:07 smithi110 ceph-mon[136032]: pgmap v1481: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.6 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 44993/274602 objects degraded (16.385%); 24477/274602 objects misplaced (8.914%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:26:07.771 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:07.920 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:26:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:26:07] "GET /metrics HTTP/1.1" 200 36637 "" "Prometheus/2.33.4" 2023-10-09T10:26:07.942 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:26:08.507 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:08.557 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:08.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:08.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:08.663 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:08.725 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:08.774 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:08.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:08.779 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 44993/274602 objects degraded (16.385%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:08.880 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:08.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 44993/274602 objects degraded (16.385%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:08.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 44993/274602 objects degraded (16.385%), 15 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:08.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:09.000 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:09.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:09.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:09.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:09.134 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:09.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:09 smithi012 ceph-mon[147926]: pgmap v1482: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.6 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 44993/274602 objects degraded (16.385%); 24477/274602 objects misplaced (8.914%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:26:09.783 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:09 smithi012 ceph-mon[152263]: pgmap v1482: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.6 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 44993/274602 objects degraded (16.385%); 24477/274602 objects misplaced (8.914%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:26:09.792 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:09.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:09.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:09.900 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:09.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:09 smithi110 ceph-mon[136032]: pgmap v1482: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.6 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 44993/274602 objects degraded (16.385%); 24477/274602 objects misplaced (8.914%); 13 MiB/s, 214 objects/s recovering 2023-10-09T10:26:10.005 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:10.113 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:10.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:10.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:10.220 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:10.277 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:10.311 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:10.327 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:10.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:10.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:10.431 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:10.471 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:10.522 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:10.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:10.630 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:10.686 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:10.760 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:10.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:10.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:10.772 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:26:10.791 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:26:10.841 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:10.867 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:10.997 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:11.259 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:11.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:11.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:11.366 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:11.423 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:11.505 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:11.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:11.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:11.615 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:11.650 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:11.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:11 smithi110 ceph-mon[136032]: pgmap v1483: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.5 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 44392/268794 objects degraded (16.515%); 24021/268794 objects misplaced (8.937%); 14 MiB/s, 218 objects/s recovering 2023-10-09T10:26:11.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:11 smithi012 ceph-mon[152263]: pgmap v1483: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.5 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 44392/268794 objects degraded (16.515%); 24021/268794 objects misplaced (8.937%); 14 MiB/s, 218 objects/s recovering 2023-10-09T10:26:11.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:11 smithi012 ceph-mon[147926]: pgmap v1483: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.5 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 44392/268794 objects degraded (16.515%); 24021/268794 objects misplaced (8.937%); 14 MiB/s, 218 objects/s recovering 2023-10-09T10:26:12.429 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:12.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:12.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:12.460 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1483,"stamp":"2023-10-09T10:26:09.795300+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5870753720,"num_objects":89598,"num_object_clones":0,"num_object_copies":268794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":44392,"num_objects_misplaced":24021,"num_objects_unfound":0,"num_objects_dirty":89598,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":121632,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12303,"num_bytes_recovered":798787324,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136446,"ondisk_log_size":136446,"up":297,"acting":282,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":23719756,"kb_used_data":18118684,"kb_used_omap":163,"kb_used_meta":5600540,"kb_avail":538751156,"statfs":{"total":575970213888,"available":551681183744,"internally_reserved":0,"allocated":18553532416,"data_stored":18544537474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":5734953971},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,3,2,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-422707200,"num_objects":-6450,"num_object_clones":0,"num_object_copies":-19350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2617,"num_objects_misplaced":-2376,"num_objects_unfound":0,"num_objects_dirty":-6450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6450,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2189,"num_bytes_recovered":143458304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003765"},"pg_stats":[{"pgid":"11.12","version":"530'5280","reported_seq":5343,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:07.767358+0000","last_change":"2023-10-09T10:25:09.709209+0000","last_active":"2023-10-09T10:26:07.767358+0000","last_peered":"2023-10-09T10:26:07.767358+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:26:07.767358+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5280,"log_dups_size":0,"ondisk_log_size":5280,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185565184,"num_objects":2832,"num_object_clones":0,"num_object_copies":8496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3119,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4332,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-287},{"shards":"0,2,5","objects":2369}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":123,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348456+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:25:55.348456+0000","last_peered":"2023-10-09T10:25:55.348456+0000","last_clean":"2023-10-09T10:25:55.348456+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:25:55.348456+0000","last_undegraded":"2023-10-09T10:25:55.348456+0000","last_fullsized":"2023-10-09T10:25:55.348456+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":484,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350472+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:25:55.350472+0000","last_peered":"2023-10-09T10:25:55.350472+0000","last_clean":"2023-10-09T10:25:55.350472+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:25:55.350472+0000","last_undegraded":"2023-10-09T10:25:55.350472+0000","last_fullsized":"2023-10-09T10:25:55.350472+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"530'5221","reported_seq":7528,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:07.782182+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:26:07.782182+0000","last_peered":"2023-10-09T10:26:07.782182+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:26:07.782182+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5221,"log_dups_size":0,"ondisk_log_size":5221,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182583296,"num_objects":2786,"num_object_clones":0,"num_object_copies":8358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4276,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":745,"num_bytes_recovered":48824320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1519},{"shards":"0,2,6","objects":845},{"shards":"0,6","objects":422}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":561,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348051+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:25:55.348051+0000","last_peered":"2023-10-09T10:25:55.348051+0000","last_clean":"2023-10-09T10:25:55.348051+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:25:55.348051+0000","last_undegraded":"2023-10-09T10:25:55.348051+0000","last_fullsized":"2023-10-09T10:25:55.348051+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":454,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.355349+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:25:55.355349+0000","last_peered":"2023-10-09T10:25:55.355349+0000","last_clean":"2023-10-09T10:25:55.355349+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:25:55.355349+0000","last_undegraded":"2023-10-09T10:25:55.355349+0000","last_fullsized":"2023-10-09T10:25:55.355349+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"530'5149","reported_seq":9689,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:07.574118+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:26:07.574118+0000","last_peered":"2023-10-09T10:26:07.574118+0000","last_clean":"2023-10-09T10:26:07.574118+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:07.574118+0000","last_undegraded":"2023-10-09T10:26:07.574118+0000","last_fullsized":"2023-10-09T10:26:07.574118+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5149,"log_dups_size":0,"ondisk_log_size":5149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181501952,"num_objects":2770,"num_object_clones":0,"num_object_copies":8310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4212,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":25,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342249+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:25:55.342249+0000","last_peered":"2023-10-09T10:25:55.342249+0000","last_clean":"2023-10-09T10:25:55.342249+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:25:55.342249+0000","last_undegraded":"2023-10-09T10:25:55.342249+0000","last_fullsized":"2023-10-09T10:25:55.342249+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":106,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.368560+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:25:55.368560+0000","last_peered":"2023-10-09T10:25:55.368560+0000","last_clean":"2023-10-09T10:25:55.368560+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:25:55.368560+0000","last_undegraded":"2023-10-09T10:25:55.368560+0000","last_fullsized":"2023-10-09T10:25:55.368560+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":488,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.368980+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:25:55.368980+0000","last_peered":"2023-10-09T10:25:55.368980+0000","last_clean":"2023-10-09T10:25:55.368980+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:25:55.368980+0000","last_undegraded":"2023-10-09T10:25:55.368980+0000","last_fullsized":"2023-10-09T10:25:55.368980+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":437,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.329117+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:25:55.329117+0000","last_peered":"2023-10-09T10:25:55.329117+0000","last_clean":"2023-10-09T10:25:55.329117+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:25:55.329117+0000","last_undegraded":"2023-10-09T10:25:55.329117+0000","last_fullsized":"2023-10-09T10:25:55.329117+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":205,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342817+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:25:55.342817+0000","last_peered":"2023-10-09T10:25:55.342817+0000","last_clean":"2023-10-09T10:25:55.342817+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:25:55.342817+0000","last_undegraded":"2023-10-09T10:25:55.342817+0000","last_fullsized":"2023-10-09T10:25:55.342817+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":437,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.332382+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:25:55.332382+0000","last_peered":"2023-10-09T10:25:55.332382+0000","last_clean":"2023-10-09T10:25:55.332382+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:25:55.332382+0000","last_undegraded":"2023-10-09T10:25:55.332382+0000","last_fullsized":"2023-10-09T10:25:55.332382+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":135,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350660+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:25:55.350660+0000","last_peered":"2023-10-09T10:25:55.350660+0000","last_clean":"2023-10-09T10:25:55.350660+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:25:55.350660+0000","last_undegraded":"2023-10-09T10:25:55.350660+0000","last_fullsized":"2023-10-09T10:25:55.350660+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":25,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.344036+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:25:55.344036+0000","last_peered":"2023-10-09T10:25:55.344036+0000","last_clean":"2023-10-09T10:25:55.344036+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:25:55.344036+0000","last_undegraded":"2023-10-09T10:25:55.344036+0000","last_fullsized":"2023-10-09T10:25:55.344036+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":181,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348132+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:25:55.348132+0000","last_peered":"2023-10-09T10:25:55.348132+0000","last_clean":"2023-10-09T10:25:55.348132+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:25:55.348132+0000","last_undegraded":"2023-10-09T10:25:55.348132+0000","last_fullsized":"2023-10-09T10:25:55.348132+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":141,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.349371+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:25:55.349371+0000","last_peered":"2023-10-09T10:25:55.349371+0000","last_clean":"2023-10-09T10:25:55.349371+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:25:55.349371+0000","last_undegraded":"2023-10-09T10:25:55.349371+0000","last_fullsized":"2023-10-09T10:25:55.349371+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":101,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.349245+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:25:55.349245+0000","last_peered":"2023-10-09T10:25:55.349245+0000","last_clean":"2023-10-09T10:25:55.349245+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:25:55.349245+0000","last_undegraded":"2023-10-09T10:25:55.349245+0000","last_fullsized":"2023-10-09T10:25:55.349245+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":55,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.346287+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:25:55.346287+0000","last_peered":"2023-10-09T10:25:55.346287+0000","last_clean":"2023-10-09T10:25:55.346287+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:25:55.346287+0000","last_undegraded":"2023-10-09T10:25:55.346287+0000","last_fullsized":"2023-10-09T10:25:55.346287+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":453,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.347247+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:25:55.347247+0000","last_peered":"2023-10-09T10:25:55.347247+0000","last_clean":"2023-10-09T10:25:55.347247+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:25:55.347247+0000","last_undegraded":"2023-10-09T10:25:55.347247+0000","last_fullsized":"2023-10-09T10:25:55.347247+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":196,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350127+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:25:55.350127+0000","last_peered":"2023-10-09T10:25:55.350127+0000","last_clean":"2023-10-09T10:25:55.350127+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:25:55.350127+0000","last_undegraded":"2023-10-09T10:25:55.350127+0000","last_fullsized":"2023-10-09T10:25:55.350127+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"530'5152","reported_seq":5949,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:04.892212+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:26:04.892212+0000","last_peered":"2023-10-09T10:26:04.892212+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:26:04.892212+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5152,"log_dups_size":0,"ondisk_log_size":5152,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188514304,"num_objects":2877,"num_object_clones":0,"num_object_copies":8631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":2877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4229,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-274},{"shards":"3,5,6","objects":843}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":141,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348184+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:25:55.348184+0000","last_peered":"2023-10-09T10:25:55.348184+0000","last_clean":"2023-10-09T10:25:55.348184+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:25:55.348184+0000","last_undegraded":"2023-10-09T10:25:55.348184+0000","last_fullsized":"2023-10-09T10:25:55.348184+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":25,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.353819+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:25:55.353819+0000","last_peered":"2023-10-09T10:25:55.353819+0000","last_clean":"2023-10-09T10:25:55.353819+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:25:55.353819+0000","last_undegraded":"2023-10-09T10:25:55.353819+0000","last_fullsized":"2023-10-09T10:25:55.353819+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"530'5207","reported_seq":8495,"reported_epoch":530,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:26:04.997065+0000","last_change":"2023-10-09T10:25:48.137669+0000","last_active":"2023-10-09T10:26:04.997065+0000","last_peered":"2023-10-09T10:26:04.997065+0000","last_clean":"2023-10-09T10:24:39.576387+0000","last_became_active":"2023-10-09T10:25:09.696470+0000","last_became_peered":"2023-10-09T10:25:09.696470+0000","last_unstale":"2023-10-09T10:26:04.997065+0000","last_undegraded":"2023-10-09T10:26:04.997065+0000","last_fullsized":"2023-10-09T10:25:09.436538+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5207,"log_dups_size":0,"ondisk_log_size":5207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187006976,"num_objects":2854,"num_object_clones":0,"num_object_copies":8562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3204,"num_objects_unfound":0,"num_objects_dirty":2854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4242,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1073,"num_bytes_recovered":70320128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1205},{"shards":"2,5,6","objects":794},{"shards":"5,6","objects":855}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":35,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342478+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:25:55.342478+0000","last_peered":"2023-10-09T10:25:55.342478+0000","last_clean":"2023-10-09T10:25:55.342478+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:25:55.342478+0000","last_undegraded":"2023-10-09T10:25:55.342478+0000","last_fullsized":"2023-10-09T10:25:55.342478+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":28,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.329280+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:25:55.329280+0000","last_peered":"2023-10-09T10:25:55.329280+0000","last_clean":"2023-10-09T10:25:55.329280+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:25:55.329280+0000","last_undegraded":"2023-10-09T10:25:55.329280+0000","last_fullsized":"2023-10-09T10:25:55.329280+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"530'5275","reported_seq":5333,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:04.966155+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:26:04.966155+0000","last_peered":"2023-10-09T10:26:04.966155+0000","last_clean":"2023-10-09T10:26:04.966155+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:26:04.966155+0000","last_undegraded":"2023-10-09T10:26:04.966155+0000","last_fullsized":"2023-10-09T10:26:04.966155+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":5275,"log_dups_size":0,"ondisk_log_size":5275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188809216,"num_objects":2881,"num_object_clones":0,"num_object_copies":8643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1987,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":31,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350194+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:25:55.350194+0000","last_peered":"2023-10-09T10:25:55.350194+0000","last_clean":"2023-10-09T10:25:55.350194+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:25:55.350194+0000","last_undegraded":"2023-10-09T10:25:55.350194+0000","last_fullsized":"2023-10-09T10:25:55.350194+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":205,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.336254+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:25:55.336254+0000","last_peered":"2023-10-09T10:25:55.336254+0000","last_clean":"2023-10-09T10:25:55.336254+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:25:55.336254+0000","last_undegraded":"2023-10-09T10:25:55.336254+0000","last_fullsized":"2023-10-09T10:25:55.336254+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"530'5035","reported_seq":7379,"reported_epoch":530,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:09.447421+0000","last_change":"2023-10-09T10:25:48.142610+0000","last_active":"2023-10-09T10:26:09.447421+0000","last_peered":"2023-10-09T10:26:09.447421+0000","last_clean":"2023-10-09T10:25:11.561932+0000","last_became_active":"2023-10-09T10:25:13.746279+0000","last_became_peered":"2023-10-09T10:25:13.746279+0000","last_unstale":"2023-10-09T10:26:09.447421+0000","last_undegraded":"2023-10-09T10:25:13.623380+0000","last_fullsized":"2023-10-09T10:25:13.604555+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5035,"log_dups_size":0,"ondisk_log_size":5035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:39.267545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172097536,"num_objects":2626,"num_object_clones":0,"num_object_copies":7878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":343,"num_objects_misplaced":1528,"num_objects_unfound":0,"num_objects_dirty":2626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2275,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1141,"num_bytes_recovered":74776576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":755},{"shards":"2,4,5","objects":1528}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":157,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350852+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:25:55.350852+0000","last_peered":"2023-10-09T10:25:55.350852+0000","last_clean":"2023-10-09T10:25:55.350852+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:25:55.350852+0000","last_undegraded":"2023-10-09T10:25:55.350852+0000","last_fullsized":"2023-10-09T10:25:55.350852+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":144,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348244+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:25:55.348244+0000","last_peered":"2023-10-09T10:25:55.348244+0000","last_clean":"2023-10-09T10:25:55.348244+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:25:55.348244+0000","last_undegraded":"2023-10-09T10:25:55.348244+0000","last_fullsized":"2023-10-09T10:25:55.348244+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"530'5174","reported_seq":7341,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:04.987437+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:26:04.987437+0000","last_peered":"2023-10-09T10:26:04.987437+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:26:04.987437+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5174,"log_dups_size":0,"ondisk_log_size":5174,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185827328,"num_objects":2835,"num_object_clones":0,"num_object_copies":8505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4055,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4229,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":698,"num_bytes_recovered":45744128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1621},{"shards":"0,5,6","objects":813},{"shards":"5,6","objects":401}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":221,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.346642+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:25:55.346642+0000","last_peered":"2023-10-09T10:25:55.346642+0000","last_clean":"2023-10-09T10:25:55.346642+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:25:55.346642+0000","last_undegraded":"2023-10-09T10:25:55.346642+0000","last_fullsized":"2023-10-09T10:25:55.346642+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.337708+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:25:55.337708+0000","last_peered":"2023-10-09T10:25:55.337708+0000","last_clean":"2023-10-09T10:25:55.337708+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:25:55.337708+0000","last_undegraded":"2023-10-09T10:25:55.337708+0000","last_fullsized":"2023-10-09T10:25:55.337708+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"530'5114","reported_seq":9297,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:04.799094+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:26:04.799094+0000","last_peered":"2023-10-09T10:26:04.799094+0000","last_clean":"2023-10-09T10:26:04.799094+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:26:04.799094+0000","last_undegraded":"2023-10-09T10:26:04.799094+0000","last_fullsized":"2023-10-09T10:26:04.799094+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":5114,"log_dups_size":0,"ondisk_log_size":5114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187629568,"num_objects":2863,"num_object_clones":0,"num_object_copies":8589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2200,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":205,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.337735+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:25:55.337735+0000","last_peered":"2023-10-09T10:25:55.337735+0000","last_clean":"2023-10-09T10:25:55.337735+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:25:55.337735+0000","last_undegraded":"2023-10-09T10:25:55.337735+0000","last_fullsized":"2023-10-09T10:25:55.337735+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":439,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342394+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:25:55.342394+0000","last_peered":"2023-10-09T10:25:55.342394+0000","last_clean":"2023-10-09T10:25:55.342394+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:25:55.342394+0000","last_undegraded":"2023-10-09T10:25:55.342394+0000","last_fullsized":"2023-10-09T10:25:55.342394+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"530'5129","reported_seq":6416,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:04.932739+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:26:04.932739+0000","last_peered":"2023-10-09T10:26:04.932739+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:26:04.932739+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5129,"log_dups_size":0,"ondisk_log_size":5129,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184188928,"num_objects":2810,"num_object_clones":0,"num_object_copies":8430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4208,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1541},{"shards":"2,5","objects":1181}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":140,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342296+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:25:55.342296+0000","last_peered":"2023-10-09T10:25:55.342296+0000","last_clean":"2023-10-09T10:25:55.342296+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:25:55.342296+0000","last_undegraded":"2023-10-09T10:25:55.342296+0000","last_fullsized":"2023-10-09T10:25:55.342296+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":141,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348041+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:25:55.348041+0000","last_peered":"2023-10-09T10:25:55.348041+0000","last_clean":"2023-10-09T10:25:55.348041+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:25:55.348041+0000","last_undegraded":"2023-10-09T10:25:55.348041+0000","last_fullsized":"2023-10-09T10:25:55.348041+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"530'5194","reported_seq":5311,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:07.806481+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:26:07.806481+0000","last_peered":"2023-10-09T10:26:07.806481+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:26:07.806481+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5194,"log_dups_size":0,"ondisk_log_size":5194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180944896,"num_objects":2761,"num_object_clones":0,"num_object_copies":8283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2306,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-297},{"shards":"0,2,5","objects":2313}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":141,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348381+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:25:55.348381+0000","last_peered":"2023-10-09T10:25:55.348381+0000","last_clean":"2023-10-09T10:25:55.348381+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:25:55.348381+0000","last_undegraded":"2023-10-09T10:25:55.348381+0000","last_fullsized":"2023-10-09T10:25:55.348381+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350561+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:25:55.350561+0000","last_peered":"2023-10-09T10:25:55.350561+0000","last_clean":"2023-10-09T10:25:55.350561+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:25:55.350561+0000","last_undegraded":"2023-10-09T10:25:55.350561+0000","last_fullsized":"2023-10-09T10:25:55.350561+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"530'5201","reported_seq":7446,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:07.769800+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:26:07.769800+0000","last_peered":"2023-10-09T10:26:07.769800+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:26:07.769800+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5201,"log_dups_size":0,"ondisk_log_size":5201,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184025088,"num_objects":2808,"num_object_clones":0,"num_object_copies":8424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4256,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":724,"num_bytes_recovered":47448064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1530},{"shards":"0,2,6","objects":806},{"shards":"0,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348325+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:25:55.348325+0000","last_peered":"2023-10-09T10:25:55.348325+0000","last_clean":"2023-10-09T10:25:55.348325+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:25:55.348325+0000","last_undegraded":"2023-10-09T10:25:55.348325+0000","last_fullsized":"2023-10-09T10:25:55.348325+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":150,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350701+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:25:55.350701+0000","last_peered":"2023-10-09T10:25:55.350701+0000","last_clean":"2023-10-09T10:25:55.350701+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:25:55.350701+0000","last_undegraded":"2023-10-09T10:25:55.350701+0000","last_fullsized":"2023-10-09T10:25:55.350701+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"530'5168","reported_seq":5599,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:07.794163+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:26:07.794163+0000","last_peered":"2023-10-09T10:26:07.794163+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:26:07.794163+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5168,"log_dups_size":0,"ondisk_log_size":5168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179961856,"num_objects":2746,"num_object_clones":0,"num_object_copies":8238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2061,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1492},{"shards":"0,3","objects":665}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":482,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.336164+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:25:55.336164+0000","last_peered":"2023-10-09T10:25:55.336164+0000","last_clean":"2023-10-09T10:25:55.336164+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:25:55.336164+0000","last_undegraded":"2023-10-09T10:25:55.336164+0000","last_fullsized":"2023-10-09T10:25:55.336164+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":544,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350398+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:25:55.350398+0000","last_peered":"2023-10-09T10:25:55.350398+0000","last_clean":"2023-10-09T10:25:55.350398+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:25:55.350398+0000","last_undegraded":"2023-10-09T10:25:55.350398+0000","last_fullsized":"2023-10-09T10:25:55.350398+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"530'8500","reported_seq":8579,"reported_epoch":530,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:08.116067+0000","last_change":"2023-10-09T10:25:17.929569+0000","last_active":"2023-10-09T10:26:08.116067+0000","last_peered":"2023-10-09T10:26:08.116067+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:26:08.116067+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"503'2600","ondisk_log_start":"503'2600","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5900,"log_dups_size":0,"ondisk_log_size":5900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366673944,"num_objects":5596,"num_object_clones":0,"num_object_copies":16788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5593,"num_objects_misplaced":5596,"num_objects_unfound":0,"num_objects_dirty":5596,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8500,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":5596}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":250,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.353106+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:25:55.353106+0000","last_peered":"2023-10-09T10:25:55.353106+0000","last_clean":"2023-10-09T10:25:55.353106+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:25:55.353106+0000","last_undegraded":"2023-10-09T10:25:55.353106+0000","last_fullsized":"2023-10-09T10:25:55.353106+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":214,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350061+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:25:55.350061+0000","last_peered":"2023-10-09T10:25:55.350061+0000","last_clean":"2023-10-09T10:25:55.350061+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:25:55.350061+0000","last_undegraded":"2023-10-09T10:25:55.350061+0000","last_fullsized":"2023-10-09T10:25:55.350061+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":494,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.337771+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:25:55.337771+0000","last_peered":"2023-10-09T10:25:55.337771+0000","last_clean":"2023-10-09T10:25:55.337771+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:25:55.337771+0000","last_undegraded":"2023-10-09T10:25:55.337771+0000","last_fullsized":"2023-10-09T10:25:55.337771+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"530'8622","reported_seq":8667,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:07.800527+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:26:07.800527+0000","last_peered":"2023-10-09T10:26:07.800527+0000","last_clean":"2023-10-09T10:26:07.800527+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:26:07.800527+0000","last_undegraded":"2023-10-09T10:26:07.800527+0000","last_fullsized":"2023-10-09T10:26:07.800527+0000","mapping_epoch":494,"log_start":"503'3606","ondisk_log_start":"503'3606","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5016,"log_dups_size":0,"ondisk_log_size":5016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370671616,"num_objects":5656,"num_object_clones":0,"num_object_copies":16968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8622,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":214,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.337625+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:25:55.337625+0000","last_peered":"2023-10-09T10:25:55.337625+0000","last_clean":"2023-10-09T10:25:55.337625+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:25:55.337625+0000","last_undegraded":"2023-10-09T10:25:55.337625+0000","last_fullsized":"2023-10-09T10:25:55.337625+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":132,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.349328+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:25:55.349328+0000","last_peered":"2023-10-09T10:25:55.349328+0000","last_clean":"2023-10-09T10:25:55.349328+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:25:55.349328+0000","last_undegraded":"2023-10-09T10:25:55.349328+0000","last_fullsized":"2023-10-09T10:25:55.349328+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":442,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.358633+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:25:55.358633+0000","last_peered":"2023-10-09T10:25:55.358633+0000","last_clean":"2023-10-09T10:25:55.358633+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:25:55.358633+0000","last_undegraded":"2023-10-09T10:25:55.358633+0000","last_fullsized":"2023-10-09T10:25:55.358633+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"530'5221","reported_seq":7544,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:04.966224+0000","last_change":"2023-10-09T10:25:09.703037+0000","last_active":"2023-10-09T10:26:04.966224+0000","last_peered":"2023-10-09T10:26:04.966224+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:25:09.696864+0000","last_became_peered":"2023-10-09T10:25:09.696864+0000","last_unstale":"2023-10-09T10:26:04.966224+0000","last_undegraded":"2023-10-09T10:25:09.545735+0000","last_fullsized":"2023-10-09T10:25:09.430540+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5221,"log_dups_size":0,"ondisk_log_size":5221,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180453376,"num_objects":2753,"num_object_clones":0,"num_object_copies":8259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3882,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4255,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":751,"num_bytes_recovered":49217536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1539},{"shards":"2,5,6","objects":804},{"shards":"5,6","objects":410}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":480,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348864+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:25:55.348864+0000","last_peered":"2023-10-09T10:25:55.348864+0000","last_clean":"2023-10-09T10:25:55.348864+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:25:55.348864+0000","last_undegraded":"2023-10-09T10:25:55.348864+0000","last_fullsized":"2023-10-09T10:25:55.348864+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":97,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342405+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:25:55.342405+0000","last_peered":"2023-10-09T10:25:55.342405+0000","last_clean":"2023-10-09T10:25:55.342405+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:25:55.342405+0000","last_undegraded":"2023-10-09T10:25:55.342405+0000","last_fullsized":"2023-10-09T10:25:55.342405+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":543,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.343419+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:25:55.343419+0000","last_peered":"2023-10-09T10:25:55.343419+0000","last_clean":"2023-10-09T10:25:55.343419+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:25:55.343419+0000","last_undegraded":"2023-10-09T10:25:55.343419+0000","last_fullsized":"2023-10-09T10:25:55.343419+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"530'8467","reported_seq":8530,"reported_epoch":530,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:04.985182+0000","last_change":"2023-10-09T10:24:41.858672+0000","last_active":"2023-10-09T10:26:04.985182+0000","last_peered":"2023-10-09T10:26:04.985182+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:26:04.985182+0000","last_undegraded":"2023-10-09T10:26:04.985182+0000","last_fullsized":"2023-10-09T10:26:04.985182+0000","mapping_epoch":507,"log_start":"503'3600","ondisk_log_start":"503'3600","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4867,"log_dups_size":0,"ondisk_log_size":4867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369950720,"num_objects":5645,"num_object_clones":0,"num_object_copies":16935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5645,"num_objects_unfound":0,"num_objects_dirty":5645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8467,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":5645}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":184,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342221+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:25:55.342221+0000","last_peered":"2023-10-09T10:25:55.342221+0000","last_clean":"2023-10-09T10:25:55.342221+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:25:55.342221+0000","last_undegraded":"2023-10-09T10:25:55.342221+0000","last_fullsized":"2023-10-09T10:25:55.342221+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":723,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342146+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:25:55.342146+0000","last_peered":"2023-10-09T10:25:55.342146+0000","last_clean":"2023-10-09T10:25:55.342146+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:25:55.342146+0000","last_undegraded":"2023-10-09T10:25:55.342146+0000","last_fullsized":"2023-10-09T10:25:55.342146+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":232,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.333808+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:25:55.333808+0000","last_peered":"2023-10-09T10:25:55.333808+0000","last_clean":"2023-10-09T10:25:55.333808+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:25:55.333808+0000","last_undegraded":"2023-10-09T10:25:55.333808+0000","last_fullsized":"2023-10-09T10:25:55.333808+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":141,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350564+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:25:55.350564+0000","last_peered":"2023-10-09T10:25:55.350564+0000","last_clean":"2023-10-09T10:25:55.350564+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:25:55.350564+0000","last_undegraded":"2023-10-09T10:25:55.350564+0000","last_fullsized":"2023-10-09T10:25:55.350564+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"530'5234","reported_seq":7329,"reported_epoch":530,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:26:04.999364+0000","last_change":"2023-10-09T10:25:13.759251+0000","last_active":"2023-10-09T10:26:04.999364+0000","last_peered":"2023-10-09T10:26:04.999364+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:26:04.999364+0000","last_undegraded":"2023-10-09T10:25:13.707357+0000","last_fullsized":"2023-10-09T10:26:04.999364+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5234,"log_dups_size":0,"ondisk_log_size":5234,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191922176,"num_objects":2928,"num_object_clones":0,"num_object_copies":8784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1613,"num_objects_misplaced":2566,"num_objects_unfound":0,"num_objects_dirty":2928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4310,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":691,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1613},{"shards":"3,5,6","objects":362}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":214,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342550+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:25:55.342550+0000","last_peered":"2023-10-09T10:25:55.342550+0000","last_clean":"2023-10-09T10:25:55.342550+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:25:55.342550+0000","last_undegraded":"2023-10-09T10:25:55.342550+0000","last_fullsized":"2023-10-09T10:25:55.342550+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":474,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.349018+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:25:55.349018+0000","last_peered":"2023-10-09T10:25:55.349018+0000","last_clean":"2023-10-09T10:25:55.349018+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:25:55.349018+0000","last_undegraded":"2023-10-09T10:25:55.349018+0000","last_fullsized":"2023-10-09T10:25:55.349018+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":25,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.351564+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:25:55.351564+0000","last_peered":"2023-10-09T10:25:55.351564+0000","last_clean":"2023-10-09T10:25:55.351564+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:25:55.351564+0000","last_undegraded":"2023-10-09T10:25:55.351564+0000","last_fullsized":"2023-10-09T10:25:55.351564+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"530'8648","reported_seq":8692,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:08.103574+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:26:08.103574+0000","last_peered":"2023-10-09T10:26:08.103574+0000","last_clean":"2023-10-09T10:26:08.103574+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:26:08.103574+0000","last_undegraded":"2023-10-09T10:26:08.103574+0000","last_fullsized":"2023-10-09T10:26:08.103574+0000","mapping_epoch":494,"log_start":"503'2700","ondisk_log_start":"503'2700","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364642304,"num_objects":5564,"num_object_clones":0,"num_object_copies":16692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8648,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.351163+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:25:55.351163+0000","last_peered":"2023-10-09T10:25:55.351163+0000","last_clean":"2023-10-09T10:25:55.351163+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:25:55.351163+0000","last_undegraded":"2023-10-09T10:25:55.351163+0000","last_fullsized":"2023-10-09T10:25:55.351163+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":511,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.343795+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:25:55.343795+0000","last_peered":"2023-10-09T10:25:55.343795+0000","last_clean":"2023-10-09T10:25:55.343795+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:25:55.343795+0000","last_undegraded":"2023-10-09T10:25:55.343795+0000","last_fullsized":"2023-10-09T10:25:55.343795+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":442,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342673+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:25:55.342673+0000","last_peered":"2023-10-09T10:25:55.342673+0000","last_clean":"2023-10-09T10:25:55.342673+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:25:55.342673+0000","last_undegraded":"2023-10-09T10:25:55.342673+0000","last_fullsized":"2023-10-09T10:25:55.342673+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"530'8328","reported_seq":8373,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:07.812457+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:26:07.812457+0000","last_peered":"2023-10-09T10:26:07.812457+0000","last_clean":"2023-10-09T10:26:07.812457+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:26:07.812457+0000","last_undegraded":"2023-10-09T10:26:07.812457+0000","last_fullsized":"2023-10-09T10:26:07.812457+0000","mapping_epoch":494,"log_start":"503'3276","ondisk_log_start":"503'3276","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5052,"log_dups_size":0,"ondisk_log_size":5052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354418688,"num_objects":5408,"num_object_clones":0,"num_object_copies":16224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8328,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":330,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.368857+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:25:55.368857+0000","last_peered":"2023-10-09T10:25:55.368857+0000","last_clean":"2023-10-09T10:25:55.368857+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:25:55.368857+0000","last_undegraded":"2023-10-09T10:25:55.368857+0000","last_fullsized":"2023-10-09T10:25:55.368857+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":511,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350629+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:25:55.350629+0000","last_peered":"2023-10-09T10:25:55.350629+0000","last_clean":"2023-10-09T10:25:55.350629+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:25:55.350629+0000","last_undegraded":"2023-10-09T10:25:55.350629+0000","last_fullsized":"2023-10-09T10:25:55.350629+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342139+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:25:55.342139+0000","last_peered":"2023-10-09T10:25:55.342139+0000","last_clean":"2023-10-09T10:25:55.342139+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:25:55.342139+0000","last_undegraded":"2023-10-09T10:25:55.342139+0000","last_fullsized":"2023-10-09T10:25:55.342139+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"530'8642","reported_seq":8686,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:08.063324+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:26:08.063324+0000","last_peered":"2023-10-09T10:26:08.063324+0000","last_clean":"2023-10-09T10:26:08.063324+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:26:08.063324+0000","last_undegraded":"2023-10-09T10:26:08.063324+0000","last_fullsized":"2023-10-09T10:26:08.063324+0000","mapping_epoch":494,"log_start":"503'2700","ondisk_log_start":"503'2700","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368443392,"num_objects":5622,"num_object_clones":0,"num_object_copies":16866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8642,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":33,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.347493+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:25:55.347493+0000","last_peered":"2023-10-09T10:25:55.347493+0000","last_clean":"2023-10-09T10:25:55.347493+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:25:55.347493+0000","last_undegraded":"2023-10-09T10:25:55.347493+0000","last_fullsized":"2023-10-09T10:25:55.347493+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":29,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.355089+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:25:55.355089+0000","last_peered":"2023-10-09T10:25:55.355089+0000","last_clean":"2023-10-09T10:25:55.355089+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:25:55.355089+0000","last_undegraded":"2023-10-09T10:25:55.355089+0000","last_fullsized":"2023-10-09T10:25:55.355089+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":210,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.335661+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:25:55.335661+0000","last_peered":"2023-10-09T10:25:55.335661+0000","last_clean":"2023-10-09T10:25:55.335661+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:25:55.335661+0000","last_undegraded":"2023-10-09T10:25:55.335661+0000","last_fullsized":"2023-10-09T10:25:55.335661+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"530'5070","reported_seq":5162,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:04.746359+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:26:04.746359+0000","last_peered":"2023-10-09T10:26:04.746359+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:26:04.746359+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5070,"log_dups_size":0,"ondisk_log_size":5070,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182288384,"num_objects":2782,"num_object_clones":0,"num_object_copies":8346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4150,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1546},{"shards":"2,5","objects":566}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":25,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.342608+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:25:55.342608+0000","last_peered":"2023-10-09T10:25:55.342608+0000","last_clean":"2023-10-09T10:25:55.342608+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:25:55.342608+0000","last_undegraded":"2023-10-09T10:25:55.342608+0000","last_fullsized":"2023-10-09T10:25:55.342608+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":34,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.350693+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:25:55.350693+0000","last_peered":"2023-10-09T10:25:55.350693+0000","last_clean":"2023-10-09T10:25:55.350693+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:25:55.350693+0000","last_undegraded":"2023-10-09T10:25:55.350693+0000","last_fullsized":"2023-10-09T10:25:55.350693+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"530'5226","reported_seq":5291,"reported_epoch":530,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:26:07.735296+0000","last_change":"2023-10-09T10:25:08.466144+0000","last_active":"2023-10-09T10:26:07.735296+0000","last_peered":"2023-10-09T10:26:07.735296+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:26:07.735296+0000","last_undegraded":"2023-10-09T10:26:07.735296+0000","last_fullsized":"2023-10-09T10:26:07.735296+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5226,"log_dups_size":0,"ondisk_log_size":5226,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180191232,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2331,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4302,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":419}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":260,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.343600+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:25:55.343600+0000","last_peered":"2023-10-09T10:25:55.343600+0000","last_clean":"2023-10-09T10:25:55.343600+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:25:55.343600+0000","last_undegraded":"2023-10-09T10:25:55.343600+0000","last_fullsized":"2023-10-09T10:25:55.343600+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.349433+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:25:55.349433+0000","last_peered":"2023-10-09T10:25:55.349433+0000","last_clean":"2023-10-09T10:25:55.349433+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:25:55.349433+0000","last_undegraded":"2023-10-09T10:25:55.349433+0000","last_fullsized":"2023-10-09T10:25:55.349433+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"530'5162","reported_seq":7245,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:04.968528+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:26:04.968528+0000","last_peered":"2023-10-09T10:26:04.968528+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:26:04.968528+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5162,"log_dups_size":0,"ondisk_log_size":5162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188317696,"num_objects":2874,"num_object_clones":0,"num_object_copies":8622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3970,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4218,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":672,"num_bytes_recovered":44040192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1568},{"shards":"0,5,6","objects":833},{"shards":"5,6","objects":472}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":243,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.349915+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:25:55.349915+0000","last_peered":"2023-10-09T10:25:55.349915+0000","last_clean":"2023-10-09T10:25:55.349915+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:25:55.349915+0000","last_undegraded":"2023-10-09T10:25:55.349915+0000","last_fullsized":"2023-10-09T10:25:55.349915+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":352,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.347130+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:25:55.347130+0000","last_peered":"2023-10-09T10:25:55.347130+0000","last_clean":"2023-10-09T10:25:55.347130+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:25:55.347130+0000","last_undegraded":"2023-10-09T10:25:55.347130+0000","last_fullsized":"2023-10-09T10:25:55.347130+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"530'5162","reported_seq":5223,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:09.281442+0000","last_change":"2023-10-09T10:25:13.762065+0000","last_active":"2023-10-09T10:26:09.281442+0000","last_peered":"2023-10-09T10:26:09.281442+0000","last_clean":"2023-10-09T10:25:11.551777+0000","last_became_active":"2023-10-09T10:25:13.754797+0000","last_became_peered":"2023-10-09T10:25:13.754797+0000","last_unstale":"2023-10-09T10:26:09.281442+0000","last_undegraded":"2023-10-09T10:25:13.625003+0000","last_fullsized":"2023-10-09T10:25:13.604794+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":5162,"log_dups_size":0,"ondisk_log_size":5162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:58.037975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180486144,"num_objects":2754,"num_object_clones":0,"num_object_copies":8262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2225,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":-383},{"shards":"2,4,5","objects":2373}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":28,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.331800+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:25:55.331800+0000","last_peered":"2023-10-09T10:25:55.331800+0000","last_clean":"2023-10-09T10:25:55.331800+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:25:55.331800+0000","last_undegraded":"2023-10-09T10:25:55.331800+0000","last_fullsized":"2023-10-09T10:25:55.331800+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":582,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.348782+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:25:55.348782+0000","last_peered":"2023-10-09T10:25:55.348782+0000","last_clean":"2023-10-09T10:25:55.348782+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:25:55.348782+0000","last_undegraded":"2023-10-09T10:25:55.348782+0000","last_fullsized":"2023-10-09T10:25:55.348782+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"530'5153","reported_seq":5210,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:04.869978+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:26:04.869978+0000","last_peered":"2023-10-09T10:26:04.869978+0000","last_clean":"2023-10-09T10:26:04.869978+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:26:04.869978+0000","last_undegraded":"2023-10-09T10:26:04.869978+0000","last_fullsized":"2023-10-09T10:26:04.869978+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":5153,"log_dups_size":0,"ondisk_log_size":5153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183042048,"num_objects":2793,"num_object_clones":0,"num_object_copies":8379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1892,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":46,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.368399+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:25:55.368399+0000","last_peered":"2023-10-09T10:25:55.368399+0000","last_clean":"2023-10-09T10:25:55.368399+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:25:55.368399+0000","last_undegraded":"2023-10-09T10:25:55.368399+0000","last_fullsized":"2023-10-09T10:25:55.368399+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":150,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-09T10:25:55.347632+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:25:55.347632+0000","last_peered":"2023-10-09T10:25:55.347632+0000","last_clean":"2023-10-09T10:25:55.347632+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:25:55.347632+0000","last_undegraded":"2023-10-09T10:25:55.347632+0000","last_fullsized":"2023-10-09T10:25:55.347632+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":5870157848,"num_objects":89574,"num_object_clones":0,"num_object_copies":268722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":44392,"num_objects_misplaced":24021,"num_objects_unfound":0,"num_objects_dirty":89574,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121372,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12143,"num_bytes_recovered":795803648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18488762368,"data_stored":18488754224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136252,"ondisk_log_size":136252,"up":78,"acting":63,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430677,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626905,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2263336,"kb_used_data":1359968,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":91481816,"statfs":{"total":95995035648,"available":93677379584,"internally_reserved":0,"allocated":1392607232,"data_stored":1391692185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:15 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.3999999999999999,"15min":1.6739999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":4.8099999999999996,"5min":9.4260000000000002,"15min":44.055},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.756,"15min":1.919},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.22},"max":{"1min":3.6760000000000002,"5min":9.2609999999999992,"15min":43.869999999999997},"last":5.6079999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":2.911,"15min":2.2400000000000002},"min":{"1min":0.29399999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.0419999999999998,"5min":155.01400000000001,"15min":155.01400000000001},"last":3.23},{"interface":"front","average":{"1min":1.383,"5min":2.9460000000000002,"15min":2.4060000000000001},"min":{"1min":0.33200000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.5709999999999997,"5min":154.71100000000001,"15min":154.71100000000001},"last":4.7149999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:25:15 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":1.7889999999999999,"15min":1.887},"min":{"1min":0.32800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.8460000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":2.8919999999999999},{"interface":"front","average":{"1min":1.6970000000000001,"5min":1.649,"15min":1.8029999999999999},"min":{"1min":0.36299999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.875,"5min":7.5629999999999997,"15min":14.542999999999999},"last":0.73699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.619,"15min":1.514},"min":{"1min":0.498,"5min":0.308,"15min":0.308},"max":{"1min":7.681,"5min":7.681,"15min":8.0229999999999997},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.496,"5min":1.927,"15min":1.6220000000000001},"min":{"1min":0.36399999999999999,"5min":0.316,"15min":0.31},"max":{"1min":7.6920000000000002,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.94099999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.881,"5min":3.4460000000000002,"15min":1.8600000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":10.618,"5min":122.041,"15min":122.041},"last":1.575},{"interface":"front","average":{"1min":1.8069999999999999,"5min":3.3130000000000002,"15min":1.75},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.9470000000000001,"5min":122.13200000000001,"15min":122.13200000000001},"last":0.311}]},{"osd":5,"last update":"Mon Oct 9 10:25:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":3.4620000000000002,"15min":1.976},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.6890000000000001,"5min":121.98999999999999,"15min":121.98999999999999},"last":1.655},{"interface":"front","average":{"1min":0.95299999999999996,"5min":2.6280000000000001,"15min":1.6200000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":5.1420000000000003,"5min":122.346,"15min":122.346},"last":0.55000000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":3.532,"15min":1.9730000000000001},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":4.9169999999999998,"5min":130.66999999999999,"15min":130.66999999999999},"last":0.495},{"interface":"front","average":{"1min":1.3779999999999999,"5min":3.573,"15min":2.016},"min":{"1min":0.374,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":4.9589999999999996,"5min":130.702,"15min":130.702},"last":0.52000000000000002}]}]},{"osd":1,"up_from":0,"seq":360777253219,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416743,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4374648,"kb_used_data":3426608,"kb_used_omap":28,"kb_used_meta":948003,"kb_avail":89370504,"statfs":{"total":95995035648,"available":91515396096,"internally_reserved":0,"allocated":3508846592,"data_stored":3506794024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":970755920},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:25:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.6200000000000001,"15min":2.0059999999999998},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":17.161999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.585,"5min":1.506,"15min":2.4689999999999999},"min":{"1min":0.218,"5min":0.214,"15min":0.192},"max":{"1min":10.786,"5min":17.033000000000001,"15min":56.950000000000003},"last":1.0720000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.9630000000000001,"15min":2.1419999999999999},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.7839999999999998,"5min":47.215000000000003,"15min":53.988999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.99,"5min":2.254,"15min":2.1339999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.173,"5min":47.000999999999998,"15min":54.119},"last":0.83899999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.609,"15min":1.4850000000000001},"min":{"1min":0.28599999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.763,"5min":11.933,"15min":56.984999999999999},"last":8.6479999999999997},{"interface":"front","average":{"1min":2.8660000000000001,"5min":1.9790000000000001,"15min":1.5640000000000001},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.067,"5min":10.067,"15min":56.475999999999999},"last":0.64100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.869,"15min":2.3159999999999998},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":3.3399999999999999,"5min":28.289000000000001,"15min":57.719000000000001},"last":8.6959999999999997},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.1230000000000002,"15min":2.246},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.32400000000000001},"max":{"1min":8.2479999999999993,"5min":28.331,"15min":56.378},"last":0.68100000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.1259999999999999,"15min":3.129},"min":{"1min":0.46800000000000003,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":6.923,"5min":11.775,"15min":229.589},"last":3.6110000000000002},{"interface":"front","average":{"1min":2.238,"5min":1.8029999999999999,"15min":2.8279999999999998},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":7.032,"5min":8.7530000000000001,"15min":228.88900000000001},"last":0.76600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.919,"15min":2.2320000000000002},"min":{"1min":0.435,"5min":0.379,"15min":0.33700000000000002},"max":{"1min":6.8230000000000004,"5min":8.7810000000000006,"15min":54.482999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.248,"5min":2.0859999999999999,"15min":2.6349999999999998},"min":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.7160000000000002,"5min":16.954999999999998,"15min":57.753},"last":8.8309999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.9930000000000001,"15min":2.391},"min":{"1min":0.54500000000000004,"5min":0.38600000000000001,"15min":0.34999999999999998},"max":{"1min":6.6079999999999997,"5min":8.0329999999999995,"15min":56.411999999999999},"last":1.466},{"interface":"front","average":{"1min":2.0070000000000001,"5min":2.0640000000000001,"15min":2.258},"min":{"1min":0.47099999999999997,"5min":0.38600000000000001,"15min":0.36499999999999999},"max":{"1min":6.891,"5min":11.175000000000001,"15min":54.451999999999998},"last":0.79800000000000004}]}]},{"osd":2,"up_from":217,"seq":932007903479,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4345904,"kb_used_data":3523496,"kb_used_omap":83,"kb_used_meta":822316,"kb_avail":89399248,"statfs":{"total":95995035648,"available":91544829952,"internally_reserved":0,"allocated":3608059904,"data_stored":3606199476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":842052079},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.778,"15min":2.0600000000000001},"min":{"1min":0.216,"5min":0.20899999999999999,"15min":0.185},"max":{"1min":5.2480000000000002,"5min":48.990000000000002,"15min":48.990000000000002},"last":7.1399999999999997},{"interface":"front","average":{"1min":1.5089999999999999,"5min":2.0649999999999999,"15min":2.3849999999999998},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.590999999999999,"5min":49.412999999999997,"15min":49.412999999999997},"last":5.0309999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.3089999999999999,"15min":2.0899999999999999},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":5.008,"5min":8.2010000000000005,"15min":34.893999999999998},"last":7.1859999999999999},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.4910000000000001,"15min":2.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":7.8029999999999999,"5min":7.8029999999999999,"15min":34.962000000000003},"last":0.94499999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.4139999999999999,"15min":1.202},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.248},"max":{"1min":9.3119999999999994,"5min":9.3119999999999994,"15min":35.043999999999997},"last":0.92100000000000004},{"interface":"front","average":{"1min":2.7909999999999999,"5min":1.2829999999999999,"15min":1.3},"min":{"1min":0.35699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.206,"5min":11.206,"15min":50.406999999999996},"last":0.85899999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:25:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.772,"15min":1.6990000000000001},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.26600000000000001},"max":{"1min":5.6749999999999998,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.8859999999999999,"15min":1.845},"min":{"1min":0.40699999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.2720000000000002,"5min":47.704000000000001,"15min":47.704000000000001},"last":0.83499999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.603,"15min":1.992},"min":{"1min":0.39100000000000001,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":3.0329999999999999,"5min":10.448,"15min":57.298000000000002},"last":0.81000000000000005},{"interface":"front","average":{"1min":1.534,"5min":1.7050000000000001,"15min":2.0310000000000001},"min":{"1min":0.374,"5min":0.31,"15min":0.31},"max":{"1min":5.8810000000000002,"5min":8.1630000000000003,"15min":57.828000000000003},"last":2.0030000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.99,"5min":2.2360000000000002,"15min":2.0329999999999999},"min":{"1min":0.45000000000000001,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":5.5510000000000002,"5min":12.411,"15min":18.943999999999999},"last":1.7290000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.744,"15min":1.95},"min":{"1min":0.442,"5min":0.38300000000000001,"15min":0.36599999999999999},"max":{"1min":3.0950000000000002,"5min":9.8399999999999999,"15min":24.379000000000001},"last":0.77900000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.3370000000000002,"15min":2.2850000000000001},"min":{"1min":0.53500000000000003,"5min":0.41799999999999998,"15min":0.308},"max":{"1min":3.5760000000000001,"5min":13.821,"15min":35.008000000000003},"last":4.976},{"interface":"front","average":{"1min":1.655,"5min":1.6619999999999999,"15min":2.1160000000000001},"min":{"1min":0.439,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":6.2140000000000004,"5min":10.507,"15min":24.323},"last":1.163}]}]},{"osd":3,"up_from":352,"seq":1511828488318,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3219120,"kb_used_data":2734248,"kb_used_omap":15,"kb_used_meta":484784,"kb_avail":90526032,"statfs":{"total":95995035648,"available":92698656768,"internally_reserved":0,"allocated":2799869952,"data_stored":2797536441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":496419270},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8259999999999996,"5min":3.1200000000000001,"15min":1.696},"min":{"1min":0.34999999999999998,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":70.004000000000005,"5min":70.004000000000005,"15min":70.004000000000005},"last":3.4569999999999999},{"interface":"front","average":{"1min":6.2519999999999998,"5min":2.8159999999999998,"15min":1.6739999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":66.802000000000007,"5min":66.802000000000007,"15min":66.802000000000007},"last":8.1899999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7889999999999999,"15min":1.8620000000000001},"min":{"1min":0.311,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.785,"5min":25.370000000000001,"15min":105.401},"last":3.4790000000000001},{"interface":"front","average":{"1min":2.7109999999999999,"5min":2.04,"15min":1.7969999999999999},"min":{"1min":0.30299999999999999,"5min":0.25900000000000001,"15min":0.23300000000000001},"max":{"1min":9.2140000000000004,"5min":14.613,"15min":104.914},"last":2.8140000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.2989999999999999,"15min":1.9490000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":11.141999999999999,"5min":11.141999999999999,"15min":11.141999999999999},"last":13.497999999999999},{"interface":"front","average":{"1min":3.1400000000000001,"5min":1.956,"15min":1.4279999999999999},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.27600000000000002},"max":{"1min":16.475000000000001,"5min":25.338999999999999,"15min":25.338999999999999},"last":3.3839999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.194,"15min":1.415},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":6.9370000000000003,"5min":21.196000000000002,"15min":21.831},"last":2.9660000000000002},{"interface":"front","average":{"1min":3.3980000000000001,"5min":2.8820000000000001,"15min":1.958},"min":{"1min":0.36399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":11.087,"5min":19.323,"15min":22.097999999999999},"last":13.577999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":2.0910000000000002,"15min":1.294},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":11.363,"5min":25.251999999999999,"15min":25.251999999999999},"last":3.7799999999999998},{"interface":"front","average":{"1min":2.46,"5min":2.411,"15min":1.5429999999999999},"min":{"1min":0.36599999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":6.9900000000000002,"5min":13.394,"15min":13.394},"last":2.8780000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.4780000000000002,"15min":1.7250000000000001},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.35599999999999998},"max":{"1min":11.114000000000001,"5min":19.114000000000001,"15min":19.114000000000001},"last":13.545999999999999},{"interface":"front","average":{"1min":2.3279999999999998,"5min":2.1230000000000002,"15min":1.468},"min":{"1min":0.47699999999999998,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":11.32,"5min":25.306999999999999,"15min":25.306999999999999},"last":3.4159999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":2.0779999999999998,"15min":1.514},"min":{"1min":0.42299999999999999,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":6.9640000000000004,"5min":9.7880000000000003,"15min":10.459},"last":2.8500000000000001},{"interface":"front","average":{"1min":2.327,"5min":2.3220000000000001,"15min":1.762},"min":{"1min":0.432,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":11.045999999999999,"5min":19.280000000000001,"15min":19.280000000000001},"last":4.0460000000000003}]}]},{"osd":4,"up_from":266,"seq":1142461300948,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4298668,"kb_used_data":3369444,"kb_used_omap":35,"kb_used_meta":928988,"kb_avail":89446484,"statfs":{"total":95995035648,"available":91593199616,"internally_reserved":0,"allocated":3450310656,"data_stored":3449389935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":951283950},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.893,"15min":2.1469999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":6.5019999999999998,"5min":14.548,"15min":97.721999999999994},"last":2.9319999999999999},{"interface":"front","average":{"1min":2.8940000000000001,"5min":2.694,"15min":2.8740000000000001},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.224},"max":{"1min":11.022,"5min":11.257,"15min":97.915999999999997},"last":0.376}]},{"osd":1,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4449999999999998,"15min":2.3700000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.24299999999999999},"max":{"1min":12.209,"5min":12.209,"15min":71.457999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.992,"5min":2.8999999999999999,"15min":2.2269999999999999},"min":{"1min":0.376,"5min":0.248,"15min":0.248},"max":{"1min":8.327,"5min":49.781999999999996,"15min":71.658000000000001},"last":2.1200000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3809999999999998,"15min":1.845},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":7.3230000000000004,"5min":15.944000000000001,"15min":15.944000000000001},"last":2.0459999999999998},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.9970000000000001,"15min":1.7989999999999999},"min":{"1min":0.36199999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.102,"5min":14.473000000000001,"15min":14.473000000000001},"last":0.51200000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.7410000000000001,"5min":2.9199999999999999,"15min":2.024},"min":{"1min":0.39800000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.347,"5min":12.074,"15min":12.074},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.907,"15min":1.7390000000000001},"min":{"1min":0.40899999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.369999999999999,"5min":16.067,"15min":16.067},"last":2.0880000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":2.5939999999999999,"15min":2.069},"min":{"1min":0.314,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":3.3279999999999998,"5min":16.094000000000001,"15min":99.585999999999999},"last":2.1469999999999998},{"interface":"front","average":{"1min":1.054,"5min":2.4969999999999999,"15min":1.831},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.464,"5min":49.915999999999997,"15min":99.384},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.881,"15min":1.3740000000000001},"min":{"1min":0.33800000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.5389999999999997,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.8220000000000001,"15min":1.7949999999999999},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":12.242000000000001,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.436}]},{"osd":7,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":3.0409999999999999,"15min":1.9239999999999999},"min":{"1min":0.39500000000000002,"5min":0.316,"15min":0.307},"max":{"1min":12.391999999999999,"5min":17.369,"15min":17.369},"last":1.3580000000000001},{"interface":"front","average":{"1min":1.3320000000000001,"5min":2.7829999999999999,"15min":1.778},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.30399999999999999},"max":{"1min":4.6689999999999996,"5min":51.026000000000003,"15min":51.026000000000003},"last":2.1699999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823132,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5218080,"kb_used_data":3704920,"kb_used_omap":0,"kb_used_meta":1513088,"kb_avail":88527072,"statfs":{"total":95995035648,"available":90651721728,"internally_reserved":0,"allocated":3793838080,"data_stored":3792925413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549402112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.754,"15min":1.9370000000000001},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":16.536999999999999,"5min":22.469999999999999,"15min":22.469999999999999},"last":3.645},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.1309999999999998,"15min":1.7270000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":8.9339999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.46999999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:25:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.6339999999999999,"15min":2.2090000000000001},"min":{"1min":0.29599999999999999,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":16.562999999999999,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.8,"5min":1.8759999999999999,"15min":1.8580000000000001},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":8.9529999999999994,"5min":13.143000000000001,"15min":29.065000000000001},"last":0.50800000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6899999999999999,"5min":2.5680000000000001,"15min":2.1269999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":16.629999999999999,"5min":16.629999999999999,"15min":16.629999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.403,"5min":2.2890000000000001,"15min":1.7270000000000001},"min":{"1min":0.30499999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.8930000000000007,"5min":13.095000000000001,"15min":15.929},"last":0.60299999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.2170000000000001,"15min":1.7889999999999999},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":10.362,"5min":10.561999999999999,"15min":10.561999999999999},"last":1.603},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.0830000000000002,"15min":1.7270000000000001},"min":{"1min":0.41599999999999998,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9130000000000003,"5min":13.624000000000001,"15min":13.624000000000001},"last":1.5820000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":2.242,"15min":1.5760000000000001},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":10.826000000000001,"5min":12.023,"15min":12.023},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.6579999999999999,"5min":2.0369999999999999,"15min":1.399},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":8.8620000000000001,"5min":11.956,"15min":11.956},"last":2.9420000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.621,"15min":1.794},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":10.664999999999999,"5min":11.686,"15min":12.772},"last":3.0179999999999998},{"interface":"front","average":{"1min":2.4500000000000002,"5min":2.1640000000000001,"15min":1.665},"min":{"1min":0.29699999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.57199999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:25:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.6429999999999998,"15min":1.798},"min":{"1min":0.34000000000000002,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":16.686,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.397,"15min":1.1719999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":8.9719999999999995,"5min":11.994,"15min":11.994},"last":0.54000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3487563776,"data_stored":3487563776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3600547840,"data_stored":3600547840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2790457344,"data_stored":2790457344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3437039616,"data_stored":3437035544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3786412032,"data_stored":3786407960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1386741760,"data_stored":1386741760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:26:12.467 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:26:12.540 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:12.585 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:12.650 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:12.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:12.742 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:12 smithi012 ceph-mon[147926]: from='client.37086 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:12.747 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:12 smithi012 ceph-mon[152263]: from='client.37086 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:12.760 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:12.842 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:12 smithi110 ceph-mon[136032]: from='client.37086 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:12.951 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:12.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:12.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:13.060 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:13.172 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:13.321 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:13.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:13.431 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:13.478 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:13.535 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:13.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:13.641 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:13.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:13 smithi012 ceph-mon[147926]: pgmap v1484: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 43351/262260 objects degraded (16.530%); 23221/262260 objects misplaced (8.854%); 15 MiB/s, 239 objects/s recovering 2023-10-09T10:26:13.659 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 43351/262260 objects degraded (16.530%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:13.659 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:13 smithi012 ceph-mon[152263]: pgmap v1484: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 43351/262260 objects degraded (16.530%); 23221/262260 objects misplaced (8.854%); 15 MiB/s, 239 objects/s recovering 2023-10-09T10:26:13.659 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 43351/262260 objects degraded (16.530%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:13.693 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:26:13.693 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:26:13.693 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:26:13.693 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:26:13.772 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:13.890 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:13.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:13.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:13 smithi110 ceph-mon[136032]: pgmap v1484: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 43351/262260 objects degraded (16.530%); 23221/262260 objects misplaced (8.854%); 15 MiB/s, 239 objects/s recovering 2023-10-09T10:26:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 43351/262260 objects degraded (16.530%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:14.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:14.069 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:14.251 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:14.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:14.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:14.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:14.398 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:26:14.409 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:14.456 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:14.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:14.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:14.560 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:14.582 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:26:14.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:14.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2732857925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:14.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:14.696 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:14.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:14.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:14.788 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:14.804 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:14.854 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:14.895 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:14.895 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2732857925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:14.895 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:14.895 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:14.896 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2732857925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:14.896 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:14.896 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:14.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:14.925 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:26:14.967 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:26:15.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:15.010 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:26:15.045 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:26:15.057 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:15.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:26:15.105 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:15.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:15.114 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:26:15.158 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:26:15.194 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:26:15.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:15.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:26:15.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:26:15.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:26:15.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:26:15.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:26:15.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:26:15.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:26:15.221 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:26:15.278 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:15.293 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:15.379 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:15.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:15.489 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:15.579 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:15.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[147926]: pgmap v1485: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 43161/260445 objects degraded (16.572%); 23031/260445 objects misplaced (8.843%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:26:15.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:15.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[147926]: failed to encode map e531 with expected crc 2023-10-09T10:26:15.826 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[147926]: failed to encode map e531 with expected crc 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[147926]: osdmap e531: 8 total, 8 up, 6 in 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/293847894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[152263]: pgmap v1485: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 43161/260445 objects degraded (16.572%); 23031/260445 objects misplaced (8.843%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[152263]: failed to encode map e531 with expected crc 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[152263]: failed to encode map e531 with expected crc 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[152263]: osdmap e531: 8 total, 8 up, 6 in 2023-10-09T10:26:15.827 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/293847894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:15.948 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:15 smithi110 ceph-mon[136032]: pgmap v1485: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 43161/260445 objects degraded (16.572%); 23031/260445 objects misplaced (8.843%); 13 MiB/s, 212 objects/s recovering 2023-10-09T10:26:15.948 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:15 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:15.949 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:15 smithi110 ceph-mon[136032]: failed to encode map e531 with expected crc 2023-10-09T10:26:15.949 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:15 smithi110 ceph-mon[136032]: failed to encode map e531 with expected crc 2023-10-09T10:26:15.949 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:15 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:26:15.949 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:15 smithi110 ceph-mon[136032]: osdmap e531: 8 total, 8 up, 6 in 2023-10-09T10:26:15.949 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/293847894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:16.013 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:16.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:16.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:16.124 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:16.180 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:16.246 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:16.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:16.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:16.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:16.435 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:16.529 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:16.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:16.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:16.708 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:16.874 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:16.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:16.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:17.089 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:17.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:26:17.262 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:17.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:17.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:17.380 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:17.426 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:17.475 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:17.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:17.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:17.579 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:17.589 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:17 smithi012 ceph-mon[147926]: pgmap v1487: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 42491/256398 objects degraded (16.572%); 21947/256398 objects misplaced (8.560%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:26:17.621 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:17.643 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:26:17.643 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:26:17.682 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:17.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:17.687 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:17 smithi110 ceph-mon[136032]: pgmap v1487: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 42491/256398 objects degraded (16.572%); 21947/256398 objects misplaced (8.560%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:26:17.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:17.788 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:17.833 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:17.868 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:26:17.868 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 42491/256398 objects degraded (16.572%), 15 pgs degraded, 14 pgs undersized","count":29},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1894,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":531,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846541,"num_in_osds":6,"osd_in_since":1696847079,"num_remapped_pgs":18},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":12},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":85466,"data_bytes":5599958968,"bytes_used":23726166016,"bytes_avail":552244047872,"bytes_total":575970213888,"degraded_objects":42491,"degraded_total":256398,"degraded_ratio":0.16572282155087012,"misplaced_objects":21947,"misplaced_total":256398,"misplaced_ratio":0.085597391555316343,"recovering_objects_per_sec":226,"recovering_bytes_per_sec":14850521,"recovering_keys_per_sec":0,"num_objects_recovered":2267,"num_bytes_recovered":148570112,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":606},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":32,"modified":"2023-10-09T10:25:43.785386+0000","services":{}},"progress_events":{"4aaa0d12-13e6-4ea3-ae4b-c27ebe383851":{"message":"Global Recovery Event (5m)\n [======================......] (remaining: 68s)","progress":0.81818181276321411,"add_to_ceph_s":true},"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T10:26:17.873 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:26:17.895 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:17.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:17.916 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:17 smithi012 ceph-mon[152263]: pgmap v1487: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 42491/256398 objects degraded (16.572%); 21947/256398 objects misplaced (8.560%); 14 MiB/s, 226 objects/s recovering 2023-10-09T10:26:17.916 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:26:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:26:17] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-10-09T10:26:18.002 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:18.047 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:18.128 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:18.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:18.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:18.233 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:18.265 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:18.307 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:18.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:18.415 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:18.484 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:18.544 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:18.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:18.624 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2779918386' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:26:18.624 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 42491/256398 objects degraded (16.572%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:18.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:18.742 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2779918386' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:26:18.742 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 42491/256398 objects degraded (16.572%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:18.742 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2779918386' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:26:18.743 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 42491/256398 objects degraded (16.572%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:18.750 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:18.823 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:26:18.834 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:26:18.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:26:18.836 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:18.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:18.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:18.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:19.039 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:19.070 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:19.168 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:19.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:19.279 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:19.331 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:19.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:19.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:19.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:19.403 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:19.503 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:19.555 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:19.615 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:19.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:19.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:19 smithi012 ceph-mon[147926]: pgmap v1488: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 711 op/s; 42080/253248 objects degraded (16.616%); 21947/253248 objects misplaced (8.666%); 15 MiB/s, 247 objects/s recovering 2023-10-09T10:26:19.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:19 smithi012 ceph-mon[152263]: pgmap v1488: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 711 op/s; 42080/253248 objects degraded (16.616%); 21947/253248 objects misplaced (8.666%); 15 MiB/s, 247 objects/s recovering 2023-10-09T10:26:19.720 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:19.788 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:19.835 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:19.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:19.940 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:19 smithi110 ceph-mon[136032]: pgmap v1488: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 711 op/s; 42080/253248 objects degraded (16.616%); 21947/253248 objects misplaced (8.666%); 15 MiB/s, 247 objects/s recovering 2023-10-09T10:26:19.984 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:20.034 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:20.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:20.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:20.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:20.259 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:20.318 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:20.346 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:20.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:20.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:20.540 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:20.874 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:20.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:20.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:20.900 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:20.985 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:21.107 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:21.481 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:21.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:21.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:21.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:21.695 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:21.742 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:21 smithi012 ceph-mon[147926]: pgmap v1489: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.1 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 41862/250899 objects degraded (16.685%); 21439/250899 objects misplaced (8.545%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:26:21.742 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:21 smithi012 ceph-mon[152263]: pgmap v1489: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.1 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 41862/250899 objects degraded (16.685%); 21439/250899 objects misplaced (8.545%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:26:21.835 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:21.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:21.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:21.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:21 smithi110 ceph-mon[136032]: pgmap v1489: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.1 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 41862/250899 objects degraded (16.685%); 21439/250899 objects misplaced (8.545%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:26:21.946 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:21.997 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:22.055 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:22.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:22.160 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:22.202 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:22.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:22.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:22.310 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:26:22.361 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:22.372 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:26:22.407 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:26:22.444 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:26:22.452 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:22.486 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:26:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:26:22.571 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:26:22.617 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:26:22.646 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1341218566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:22.646 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:22 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2505864309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:22.647 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:26:22.647 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:26:22.647 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:26:22.647 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:26:22.647 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:26:22.647 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:26:22.647 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:26:22.647 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:26:22.648 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1341218566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:22.648 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:22 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2505864309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:22.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1341218566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:22.648 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:22 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2505864309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:22.695 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:26:22.732 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:22.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:22.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:22.842 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:22.896 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:22.967 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:22.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:23.073 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:23.130 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:23.207 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:23.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:23.317 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:23.381 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:23.446 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:23.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:23.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:23.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:23.609 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:23.616 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:26:23.616 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:195679 9.3 9:c12d9ed3:::benchmark_data_smithi012_313365_object83204:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 468.17211104, 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9465798859999999, 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:26:23.617 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195679 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:23.618 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436218+0000", 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436217+0000", 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436223+0000", 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9850000000000003e-06 2023-10-09T10:26:23.619 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436235+0000", 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1229e-05 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436243+0000", 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.9789999999999993e-06 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.620 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436300+0000", 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7265999999999997e-05 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.436951+0000", 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00065154099999999995 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345069+0000", 2023-10-09T10:26:23.621 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.90811741 2023-10-09T10:26:23.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.622 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.622 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:23.622 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345197+0000", 2023-10-09T10:26:23.622 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000128496 2023-10-09T10:26:23.622 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.345622+0000", 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042431400000000002 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.347037+0000", 2023-10-09T10:26:23.623 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014153449999999999 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.348160+0000", 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001123097 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382157+0000", 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.033996849000000003 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.624 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382214+0000", 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.7012000000000002e-05 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.382798+0000", 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00058430099999999996 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:23.625 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.45908.0:195683 9.13 9:c8f1253c:::benchmark_data_smithi012_313365_object83208:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 468.16688899899998, 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9973604489999999, 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.45908", 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/2712059247", 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 195683 2023-10-09T10:26:23.626 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441440+0000", 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.627 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441439+0000", 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441447+0000", 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4549999999999995e-06 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441463+0000", 2023-10-09T10:26:23.628 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5675000000000002e-05 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441471+0000", 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3100000000000001e-06 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.441521+0000", 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9709999999999997e-05 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.629 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for degraded object", 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:35.442142+0000", 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00062057699999999998 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383458+0000", 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.941316144 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:26:23.630 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383530+0000", 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2574e-05 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.383875+0000", 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00034479399999999999 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.386609+0000", 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027342930000000001 2023-10-09T10:26:23.631 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.386879+0000", 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026999800000000003 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.437748+0000", 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.050869079999999997 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.632 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.437820+0000", 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1427999999999994e-05 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:39.438801+0000", 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00098100300000000004 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:26:23.633 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:26:23.634 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:26:23.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: pgmap v1490: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 40822/243855 objects degraded (16.740%); 20626/243855 objects misplaced (8.458%); 14 MiB/s, 228 objects/s recovering 2023-10-09T10:26:23.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:26:23.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2505864309' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:23.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: osdmap e532: 8 total, 8 up, 6 in 2023-10-09T10:26:23.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.647 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 40822/243855 objects degraded (16.740%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:23.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: pgmap v1490: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 40822/243855 objects degraded (16.740%); 20626/243855 objects misplaced (8.458%); 14 MiB/s, 228 objects/s recovering 2023-10-09T10:26:23.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:26:23.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.648 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2505864309' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:23.668 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:23.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:23.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:23.773 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:23.822 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:23.882 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:23.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:23.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: pgmap v1490: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 5.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 40822/243855 objects degraded (16.740%); 20626/243855 objects misplaced (8.458%); 14 MiB/s, 228 objects/s recovering 2023-10-09T10:26:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:26:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2505864309' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:23.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: osdmap e532: 8 total, 8 up, 6 in 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 40822/243855 objects degraded (16.740%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: osdmap e532: 8 total, 8 up, 6 in 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: failed to encode map e532 with expected crc 2023-10-09T10:26:23.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 40822/243855 objects degraded (16.740%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:23.987 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:24.098 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:24.433 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:24.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:24.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:24.543 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:24.597 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:24.662 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:24.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:24.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:24.835 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:25.449 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:25.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:25.554 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:25.555 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:26:25.608 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:25.693 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:25.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:25.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:25.796 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:26:25.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:25.854 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:25 smithi012 ceph-mon[147926]: pgmap v1492: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 40633/241929 objects degraded (16.795%); 20437/241929 objects misplaced (8.448%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:26:25.856 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:26:25.856 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:25 smithi012 ceph-mon[152263]: pgmap v1492: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 40633/241929 objects degraded (16.795%); 20437/241929 objects misplaced (8.448%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:26:25.881 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:26:25.938 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:25.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:25 smithi110 ceph-mon[136032]: pgmap v1492: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 40633/241929 objects degraded (16.795%); 20437/241929 objects misplaced (8.448%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:26:26.067 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:26.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:26.080 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:26:26.181 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:26.255 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:26.311 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:26.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:26.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:26.424 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:26.463 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:26.479 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1493,"stamp":"2023-10-09T10:26:25.805486+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5181380536,"num_objects":79079,"num_object_clones":0,"num_object_copies":237237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":39897,"num_objects_misplaced":19208,"num_objects_unfound":0,"num_objects_dirty":79079,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":132151,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16467,"num_bytes_recovered":1071679228,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143465,"ondisk_log_size":143465,"up":297,"acting":282,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":22356748,"kb_used_data":16713180,"kb_used_omap":163,"kb_used_meta":5643164,"kb_avail":540114164,"statfs":{"total":575970213888,"available":553076903936,"internally_reserved":0,"allocated":17114296320,"data_stored":17105254920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":5778600947},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2,4,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-418578432,"num_objects":-6387,"num_object_clones":0,"num_object_copies":-19161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2594,"num_objects_misplaced":-2739,"num_objects_unfound":0,"num_objects_dirty":-6387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6387,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2381,"num_bytes_recovered":156041216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007213"},"pg_stats":[{"pgid":"11.12","version":"531'5547","reported_seq":5611,"reported_epoch":531,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:21.455215+0000","last_change":"2023-10-09T10:25:09.709209+0000","last_active":"2023-10-09T10:26:21.455215+0000","last_peered":"2023-10-09T10:26:21.455215+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:26:21.455215+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5547,"log_dups_size":0,"ondisk_log_size":5547,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168067072,"num_objects":2565,"num_object_clones":0,"num_object_copies":7695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3119,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4599,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-554},{"shards":"0,2,5","objects":2102}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":125,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562556+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:26:14.562556+0000","last_peered":"2023-10-09T10:26:14.562556+0000","last_clean":"2023-10-09T10:26:14.562556+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:26:14.562556+0000","last_undegraded":"2023-10-09T10:26:14.562556+0000","last_fullsized":"2023-10-09T10:26:14.562556+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":486,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.563037+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:26:14.563037+0000","last_peered":"2023-10-09T10:26:14.563037+0000","last_clean":"2023-10-09T10:26:14.563037+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:26:14.563037+0000","last_undegraded":"2023-10-09T10:26:14.563037+0000","last_fullsized":"2023-10-09T10:26:14.563037+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"531'5506","reported_seq":8669,"reported_epoch":531,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:21.519898+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:26:21.519898+0000","last_peered":"2023-10-09T10:26:21.519898+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:26:21.519898+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5506,"log_dups_size":0,"ondisk_log_size":5506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163905536,"num_objects":2501,"num_object_clones":0,"num_object_copies":7503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3313,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4561,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1030,"num_bytes_recovered":67502080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1234},{"shards":"0,2,6","objects":845},{"shards":"0,6","objects":422}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":563,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562024+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:26:14.562024+0000","last_peered":"2023-10-09T10:26:14.562024+0000","last_clean":"2023-10-09T10:26:14.562024+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:26:14.562024+0000","last_undegraded":"2023-10-09T10:26:14.562024+0000","last_fullsized":"2023-10-09T10:26:14.562024+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":455,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.421828+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:26:03.421828+0000","last_peered":"2023-10-09T10:26:03.421828+0000","last_clean":"2023-10-09T10:26:03.421828+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:26:03.421828+0000","last_undegraded":"2023-10-09T10:26:03.421828+0000","last_fullsized":"2023-10-09T10:26:03.421828+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"531'5457","reported_seq":9998,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:21.539968+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:26:21.539968+0000","last_peered":"2023-10-09T10:26:21.539968+0000","last_clean":"2023-10-09T10:26:21.539968+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:21.539968+0000","last_undegraded":"2023-10-09T10:26:21.539968+0000","last_fullsized":"2023-10-09T10:26:21.539968+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161316864,"num_objects":2462,"num_object_clones":0,"num_object_copies":7386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4520,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.414855+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:26:03.414855+0000","last_peered":"2023-10-09T10:26:03.414855+0000","last_clean":"2023-10-09T10:26:03.414855+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:26:03.414855+0000","last_undegraded":"2023-10-09T10:26:03.414855+0000","last_fullsized":"2023-10-09T10:26:03.414855+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.415010+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:26:03.415010+0000","last_peered":"2023-10-09T10:26:03.415010+0000","last_clean":"2023-10-09T10:26:03.415010+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:26:03.415010+0000","last_undegraded":"2023-10-09T10:26:03.415010+0000","last_fullsized":"2023-10-09T10:26:03.415010+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":489,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.417444+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:26:03.417444+0000","last_peered":"2023-10-09T10:26:03.417444+0000","last_clean":"2023-10-09T10:26:03.417444+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:26:03.417444+0000","last_undegraded":"2023-10-09T10:26:03.417444+0000","last_fullsized":"2023-10-09T10:26:03.417444+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":439,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.578133+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:26:14.578133+0000","last_peered":"2023-10-09T10:26:14.578133+0000","last_clean":"2023-10-09T10:26:14.578133+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:26:14.578133+0000","last_undegraded":"2023-10-09T10:26:14.578133+0000","last_fullsized":"2023-10-09T10:26:14.578133+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":207,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.566421+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:26:14.566421+0000","last_peered":"2023-10-09T10:26:14.566421+0000","last_clean":"2023-10-09T10:26:14.566421+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:26:14.566421+0000","last_undegraded":"2023-10-09T10:26:14.566421+0000","last_fullsized":"2023-10-09T10:26:14.566421+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":439,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.578715+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:26:14.578715+0000","last_peered":"2023-10-09T10:26:14.578715+0000","last_clean":"2023-10-09T10:26:14.578715+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:26:14.578715+0000","last_undegraded":"2023-10-09T10:26:14.578715+0000","last_fullsized":"2023-10-09T10:26:14.578715+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":137,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.563276+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:26:14.563276+0000","last_peered":"2023-10-09T10:26:14.563276+0000","last_clean":"2023-10-09T10:26:14.563276+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:26:14.563276+0000","last_undegraded":"2023-10-09T10:26:14.563276+0000","last_fullsized":"2023-10-09T10:26:14.563276+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.428146+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:26:03.428146+0000","last_peered":"2023-10-09T10:26:03.428146+0000","last_clean":"2023-10-09T10:26:03.428146+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:26:03.428146+0000","last_undegraded":"2023-10-09T10:26:03.428146+0000","last_fullsized":"2023-10-09T10:26:03.428146+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":183,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.568042+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:26:14.568042+0000","last_peered":"2023-10-09T10:26:14.568042+0000","last_clean":"2023-10-09T10:26:14.568042+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:26:14.568042+0000","last_undegraded":"2023-10-09T10:26:14.568042+0000","last_fullsized":"2023-10-09T10:26:14.568042+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":143,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.563018+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:26:14.563018+0000","last_peered":"2023-10-09T10:26:14.563018+0000","last_clean":"2023-10-09T10:26:14.563018+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:26:14.563018+0000","last_undegraded":"2023-10-09T10:26:14.563018+0000","last_fullsized":"2023-10-09T10:26:14.563018+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":103,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562814+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:26:14.562814+0000","last_peered":"2023-10-09T10:26:14.562814+0000","last_clean":"2023-10-09T10:26:14.562814+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:26:14.562814+0000","last_undegraded":"2023-10-09T10:26:14.562814+0000","last_fullsized":"2023-10-09T10:26:14.562814+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":57,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.570400+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:26:14.570400+0000","last_peered":"2023-10-09T10:26:14.570400+0000","last_clean":"2023-10-09T10:26:14.570400+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:26:14.570400+0000","last_undegraded":"2023-10-09T10:26:14.570400+0000","last_fullsized":"2023-10-09T10:26:14.570400+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":454,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.437505+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:26:03.437505+0000","last_peered":"2023-10-09T10:26:03.437505+0000","last_clean":"2023-10-09T10:26:03.437505+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:26:03.437505+0000","last_undegraded":"2023-10-09T10:26:03.437505+0000","last_fullsized":"2023-10-09T10:26:03.437505+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":198,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.576357+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:26:14.576357+0000","last_peered":"2023-10-09T10:26:14.576357+0000","last_clean":"2023-10-09T10:26:14.576357+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:26:14.576357+0000","last_undegraded":"2023-10-09T10:26:14.576357+0000","last_fullsized":"2023-10-09T10:26:14.576357+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"532'5552","reported_seq":6351,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:24.983887+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:26:24.983887+0000","last_peered":"2023-10-09T10:26:24.983887+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:26:24.983887+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162299904,"num_objects":2477,"num_object_clones":0,"num_object_copies":7431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":2477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4629,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-674},{"shards":"3,5,6","objects":843}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":143,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562149+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:26:14.562149+0000","last_peered":"2023-10-09T10:26:14.562149+0000","last_clean":"2023-10-09T10:26:14.562149+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:26:14.562149+0000","last_undegraded":"2023-10-09T10:26:14.562149+0000","last_fullsized":"2023-10-09T10:26:14.562149+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.421625+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:26:03.421625+0000","last_peered":"2023-10-09T10:26:03.421625+0000","last_clean":"2023-10-09T10:26:03.421625+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:26:03.421625+0000","last_undegraded":"2023-10-09T10:26:03.421625+0000","last_fullsized":"2023-10-09T10:26:03.421625+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"532'5631","reported_seq":12038,"reported_epoch":532,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:26:25.023042+0000","last_change":"2023-10-09T10:25:48.137669+0000","last_active":"2023-10-09T10:26:25.023042+0000","last_peered":"2023-10-09T10:26:25.023042+0000","last_clean":"2023-10-09T10:24:39.576387+0000","last_became_active":"2023-10-09T10:25:09.696470+0000","last_became_peered":"2023-10-09T10:25:09.696470+0000","last_unstale":"2023-10-09T10:26:25.023042+0000","last_undegraded":"2023-10-09T10:26:25.023042+0000","last_fullsized":"2023-10-09T10:25:09.436538+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159219712,"num_objects":2430,"num_object_clones":0,"num_object_copies":7290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1126,"num_objects_unfound":0,"num_objects_dirty":2430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4666,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":138412032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":166},{"shards":"2,5,6","objects":794},{"shards":"5,6","objects":1470}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":36,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.418043+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:26:03.418043+0000","last_peered":"2023-10-09T10:26:03.418043+0000","last_clean":"2023-10-09T10:26:03.418043+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:26:03.418043+0000","last_undegraded":"2023-10-09T10:26:03.418043+0000","last_fullsized":"2023-10-09T10:26:03.418043+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":30,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.578899+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:26:14.578899+0000","last_peered":"2023-10-09T10:26:14.578899+0000","last_clean":"2023-10-09T10:26:14.578899+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:26:14.578899+0000","last_undegraded":"2023-10-09T10:26:14.578899+0000","last_fullsized":"2023-10-09T10:26:14.578899+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"532'5671","reported_seq":5731,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:24.923807+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:26:24.923807+0000","last_peered":"2023-10-09T10:26:24.923807+0000","last_clean":"2023-10-09T10:26:24.923807+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:26:24.923807+0000","last_undegraded":"2023-10-09T10:26:24.923807+0000","last_fullsized":"2023-10-09T10:26:24.923807+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":5671,"log_dups_size":0,"ondisk_log_size":5671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162856960,"num_objects":2485,"num_object_clones":0,"num_object_copies":7455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2383,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":33,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.564478+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:26:14.564478+0000","last_peered":"2023-10-09T10:26:14.564478+0000","last_clean":"2023-10-09T10:26:14.564478+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:26:14.564478+0000","last_undegraded":"2023-10-09T10:26:14.564478+0000","last_fullsized":"2023-10-09T10:26:14.564478+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":207,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.566379+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:26:14.566379+0000","last_peered":"2023-10-09T10:26:14.566379+0000","last_clean":"2023-10-09T10:26:14.566379+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:26:14.566379+0000","last_undegraded":"2023-10-09T10:26:14.566379+0000","last_fullsized":"2023-10-09T10:26:14.566379+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"532'5315","reported_seq":9605,"reported_epoch":532,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:24.450881+0000","last_change":"2023-10-09T10:25:48.142610+0000","last_active":"2023-10-09T10:26:24.450881+0000","last_peered":"2023-10-09T10:26:24.450881+0000","last_clean":"2023-10-09T10:25:11.561932+0000","last_became_active":"2023-10-09T10:25:13.746279+0000","last_became_peered":"2023-10-09T10:25:13.746279+0000","last_unstale":"2023-10-09T10:26:24.450881+0000","last_undegraded":"2023-10-09T10:25:13.623380+0000","last_fullsized":"2023-10-09T10:25:13.604555+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5315,"log_dups_size":0,"ondisk_log_size":5315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:39.267545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153747456,"num_objects":2346,"num_object_clones":0,"num_object_copies":7038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":343,"num_objects_misplaced":556,"num_objects_unfound":0,"num_objects_dirty":2346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2555,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2113,"num_bytes_recovered":138477568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":1447},{"shards":"2,4,5","objects":556}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":159,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562893+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:26:14.562893+0000","last_peered":"2023-10-09T10:26:14.562893+0000","last_clean":"2023-10-09T10:26:14.562893+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:26:14.562893+0000","last_undegraded":"2023-10-09T10:26:14.562893+0000","last_fullsized":"2023-10-09T10:26:14.562893+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":146,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562218+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:26:14.562218+0000","last_peered":"2023-10-09T10:26:14.562218+0000","last_clean":"2023-10-09T10:26:14.562218+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:26:14.562218+0000","last_undegraded":"2023-10-09T10:26:14.562218+0000","last_fullsized":"2023-10-09T10:26:14.562218+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"532'5575","reported_seq":8944,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:25.003207+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:26:25.003207+0000","last_peered":"2023-10-09T10:26:25.003207+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:26:25.003207+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159547392,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3255,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4630,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1221},{"shards":"0,5,6","objects":813},{"shards":"5,6","objects":400}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":223,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.574220+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:26:14.574220+0000","last_peered":"2023-10-09T10:26:14.574220+0000","last_clean":"2023-10-09T10:26:14.574220+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:26:14.574220+0000","last_undegraded":"2023-10-09T10:26:14.574220+0000","last_fullsized":"2023-10-09T10:26:14.574220+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":124,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.568965+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:26:14.568965+0000","last_peered":"2023-10-09T10:26:14.568965+0000","last_clean":"2023-10-09T10:26:14.568965+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:26:14.568965+0000","last_undegraded":"2023-10-09T10:26:14.568965+0000","last_fullsized":"2023-10-09T10:26:14.568965+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"532'5515","reported_seq":9700,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:24.876812+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:26:24.876812+0000","last_peered":"2023-10-09T10:26:24.876812+0000","last_clean":"2023-10-09T10:26:24.876812+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:26:24.876812+0000","last_undegraded":"2023-10-09T10:26:24.876812+0000","last_fullsized":"2023-10-09T10:26:24.876812+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161349632,"num_objects":2462,"num_object_clones":0,"num_object_copies":7386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2601,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":207,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.566911+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:26:14.566911+0000","last_peered":"2023-10-09T10:26:14.566911+0000","last_clean":"2023-10-09T10:26:14.566911+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:26:14.566911+0000","last_undegraded":"2023-10-09T10:26:14.566911+0000","last_fullsized":"2023-10-09T10:26:14.566911+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":440,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.417564+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:26:03.417564+0000","last_peered":"2023-10-09T10:26:03.417564+0000","last_clean":"2023-10-09T10:26:03.417564+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:26:03.417564+0000","last_undegraded":"2023-10-09T10:26:03.417564+0000","last_fullsized":"2023-10-09T10:26:03.417564+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"532'5503","reported_seq":6792,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:24.934884+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:26:24.934884+0000","last_peered":"2023-10-09T10:26:24.934884+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:26:24.934884+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159678464,"num_objects":2436,"num_object_clones":0,"num_object_copies":7308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4582,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1167},{"shards":"2,5","objects":807}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":141,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.415490+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:26:03.415490+0000","last_peered":"2023-10-09T10:26:03.415490+0000","last_clean":"2023-10-09T10:26:03.415490+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:26:03.415490+0000","last_undegraded":"2023-10-09T10:26:03.415490+0000","last_fullsized":"2023-10-09T10:26:03.415490+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":143,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.567950+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:26:14.567950+0000","last_peered":"2023-10-09T10:26:14.567950+0000","last_clean":"2023-10-09T10:26:14.567950+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:26:14.567950+0000","last_undegraded":"2023-10-09T10:26:14.567950+0000","last_fullsized":"2023-10-09T10:26:14.567950+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"531'5466","reported_seq":5584,"reported_epoch":531,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:21.577914+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:26:21.577914+0000","last_peered":"2023-10-09T10:26:21.577914+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:26:21.577914+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2578,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-569},{"shards":"0,2,5","objects":2041}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":143,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.561911+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:26:14.561911+0000","last_peered":"2023-10-09T10:26:14.561911+0000","last_clean":"2023-10-09T10:26:14.561911+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:26:14.561911+0000","last_undegraded":"2023-10-09T10:26:14.561911+0000","last_fullsized":"2023-10-09T10:26:14.561911+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":109,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.563114+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:26:14.563114+0000","last_peered":"2023-10-09T10:26:14.563114+0000","last_clean":"2023-10-09T10:26:14.563114+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:26:14.563114+0000","last_undegraded":"2023-10-09T10:26:14.563114+0000","last_fullsized":"2023-10-09T10:26:14.563114+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"531'5488","reported_seq":8595,"reported_epoch":531,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:21.613186+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:26:21.613186+0000","last_peered":"2023-10-09T10:26:21.613186+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:26:21.613186+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5488,"log_dups_size":0,"ondisk_log_size":5488,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165216256,"num_objects":2521,"num_object_clones":0,"num_object_copies":7563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4543,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1011,"num_bytes_recovered":66256896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1243},{"shards":"0,2,6","objects":806},{"shards":"0,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":125,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562347+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:26:14.562347+0000","last_peered":"2023-10-09T10:26:14.562347+0000","last_clean":"2023-10-09T10:26:14.562347+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:26:14.562347+0000","last_undegraded":"2023-10-09T10:26:14.562347+0000","last_fullsized":"2023-10-09T10:26:14.562347+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":152,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562798+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:26:14.562798+0000","last_peered":"2023-10-09T10:26:14.562798+0000","last_clean":"2023-10-09T10:26:14.562798+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:26:14.562798+0000","last_undegraded":"2023-10-09T10:26:14.562798+0000","last_fullsized":"2023-10-09T10:26:14.562798+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"531'5449","reported_seq":5881,"reported_epoch":531,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:21.440818+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:26:21.440818+0000","last_peered":"2023-10-09T10:26:21.440818+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:26:21.440818+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5449,"log_dups_size":0,"ondisk_log_size":5449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161546240,"num_objects":2465,"num_object_clones":0,"num_object_copies":7395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2342,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1211},{"shards":"0,3","objects":384}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":484,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.566293+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:26:14.566293+0000","last_peered":"2023-10-09T10:26:14.566293+0000","last_clean":"2023-10-09T10:26:14.566293+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:26:14.566293+0000","last_undegraded":"2023-10-09T10:26:14.566293+0000","last_fullsized":"2023-10-09T10:26:14.566293+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":546,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.564738+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:26:14.564738+0000","last_peered":"2023-10-09T10:26:14.564738+0000","last_clean":"2023-10-09T10:26:14.564738+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:26:14.564738+0000","last_undegraded":"2023-10-09T10:26:14.564738+0000","last_fullsized":"2023-10-09T10:26:14.564738+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"532'9097","reported_seq":9178,"reported_epoch":532,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:23.077299+0000","last_change":"2023-10-09T10:25:17.929569+0000","last_active":"2023-10-09T10:26:23.077299+0000","last_peered":"2023-10-09T10:26:23.077299+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:26:23.077299+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"503'3200","ondisk_log_start":"503'3200","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5897,"log_dups_size":0,"ondisk_log_size":5897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327548952,"num_objects":4999,"num_object_clones":0,"num_object_copies":14997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4996,"num_objects_misplaced":4999,"num_objects_unfound":0,"num_objects_dirty":4999,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9097,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":4999}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":251,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.421526+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:26:03.421526+0000","last_peered":"2023-10-09T10:26:03.421526+0000","last_clean":"2023-10-09T10:26:03.421526+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:26:03.421526+0000","last_undegraded":"2023-10-09T10:26:03.421526+0000","last_fullsized":"2023-10-09T10:26:03.421526+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":216,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.576275+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:26:14.576275+0000","last_peered":"2023-10-09T10:26:14.576275+0000","last_clean":"2023-10-09T10:26:14.576275+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:26:14.576275+0000","last_undegraded":"2023-10-09T10:26:14.576275+0000","last_fullsized":"2023-10-09T10:26:14.576275+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":496,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.569037+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:26:14.569037+0000","last_peered":"2023-10-09T10:26:14.569037+0000","last_clean":"2023-10-09T10:26:14.569037+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:26:14.569037+0000","last_undegraded":"2023-10-09T10:26:14.569037+0000","last_fullsized":"2023-10-09T10:26:14.569037+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"531'9169","reported_seq":9216,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:21.574691+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:26:21.574691+0000","last_peered":"2023-10-09T10:26:21.574691+0000","last_clean":"2023-10-09T10:26:21.574691+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:26:21.574691+0000","last_undegraded":"2023-10-09T10:26:21.574691+0000","last_fullsized":"2023-10-09T10:26:21.574691+0000","mapping_epoch":494,"log_start":"504'4106","ondisk_log_start":"504'4106","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5063,"log_dups_size":0,"ondisk_log_size":5063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334823424,"num_objects":5109,"num_object_clones":0,"num_object_copies":15327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9169,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":216,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.568891+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:26:14.568891+0000","last_peered":"2023-10-09T10:26:14.568891+0000","last_clean":"2023-10-09T10:26:14.568891+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:26:14.568891+0000","last_undegraded":"2023-10-09T10:26:14.568891+0000","last_fullsized":"2023-10-09T10:26:14.568891+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":134,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562949+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:26:14.562949+0000","last_peered":"2023-10-09T10:26:14.562949+0000","last_clean":"2023-10-09T10:26:14.562949+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:26:14.562949+0000","last_undegraded":"2023-10-09T10:26:14.562949+0000","last_fullsized":"2023-10-09T10:26:14.562949+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":444,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.576180+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:26:14.576180+0000","last_peered":"2023-10-09T10:26:14.576180+0000","last_clean":"2023-10-09T10:26:14.576180+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:26:14.576180+0000","last_undegraded":"2023-10-09T10:26:14.576180+0000","last_fullsized":"2023-10-09T10:26:14.576180+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"532'5600","reported_seq":9062,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:24.978134+0000","last_change":"2023-10-09T10:25:09.703037+0000","last_active":"2023-10-09T10:26:24.978134+0000","last_peered":"2023-10-09T10:26:24.978134+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:25:09.696864+0000","last_became_peered":"2023-10-09T10:25:09.696864+0000","last_unstale":"2023-10-09T10:26:24.978134+0000","last_undegraded":"2023-10-09T10:25:09.545735+0000","last_fullsized":"2023-10-09T10:25:09.430540+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5600,"log_dups_size":0,"ondisk_log_size":5600,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155615232,"num_objects":2374,"num_object_clones":0,"num_object_copies":7122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3124,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4634,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1130,"num_bytes_recovered":74055680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1160},{"shards":"2,5,6","objects":804},{"shards":"5,6","objects":410}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":481,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.420094+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:26:03.420094+0000","last_peered":"2023-10-09T10:26:03.420094+0000","last_clean":"2023-10-09T10:26:03.420094+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:26:03.420094+0000","last_undegraded":"2023-10-09T10:26:03.420094+0000","last_fullsized":"2023-10-09T10:26:03.420094+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":99,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.581082+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:26:14.581082+0000","last_peered":"2023-10-09T10:26:14.581082+0000","last_clean":"2023-10-09T10:26:14.581082+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:26:14.581082+0000","last_undegraded":"2023-10-09T10:26:14.581082+0000","last_fullsized":"2023-10-09T10:26:14.581082+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":545,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.566746+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:26:14.566746+0000","last_peered":"2023-10-09T10:26:14.566746+0000","last_clean":"2023-10-09T10:26:14.566746+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:26:14.566746+0000","last_undegraded":"2023-10-09T10:26:14.566746+0000","last_fullsized":"2023-10-09T10:26:14.566746+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"532'9227","reported_seq":9294,"reported_epoch":532,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:24.926173+0000","last_change":"2023-10-09T10:24:41.858672+0000","last_active":"2023-10-09T10:26:24.926173+0000","last_peered":"2023-10-09T10:26:24.926173+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:26:24.926173+0000","last_undegraded":"2023-10-09T10:26:24.926173+0000","last_fullsized":"2023-10-09T10:26:24.926173+0000","mapping_epoch":507,"log_start":"504'4300","ondisk_log_start":"504'4300","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320143360,"num_objects":4885,"num_object_clones":0,"num_object_copies":14655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4885,"num_objects_unfound":0,"num_objects_dirty":4885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9227,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":4885}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":186,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.570852+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:26:14.570852+0000","last_peered":"2023-10-09T10:26:14.570852+0000","last_clean":"2023-10-09T10:26:14.570852+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:26:14.570852+0000","last_undegraded":"2023-10-09T10:26:14.570852+0000","last_fullsized":"2023-10-09T10:26:14.570852+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":724,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.414672+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:26:03.414672+0000","last_peered":"2023-10-09T10:26:03.414672+0000","last_clean":"2023-10-09T10:26:03.414672+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:03.414672+0000","last_undegraded":"2023-10-09T10:26:03.414672+0000","last_fullsized":"2023-10-09T10:26:03.414672+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":234,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.568688+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:26:14.568688+0000","last_peered":"2023-10-09T10:26:14.568688+0000","last_clean":"2023-10-09T10:26:14.568688+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:26:14.568688+0000","last_undegraded":"2023-10-09T10:26:14.568688+0000","last_fullsized":"2023-10-09T10:26:14.568688+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":143,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.563189+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:26:14.563189+0000","last_peered":"2023-10-09T10:26:14.563189+0000","last_clean":"2023-10-09T10:26:14.563189+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:26:14.563189+0000","last_undegraded":"2023-10-09T10:26:14.563189+0000","last_fullsized":"2023-10-09T10:26:14.563189+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"532'5640","reported_seq":8952,"reported_epoch":532,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:26:25.007246+0000","last_change":"2023-10-09T10:25:13.759251+0000","last_active":"2023-10-09T10:26:25.007246+0000","last_peered":"2023-10-09T10:26:25.007246+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:26:25.007246+0000","last_undegraded":"2023-10-09T10:25:13.707357+0000","last_fullsized":"2023-10-09T10:26:25.007246+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5640,"log_dups_size":0,"ondisk_log_size":5640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165314560,"num_objects":2522,"num_object_clones":0,"num_object_copies":7566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1209,"num_objects_misplaced":2160,"num_objects_unfound":0,"num_objects_dirty":2522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4716,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1097,"num_bytes_recovered":71892992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1209},{"shards":"3,5,6","objects":362}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":216,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.581256+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:26:14.581256+0000","last_peered":"2023-10-09T10:26:14.581256+0000","last_clean":"2023-10-09T10:26:14.581256+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:26:14.581256+0000","last_undegraded":"2023-10-09T10:26:14.581256+0000","last_fullsized":"2023-10-09T10:26:14.581256+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":475,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.420234+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:26:03.420234+0000","last_peered":"2023-10-09T10:26:03.420234+0000","last_clean":"2023-10-09T10:26:03.420234+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:26:03.420234+0000","last_undegraded":"2023-10-09T10:26:03.420234+0000","last_fullsized":"2023-10-09T10:26:03.420234+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.421527+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:26:03.421527+0000","last_peered":"2023-10-09T10:26:03.421527+0000","last_clean":"2023-10-09T10:26:03.421527+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:26:03.421527+0000","last_undegraded":"2023-10-09T10:26:03.421527+0000","last_fullsized":"2023-10-09T10:26:03.421527+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"532'9244","reported_seq":9290,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:23.127724+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:26:23.127724+0000","last_peered":"2023-10-09T10:26:23.127724+0000","last_clean":"2023-10-09T10:26:23.127724+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:26:23.127724+0000","last_undegraded":"2023-10-09T10:26:23.127724+0000","last_fullsized":"2023-10-09T10:26:23.127724+0000","mapping_epoch":494,"log_start":"503'3300","ondisk_log_start":"503'3300","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5944,"log_dups_size":0,"ondisk_log_size":5944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325582848,"num_objects":4968,"num_object_clones":0,"num_object_copies":14904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9244,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.414574+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:26:03.414574+0000","last_peered":"2023-10-09T10:26:03.414574+0000","last_clean":"2023-10-09T10:26:03.414574+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:26:03.414574+0000","last_undegraded":"2023-10-09T10:26:03.414574+0000","last_fullsized":"2023-10-09T10:26:03.414574+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":513,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.570249+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:26:14.570249+0000","last_peered":"2023-10-09T10:26:14.570249+0000","last_clean":"2023-10-09T10:26:14.570249+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:26:14.570249+0000","last_undegraded":"2023-10-09T10:26:14.570249+0000","last_fullsized":"2023-10-09T10:26:14.570249+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":443,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.417868+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:26:03.417868+0000","last_peered":"2023-10-09T10:26:03.417868+0000","last_clean":"2023-10-09T10:26:03.417868+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:26:03.417868+0000","last_undegraded":"2023-10-09T10:26:03.417868+0000","last_fullsized":"2023-10-09T10:26:03.417868+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"531'8842","reported_seq":8889,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:21.605376+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:26:21.605376+0000","last_peered":"2023-10-09T10:26:21.605376+0000","last_clean":"2023-10-09T10:26:21.605376+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:26:21.605376+0000","last_undegraded":"2023-10-09T10:26:21.605376+0000","last_fullsized":"2023-10-09T10:26:21.605376+0000","mapping_epoch":494,"log_start":"504'3776","ondisk_log_start":"504'3776","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320733184,"num_objects":4894,"num_object_clones":0,"num_object_copies":14682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8842,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":331,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.415294+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:26:03.415294+0000","last_peered":"2023-10-09T10:26:03.415294+0000","last_clean":"2023-10-09T10:26:03.415294+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:26:03.415294+0000","last_undegraded":"2023-10-09T10:26:03.415294+0000","last_fullsized":"2023-10-09T10:26:03.415294+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":513,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.562747+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:26:14.562747+0000","last_peered":"2023-10-09T10:26:14.562747+0000","last_clean":"2023-10-09T10:26:14.562747+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:26:14.562747+0000","last_undegraded":"2023-10-09T10:26:14.562747+0000","last_fullsized":"2023-10-09T10:26:14.562747+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.414763+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:26:03.414763+0000","last_peered":"2023-10-09T10:26:03.414763+0000","last_clean":"2023-10-09T10:26:03.414763+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:26:03.414763+0000","last_undegraded":"2023-10-09T10:26:03.414763+0000","last_fullsized":"2023-10-09T10:26:03.414763+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"532'9266","reported_seq":9312,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:23.102706+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:26:23.102706+0000","last_peered":"2023-10-09T10:26:23.102706+0000","last_clean":"2023-10-09T10:26:23.102706+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:26:23.102706+0000","last_undegraded":"2023-10-09T10:26:23.102706+0000","last_fullsized":"2023-10-09T10:26:23.102706+0000","mapping_epoch":494,"log_start":"503'3300","ondisk_log_start":"503'3300","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5966,"log_dups_size":0,"ondisk_log_size":5966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327548928,"num_objects":4998,"num_object_clones":0,"num_object_copies":14994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9266,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":35,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.577779+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:26:14.577779+0000","last_peered":"2023-10-09T10:26:14.577779+0000","last_clean":"2023-10-09T10:26:14.577779+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:26:14.577779+0000","last_undegraded":"2023-10-09T10:26:14.577779+0000","last_fullsized":"2023-10-09T10:26:14.577779+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":30,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.421713+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:26:03.421713+0000","last_peered":"2023-10-09T10:26:03.421713+0000","last_clean":"2023-10-09T10:26:03.421713+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:26:03.421713+0000","last_undegraded":"2023-10-09T10:26:03.421713+0000","last_fullsized":"2023-10-09T10:26:03.421713+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":212,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.576523+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:26:14.576523+0000","last_peered":"2023-10-09T10:26:14.576523+0000","last_clean":"2023-10-09T10:26:14.576523+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:26:14.576523+0000","last_undegraded":"2023-10-09T10:26:14.576523+0000","last_fullsized":"2023-10-09T10:26:14.576523+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"532'5456","reported_seq":5550,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:24.883003+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:26:24.883003+0000","last_peered":"2023-10-09T10:26:24.883003+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:26:24.883003+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156991488,"num_objects":2396,"num_object_clones":0,"num_object_copies":7188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4536,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1160},{"shards":"2,5","objects":180}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.417792+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:26:03.417792+0000","last_peered":"2023-10-09T10:26:03.417792+0000","last_clean":"2023-10-09T10:26:03.417792+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:26:03.417792+0000","last_undegraded":"2023-10-09T10:26:03.417792+0000","last_fullsized":"2023-10-09T10:26:03.417792+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":35,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.415141+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:26:03.415141+0000","last_peered":"2023-10-09T10:26:03.415141+0000","last_clean":"2023-10-09T10:26:03.415141+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:26:03.415141+0000","last_undegraded":"2023-10-09T10:26:03.415141+0000","last_fullsized":"2023-10-09T10:26:03.415141+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"531'5476","reported_seq":5543,"reported_epoch":531,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:26:21.515736+0000","last_change":"2023-10-09T10:25:08.466144+0000","last_active":"2023-10-09T10:26:21.515736+0000","last_peered":"2023-10-09T10:26:21.515736+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:26:21.515736+0000","last_undegraded":"2023-10-09T10:26:21.515736+0000","last_fullsized":"2023-10-09T10:26:21.515736+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5476,"log_dups_size":0,"ondisk_log_size":5476,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163807232,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2331,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4552,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":169}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":262,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.567000+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:26:14.567000+0000","last_peered":"2023-10-09T10:26:14.567000+0000","last_clean":"2023-10-09T10:26:14.567000+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:26:14.567000+0000","last_undegraded":"2023-10-09T10:26:14.567000+0000","last_fullsized":"2023-10-09T10:26:14.567000+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.563091+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:26:14.563091+0000","last_peered":"2023-10-09T10:26:14.563091+0000","last_clean":"2023-10-09T10:26:14.563091+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:26:14.563091+0000","last_undegraded":"2023-10-09T10:26:14.563091+0000","last_fullsized":"2023-10-09T10:26:14.563091+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"532'5558","reported_seq":8831,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:24.917561+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:26:24.917561+0000","last_peered":"2023-10-09T10:26:24.917561+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:26:24.917561+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5558,"log_dups_size":0,"ondisk_log_size":5558,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162365440,"num_objects":2478,"num_object_clones":0,"num_object_copies":7434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3178,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4614,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1068,"num_bytes_recovered":69992448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1172},{"shards":"0,5,6","objects":833},{"shards":"5,6","objects":472}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":245,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.576068+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:26:14.576068+0000","last_peered":"2023-10-09T10:26:14.576068+0000","last_clean":"2023-10-09T10:26:14.576068+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:26:14.576068+0000","last_undegraded":"2023-10-09T10:26:14.576068+0000","last_fullsized":"2023-10-09T10:26:14.576068+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":354,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.586544+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:26:14.586544+0000","last_peered":"2023-10-09T10:26:14.586544+0000","last_clean":"2023-10-09T10:26:14.586544+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:26:14.586544+0000","last_undegraded":"2023-10-09T10:26:14.586544+0000","last_fullsized":"2023-10-09T10:26:14.586544+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"532'5473","reported_seq":5536,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:24.369588+0000","last_change":"2023-10-09T10:25:13.762065+0000","last_active":"2023-10-09T10:26:24.369588+0000","last_peered":"2023-10-09T10:26:24.369588+0000","last_clean":"2023-10-09T10:25:11.551777+0000","last_became_active":"2023-10-09T10:25:13.754797+0000","last_became_peered":"2023-10-09T10:25:13.754797+0000","last_unstale":"2023-10-09T10:26:24.369588+0000","last_undegraded":"2023-10-09T10:25:13.625003+0000","last_fullsized":"2023-10-09T10:25:13.604794+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:58.037975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160104448,"num_objects":2443,"num_object_clones":0,"num_object_copies":7329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2536,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":-694},{"shards":"2,4,5","objects":2062}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":30,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.578764+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:26:14.578764+0000","last_peered":"2023-10-09T10:26:14.578764+0000","last_clean":"2023-10-09T10:26:14.578764+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:26:14.578764+0000","last_undegraded":"2023-10-09T10:26:14.578764+0000","last_fullsized":"2023-10-09T10:26:14.578764+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":583,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.420024+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:26:03.420024+0000","last_peered":"2023-10-09T10:26:03.420024+0000","last_clean":"2023-10-09T10:26:03.420024+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:26:03.420024+0000","last_undegraded":"2023-10-09T10:26:03.420024+0000","last_fullsized":"2023-10-09T10:26:03.420024+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"532'5530","reported_seq":5589,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:24.990026+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:26:24.990026+0000","last_peered":"2023-10-09T10:26:24.990026+0000","last_clean":"2023-10-09T10:26:24.990026+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:26:24.990026+0000","last_undegraded":"2023-10-09T10:26:24.990026+0000","last_fullsized":"2023-10-09T10:26:24.990026+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":5530,"log_dups_size":0,"ondisk_log_size":5530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158334976,"num_objects":2416,"num_object_clones":0,"num_object_copies":7248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2269,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":47,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T10:26:03.414862+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:26:03.414862+0000","last_peered":"2023-10-09T10:26:03.414862+0000","last_clean":"2023-10-09T10:26:03.414862+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:26:03.414862+0000","last_undegraded":"2023-10-09T10:26:03.414862+0000","last_fullsized":"2023-10-09T10:26:03.414862+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":152,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T10:26:14.561945+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:26:14.561945+0000","last_peered":"2023-10-09T10:26:14.561945+0000","last_clean":"2023-10-09T10:26:14.561945+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:26:14.561945+0000","last_undegraded":"2023-10-09T10:26:14.561945+0000","last_fullsized":"2023-10-09T10:26:14.561945+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":5180784664,"num_objects":79055,"num_object_clones":0,"num_object_copies":237165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":39897,"num_objects_misplaced":19208,"num_objects_unfound":0,"num_objects_dirty":79055,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131891,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16307,"num_bytes_recovered":1068695552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17022263296,"data_stored":17022255152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143271,"ondisk_log_size":143271,"up":78,"acting":63,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430681,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626909,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2254976,"kb_used_data":1347512,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91490176,"statfs":{"total":95995035648,"available":93685940224,"internally_reserved":0,"allocated":1379852288,"data_stored":1378929467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.4330000000000001,"15min":1.6339999999999999},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":4.7990000000000004,"5min":9.4260000000000002,"15min":44.055},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.591,"15min":1.8600000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":5.6079999999999997,"5min":9.2609999999999992,"15min":43.869999999999997},"last":3.4399999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":2.911,"15min":2.2400000000000002},"min":{"1min":0.29399999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.0419999999999998,"5min":155.01400000000001,"15min":155.01400000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.383,"5min":2.9460000000000002,"15min":2.4060000000000001},"min":{"1min":0.33200000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.5709999999999997,"5min":154.71100000000001,"15min":154.71100000000001},"last":0.42399999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":1.821,"15min":1.865},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":8.5069999999999997,"5min":8.6280000000000001,"15min":29.367999999999999},"last":1.4710000000000001},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.617,"15min":1.752},"min":{"1min":0.32600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.0640000000000001,"5min":7.5629999999999997,"15min":14.542999999999999},"last":0.435}]},{"osd":3,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.76,"15min":1.573},"min":{"1min":0.34599999999999997,"5min":0.32700000000000001,"15min":0.308},"max":{"1min":9.4619999999999997,"5min":9.4619999999999997,"15min":9.4619999999999997},"last":0.373},{"interface":"front","average":{"1min":2.3130000000000002,"5min":1.946,"15min":1.6759999999999999},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.31},"max":{"1min":7.9939999999999998,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.39000000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.3610000000000002,"15min":1.9430000000000001},"min":{"1min":0.30299999999999999,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":7.3289999999999997,"5min":10.618,"15min":122.041},"last":1.4399999999999999},{"interface":"front","average":{"1min":1.478,"5min":2.04,"15min":1.7809999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.4729999999999999,"5min":11.266,"15min":122.13200000000001},"last":4.1059999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":2.4100000000000001,"15min":2.0350000000000001},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1980000000000004,"5min":12.894,"15min":121.98999999999999},"last":4.7859999999999996},{"interface":"front","average":{"1min":1.9019999999999999,"5min":1.5960000000000001,"15min":1.6919999999999999},"min":{"1min":0.33300000000000002,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":7.0060000000000002,"5min":15.041,"15min":122.346},"last":0.40500000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":3.532,"15min":1.9730000000000001},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":4.9169999999999998,"5min":130.66999999999999,"15min":130.66999999999999},"last":4.1550000000000002},{"interface":"front","average":{"1min":1.3779999999999999,"5min":3.573,"15min":2.016},"min":{"1min":0.374,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":4.9589999999999996,"5min":130.702,"15min":130.702},"last":4.1879999999999997}]}]},{"osd":1,"up_from":0,"seq":360777253222,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416746,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4150276,"kb_used_data":3198012,"kb_used_omap":28,"kb_used_meta":952099,"kb_avail":89594876,"statfs":{"total":95995035648,"available":91745153024,"internally_reserved":0,"allocated":3274764288,"data_stored":3272707833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":974950224},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:25:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.6200000000000001,"15min":2.0059999999999998},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":17.161999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":2.2250000000000001},{"interface":"front","average":{"1min":1.585,"5min":1.506,"15min":2.4689999999999999},"min":{"1min":0.218,"5min":0.214,"15min":0.192},"max":{"1min":10.786,"5min":17.033000000000001,"15min":56.950000000000003},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:25:21 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.9630000000000001,"15min":2.1419999999999999},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.7839999999999998,"5min":47.215000000000003,"15min":53.988999999999997},"last":2.3250000000000002},{"interface":"front","average":{"1min":1.99,"5min":2.254,"15min":2.1339999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.173,"5min":47.000999999999998,"15min":54.119},"last":10.205}]},{"osd":3,"last update":"Mon Oct 9 10:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.609,"15min":1.4850000000000001},"min":{"1min":0.28599999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.763,"5min":11.933,"15min":56.984999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.8660000000000001,"5min":1.9790000000000001,"15min":1.5640000000000001},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.067,"5min":10.067,"15min":56.475999999999999},"last":7.8570000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":1.974,"15min":2.254},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":10.941000000000001,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.7709999999999999,"5min":2.3420000000000001,"15min":2.2650000000000001},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.32400000000000001},"max":{"1min":9.9499999999999993,"5min":28.331,"15min":56.378},"last":7.7400000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":8.6020000000000003,"5min":3.4510000000000001,"15min":3.544},"min":{"1min":0.51700000000000002,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":90.412000000000006,"5min":90.412000000000006,"15min":229.589},"last":7.7880000000000003},{"interface":"front","average":{"1min":6.9400000000000004,"5min":2.7509999999999999,"15min":3.1909999999999998},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":90.393000000000001,"5min":90.393000000000001,"15min":228.88900000000001},"last":2.2599999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.919,"15min":2.29},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":54.482999999999997},"last":2.282},{"interface":"front","average":{"1min":2.806,"5min":2.1240000000000001,"15min":2.5230000000000001},"min":{"1min":0.47899999999999998,"5min":0.39700000000000002,"15min":0.34799999999999998},"max":{"1min":8.8309999999999995,"5min":8.8309999999999995,"15min":57.753},"last":0.91900000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0470000000000002,"5min":2.2679999999999998,"15min":2.4220000000000002},"min":{"1min":0.48299999999999998,"5min":0.40500000000000003,"15min":0.34999999999999998},"max":{"1min":12.029999999999999,"5min":12.029999999999999,"15min":56.411999999999999},"last":7.8170000000000002},{"interface":"front","average":{"1min":3.0350000000000001,"5min":2.1869999999999998,"15min":2.3530000000000002},"min":{"1min":0.44800000000000001,"5min":0.38600000000000001,"15min":0.38300000000000001},"max":{"1min":17.129999999999999,"5min":17.129999999999999,"15min":54.451999999999998},"last":2.306}]}]},{"osd":2,"up_from":217,"seq":932007903482,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4122444,"kb_used_data":3283588,"kb_used_omap":83,"kb_used_meta":838700,"kb_avail":89622708,"statfs":{"total":95995035648,"available":91773652992,"internally_reserved":0,"allocated":3362394112,"data_stored":3360522116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":858829295},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2000000000000002,"5min":2.0579999999999998,"15min":2.0049999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.185},"max":{"1min":11.093999999999999,"5min":48.990000000000002,"15min":48.990000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.1160000000000001,"15min":2.3220000000000001},"min":{"1min":0.45600000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.2699999999999996,"5min":49.412999999999997,"15min":49.412999999999997},"last":0.73599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":1.893,"15min":2.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":13.303000000000001,"5min":13.303000000000001,"15min":34.893999999999998},"last":1.468},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.5740000000000001,"15min":2.0339999999999998},"min":{"1min":0.35099999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":9.9239999999999995,"5min":9.9239999999999995,"15min":34.962000000000003},"last":3.3380000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.4139999999999999,"15min":1.202},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.248},"max":{"1min":9.3119999999999994,"5min":9.3119999999999994,"15min":35.043999999999997},"last":0.33100000000000002},{"interface":"front","average":{"1min":2.7909999999999999,"5min":1.2829999999999999,"15min":1.3},"min":{"1min":0.35699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.206,"5min":11.206,"15min":50.406999999999996},"last":0.376}]},{"osd":4,"last update":"Mon Oct 9 10:26:14 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.8149999999999999,"15min":1.6870000000000001},"min":{"1min":0.31900000000000001,"5min":0.29799999999999999,"15min":0.28999999999999998},"max":{"1min":4.4710000000000001,"5min":47.503999999999998,"15min":47.503999999999998},"last":8.1859999999999999},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.0640000000000001,"15min":1.835},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":11.298,"5min":47.704000000000001,"15min":47.704000000000001},"last":4.7350000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.8340000000000001,"15min":2.012},"min":{"1min":0.40500000000000003,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":57.298000000000002},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.8639999999999999,"5min":2.0430000000000001,"15min":2.0169999999999999},"min":{"1min":0.442,"5min":0.31,"15min":0.31},"max":{"1min":11.195,"5min":11.195,"15min":57.828000000000003},"last":6.3860000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":2.0590000000000002,"15min":2.0230000000000001},"min":{"1min":0.35399999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":4.5579999999999998,"5min":12.411,"15min":18.943999999999999},"last":3.375},{"interface":"front","average":{"1min":2.4860000000000002,"5min":1.855,"15min":1.97},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":13.975,"5min":13.975,"15min":24.379000000000001},"last":1.077}]},{"osd":7,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.1349999999999998,"15min":2.2949999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.308},"max":{"1min":5.415,"5min":11.704000000000001,"15min":35.008000000000003},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.907,"5min":1.99,"15min":2.1579999999999999},"min":{"1min":0.47099999999999997,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":15.548,"5min":15.548,"15min":24.323},"last":3.3109999999999999}]}]},{"osd":3,"up_from":352,"seq":1511828488322,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2979532,"kb_used_data":2476676,"kb_used_omap":15,"kb_used_meta":502832,"kb_avail":90765620,"statfs":{"total":95995035648,"available":92943994880,"internally_reserved":0,"allocated":2536116224,"data_stored":2533771145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":514900422},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8259999999999996,"5min":3.1200000000000001,"15min":1.696},"min":{"1min":0.34999999999999998,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":70.004000000000005,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.25800000000000001},{"interface":"front","average":{"1min":6.2519999999999998,"5min":2.8159999999999998,"15min":1.6739999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":66.802000000000007,"5min":66.802000000000007,"15min":66.802000000000007},"last":8.9559999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7889999999999999,"15min":1.8620000000000001},"min":{"1min":0.311,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.785,"5min":25.370000000000001,"15min":105.401},"last":0.745},{"interface":"front","average":{"1min":2.7109999999999999,"5min":2.04,"15min":1.7969999999999999},"min":{"1min":0.30299999999999999,"5min":0.25900000000000001,"15min":0.23300000000000001},"max":{"1min":9.2140000000000004,"5min":14.613,"15min":104.914},"last":8.9100000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.2989999999999999,"15min":1.9490000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":11.141999999999999,"5min":11.141999999999999,"15min":11.141999999999999},"last":3.0600000000000001},{"interface":"front","average":{"1min":3.1400000000000001,"5min":1.956,"15min":1.4279999999999999},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.27600000000000002},"max":{"1min":16.475000000000001,"5min":25.338999999999999,"15min":25.338999999999999},"last":1.8480000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.194,"15min":1.415},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":6.9370000000000003,"5min":21.196000000000002,"15min":21.831},"last":5.952},{"interface":"front","average":{"1min":3.3980000000000001,"5min":2.8820000000000001,"15min":1.958},"min":{"1min":0.36399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":11.087,"5min":19.323,"15min":22.097999999999999},"last":5.4169999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":2.0910000000000002,"15min":1.294},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":11.363,"5min":25.251999999999999,"15min":25.251999999999999},"last":1.8819999999999999},{"interface":"front","average":{"1min":2.46,"5min":2.411,"15min":1.5429999999999999},"min":{"1min":0.36599999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":6.9900000000000002,"5min":13.394,"15min":13.394},"last":6.1420000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.4780000000000002,"15min":1.7250000000000001},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.35599999999999998},"max":{"1min":11.114000000000001,"5min":19.114000000000001,"15min":19.114000000000001},"last":3.1320000000000001},{"interface":"front","average":{"1min":2.3279999999999998,"5min":2.1230000000000002,"15min":1.468},"min":{"1min":0.47699999999999998,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":11.32,"5min":25.306999999999999,"15min":25.306999999999999},"last":1.9059999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":2.0779999999999998,"15min":1.514},"min":{"1min":0.42299999999999999,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":6.9640000000000004,"5min":9.7880000000000003,"15min":10.459},"last":6.181},{"interface":"front","average":{"1min":2.327,"5min":2.3220000000000001,"15min":1.762},"min":{"1min":0.432,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":11.045999999999999,"5min":19.280000000000001,"15min":19.280000000000001},"last":3.3460000000000001}]}]},{"osd":4,"up_from":266,"seq":1142461300951,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4047672,"kb_used_data":3106352,"kb_used_omap":35,"kb_used_meta":941276,"kb_avail":89697480,"statfs":{"total":95995035648,"available":91850219520,"internally_reserved":0,"allocated":3180904448,"data_stored":3179979840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":963866862},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.893,"15min":2.1469999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":6.5019999999999998,"5min":14.548,"15min":97.721999999999994},"last":5.0570000000000004},{"interface":"front","average":{"1min":2.8940000000000001,"5min":2.694,"15min":2.8740000000000001},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.224},"max":{"1min":11.022,"5min":11.257,"15min":97.915999999999997},"last":0.77200000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4449999999999998,"15min":2.3700000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.24299999999999999},"max":{"1min":12.209,"5min":12.209,"15min":71.457999999999998},"last":1.3160000000000001},{"interface":"front","average":{"1min":2.992,"5min":2.8999999999999999,"15min":2.2269999999999999},"min":{"1min":0.376,"5min":0.248,"15min":0.248},"max":{"1min":8.327,"5min":49.781999999999996,"15min":71.658000000000001},"last":0.83499999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3809999999999998,"15min":1.845},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":7.3230000000000004,"5min":15.944000000000001,"15min":15.944000000000001},"last":1.9610000000000001},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.9970000000000001,"15min":1.7989999999999999},"min":{"1min":0.36199999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.102,"5min":14.473000000000001,"15min":14.473000000000001},"last":1.1859999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.7410000000000001,"5min":2.9199999999999999,"15min":2.024},"min":{"1min":0.39800000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.347,"5min":12.074,"15min":12.074},"last":0.69599999999999995},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.907,"15min":1.7390000000000001},"min":{"1min":0.40899999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.369999999999999,"5min":16.067,"15min":16.067},"last":1.0469999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":2.5939999999999999,"15min":2.069},"min":{"1min":0.314,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":3.3279999999999998,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.054,"5min":2.4969999999999999,"15min":1.831},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.464,"5min":49.915999999999997,"15min":99.384},"last":0.317}]},{"osd":6,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.881,"15min":1.3740000000000001},"min":{"1min":0.33800000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.5389999999999997,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.8220000000000001,"15min":1.7949999999999999},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":12.242000000000001,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.40699999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":3.0409999999999999,"15min":1.9239999999999999},"min":{"1min":0.39500000000000002,"5min":0.316,"15min":0.307},"max":{"1min":12.391999999999999,"5min":17.369,"15min":17.369},"last":0.44},{"interface":"front","average":{"1min":1.3320000000000001,"5min":2.7829999999999999,"15min":1.778},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.30399999999999999},"max":{"1min":4.6689999999999996,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.39900000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823136,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4801848,"kb_used_data":3301040,"kb_used_omap":0,"kb_used_meta":1500800,"kb_avail":88943304,"statfs":{"total":95995035648,"available":91077943296,"internally_reserved":0,"allocated":3380264960,"data_stored":3379344519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536819200},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.754,"15min":1.9370000000000001},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":16.536999999999999,"5min":22.469999999999999,"15min":22.469999999999999},"last":2.9820000000000002},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.1309999999999998,"15min":1.7270000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":8.9339999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":4.7480000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:25:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.6339999999999999,"15min":2.2090000000000001},"min":{"1min":0.29599999999999999,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":16.562999999999999,"5min":16.562999999999999,"15min":29.370000000000001},"last":3.9540000000000002},{"interface":"front","average":{"1min":1.8,"5min":1.8759999999999999,"15min":1.8580000000000001},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":8.9529999999999994,"5min":13.143000000000001,"15min":29.065000000000001},"last":0.51000000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":2.5990000000000002,"15min":2.1309999999999998},"min":{"1min":0.33000000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":12.247999999999999,"5min":16.629999999999999,"15min":16.629999999999999},"last":4.048},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.907,"15min":1.6919999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":3.1389999999999998,"5min":13.095000000000001,"15min":15.929},"last":1.444}]},{"osd":3,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.2170000000000001,"15min":1.7889999999999999},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":10.362,"5min":10.561999999999999,"15min":10.561999999999999},"last":3.0390000000000001},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.0830000000000002,"15min":1.7270000000000001},"min":{"1min":0.41599999999999998,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9130000000000003,"5min":13.624000000000001,"15min":13.624000000000001},"last":0.55400000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":2.242,"15min":1.5760000000000001},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":10.826000000000001,"5min":12.023,"15min":12.023},"last":2.9660000000000002},{"interface":"front","average":{"1min":1.6579999999999999,"5min":2.0369999999999999,"15min":1.399},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":8.8620000000000001,"5min":11.956,"15min":11.956},"last":0.48099999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.621,"15min":1.794},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":10.664999999999999,"5min":11.686,"15min":12.772},"last":3.0089999999999999},{"interface":"front","average":{"1min":2.4500000000000002,"5min":2.1640000000000001,"15min":1.665},"min":{"1min":0.29699999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":1.2589999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:25:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.6429999999999998,"15min":1.798},"min":{"1min":0.34000000000000002,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":16.686,"5min":21.109000000000002,"15min":21.109000000000002},"last":1.3660000000000001},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.397,"15min":1.1719999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":8.9719999999999995,"5min":11.994,"15min":11.994},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3256877056,"data_stored":3256877056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339649024,"data_stored":3339649024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2513108992,"data_stored":2513108992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3167817728,"data_stored":3167813656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3370979328,"data_stored":3370975256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1373831168,"data_stored":1373831168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:26:26.490 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:26.525 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:26.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:26.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:26.633 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:26.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:26.825 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:26.987 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:27.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:27 smithi012 ceph-mon[147926]: pgmap v1493: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 638 op/s; 39897/237237 objects degraded (16.817%); 19208/237237 objects misplaced (8.097%); 15 MiB/s, 237 objects/s recovering 2023-10-09T10:26:27.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:27 smithi012 ceph-mon[147926]: from='client.37119 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:27.663 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:27.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:27.770 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:27.795 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:27.840 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:27.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:27.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:27.917 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:26:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:26:27] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-10-09T10:26:27.917 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:27 smithi012 ceph-mon[152263]: pgmap v1493: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 638 op/s; 39897/237237 objects degraded (16.817%); 19208/237237 objects misplaced (8.097%); 15 MiB/s, 237 objects/s recovering 2023-10-09T10:26:27.917 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:27 smithi012 ceph-mon[152263]: from='client.37119 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:27.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:27 smithi110 ceph-mon[136032]: pgmap v1493: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 638 op/s; 39897/237237 objects degraded (16.817%); 19208/237237 objects misplaced (8.097%); 15 MiB/s, 237 objects/s recovering 2023-10-09T10:26:27.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:27 smithi110 ceph-mon[136032]: from='client.37119 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:27.947 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:28.013 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:28.779 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:28.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:28.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:28.890 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:28.925 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:28.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 39897/237237 objects degraded (16.817%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:28.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 39897/237237 objects degraded (16.817%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 39897/237237 objects degraded (16.817%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:28.979 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:28.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:28.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:29.029 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:26:29.061 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:26:29.091 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:29.103 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:26:29.143 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:26:29.144 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:29.186 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:26:29.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:26:29.280 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:26:29.334 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:26:29.350 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:26:29.350 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:26:29.350 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:26:29.350 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:26:29.350 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:26:29.350 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:26:29.351 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:26:29.351 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:26:29.489 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:26:29.506 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:29.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:29.616 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:29.664 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:29.777 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:29.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:29.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:29 smithi110 ceph-mon[136032]: pgmap v1494: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 642 op/s; 39469/233946 objects degraded (16.871%); 19208/233946 objects misplaced (8.210%); 15 MiB/s, 238 objects/s recovering 2023-10-09T10:26:29.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:29.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:29.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4048945237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:29.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/218580541' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:26:29.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[147926]: pgmap v1494: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 642 op/s; 39469/233946 objects degraded (16.871%); 19208/233946 objects misplaced (8.210%); 15 MiB/s, 238 objects/s recovering 2023-10-09T10:26:29.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:29.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:29.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4048945237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:29.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/218580541' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:26:29.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[152263]: pgmap v1494: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.8 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 642 op/s; 39469/233946 objects degraded (16.871%); 19208/233946 objects misplaced (8.210%); 15 MiB/s, 238 objects/s recovering 2023-10-09T10:26:29.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:29.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:29.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4048945237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:29.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/218580541' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:26:29.978 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:30.012 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:26:30.052 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:30.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:30.124 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:26:30.125 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:26:30.164 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:30.235 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:30.485 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:30.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:30.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:30.528 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:26:30.601 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:30.670 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[147926]: pgmap v1495: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 39255/231315 objects degraded (16.970%); 18581/231315 objects misplaced (8.033%); 16 MiB/s, 251 objects/s recovering 2023-10-09T10:26:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/218580541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:26:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[147926]: osdmap e533: 8 total, 8 up, 6 in 2023-10-09T10:26:30.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[147926]: failed to encode map e533 with expected crc 2023-10-09T10:26:30.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[147926]: failed to encode map e533 with expected crc 2023-10-09T10:26:30.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[152263]: pgmap v1495: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 39255/231315 objects degraded (16.970%); 18581/231315 objects misplaced (8.033%); 16 MiB/s, 251 objects/s recovering 2023-10-09T10:26:30.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:30.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/218580541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:26:30.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[152263]: osdmap e533: 8 total, 8 up, 6 in 2023-10-09T10:26:30.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[152263]: failed to encode map e533 with expected crc 2023-10-09T10:26:30.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:30 smithi012 ceph-mon[152263]: failed to encode map e533 with expected crc 2023-10-09T10:26:30.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:30 smithi110 ceph-mon[136032]: pgmap v1495: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 39255/231315 objects degraded (16.970%); 18581/231315 objects misplaced (8.033%); 16 MiB/s, 251 objects/s recovering 2023-10-09T10:26:30.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:30 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:30.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/218580541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:26:30.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:30 smithi110 ceph-mon[136032]: osdmap e533: 8 total, 8 up, 6 in 2023-10-09T10:26:30.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:30 smithi110 ceph-mon[136032]: failed to encode map e533 with expected crc 2023-10-09T10:26:30.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:30 smithi110 ceph-mon[136032]: failed to encode map e533 with expected crc 2023-10-09T10:26:31.384 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:31.871 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:26:32.148 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:32.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:32.257 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:32.355 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:32.391 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:32.451 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:32.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:32.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:32.557 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:32.626 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:32.745 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:26:32.745 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:26:33.173 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:32 smithi110 ceph-mon[136032]: pgmap v1497: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 38158/223797 objects degraded (17.050%); 17783/223797 objects misplaced (7.946%); 18 MiB/s, 282 objects/s recovering 2023-10-09T10:26:33.175 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:26:33.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:32 smithi012 ceph-mon[147926]: pgmap v1497: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 38158/223797 objects degraded (17.050%); 17783/223797 objects misplaced (7.946%); 18 MiB/s, 282 objects/s recovering 2023-10-09T10:26:33.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:32 smithi012 ceph-mon[152263]: pgmap v1497: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 38158/223797 objects degraded (17.050%); 17783/223797 objects misplaced (7.946%); 18 MiB/s, 282 objects/s recovering 2023-10-09T10:26:33.505 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:33.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:33.610 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:33.675 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:33.730 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:33.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:33.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:33.766 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:33.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:33.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:33 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 38158/223797 objects degraded (17.050%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:33.896 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:33.932 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:33 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 38158/223797 objects degraded (17.050%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:34.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:33 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 38158/223797 objects degraded (17.050%), 15 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:34.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:26:34.491 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:34.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:34.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:34.597 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:34.650 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:34.699 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:34.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:34.803 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:34.842 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:34.949 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:34.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:34.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:35.055 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:35.123 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:35.190 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:35.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:35.298 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:35.341 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:35.388 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:35.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:35.438 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:26:35.438 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:26:35.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:35.541 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:35.592 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:35 smithi110 ceph-mon[136032]: pgmap v1498: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 37967/221973 objects degraded (17.104%); 17592/221973 objects misplaced (7.925%); 16 MiB/s, 258 objects/s recovering 2023-10-09T10:26:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:35 smithi110 ceph-mon[136032]: failed to encode map e534 with expected crc 2023-10-09T10:26:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:35 smithi110 ceph-mon[136032]: osdmap e534: 8 total, 8 up, 6 in 2023-10-09T10:26:35.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:35 smithi110 ceph-mon[136032]: failed to encode map e534 with expected crc 2023-10-09T10:26:35.723 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:35.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:35 smithi012 ceph-mon[152263]: pgmap v1498: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 37967/221973 objects degraded (17.104%); 17592/221973 objects misplaced (7.925%); 16 MiB/s, 258 objects/s recovering 2023-10-09T10:26:35.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:35 smithi012 ceph-mon[152263]: failed to encode map e534 with expected crc 2023-10-09T10:26:35.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:35 smithi012 ceph-mon[152263]: osdmap e534: 8 total, 8 up, 6 in 2023-10-09T10:26:35.735 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:35 smithi012 ceph-mon[152263]: failed to encode map e534 with expected crc 2023-10-09T10:26:35.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:35 smithi012 ceph-mon[147926]: pgmap v1498: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 37967/221973 objects degraded (17.104%); 17592/221973 objects misplaced (7.925%); 16 MiB/s, 258 objects/s recovering 2023-10-09T10:26:35.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:35 smithi012 ceph-mon[147926]: failed to encode map e534 with expected crc 2023-10-09T10:26:35.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:35 smithi012 ceph-mon[147926]: osdmap e534: 8 total, 8 up, 6 in 2023-10-09T10:26:35.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:35 smithi012 ceph-mon[147926]: failed to encode map e534 with expected crc 2023-10-09T10:26:35.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:35.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:35.785 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:26:35.837 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:35.883 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:35.936 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:35.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:35.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:36.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:36.153 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:36.243 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:36.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:36.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:36.351 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:36.442 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:36.489 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:26:36.514 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:26:36.520 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:36.603 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:36.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:36.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:36.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:36 smithi012 ceph-mon[147926]: osdmap e535: 8 total, 8 up, 6 in 2023-10-09T10:26:36.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:36 smithi012 ceph-mon[147926]: failed to encode map e535 with expected crc 2023-10-09T10:26:36.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:36 smithi012 ceph-mon[147926]: failed to encode map e535 with expected crc 2023-10-09T10:26:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:36 smithi012 ceph-mon[152263]: osdmap e535: 8 total, 8 up, 6 in 2023-10-09T10:26:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:36 smithi012 ceph-mon[152263]: failed to encode map e535 with expected crc 2023-10-09T10:26:36.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:36 smithi012 ceph-mon[152263]: failed to encode map e535 with expected crc 2023-10-09T10:26:36.719 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:36.762 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:36.795 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:26:36.807 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:36 smithi110 ceph-mon[136032]: osdmap e535: 8 total, 8 up, 6 in 2023-10-09T10:26:36.807 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:36 smithi110 ceph-mon[136032]: failed to encode map e535 with expected crc 2023-10-09T10:26:36.807 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:36 smithi110 ceph-mon[136032]: failed to encode map e535 with expected crc 2023-10-09T10:26:36.814 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:36.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:36.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:36.827 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:26:36.877 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:26:36.916 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:26:36.923 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:36.951 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:26:36.989 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:26:36.992 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:37.034 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:26:37.070 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:26:37.078 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:26:37.078 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:26:37.078 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:26:37.078 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:26:37.078 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:26:37.079 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:26:37.079 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:26:37.079 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:26:37.154 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1501,"stamp":"2023-10-09T10:26:35.809454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4742289336,"num_objects":72379,"num_object_clones":0,"num_object_copies":217137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35012,"num_objects_misplaced":18577,"num_objects_unfound":0,"num_objects_dirty":72379,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":138851,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19337,"num_bytes_recovered":1237288700,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147030,"ondisk_log_size":147030,"up":297,"acting":282,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":21505684,"kb_used_data":15797476,"kb_used_omap":163,"kb_used_meta":5707868,"kb_avail":540965228,"statfs":{"total":575970213888,"available":553948393472,"internally_reserved":0,"allocated":16176615424,"data_stored":16167519920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":5844857843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-309723136,"num_objects":-4726,"num_object_clones":0,"num_object_copies":-14178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-4243,"num_objects_misplaced":-4,"num_objects_unfound":0,"num_objects_dirty":-4726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4726,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2243,"num_bytes_recovered":124518400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002361"},"pg_stats":[{"pgid":"11.12","version":"533'5740","reported_seq":5806,"reported_epoch":533,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:31.630427+0000","last_change":"2023-10-09T10:25:09.709209+0000","last_active":"2023-10-09T10:26:31.630427+0000","last_peered":"2023-10-09T10:26:31.630427+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:26:31.630427+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155418624,"num_objects":2372,"num_object_clones":0,"num_object_copies":7116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3119,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4792,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-747},{"shards":"0,2,5","objects":1909}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":126,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679653+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:26:22.679653+0000","last_peered":"2023-10-09T10:26:22.679653+0000","last_clean":"2023-10-09T10:26:22.679653+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:26:22.679653+0000","last_undegraded":"2023-10-09T10:26:22.679653+0000","last_fullsized":"2023-10-09T10:26:22.679653+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":487,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679749+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:26:22.679749+0000","last_peered":"2023-10-09T10:26:22.679749+0000","last_clean":"2023-10-09T10:26:22.679749+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:26:22.679749+0000","last_undegraded":"2023-10-09T10:26:22.679749+0000","last_fullsized":"2023-10-09T10:26:22.679749+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"533'5703","reported_seq":9460,"reported_epoch":533,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:31.625005+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:26:31.625005+0000","last_peered":"2023-10-09T10:26:31.625005+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:26:31.625005+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150994944,"num_objects":2304,"num_object_clones":0,"num_object_copies":6912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2919,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4758,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1227,"num_bytes_recovered":80412672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1037},{"shards":"0,2,6","objects":845},{"shards":"0,6","objects":422}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":564,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679385+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:26:22.679385+0000","last_peered":"2023-10-09T10:26:22.679385+0000","last_clean":"2023-10-09T10:26:22.679385+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:26:22.679385+0000","last_undegraded":"2023-10-09T10:26:22.679385+0000","last_fullsized":"2023-10-09T10:26:22.679385+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":457,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.666881+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:26:22.666881+0000","last_peered":"2023-10-09T10:26:22.666881+0000","last_clean":"2023-10-09T10:26:22.666881+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:26:22.666881+0000","last_undegraded":"2023-10-09T10:26:22.666881+0000","last_fullsized":"2023-10-09T10:26:22.666881+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"533'5627","reported_seq":10171,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:31.644879+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:26:31.644879+0000","last_peered":"2023-10-09T10:26:31.644879+0000","last_clean":"2023-10-09T10:26:31.644879+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:31.644879+0000","last_undegraded":"2023-10-09T10:26:31.644879+0000","last_fullsized":"2023-10-09T10:26:31.644879+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5627,"log_dups_size":0,"ondisk_log_size":5627,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150175744,"num_objects":2292,"num_object_clones":0,"num_object_copies":6876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4690,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665080+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:26:22.665080+0000","last_peered":"2023-10-09T10:26:22.665080+0000","last_clean":"2023-10-09T10:26:22.665080+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:26:22.665080+0000","last_undegraded":"2023-10-09T10:26:22.665080+0000","last_fullsized":"2023-10-09T10:26:22.665080+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:26.633442+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:26:26.633442+0000","last_peered":"2023-10-09T10:26:26.633442+0000","last_clean":"2023-10-09T10:26:26.633442+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:26:26.633442+0000","last_undegraded":"2023-10-09T10:26:26.633442+0000","last_fullsized":"2023-10-09T10:26:26.633442+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":491,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665986+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:26:22.665986+0000","last_peered":"2023-10-09T10:26:22.665986+0000","last_clean":"2023-10-09T10:26:22.665986+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:26:22.665986+0000","last_undegraded":"2023-10-09T10:26:22.665986+0000","last_fullsized":"2023-10-09T10:26:22.665986+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":442,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T10:26:34.529245+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:26:34.529245+0000","last_peered":"2023-10-09T10:26:34.529245+0000","last_clean":"2023-10-09T10:26:34.529245+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:26:34.529245+0000","last_undegraded":"2023-10-09T10:26:34.529245+0000","last_fullsized":"2023-10-09T10:26:34.529245+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":209,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.003422+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:26:30.003422+0000","last_peered":"2023-10-09T10:26:30.003422+0000","last_clean":"2023-10-09T10:26:30.003422+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:26:30.003422+0000","last_undegraded":"2023-10-09T10:26:30.003422+0000","last_fullsized":"2023-10-09T10:26:30.003422+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":442,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T10:26:34.525174+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:26:34.525174+0000","last_peered":"2023-10-09T10:26:34.525174+0000","last_clean":"2023-10-09T10:26:34.525174+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:26:34.525174+0000","last_undegraded":"2023-10-09T10:26:34.525174+0000","last_fullsized":"2023-10-09T10:26:34.525174+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":138,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679911+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:26:22.679911+0000","last_peered":"2023-10-09T10:26:22.679911+0000","last_clean":"2023-10-09T10:26:22.679911+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:26:22.679911+0000","last_undegraded":"2023-10-09T10:26:22.679911+0000","last_fullsized":"2023-10-09T10:26:22.679911+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":28,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.668911+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:26:22.668911+0000","last_peered":"2023-10-09T10:26:22.668911+0000","last_clean":"2023-10-09T10:26:22.668911+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:26:22.668911+0000","last_undegraded":"2023-10-09T10:26:22.668911+0000","last_fullsized":"2023-10-09T10:26:22.668911+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":184,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.681105+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:26:22.681105+0000","last_peered":"2023-10-09T10:26:22.681105+0000","last_clean":"2023-10-09T10:26:22.681105+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:26:22.681105+0000","last_undegraded":"2023-10-09T10:26:22.681105+0000","last_fullsized":"2023-10-09T10:26:22.681105+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":144,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.680174+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:26:22.680174+0000","last_peered":"2023-10-09T10:26:22.680174+0000","last_clean":"2023-10-09T10:26:22.680174+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:26:22.680174+0000","last_undegraded":"2023-10-09T10:26:22.680174+0000","last_fullsized":"2023-10-09T10:26:22.680174+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":104,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679923+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:26:22.679923+0000","last_peered":"2023-10-09T10:26:22.679923+0000","last_clean":"2023-10-09T10:26:22.679923+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:26:22.679923+0000","last_undegraded":"2023-10-09T10:26:22.679923+0000","last_fullsized":"2023-10-09T10:26:22.679923+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":59,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.006485+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:26:30.006485+0000","last_peered":"2023-10-09T10:26:30.006485+0000","last_clean":"2023-10-09T10:26:30.006485+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:26:30.006485+0000","last_undegraded":"2023-10-09T10:26:30.006485+0000","last_fullsized":"2023-10-09T10:26:30.006485+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":456,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.669235+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:26:22.669235+0000","last_peered":"2023-10-09T10:26:22.669235+0000","last_clean":"2023-10-09T10:26:22.669235+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:26:22.669235+0000","last_undegraded":"2023-10-09T10:26:22.669235+0000","last_fullsized":"2023-10-09T10:26:22.669235+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":200,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.009566+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:26:30.009566+0000","last_peered":"2023-10-09T10:26:30.009566+0000","last_clean":"2023-10-09T10:26:30.009566+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:26:30.009566+0000","last_undegraded":"2023-10-09T10:26:30.009566+0000","last_fullsized":"2023-10-09T10:26:30.009566+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"534'5785","reported_seq":6586,"reported_epoch":534,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:34.885592+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:26:34.885592+0000","last_peered":"2023-10-09T10:26:34.885592+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:26:34.885592+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"498'15","ondisk_log_start":"498'15","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147030016,"num_objects":2244,"num_object_clones":0,"num_object_copies":6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4862,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-907},{"shards":"3,5,6","objects":843}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":144,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679470+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:26:22.679470+0000","last_peered":"2023-10-09T10:26:22.679470+0000","last_clean":"2023-10-09T10:26:22.679470+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:26:22.679470+0000","last_undegraded":"2023-10-09T10:26:22.679470+0000","last_fullsized":"2023-10-09T10:26:22.679470+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":28,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.666570+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:26:22.666570+0000","last_peered":"2023-10-09T10:26:22.666570+0000","last_clean":"2023-10-09T10:26:22.666570+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:26:22.666570+0000","last_undegraded":"2023-10-09T10:26:22.666570+0000","last_fullsized":"2023-10-09T10:26:22.666570+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"534'5844","reported_seq":13833,"reported_epoch":534,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:26:35.030200+0000","last_change":"2023-10-09T10:25:48.137669+0000","last_active":"2023-10-09T10:26:35.030200+0000","last_peered":"2023-10-09T10:26:35.030200+0000","last_clean":"2023-10-09T10:24:39.576387+0000","last_became_active":"2023-10-09T10:25:09.696470+0000","last_became_peered":"2023-10-09T10:25:09.696470+0000","last_unstale":"2023-10-09T10:26:35.030200+0000","last_undegraded":"2023-10-09T10:26:35.030200+0000","last_fullsized":"2023-10-09T10:25:09.436538+0000","mapping_epoch":513,"log_start":"498'51","ondisk_log_start":"498'51","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5793,"log_dups_size":0,"ondisk_log_size":5793,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145260544,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":254,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4879,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2818,"num_bytes_recovered":184680448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4","6"],"object_location_counts":[{"shards":"2,5,6","objects":254},{"shards":"5,6","objects":1963}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":38,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.667871+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:26:22.667871+0000","last_peered":"2023-10-09T10:26:22.667871+0000","last_clean":"2023-10-09T10:26:22.667871+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:26:22.667871+0000","last_undegraded":"2023-10-09T10:26:22.667871+0000","last_fullsized":"2023-10-09T10:26:22.667871+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T10:26:34.533547+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:26:34.533547+0000","last_peered":"2023-10-09T10:26:34.533547+0000","last_clean":"2023-10-09T10:26:34.533547+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:26:34.533547+0000","last_undegraded":"2023-10-09T10:26:34.533547+0000","last_fullsized":"2023-10-09T10:26:34.533547+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"534'5902","reported_seq":5964,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T10:26:34.986829+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:26:34.986829+0000","last_peered":"2023-10-09T10:26:34.986829+0000","last_clean":"2023-10-09T10:26:34.986829+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:26:34.986829+0000","last_undegraded":"2023-10-09T10:26:34.986829+0000","last_fullsized":"2023-10-09T10:26:34.986829+0000","mapping_epoch":494,"log_start":"498'247","ondisk_log_start":"498'247","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":5655,"log_dups_size":0,"ondisk_log_size":5655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147718144,"num_objects":2254,"num_object_clones":0,"num_object_copies":6762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2614,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":34,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.680040+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:26:22.680040+0000","last_peered":"2023-10-09T10:26:22.680040+0000","last_clean":"2023-10-09T10:26:22.680040+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:26:22.680040+0000","last_undegraded":"2023-10-09T10:26:22.680040+0000","last_fullsized":"2023-10-09T10:26:22.680040+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":209,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.002866+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:26:30.002866+0000","last_peered":"2023-10-09T10:26:30.002866+0000","last_clean":"2023-10-09T10:26:30.002866+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:26:30.002866+0000","last_undegraded":"2023-10-09T10:26:30.002866+0000","last_fullsized":"2023-10-09T10:26:30.002866+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"533'5522","reported_seq":11612,"reported_epoch":533,"state":"active+undersized+remapped","last_fresh":"2023-10-09T10:26:34.417032+0000","last_change":"2023-10-09T10:26:33.850878+0000","last_active":"2023-10-09T10:26:34.417032+0000","last_peered":"2023-10-09T10:26:34.417032+0000","last_clean":"2023-10-09T10:25:11.561932+0000","last_became_active":"2023-10-09T10:25:13.746279+0000","last_became_peered":"2023-10-09T10:25:13.746279+0000","last_unstale":"2023-10-09T10:26:34.417032+0000","last_undegraded":"2023-10-09T10:26:34.417032+0000","last_fullsized":"2023-10-09T10:25:13.604555+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5522,"log_dups_size":0,"ondisk_log_size":5522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:39.267545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140181504,"num_objects":2139,"num_object_clones":0,"num_object_copies":6417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2762,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3012,"num_bytes_recovered":174915584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","0","4","5"],"object_location_counts":[{"shards":"2,4","objects":2139}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":160,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.680055+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:26:22.680055+0000","last_peered":"2023-10-09T10:26:22.680055+0000","last_clean":"2023-10-09T10:26:22.680055+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:26:22.680055+0000","last_undegraded":"2023-10-09T10:26:22.680055+0000","last_fullsized":"2023-10-09T10:26:22.680055+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":147,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679490+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:26:22.679490+0000","last_peered":"2023-10-09T10:26:22.679490+0000","last_clean":"2023-10-09T10:26:22.679490+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:26:22.679490+0000","last_undegraded":"2023-10-09T10:26:22.679490+0000","last_fullsized":"2023-10-09T10:26:22.679490+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"534'5802","reported_seq":9857,"reported_epoch":534,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:35.019573+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:26:35.019573+0000","last_peered":"2023-10-09T10:26:35.019573+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:26:35.019573+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"498'63","ondisk_log_start":"498'63","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144670720,"num_objects":2207,"num_object_clones":0,"num_object_copies":6621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2799,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4857,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1326,"num_bytes_recovered":86900736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":993},{"shards":"0,5,6","objects":813},{"shards":"5,6","objects":401}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":225,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.008579+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:26:30.008579+0000","last_peered":"2023-10-09T10:26:30.008579+0000","last_clean":"2023-10-09T10:26:30.008579+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:26:30.008579+0000","last_undegraded":"2023-10-09T10:26:30.008579+0000","last_fullsized":"2023-10-09T10:26:30.008579+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":126,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.003414+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:26:30.003414+0000","last_peered":"2023-10-09T10:26:30.003414+0000","last_clean":"2023-10-09T10:26:30.003414+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:26:30.003414+0000","last_undegraded":"2023-10-09T10:26:30.003414+0000","last_fullsized":"2023-10-09T10:26:30.003414+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"534'5718","reported_seq":9905,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T10:26:34.894198+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:26:34.894198+0000","last_peered":"2023-10-09T10:26:34.894198+0000","last_clean":"2023-10-09T10:26:34.894198+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:26:34.894198+0000","last_undegraded":"2023-10-09T10:26:34.894198+0000","last_fullsized":"2023-10-09T10:26:34.894198+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2804,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":209,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.006076+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:26:30.006076+0000","last_peered":"2023-10-09T10:26:30.006076+0000","last_clean":"2023-10-09T10:26:30.006076+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:26:30.006076+0000","last_undegraded":"2023-10-09T10:26:30.006076+0000","last_fullsized":"2023-10-09T10:26:30.006076+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":442,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665197+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:26:22.665197+0000","last_peered":"2023-10-09T10:26:22.665197+0000","last_clean":"2023-10-09T10:26:22.665197+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:26:22.665197+0000","last_undegraded":"2023-10-09T10:26:22.665197+0000","last_fullsized":"2023-10-09T10:26:22.665197+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"534'5682","reported_seq":6973,"reported_epoch":534,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:34.999783+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:26:34.999783+0000","last_peered":"2023-10-09T10:26:34.999783+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:26:34.999783+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147947520,"num_objects":2257,"num_object_clones":0,"num_object_copies":6771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4761,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":988},{"shards":"2,5","objects":628}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":143,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665111+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:26:22.665111+0000","last_peered":"2023-10-09T10:26:22.665111+0000","last_clean":"2023-10-09T10:26:22.665111+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:26:22.665111+0000","last_undegraded":"2023-10-09T10:26:22.665111+0000","last_fullsized":"2023-10-09T10:26:22.665111+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":144,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.681187+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:26:22.681187+0000","last_peered":"2023-10-09T10:26:22.681187+0000","last_clean":"2023-10-09T10:26:22.681187+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:26:22.681187+0000","last_undegraded":"2023-10-09T10:26:22.681187+0000","last_fullsized":"2023-10-09T10:26:22.681187+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"533'5684","reported_seq":5804,"reported_epoch":533,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:31.558498+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:26:31.558498+0000","last_peered":"2023-10-09T10:26:31.558498+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:26:31.558498+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148832256,"num_objects":2271,"num_object_clones":0,"num_object_copies":6813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2796,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-787},{"shards":"0,2,5","objects":1823}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":144,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679518+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:26:22.679518+0000","last_peered":"2023-10-09T10:26:22.679518+0000","last_clean":"2023-10-09T10:26:22.679518+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:26:22.679518+0000","last_undegraded":"2023-10-09T10:26:22.679518+0000","last_fullsized":"2023-10-09T10:26:22.679518+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":110,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679803+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:26:22.679803+0000","last_peered":"2023-10-09T10:26:22.679803+0000","last_clean":"2023-10-09T10:26:22.679803+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:26:22.679803+0000","last_undegraded":"2023-10-09T10:26:22.679803+0000","last_fullsized":"2023-10-09T10:26:22.679803+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"533'5709","reported_seq":9481,"reported_epoch":533,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:31.647768+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:26:31.647768+0000","last_peered":"2023-10-09T10:26:31.647768+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:26:31.647768+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150732800,"num_objects":2300,"num_object_clones":0,"num_object_copies":6900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4764,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1232,"num_bytes_recovered":80740352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1022},{"shards":"0,2,6","objects":806},{"shards":"0,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":126,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679595+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:26:22.679595+0000","last_peered":"2023-10-09T10:26:22.679595+0000","last_clean":"2023-10-09T10:26:22.679595+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:26:22.679595+0000","last_undegraded":"2023-10-09T10:26:22.679595+0000","last_fullsized":"2023-10-09T10:26:22.679595+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":153,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679968+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:26:22.679968+0000","last_peered":"2023-10-09T10:26:22.679968+0000","last_clean":"2023-10-09T10:26:22.679968+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:26:22.679968+0000","last_undegraded":"2023-10-09T10:26:22.679968+0000","last_fullsized":"2023-10-09T10:26:22.679968+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"533'5647","reported_seq":6081,"reported_epoch":533,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:31.644902+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:26:31.644902+0000","last_peered":"2023-10-09T10:26:31.644902+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:26:31.644902+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148570112,"num_objects":2267,"num_object_clones":0,"num_object_copies":6801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2540,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1013},{"shards":"0,3","objects":186}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":486,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.002783+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:26:30.002783+0000","last_peered":"2023-10-09T10:26:30.002783+0000","last_clean":"2023-10-09T10:26:30.002783+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:26:30.002783+0000","last_undegraded":"2023-10-09T10:26:30.002783+0000","last_fullsized":"2023-10-09T10:26:30.002783+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":547,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.680709+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:26:22.680709+0000","last_peered":"2023-10-09T10:26:22.680709+0000","last_clean":"2023-10-09T10:26:22.680709+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:26:22.680709+0000","last_undegraded":"2023-10-09T10:26:22.680709+0000","last_fullsized":"2023-10-09T10:26:22.680709+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"533'9502","reported_seq":9584,"reported_epoch":533,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:33.079679+0000","last_change":"2023-10-09T10:25:17.929569+0000","last_active":"2023-10-09T10:26:33.079679+0000","last_peered":"2023-10-09T10:26:33.079679+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:26:33.079679+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"504'3600","ondisk_log_start":"504'3600","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301006872,"num_objects":4594,"num_object_clones":0,"num_object_copies":13782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4591,"num_objects_misplaced":4594,"num_objects_unfound":0,"num_objects_dirty":4594,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9502,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":4594}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":254,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.027769+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:26:30.027769+0000","last_peered":"2023-10-09T10:26:30.027769+0000","last_clean":"2023-10-09T10:26:30.027769+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:26:30.027769+0000","last_undegraded":"2023-10-09T10:26:30.027769+0000","last_fullsized":"2023-10-09T10:26:30.027769+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":218,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.009491+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:26:30.009491+0000","last_peered":"2023-10-09T10:26:30.009491+0000","last_clean":"2023-10-09T10:26:30.009491+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:26:30.009491+0000","last_undegraded":"2023-10-09T10:26:30.009491+0000","last_fullsized":"2023-10-09T10:26:30.009491+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":498,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.003477+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:26:30.003477+0000","last_peered":"2023-10-09T10:26:30.003477+0000","last_clean":"2023-10-09T10:26:30.003477+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:26:30.003477+0000","last_undegraded":"2023-10-09T10:26:30.003477+0000","last_fullsized":"2023-10-09T10:26:30.003477+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"533'9589","reported_seq":9636,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:31.652363+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:26:31.652363+0000","last_peered":"2023-10-09T10:26:31.652363+0000","last_clean":"2023-10-09T10:26:31.652363+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:26:31.652363+0000","last_undegraded":"2023-10-09T10:26:31.652363+0000","last_fullsized":"2023-10-09T10:26:31.652363+0000","mapping_epoch":494,"log_start":"505'4506","ondisk_log_start":"505'4506","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307298304,"num_objects":4689,"num_object_clones":0,"num_object_copies":14067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9589,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":218,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.003345+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:26:30.003345+0000","last_peered":"2023-10-09T10:26:30.003345+0000","last_clean":"2023-10-09T10:26:30.003345+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:26:30.003345+0000","last_undegraded":"2023-10-09T10:26:30.003345+0000","last_fullsized":"2023-10-09T10:26:30.003345+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":135,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.680094+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:26:22.680094+0000","last_peered":"2023-10-09T10:26:22.680094+0000","last_clean":"2023-10-09T10:26:22.680094+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:26:22.680094+0000","last_undegraded":"2023-10-09T10:26:22.680094+0000","last_fullsized":"2023-10-09T10:26:22.680094+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":445,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679967+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:26:22.679967+0000","last_peered":"2023-10-09T10:26:22.679967+0000","last_clean":"2023-10-09T10:26:22.679967+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:26:22.679967+0000","last_undegraded":"2023-10-09T10:26:22.679967+0000","last_fullsized":"2023-10-09T10:26:22.679967+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"534'5781","reported_seq":9788,"reported_epoch":534,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:35.000171+0000","last_change":"2023-10-09T10:25:09.703037+0000","last_active":"2023-10-09T10:26:35.000171+0000","last_peered":"2023-10-09T10:26:35.000171+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:25:09.696864+0000","last_became_peered":"2023-10-09T10:25:09.696864+0000","last_unstale":"2023-10-09T10:26:35.000171+0000","last_undegraded":"2023-10-09T10:25:09.545735+0000","last_fullsized":"2023-10-09T10:25:09.430540+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5781,"log_dups_size":0,"ondisk_log_size":5781,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143753216,"num_objects":2193,"num_object_clones":0,"num_object_copies":6579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2762,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4815,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1311,"num_bytes_recovered":85917696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":979},{"shards":"2,5,6","objects":804},{"shards":"5,6","objects":410}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":483,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.676829+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:26:22.676829+0000","last_peered":"2023-10-09T10:26:22.676829+0000","last_clean":"2023-10-09T10:26:22.676829+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:26:22.676829+0000","last_undegraded":"2023-10-09T10:26:22.676829+0000","last_fullsized":"2023-10-09T10:26:22.676829+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":101,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.667768+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:26:22.667768+0000","last_peered":"2023-10-09T10:26:22.667768+0000","last_clean":"2023-10-09T10:26:22.667768+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:26:22.667768+0000","last_undegraded":"2023-10-09T10:26:22.667768+0000","last_fullsized":"2023-10-09T10:26:22.667768+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":547,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.004049+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:26:30.004049+0000","last_peered":"2023-10-09T10:26:30.004049+0000","last_clean":"2023-10-09T10:26:30.004049+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:26:30.004049+0000","last_undegraded":"2023-10-09T10:26:30.004049+0000","last_fullsized":"2023-10-09T10:26:30.004049+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"534'9658","reported_seq":9727,"reported_epoch":534,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:34.965189+0000","last_change":"2023-10-09T10:24:41.858672+0000","last_active":"2023-10-09T10:26:34.965189+0000","last_peered":"2023-10-09T10:26:34.965189+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:26:34.965189+0000","last_undegraded":"2023-10-09T10:26:34.965189+0000","last_fullsized":"2023-10-09T10:26:34.965189+0000","mapping_epoch":507,"log_start":"508'4800","ondisk_log_start":"508'4800","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4858,"log_dups_size":0,"ondisk_log_size":4858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291897344,"num_objects":4454,"num_object_clones":0,"num_object_copies":13362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4454,"num_objects_unfound":0,"num_objects_dirty":4454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9658,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":4454}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":188,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.005634+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:26:30.005634+0000","last_peered":"2023-10-09T10:26:30.005634+0000","last_clean":"2023-10-09T10:26:30.005634+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:26:30.005634+0000","last_undegraded":"2023-10-09T10:26:30.005634+0000","last_fullsized":"2023-10-09T10:26:30.005634+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":726,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.664865+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:26:22.664865+0000","last_peered":"2023-10-09T10:26:22.664865+0000","last_clean":"2023-10-09T10:26:22.664865+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:22.664865+0000","last_undegraded":"2023-10-09T10:26:22.664865+0000","last_fullsized":"2023-10-09T10:26:22.664865+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":236,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.003158+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:26:30.003158+0000","last_peered":"2023-10-09T10:26:30.003158+0000","last_clean":"2023-10-09T10:26:30.003158+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:26:30.003158+0000","last_undegraded":"2023-10-09T10:26:30.003158+0000","last_fullsized":"2023-10-09T10:26:30.003158+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":144,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679824+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:26:22.679824+0000","last_peered":"2023-10-09T10:26:22.679824+0000","last_clean":"2023-10-09T10:26:22.679824+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:26:22.679824+0000","last_undegraded":"2023-10-09T10:26:22.679824+0000","last_fullsized":"2023-10-09T10:26:22.679824+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"534'5844","reported_seq":9770,"reported_epoch":534,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:26:34.965001+0000","last_change":"2023-10-09T10:25:13.759251+0000","last_active":"2023-10-09T10:26:34.965001+0000","last_peered":"2023-10-09T10:26:34.965001+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:26:34.965001+0000","last_undegraded":"2023-10-09T10:25:13.707357+0000","last_fullsized":"2023-10-09T10:26:34.965001+0000","mapping_epoch":516,"log_start":"498'159","ondisk_log_start":"498'159","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151945216,"num_objects":2318,"num_object_clones":0,"num_object_copies":6954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1005,"num_objects_misplaced":1956,"num_objects_unfound":0,"num_objects_dirty":2318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4920,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1301,"num_bytes_recovered":85262336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1005},{"shards":"3,5,6","objects":362}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":218,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.667961+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:26:22.667961+0000","last_peered":"2023-10-09T10:26:22.667961+0000","last_clean":"2023-10-09T10:26:22.667961+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:26:22.667961+0000","last_undegraded":"2023-10-09T10:26:22.667961+0000","last_fullsized":"2023-10-09T10:26:22.667961+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":477,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.676960+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:26:22.676960+0000","last_peered":"2023-10-09T10:26:22.676960+0000","last_clean":"2023-10-09T10:26:22.676960+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:26:22.676960+0000","last_undegraded":"2023-10-09T10:26:22.676960+0000","last_fullsized":"2023-10-09T10:26:22.676960+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":28,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.666117+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:26:22.666117+0000","last_peered":"2023-10-09T10:26:22.666117+0000","last_clean":"2023-10-09T10:26:22.666117+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:26:22.666117+0000","last_undegraded":"2023-10-09T10:26:22.666117+0000","last_fullsized":"2023-10-09T10:26:22.666117+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"533'9669","reported_seq":9716,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:33.121629+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:26:33.121629+0000","last_peered":"2023-10-09T10:26:33.121629+0000","last_clean":"2023-10-09T10:26:33.121629+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:26:33.121629+0000","last_undegraded":"2023-10-09T10:26:33.121629+0000","last_fullsized":"2023-10-09T10:26:33.121629+0000","mapping_epoch":494,"log_start":"503'3700","ondisk_log_start":"503'3700","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5969,"log_dups_size":0,"ondisk_log_size":5969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297730048,"num_objects":4543,"num_object_clones":0,"num_object_copies":13629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9669,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665142+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:26:22.665142+0000","last_peered":"2023-10-09T10:26:22.665142+0000","last_clean":"2023-10-09T10:26:22.665142+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:26:22.665142+0000","last_undegraded":"2023-10-09T10:26:22.665142+0000","last_fullsized":"2023-10-09T10:26:22.665142+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":515,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.006340+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:26:30.006340+0000","last_peered":"2023-10-09T10:26:30.006340+0000","last_clean":"2023-10-09T10:26:30.006340+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:26:30.006340+0000","last_undegraded":"2023-10-09T10:26:30.006340+0000","last_fullsized":"2023-10-09T10:26:30.006340+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":445,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665516+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:26:22.665516+0000","last_peered":"2023-10-09T10:26:22.665516+0000","last_clean":"2023-10-09T10:26:22.665516+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:26:22.665516+0000","last_undegraded":"2023-10-09T10:26:22.665516+0000","last_fullsized":"2023-10-09T10:26:22.665516+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"533'9248","reported_seq":9297,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:31.655231+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:26:31.655231+0000","last_peered":"2023-10-09T10:26:31.655231+0000","last_clean":"2023-10-09T10:26:31.655231+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:26:31.655231+0000","last_undegraded":"2023-10-09T10:26:31.655231+0000","last_fullsized":"2023-10-09T10:26:31.655231+0000","mapping_epoch":494,"log_start":"504'4176","ondisk_log_start":"504'4176","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5072,"log_dups_size":0,"ondisk_log_size":5072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294125568,"num_objects":4488,"num_object_clones":0,"num_object_copies":13464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9248,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":333,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665871+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:26:22.665871+0000","last_peered":"2023-10-09T10:26:22.665871+0000","last_clean":"2023-10-09T10:26:22.665871+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:26:22.665871+0000","last_undegraded":"2023-10-09T10:26:22.665871+0000","last_fullsized":"2023-10-09T10:26:22.665871+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":514,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679664+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:26:22.679664+0000","last_peered":"2023-10-09T10:26:22.679664+0000","last_clean":"2023-10-09T10:26:22.679664+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:26:22.679664+0000","last_undegraded":"2023-10-09T10:26:22.679664+0000","last_fullsized":"2023-10-09T10:26:22.679664+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":28,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.664983+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:26:22.664983+0000","last_peered":"2023-10-09T10:26:22.664983+0000","last_clean":"2023-10-09T10:26:22.664983+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:26:22.664983+0000","last_undegraded":"2023-10-09T10:26:22.664983+0000","last_fullsized":"2023-10-09T10:26:22.664983+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"533'9718","reported_seq":9765,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:33.074632+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:26:33.074632+0000","last_peered":"2023-10-09T10:26:33.074632+0000","last_clean":"2023-10-09T10:26:33.074632+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:26:33.074632+0000","last_undegraded":"2023-10-09T10:26:33.074632+0000","last_fullsized":"2023-10-09T10:26:33.074632+0000","mapping_epoch":494,"log_start":"504'3800","ondisk_log_start":"504'3800","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297926656,"num_objects":4546,"num_object_clones":0,"num_object_copies":13638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9718,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":37,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.013819+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:26:30.013819+0000","last_peered":"2023-10-09T10:26:30.013819+0000","last_clean":"2023-10-09T10:26:30.013819+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:26:30.013819+0000","last_undegraded":"2023-10-09T10:26:30.013819+0000","last_fullsized":"2023-10-09T10:26:30.013819+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":32,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.666647+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:26:22.666647+0000","last_peered":"2023-10-09T10:26:22.666647+0000","last_clean":"2023-10-09T10:26:22.666647+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:26:22.666647+0000","last_undegraded":"2023-10-09T10:26:22.666647+0000","last_fullsized":"2023-10-09T10:26:22.666647+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":214,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.004112+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:26:30.004112+0000","last_peered":"2023-10-09T10:26:30.004112+0000","last_clean":"2023-10-09T10:26:30.004112+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:26:30.004112+0000","last_undegraded":"2023-10-09T10:26:30.004112+0000","last_fullsized":"2023-10-09T10:26:30.004112+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"534'5655","reported_seq":5751,"reported_epoch":534,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:34.936799+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:26:34.936799+0000","last_peered":"2023-10-09T10:26:34.936799+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:26:34.936799+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5655,"log_dups_size":0,"ondisk_log_size":5655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143949824,"num_objects":2197,"num_object_clones":0,"num_object_copies":6591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4735,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":961},{"shards":"2,5","objects":-19}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":28,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665437+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:26:22.665437+0000","last_peered":"2023-10-09T10:26:22.665437+0000","last_clean":"2023-10-09T10:26:22.665437+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:26:22.665437+0000","last_undegraded":"2023-10-09T10:26:22.665437+0000","last_fullsized":"2023-10-09T10:26:22.665437+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":37,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.668099+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:26:22.668099+0000","last_peered":"2023-10-09T10:26:22.668099+0000","last_clean":"2023-10-09T10:26:22.668099+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:26:22.668099+0000","last_undegraded":"2023-10-09T10:26:22.668099+0000","last_fullsized":"2023-10-09T10:26:22.668099+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"533'5717","reported_seq":5786,"reported_epoch":533,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:26:31.639943+0000","last_change":"2023-10-09T10:25:08.466144+0000","last_active":"2023-10-09T10:26:31.639943+0000","last_peered":"2023-10-09T10:26:31.639943+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:26:31.639943+0000","last_undegraded":"2023-10-09T10:26:31.639943+0000","last_fullsized":"2023-10-09T10:26:31.639943+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5717,"log_dups_size":0,"ondisk_log_size":5717,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148013056,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2331,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4793,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":-72}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":264,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.006155+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:26:30.006155+0000","last_peered":"2023-10-09T10:26:30.006155+0000","last_clean":"2023-10-09T10:26:30.006155+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:26:30.006155+0000","last_undegraded":"2023-10-09T10:26:30.006155+0000","last_fullsized":"2023-10-09T10:26:30.006155+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":126,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.680250+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:26:22.680250+0000","last_peered":"2023-10-09T10:26:22.680250+0000","last_clean":"2023-10-09T10:26:22.680250+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:26:22.680250+0000","last_undegraded":"2023-10-09T10:26:22.680250+0000","last_fullsized":"2023-10-09T10:26:22.680250+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"534'5766","reported_seq":9665,"reported_epoch":534,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:34.965185+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:26:34.965185+0000","last_peered":"2023-10-09T10:26:34.965185+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:26:34.965185+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5766,"log_dups_size":0,"ondisk_log_size":5766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148733952,"num_objects":2270,"num_object_clones":0,"num_object_copies":6810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2762,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4822,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1276,"num_bytes_recovered":83623936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":964},{"shards":"0,5,6","objects":833},{"shards":"5,6","objects":472}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":247,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T10:26:30.009359+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:26:30.009359+0000","last_peered":"2023-10-09T10:26:30.009359+0000","last_clean":"2023-10-09T10:26:30.009359+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:26:30.009359+0000","last_undegraded":"2023-10-09T10:26:30.009359+0000","last_fullsized":"2023-10-09T10:26:30.009359+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":356,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.669080+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:26:22.669080+0000","last_peered":"2023-10-09T10:26:22.669080+0000","last_clean":"2023-10-09T10:26:22.669080+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:26:22.669080+0000","last_undegraded":"2023-10-09T10:26:22.669080+0000","last_fullsized":"2023-10-09T10:26:22.669080+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"533'5686","reported_seq":5803,"reported_epoch":533,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:34.447769+0000","last_change":"2023-10-09T10:26:33.871249+0000","last_active":"2023-10-09T10:26:34.447769+0000","last_peered":"2023-10-09T10:26:34.447769+0000","last_clean":"2023-10-09T10:25:11.551777+0000","last_became_active":"2023-10-09T10:25:13.754797+0000","last_became_peered":"2023-10-09T10:25:13.754797+0000","last_unstale":"2023-10-09T10:26:34.447769+0000","last_undegraded":"2023-10-09T10:25:13.625003+0000","last_fullsized":"2023-10-09T10:25:13.604794+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:58.037975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146145280,"num_objects":2230,"num_object_clones":0,"num_object_copies":6690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1274,"num_objects_misplaced":1837,"num_objects_unfound":0,"num_objects_dirty":2230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2749,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1703936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":-881},{"shards":"2,4,5","objects":1837}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":33,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T10:26:34.533864+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:26:34.533864+0000","last_peered":"2023-10-09T10:26:34.533864+0000","last_clean":"2023-10-09T10:26:34.533864+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:26:34.533864+0000","last_undegraded":"2023-10-09T10:26:34.533864+0000","last_fullsized":"2023-10-09T10:26:34.533864+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":585,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.676762+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:26:22.676762+0000","last_peered":"2023-10-09T10:26:22.676762+0000","last_clean":"2023-10-09T10:26:22.676762+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:26:22.676762+0000","last_undegraded":"2023-10-09T10:26:22.676762+0000","last_fullsized":"2023-10-09T10:26:22.676762+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"534'5755","reported_seq":5816,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T10:26:35.020030+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:26:35.020030+0000","last_peered":"2023-10-09T10:26:35.020030+0000","last_clean":"2023-10-09T10:26:35.020030+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:26:35.020030+0000","last_undegraded":"2023-10-09T10:26:35.020030+0000","last_fullsized":"2023-10-09T10:26:35.020030+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143589376,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2494,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":49,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.665490+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:26:22.665490+0000","last_peered":"2023-10-09T10:26:22.665490+0000","last_clean":"2023-10-09T10:26:22.665490+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:26:22.665490+0000","last_undegraded":"2023-10-09T10:26:22.665490+0000","last_fullsized":"2023-10-09T10:26:22.665490+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":153,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T10:26:22.679345+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:26:22.679345+0000","last_peered":"2023-10-09T10:26:22.679345+0000","last_clean":"2023-10-09T10:26:22.679345+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:26:22.679345+0000","last_undegraded":"2023-10-09T10:26:22.679345+0000","last_fullsized":"2023-10-09T10:26:22.679345+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":4741693464,"num_objects":72355,"num_object_clones":0,"num_object_copies":217065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35012,"num_objects_misplaced":18577,"num_objects_unfound":0,"num_objects_dirty":72355,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138591,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19177,"num_bytes_recovered":1234305024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16088702976,"data_stored":16088694832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146836,"ondisk_log_size":146836,"up":78,"acting":63,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430683,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626911,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2228108,"kb_used_data":1320644,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91517044,"statfs":{"total":95995035648,"available":93713453056,"internally_reserved":0,"allocated":1352339456,"data_stored":1351412748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.4330000000000001,"15min":1.6339999999999999},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":4.7990000000000004,"5min":9.4260000000000002,"15min":44.055},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.591,"15min":1.8600000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":5.6079999999999997,"5min":9.2609999999999992,"15min":43.869999999999997},"last":0.49299999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":2.911,"15min":2.2400000000000002},"min":{"1min":0.29399999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.0419999999999998,"5min":155.01400000000001,"15min":155.01400000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.383,"5min":2.9460000000000002,"15min":2.4060000000000001},"min":{"1min":0.33200000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.5709999999999997,"5min":154.71100000000001,"15min":154.71100000000001},"last":0.32900000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":1.821,"15min":1.865},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":8.5069999999999997,"5min":8.6280000000000001,"15min":29.367999999999999},"last":4.6459999999999999},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.617,"15min":1.752},"min":{"1min":0.32600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.0640000000000001,"5min":7.5629999999999997,"15min":14.542999999999999},"last":2.5150000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.76,"15min":1.573},"min":{"1min":0.34599999999999997,"5min":0.32700000000000001,"15min":0.308},"max":{"1min":9.4619999999999997,"5min":9.4619999999999997,"15min":9.4619999999999997},"last":2.6429999999999998},{"interface":"front","average":{"1min":2.3130000000000002,"5min":1.946,"15min":1.6759999999999999},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.31},"max":{"1min":7.9939999999999998,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":2.6640000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.3610000000000002,"15min":1.9430000000000001},"min":{"1min":0.30299999999999999,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":7.3289999999999997,"5min":10.618,"15min":122.041},"last":4.6799999999999997},{"interface":"front","average":{"1min":1.478,"5min":2.04,"15min":1.7809999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.4729999999999999,"5min":11.266,"15min":122.13200000000001},"last":0.499}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":2.4100000000000001,"15min":2.0350000000000001},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1980000000000004,"5min":12.894,"15min":121.98999999999999},"last":0.433},{"interface":"front","average":{"1min":1.9019999999999999,"5min":1.5960000000000001,"15min":1.6919999999999999},"min":{"1min":0.33300000000000002,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":7.0060000000000002,"5min":15.041,"15min":122.346},"last":4.1310000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":3.532,"15min":1.9730000000000001},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":4.9169999999999998,"5min":130.66999999999999,"15min":130.66999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.3779999999999999,"5min":3.573,"15min":2.016},"min":{"1min":0.374,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":4.9589999999999996,"5min":130.702,"15min":130.702},"last":0.54700000000000004}]}]},{"osd":1,"up_from":0,"seq":360777253224,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416748,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4004004,"kb_used_data":3037916,"kb_used_omap":28,"kb_used_meta":966051,"kb_avail":89741148,"statfs":{"total":95995035648,"available":91894935552,"internally_reserved":0,"allocated":3110825984,"data_stored":3108754167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":989237072},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:25:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.6200000000000001,"15min":2.0059999999999998},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":17.161999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":3.3860000000000001},{"interface":"front","average":{"1min":1.585,"5min":1.506,"15min":2.4689999999999999},"min":{"1min":0.218,"5min":0.214,"15min":0.192},"max":{"1min":10.786,"5min":17.033000000000001,"15min":56.950000000000003},"last":0.28199999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:26:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":1.4970000000000001,"15min":2.1190000000000002},"min":{"1min":0.26600000000000001,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":17.093,"5min":17.093,"15min":53.988999999999997},"last":3.4580000000000002},{"interface":"front","average":{"1min":3.3370000000000002,"5min":1.8759999999999999,"15min":2.161},"min":{"1min":0.32100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.053999999999998,"5min":17.053999999999998,"15min":54.119},"last":3.4289999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.609,"15min":1.4850000000000001},"min":{"1min":0.28599999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.763,"5min":11.933,"15min":56.984999999999999},"last":0.248},{"interface":"front","average":{"1min":2.8660000000000001,"5min":1.9790000000000001,"15min":1.5640000000000001},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.067,"5min":10.067,"15min":56.475999999999999},"last":2.2690000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":1.974,"15min":2.254},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":10.941000000000001,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.318},{"interface":"front","average":{"1min":2.7709999999999999,"5min":2.3420000000000001,"15min":2.2650000000000001},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.32400000000000001},"max":{"1min":9.9499999999999993,"5min":28.331,"15min":56.378},"last":1.2390000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":8.6020000000000003,"5min":3.4510000000000001,"15min":3.544},"min":{"1min":0.51700000000000002,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":90.412000000000006,"5min":90.412000000000006,"15min":229.589},"last":2.476},{"interface":"front","average":{"1min":6.9400000000000004,"5min":2.7509999999999999,"15min":3.1909999999999998},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":90.393000000000001,"5min":90.393000000000001,"15min":228.88900000000001},"last":3.5379999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.919,"15min":2.29},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":54.482999999999997},"last":3.5139999999999998},{"interface":"front","average":{"1min":2.806,"5min":2.1240000000000001,"15min":2.5230000000000001},"min":{"1min":0.47899999999999998,"5min":0.39700000000000002,"15min":0.34799999999999998},"max":{"1min":8.8309999999999995,"5min":8.8309999999999995,"15min":57.753},"last":1.591}]},{"osd":7,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0470000000000002,"5min":2.2679999999999998,"15min":2.4220000000000002},"min":{"1min":0.48299999999999998,"5min":0.40500000000000003,"15min":0.34999999999999998},"max":{"1min":12.029999999999999,"5min":12.029999999999999,"15min":56.411999999999999},"last":1.2070000000000001},{"interface":"front","average":{"1min":3.0350000000000001,"5min":2.1869999999999998,"15min":2.3530000000000002},"min":{"1min":0.44800000000000001,"5min":0.38600000000000001,"15min":0.38300000000000001},"max":{"1min":17.129999999999999,"5min":17.129999999999999,"15min":54.451999999999998},"last":3.4849999999999999}]}]},{"osd":2,"up_from":217,"seq":932007903484,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3943132,"kb_used_data":3090388,"kb_used_omap":83,"kb_used_meta":852652,"kb_avail":89802020,"statfs":{"total":95995035648,"available":91957268480,"internally_reserved":0,"allocated":3164557312,"data_stored":3162677635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":873116143},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2000000000000002,"5min":2.0579999999999998,"15min":2.0049999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.185},"max":{"1min":11.093999999999999,"5min":48.990000000000002,"15min":48.990000000000002},"last":1.944},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.1160000000000001,"15min":2.3220000000000001},"min":{"1min":0.45600000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.2699999999999996,"5min":49.412999999999997,"15min":49.412999999999997},"last":0.23599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":1.893,"15min":2.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":13.303000000000001,"5min":13.303000000000001,"15min":34.893999999999998},"last":4.6680000000000001},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.5740000000000001,"15min":2.0339999999999998},"min":{"1min":0.35099999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":9.9239999999999995,"5min":9.9239999999999995,"15min":34.962000000000003},"last":3.113}]},{"osd":3,"last update":"Mon Oct 9 10:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.4139999999999999,"15min":1.202},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.248},"max":{"1min":9.3119999999999994,"5min":9.3119999999999994,"15min":35.043999999999997},"last":2.0169999999999999},{"interface":"front","average":{"1min":2.7909999999999999,"5min":1.2829999999999999,"15min":1.3},"min":{"1min":0.35699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.206,"5min":11.206,"15min":50.406999999999996},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:26:14 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.8149999999999999,"15min":1.6870000000000001},"min":{"1min":0.31900000000000001,"5min":0.29799999999999999,"15min":0.28999999999999998},"max":{"1min":4.4710000000000001,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.0640000000000001,"15min":1.835},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":11.298,"5min":47.704000000000001,"15min":47.704000000000001},"last":0.60299999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.8340000000000001,"15min":2.012},"min":{"1min":0.40500000000000003,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":57.298000000000002},"last":1.5549999999999999},{"interface":"front","average":{"1min":2.8639999999999999,"5min":2.0430000000000001,"15min":2.0169999999999999},"min":{"1min":0.442,"5min":0.31,"15min":0.31},"max":{"1min":11.195,"5min":11.195,"15min":57.828000000000003},"last":1.901}]},{"osd":6,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":2.0590000000000002,"15min":2.0230000000000001},"min":{"1min":0.35399999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":4.5579999999999998,"5min":12.411,"15min":18.943999999999999},"last":2.2690000000000001},{"interface":"front","average":{"1min":2.4860000000000002,"5min":1.855,"15min":1.97},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":13.975,"5min":13.975,"15min":24.379000000000001},"last":1.964}]},{"osd":7,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.1349999999999998,"15min":2.2949999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.308},"max":{"1min":5.415,"5min":11.704000000000001,"15min":35.008000000000003},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.907,"5min":1.99,"15min":2.1579999999999999},"min":{"1min":0.47099999999999997,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":15.548,"5min":15.548,"15min":24.323},"last":2.2210000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488324,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2872552,"kb_used_data":2340960,"kb_used_omap":15,"kb_used_meta":531440,"kb_avail":90872600,"statfs":{"total":95995035648,"available":93053542400,"internally_reserved":0,"allocated":2397143040,"data_stored":2394786364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":544195014},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8259999999999996,"5min":3.1200000000000001,"15min":1.696},"min":{"1min":0.34999999999999998,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":70.004000000000005,"5min":70.004000000000005,"15min":70.004000000000005},"last":4.1719999999999997},{"interface":"front","average":{"1min":6.2519999999999998,"5min":2.8159999999999998,"15min":1.6739999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":66.802000000000007,"5min":66.802000000000007,"15min":66.802000000000007},"last":5.0309999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7889999999999999,"15min":1.8620000000000001},"min":{"1min":0.311,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.785,"5min":25.370000000000001,"15min":105.401},"last":1.1379999999999999},{"interface":"front","average":{"1min":2.7109999999999999,"5min":2.04,"15min":1.7969999999999999},"min":{"1min":0.30299999999999999,"5min":0.25900000000000001,"15min":0.23300000000000001},"max":{"1min":9.2140000000000004,"5min":14.613,"15min":104.914},"last":1.494}]},{"osd":2,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.2989999999999999,"15min":1.9490000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":11.141999999999999,"5min":11.141999999999999,"15min":11.141999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.1400000000000001,"5min":1.956,"15min":1.4279999999999999},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.27600000000000002},"max":{"1min":16.475000000000001,"5min":25.338999999999999,"15min":25.338999999999999},"last":0.40899999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.194,"15min":1.415},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":6.9370000000000003,"5min":21.196000000000002,"15min":21.831},"last":1.528},{"interface":"front","average":{"1min":3.3980000000000001,"5min":2.8820000000000001,"15min":1.958},"min":{"1min":0.36399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":11.087,"5min":19.323,"15min":22.097999999999999},"last":0.51100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":2.0910000000000002,"15min":1.294},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":11.363,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.46,"5min":2.411,"15min":1.5429999999999999},"min":{"1min":0.36599999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":6.9900000000000002,"5min":13.394,"15min":13.394},"last":1.5780000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.4780000000000002,"15min":1.7250000000000001},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.35599999999999998},"max":{"1min":11.114000000000001,"5min":19.114000000000001,"15min":19.114000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.3279999999999998,"5min":2.1230000000000002,"15min":1.468},"min":{"1min":0.47699999999999998,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":11.32,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":2.0779999999999998,"15min":1.514},"min":{"1min":0.42299999999999999,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":6.9640000000000004,"5min":9.7880000000000003,"15min":10.459},"last":1.5529999999999999},{"interface":"front","average":{"1min":2.327,"5min":2.3220000000000001,"15min":1.762},"min":{"1min":0.432,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":11.045999999999999,"5min":19.280000000000001,"15min":19.280000000000001},"last":0.54800000000000004}]}]},{"osd":4,"up_from":266,"seq":1142461300953,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3857488,"kb_used_data":2920264,"kb_used_omap":35,"kb_used_meta":937180,"kb_avail":89887664,"statfs":{"total":95995035648,"available":92044967936,"internally_reserved":0,"allocated":2990350336,"data_stored":2989417954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":959672558},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.893,"15min":2.1469999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":6.5019999999999998,"5min":14.548,"15min":97.721999999999994},"last":5.5060000000000002},{"interface":"front","average":{"1min":2.8940000000000001,"5min":2.694,"15min":2.8740000000000001},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.224},"max":{"1min":11.022,"5min":11.257,"15min":97.915999999999997},"last":4.3659999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4449999999999998,"15min":2.3700000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.24299999999999999},"max":{"1min":12.209,"5min":12.209,"15min":71.457999999999998},"last":2.7949999999999999},{"interface":"front","average":{"1min":2.992,"5min":2.8999999999999999,"15min":2.2269999999999999},"min":{"1min":0.376,"5min":0.248,"15min":0.248},"max":{"1min":8.327,"5min":49.781999999999996,"15min":71.658000000000001},"last":5.694}]},{"osd":2,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3809999999999998,"15min":1.845},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":7.3230000000000004,"5min":15.944000000000001,"15min":15.944000000000001},"last":5.2279999999999998},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.9970000000000001,"15min":1.7989999999999999},"min":{"1min":0.36199999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.102,"5min":14.473000000000001,"15min":14.473000000000001},"last":0.314}]},{"osd":3,"last update":"Mon Oct 9 10:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.7410000000000001,"5min":2.9199999999999999,"15min":2.024},"min":{"1min":0.39800000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.347,"5min":12.074,"15min":12.074},"last":1.2609999999999999},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.907,"15min":1.7390000000000001},"min":{"1min":0.40899999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.369999999999999,"5min":16.067,"15min":16.067},"last":3.3570000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":2.5939999999999999,"15min":2.069},"min":{"1min":0.314,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":3.3279999999999998,"5min":16.094000000000001,"15min":99.585999999999999},"last":1.4790000000000001},{"interface":"front","average":{"1min":1.054,"5min":2.4969999999999999,"15min":1.831},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.464,"5min":49.915999999999997,"15min":99.384},"last":1.615}]},{"osd":6,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.881,"15min":1.3740000000000001},"min":{"1min":0.33800000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.5389999999999997,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.8220000000000001,"15min":1.7949999999999999},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":12.242000000000001,"5min":18.952999999999999,"15min":18.952999999999999},"last":1.1759999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":3.0409999999999999,"15min":1.9239999999999999},"min":{"1min":0.39500000000000002,"5min":0.316,"15min":0.307},"max":{"1min":12.391999999999999,"5min":17.369,"15min":17.369},"last":1.214},{"interface":"front","average":{"1min":1.3320000000000001,"5min":2.7829999999999999,"15min":1.778},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.30399999999999999},"max":{"1min":4.6689999999999996,"5min":51.026000000000003,"15min":51.026000000000003},"last":1.506}]}]},{"osd":5,"up_from":36,"seq":154618823138,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4600400,"kb_used_data":3087304,"kb_used_omap":0,"kb_used_meta":1513088,"kb_avail":89144752,"statfs":{"total":95995035648,"available":91284226048,"internally_reserved":0,"allocated":3161399296,"data_stored":3160471052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549402112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.754,"15min":1.9370000000000001},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":16.536999999999999,"5min":22.469999999999999,"15min":22.469999999999999},"last":3.028},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.1309999999999998,"15min":1.7270000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":8.9339999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":4.6710000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:25:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.6339999999999999,"15min":2.2090000000000001},"min":{"1min":0.29599999999999999,"5min":0.28799999999999998,"15min":0.23999999999999999},"max":{"1min":16.562999999999999,"5min":16.562999999999999,"15min":29.370000000000001},"last":2.7909999999999999},{"interface":"front","average":{"1min":1.8,"5min":1.8759999999999999,"15min":1.8580000000000001},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":8.9529999999999994,"5min":13.143000000000001,"15min":29.065000000000001},"last":4.593}]},{"osd":2,"last update":"Mon Oct 9 10:26:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":2.5990000000000002,"15min":2.1309999999999998},"min":{"1min":0.33000000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":12.247999999999999,"5min":16.629999999999999,"15min":16.629999999999999},"last":7.8849999999999998},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.907,"15min":1.6919999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":3.1389999999999998,"5min":13.095000000000001,"15min":15.929},"last":7.0910000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.2170000000000001,"15min":1.7889999999999999},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":10.362,"5min":10.561999999999999,"15min":10.561999999999999},"last":3.04},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.0830000000000002,"15min":1.7270000000000001},"min":{"1min":0.41599999999999998,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9130000000000003,"5min":13.624000000000001,"15min":13.624000000000001},"last":0.77400000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":2.242,"15min":1.5760000000000001},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":10.826000000000001,"5min":12.023,"15min":12.023},"last":2.9860000000000002},{"interface":"front","average":{"1min":1.6579999999999999,"5min":2.0369999999999999,"15min":1.399},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":8.8620000000000001,"5min":11.956,"15min":11.956},"last":1.306}]},{"osd":6,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.621,"15min":1.794},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":10.664999999999999,"5min":11.686,"15min":12.772},"last":3.0169999999999999},{"interface":"front","average":{"1min":2.4500000000000002,"5min":2.1640000000000001,"15min":1.665},"min":{"1min":0.29699999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":2.3900000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:26:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.3719999999999999,"15min":1.7709999999999999},"min":{"1min":0.378,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":4.25,"5min":21.109000000000002,"15min":21.109000000000002},"last":2.536},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.1990000000000001,"15min":1.1319999999999999},"min":{"1min":0.31900000000000001,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":2.0299999999999998,"5min":11.994,"15min":11.994},"last":0.31900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3087925248,"data_stored":3087925248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3143499776,"data_stored":3143499776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2388656128,"data_stored":2388656128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2971734016,"data_stored":2971729944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3151826944,"data_stored":3151822872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1345060864,"data_stored":1345060864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:26:37.163 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:26:37.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:37.164 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:37.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:37.266 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:37.303 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:37.397 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:37.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:37.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:37.613 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:37 smithi110 ceph-mon[136032]: pgmap v1501: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 35012/217137 objects degraded (16.124%); 18577/217137 objects misplaced (8.555%); 20 MiB/s, 373 objects/s recovering 2023-10-09T10:26:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:37 smithi110 ceph-mon[136032]: from='client.47087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:37.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3141700397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:37.706 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:37.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:37.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:37.808 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:37 smithi012 ceph-mon[147926]: pgmap v1501: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 35012/217137 objects degraded (16.124%); 18577/217137 objects misplaced (8.555%); 20 MiB/s, 373 objects/s recovering 2023-10-09T10:26:37.808 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:37 smithi012 ceph-mon[147926]: from='client.47087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:37.808 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3141700397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:37.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:37 smithi012 ceph-mon[152263]: pgmap v1501: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 35012/217137 objects degraded (16.124%); 18577/217137 objects misplaced (8.555%); 20 MiB/s, 373 objects/s recovering 2023-10-09T10:26:37.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:37 smithi012 ceph-mon[152263]: from='client.47087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:37.809 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3141700397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:37.824 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:37.890 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:37.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:37.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:37.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:37.978 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:155887 11.d e535/518 11:b97b2d35:::benchmark_data_smithi012_404402_object43104:head v 535'9686, mlcod=535'9684)", 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:37.972847+0000", 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.002604608, 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026319759999999998, 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.972847+0000", 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.972847+0000", 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.972846+0000", 2023-10-09T10:26:37.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6890000000000004e-06 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.972854+0000", 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3518e-05 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.972868+0000", 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3690000000000003e-06 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.972874+0000", 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1171e-05 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.972945+0000", 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8351e-05 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:37.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.972973+0000", 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000126172 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:155884 11.14 e535/507 11:29470013:::benchmark_data_smithi012_404402_object43101:head v 535'5833, mlcod=503'2290)", 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:37.973710+0000", 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.001742012, 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018464880000000001, 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:37.983 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973710+0000", 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973710+0000", 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973709+0000", 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8699999999999992e-06 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973717+0000", 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4487e-05 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:37.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973731+0000", 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9630000000000003e-06 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973737+0000", 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8373000000000002e-05 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973836+0000", 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4525999999999999e-05 2023-10-09T10:26:37.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973860+0000", 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015037399999999999 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:155866 11.11 e535/513 11:8ef08c87:::benchmark_data_smithi012_404402_object43083:head v 535'5835, mlcod=503'2282)", 2023-10-09T10:26:37.987 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:37.973753+0000", 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.0016990709999999999, 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0018574010000000001, 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973753+0000", 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:37.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973753+0000", 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973752+0000", 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7489999999999998e-06 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973757+0000", 2023-10-09T10:26:37.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6170000000000003e-06 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973764+0000", 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0280000000000002e-06 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973768+0000", 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4598999999999998e-05 2023-10-09T10:26:37.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973812+0000", 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5704999999999999e-05 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.973838+0000", 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5337e-05 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:37.991 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:155882 11.10 e535/527)", 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:37.974856+0000", 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 0.00059620400000000003, 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00080380399999999998, 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "queued for pg", 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.974856+0000", 2023-10-09T10:26:37.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.974856+0000", 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.974854+0000", 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3134e-05 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.974867+0000", 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1763000000000001e-05 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.974879+0000", 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:37.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:37.974885+0000", 2023-10-09T10:26:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9458000000000002e-05 2023-10-09T10:26:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: ], 2023-10-09T10:26:37.995 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 4 2023-10-09T10:26:37.995 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:26:38.060 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:38.136 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:26:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:26:37] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.33.4" 2023-10-09T10:26:38.205 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:38.408 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:38.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:38.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:38.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:26:38.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:38.542 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:26:38.600 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:38.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1556970854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:38.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 35012/217137 objects degraded (16.124%), 14 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:38.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:38 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:38.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1556970854' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:38.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:38 smithi110 ceph-mon[136032]: osdmap e536: 8 total, 8 up, 6 in 2023-10-09T10:26:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1556970854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:38.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 35012/217137 objects degraded (16.124%), 14 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1556970854' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[152263]: osdmap e536: 8 total, 8 up, 6 in 2023-10-09T10:26:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1556970854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 35012/217137 objects degraded (16.124%), 14 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1556970854' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:38 smithi012 ceph-mon[147926]: osdmap e536: 8 total, 8 up, 6 in 2023-10-09T10:26:39.033 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:39.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:39.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:39.142 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:39.181 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:39.218 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:39.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:39.226 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:39.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:39.366 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:39.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:26:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: pgmap v1502: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 34612/213834 objects degraded (16.186%); 18577/213834 objects misplaced (8.688%); 17 MiB/s, 312 objects/s recovering 2023-10-09T10:26:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[147926]: osdmap e537: 8 total, 8 up, 6 in 2023-10-09T10:26:39.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: pgmap v1502: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 34612/213834 objects degraded (16.186%); 18577/213834 objects misplaced (8.688%); 17 MiB/s, 312 objects/s recovering 2023-10-09T10:26:39.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:39 smithi012 ceph-mon[152263]: osdmap e537: 8 total, 8 up, 6 in 2023-10-09T10:26:39.823 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: pgmap v1502: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 34612/213834 objects degraded (16.186%); 18577/213834 objects misplaced (8.688%); 17 MiB/s, 312 objects/s recovering 2023-10-09T10:26:39.823 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.823 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.823 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.823 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.823 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.824 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.824 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: failed to encode map e536 with expected crc 2023-10-09T10:26:39.824 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:39 smithi110 ceph-mon[136032]: osdmap e537: 8 total, 8 up, 6 in 2023-10-09T10:26:40.058 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:40.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:40.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:40.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:40.244 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:40.622 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:26:40.622 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:26:40.622 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:26:40.622 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:26:40.622 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:26:40.650 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:40 smithi110 ceph-mon[136032]: failed to encode map e537 with expected crc 2023-10-09T10:26:40.650 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:40 smithi110 ceph-mon[136032]: failed to encode map e537 with expected crc 2023-10-09T10:26:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:40 smithi012 ceph-mon[147926]: failed to encode map e537 with expected crc 2023-10-09T10:26:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:40 smithi012 ceph-mon[147926]: failed to encode map e537 with expected crc 2023-10-09T10:26:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:40 smithi012 ceph-mon[152263]: failed to encode map e537 with expected crc 2023-10-09T10:26:40.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:40 smithi012 ceph-mon[152263]: failed to encode map e537 with expected crc 2023-10-09T10:26:40.988 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:26:41.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:26:41.137 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:41.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:41.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:41.250 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:41.300 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:41.364 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:41.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:41.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:41.474 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:41.514 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:41.569 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:41.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:41.622 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:41.673 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:41.719 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:41.756 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:41.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:41.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:41.800 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:41.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:41.901 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:41 smithi110 ceph-mon[136032]: pgmap v1505: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 34419/211896 objects degraded (16.243%); 18103/211896 objects misplaced (8.543%); 15 MiB/s, 246 objects/s recovering 2023-10-09T10:26:41.906 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:41.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:41 smithi012 ceph-mon[152263]: pgmap v1505: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 34419/211896 objects degraded (16.243%); 18103/211896 objects misplaced (8.543%); 15 MiB/s, 246 objects/s recovering 2023-10-09T10:26:41.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:41 smithi012 ceph-mon[147926]: pgmap v1505: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 34419/211896 objects degraded (16.243%); 18103/211896 objects misplaced (8.543%); 15 MiB/s, 246 objects/s recovering 2023-10-09T10:26:41.954 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:41.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:41.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:41.969 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:26:41.969 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 34419/211896 objects degraded (16.243%), 14 pgs degraded, 14 pgs undersized","count":28},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1918,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":537,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846541,"num_in_osds":6,"osd_in_since":1696847079,"num_remapped_pgs":16},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":82},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":11},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":70632,"data_bytes":4627797944,"bytes_used":21735133184,"bytes_avail":554235080704,"bytes_total":575970213888,"degraded_objects":34419,"degraded_total":211896,"degraded_ratio":0.16243345792275454,"misplaced_objects":18103,"misplaced_total":211896,"misplaced_ratio":0.085433420168384497,"recovering_objects_per_sec":246,"recovering_bytes_per_sec":16124423,"recovering_keys_per_sec":0,"num_objects_recovered":1306,"num_bytes_recovered":85590016,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":591},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":32,"modified":"2023-10-09T10:25:43.785386+0000","services":{}},"progress_events":{"4aaa0d12-13e6-4ea3-ae4b-c27ebe383851":{"message":"Global Recovery Event (5m)\n [=======================.....] (remaining: 69s)","progress":0.82828283309936523,"add_to_ceph_s":true},"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T10:26:42.058 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:42.109 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:42.156 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:42.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:42.262 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:42.337 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:42.391 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:42.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:42 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3114811320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:26:42.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:42 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:26:42.614 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:42.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:42.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:42.665 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:26:42.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:26:42.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:42.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:42 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3114811320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:26:42.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:42 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:26:42.885 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:42.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:42 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3114811320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:26:42.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:42 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:26:42.977 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:42.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:43.086 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:43.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:43.189 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:43.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:43.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:43.279 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:26:43.279 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:26:43.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:26:43.279 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:26:43.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.279 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.45908.0:156578 9.11 e377/365 9:8e383e1c:::benchmark_data_smithi012_313365_object44103:head v 377'8329, mlcod=376'8326)", 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 574.75201522700002, 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178583306, 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525566+0000", 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525564+0000", 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7572e-05 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525581+0000", 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3825999999999999e-05 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525605+0000", 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4969e-05 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.525620+0000", 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13948444700000001 2023-10-09T10:26:43.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.665105+0000", 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8316000000000001e-05 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.665133+0000", 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.038969654999999999 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.704103+0000", 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6774000000000003e-05 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:17:08.704149+0000", 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.178583306 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:27078 10.f 10:f852e76d:::benchmark_data_smithi012_366312_object27077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.686967+0000", 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 404.59061432300001, 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16440834700000001, 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27078 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.285 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.686967+0000", 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.686967+0000", 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.686965+0000", 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4931999999999997e-05 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.687000+0000", 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3113e-05 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.687013+0000", 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8889999999999999e-06 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.687019+0000", 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001864283 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.688884+0000", 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000125807 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.689009+0000", 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8033000000000001e-05 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.689107+0000", 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002756816 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.691864+0000", 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000158092 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.692022+0000", 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15928362700000001 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851306+0000", 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9937000000000001e-05 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851346+0000", 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9476e-05 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851375+0000", 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.16440834700000001 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:27099 10.f 10:f4e4b7e3:::benchmark_data_smithi012_366312_object27098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.698261+0000", 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 404.57931986699998, 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15328217499999999, 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:43.292 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27099 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698261+0000", 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698261+0000", 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698260+0000", 2023-10-09T10:26:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8139e-05 2023-10-09T10:26:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698318+0000", 2023-10-09T10:26:43.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2685e-05 2023-10-09T10:26:43.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698331+0000", 2023-10-09T10:26:43.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0840000000000003e-06 2023-10-09T10:26:43.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.296 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:43.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698338+0000", 2023-10-09T10:26:43.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4275000000000004e-05 2023-10-09T10:26:43.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698402+0000", 2023-10-09T10:26:43.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000146921 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698549+0000", 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013566899999999999 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.698685+0000", 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0011869719999999999 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.699872+0000", 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015024600000000001 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.700022+0000", 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.151410247 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851432+0000", 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5823999999999999e-05 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851488+0000", 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5804999999999998e-05 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851544+0000", 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.15328217499999999 2023-10-09T10:26:43.300 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:27106 10.f 10:f3b8aa96:::benchmark_data_smithi012_366312_object27105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.703416+0000", 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 404.574165639, 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.148246932, 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:26:43.301 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:43.302 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27106 2023-10-09T10:26:43.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.302 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703416+0000", 2023-10-09T10:26:43.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703416+0000", 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703414+0000", 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101159 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703515+0000", 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.649e-05 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703531+0000", 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6210000000000004e-06 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703539+0000", 2023-10-09T10:26:43.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1502999999999998e-05 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703590+0000", 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000145067 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703736+0000", 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014309699999999999 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703879+0000", 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021440740000000002 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.706023+0000", 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029154099999999998 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.706314+0000", 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14527401100000001 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851588+0000", 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2505999999999997e-05 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851621+0000", 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1841999999999997e-05 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851663+0000", 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.148246932 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:27110 10.f 10:f11365c8:::benchmark_data_smithi012_366312_object27109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e428)", 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:19:58.703981+0000", 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 404.57360026399999, 2023-10-09T10:26:43.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14781381499999999, 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27110 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703981+0000", 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703981+0000", 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.703980+0000", 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016381699999999999 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.704144+0000", 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9469999999999992e-06 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.704153+0000", 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3970000000000002e-06 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.704158+0000", 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00079055900000000003 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.704949+0000", 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010478 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.705053+0000", 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013406799999999999 2023-10-09T10:26:43.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.705187+0000", 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0024943090000000001 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.707682+0000", 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085955199999999997 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.708541+0000", 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14316135599999999 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851703+0000", 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9663999999999997e-05 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851752+0000", 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2533000000000001e-05 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:19:58.851795+0000", 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.14781381499999999 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:53370 10.f 10:f9fec699:::benchmark_data_smithi012_366312_object53369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e433)", 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:20:16.349214+0000", 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 386.928367697, 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098253160000000006, 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:43.316 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 53370 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349214+0000", 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349214+0000", 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:43.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349212+0000", 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9627000000000003e-05 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349281+0000", 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7509000000000001e-05 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349299+0000", 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9070000000000001e-06 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349306+0000", 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0889e-05 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349357+0000", 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012534300000000001 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349482+0000", 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000175348 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.349657+0000", 2023-10-09T10:26:43.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021320929999999998 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.351790+0000", 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019880399999999999 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.351988+0000", 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.095375489999999993 2023-10-09T10:26:43.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447364+0000", 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9750999999999998e-05 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447424+0000", 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3198999999999998e-05 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:20:16.447467+0000", 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.098253160000000006 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:133090 10.f 10:f7fea7b9:::benchmark_data_smithi012_366312_object21761:head [delete] snapc 0=[] ondisk+write+known_if_redirected e457)", 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:21:47.116788+0000", 2023-10-09T10:26:43.323 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 296.160793238, 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060180649000000003, 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 133090 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.116788+0000", 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.116788+0000", 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.116788+0000", 2023-10-09T10:26:43.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2180000000000002e-06 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.116790+0000", 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9019999999999998e-06 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.116795+0000", 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.339e-06 2023-10-09T10:26:43.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.116797+0000", 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.044116180999999997 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.160913+0000", 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118857 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.161032+0000", 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000139073 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.161171+0000", 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.015514198999999999 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.176686+0000", 2023-10-09T10:26:43.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013259599999999999 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.176818+0000", 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9638e-05 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.176888+0000", 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4759000000000003e-05 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.176933+0000", 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6158000000000002e-05 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:21:47.176969+0000", 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.060180649000000003 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.330 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:171133 10.b e470/466 10:d4c72074:::benchmark_data_smithi012_366312_object59804:head v 470'10891, mlcod=470'10887)", 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:31.566540+0000", 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.711041198, 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.068873286000000006, 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.566540+0000", 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.566540+0000", 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.566539+0000", 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5840000000000001e-06 2023-10-09T10:26:43.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.566549+0000", 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1335000000000001e-05 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.566560+0000", 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1290000000000004e-06 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.566565+0000", 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.897e-05 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.566614+0000", 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0179e-05 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.566644+0000", 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.068741621000000003 2023-10-09T10:26:43.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.635386+0000", 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7404000000000001e-05 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.635413+0000", 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.068873286000000006 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.335 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:171136 10.10 e470/466 10:0c030a4d:::benchmark_data_smithi012_366312_object59807:head v 470'9374, mlcod=470'9372)", 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:31.570022+0000", 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.707559041, 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.063459293, 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.570022+0000", 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.570022+0000", 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.570021+0000", 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7779999999999996e-06 2023-10-09T10:26:43.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.570030+0000", 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7022999999999999e-05 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.570047+0000", 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.554e-06 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.570055+0000", 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9424000000000005e-05 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.570124+0000", 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0886999999999998e-05 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.570155+0000", 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.063294128000000005 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.633449+0000", 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2594e-05 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.633482+0000", 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.063459293 2023-10-09T10:26:43.340 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36474.0:171139 10.10 e470/466 10:0b7746ca:::benchmark_data_smithi012_366312_object59810:head v 470'9375, mlcod=470'9372)", 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:31.571398+0000", 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 251.70618357800001, 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.06213105, 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.571398+0000", 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.571398+0000", 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.342 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.571397+0000", 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6420000000000001e-06 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.571404+0000", 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.307e-05 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.571417+0000", 2023-10-09T10:26:43.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4160000000000001e-06 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.571423+0000", 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.057424164 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.628847+0000", 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3890999999999999e-05 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.628871+0000", 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0046332789999999997 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.633505+0000", 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4278999999999999e-05 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:31.633529+0000", 2023-10-09T10:26:43.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.06213105 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72716 11.4 11.2af64ba4 (undecoded) ondisk+retry+write+known_if_redirected e506)", 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.611957+0000", 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 123.665624365, 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0282683029999999, 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:26:43.346 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72716 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.611957+0000", 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.611957+0000", 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.611955+0000", 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022969399999999999 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612185+0000", 2023-10-09T10:26:43.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5172e-05 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612200+0000", 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3059999999999998e-06 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612208+0000", 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9839000000000002e-05 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612258+0000", 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3360000000000001e-06 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612268+0000", 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0279472949999999 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:40.640215+0000", 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.044e-05 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:40.640225+0000", 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0282683029999999 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.351 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72741 11.4 11.c1ae8004 (undecoded) ondisk+write+known_if_redirected e506)", 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.628049+0000", 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 123.649532104, 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0122071070000001, 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72741 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628049+0000", 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628049+0000", 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628047+0000", 2023-10-09T10:26:43.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8661000000000001e-05 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628086+0000", 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6623e-05 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628102+0000", 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5379999999999999e-06 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628109+0000", 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1010999999999997e-05 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628160+0000", 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0148e-05 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:26:43.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628170+0000", 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0120785800000001 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:40.640249+0000", 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6459999999999993e-06 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:40.640256+0000", 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0122071070000001 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.356 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:107692 11.9 e521/516 11:93fd1f3d:::benchmark_data_smithi012_404402_object107691:head v 521'4375, mlcod=0'0)", 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:25:23.667665+0000", 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 79.609916353000003, 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.063690614000000007, 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.667665+0000", 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.667665+0000", 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.667664+0000", 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4261000000000002e-05 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.667708+0000", 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5109999999999992e-06 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.667718+0000", 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1749999999999996e-06 2023-10-09T10:26:43.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.667723+0000", 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1934999999999997e-05 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.667765+0000", 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9837999999999999e-05 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.667785+0000", 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.063529252999999994 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.731314+0000", 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1394999999999999e-05 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.731356+0000", 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.063690614000000007 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:107681 11.1 e521/513 11:83eca42c:::benchmark_data_smithi012_404402_object107680:head v 521'4316, mlcod=0'0)", 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:25:23.669431+0000", 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 79.608150476000006, 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.062169704999999999, 2023-10-09T10:26:43.362 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.669431+0000", 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.669431+0000", 2023-10-09T10:26:43.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.669430+0000", 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1223000000000006e-05 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.669501+0000", 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9782000000000001e-05 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.669521+0000", 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7830000000000001e-06 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.669528+0000", 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2353e-05 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.669560+0000", 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8142999999999999e-05 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.669578+0000", 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.061983990000000003 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.731562+0000", 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8260000000000003e-05 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:25:23.731601+0000", 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.062169704999999999 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.367 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:149155 11.4 11:218fae09:::benchmark_data_smithi012_404402_object36372:head [delete] snapc 0=[] ondisk+write+known_if_redirected e532)", 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:27.429754+0000", 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.847827424, 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.097385777000000007, 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 149155 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.368 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.429754+0000", 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.429754+0000", 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.429751+0000", 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3553999999999999e-05 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.429765+0000", 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4335e-05 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.429789+0000", 2023-10-09T10:26:43.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2344000000000001e-05 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.429802+0000", 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120501 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.429922+0000", 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031210199999999997 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.430234+0000", 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000133588 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.430368+0000", 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019676530000000002 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.432335+0000", 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0052895219999999996 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.437625+0000", 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.089436003 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.527061+0000", 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1108e-05 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.527102+0000", 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7594e-05 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.527140+0000", 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.097385777000000007 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.374 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:149156 11.15 e532/513 11:ac89046e:::benchmark_data_smithi012_404402_object36373:head v 532'5605, mlcod=503'2072)", 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:27.443091+0000", 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.834490608999999, 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083634246999999995, 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.443091+0000", 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.443091+0000", 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.443090+0000", 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2771e-05 2023-10-09T10:26:43.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.443102+0000", 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.345e-05 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.443116+0000", 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4119999999999998e-06 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.377 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.443122+0000", 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7132999999999997e-05 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.443179+0000", 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7413999999999999e-05 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.443207+0000", 2023-10-09T10:26:43.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083481174000000005 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526688+0000", 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7014000000000002e-05 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526725+0000", 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.083634246999999995 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.379 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:149164 11.14 e532/507 11:28c1c827:::benchmark_data_smithi012_404402_object36381:head v 532'5602, mlcod=498'1)", 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:27.444566+0000", 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.833015361999999, 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.087029488000000002, 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.444566+0000", 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.444566+0000", 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.444565+0000", 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9600000000000003e-06 2023-10-09T10:26:43.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.444572+0000", 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.082112847000000003 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526685+0000", 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0524e-05 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526696+0000", 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0010999999999999e-05 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526746+0000", 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7987000000000002e-05 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526774+0000", 2023-10-09T10:26:43.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047922599999999996 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.531566+0000", 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9669999999999999e-05 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.531595+0000", 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.087029488000000002 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.384 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.36870.0:149165 11.13 e532/507 11:c8714d96:::benchmark_data_smithi012_404402_object36382:head v 532'5502, mlcod=498'30)", 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:27.445421+0000", 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.83216002, 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086145073000000003, 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.385 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.445421+0000", 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.445421+0000", 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.386 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.445420+0000", 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1920000000000005e-06 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.445429+0000", 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.081238452000000003 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526667+0000", 2023-10-09T10:26:43.387 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4989e-05 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526682+0000", 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7701999999999997e-05 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526740+0000", 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4366e-05 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.388 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.526764+0000", 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0047721910000000003 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.531536+0000", 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0015999999999999e-05 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.389 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:27.531566+0000", 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086145073000000003 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:157305 11.4 11:208b8c8b:::benchmark_data_smithi012_404402_object44522:head [delete] snapc 0=[] ondisk+write+known_if_redirected e537)", 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:40.571713+0000", 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.7058681130000002, 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.47087400299999999, 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.390 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 157305 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571713+0000", 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.391 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571713+0000", 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571712+0000", 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5790000000000003e-06 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571717+0000", 2023-10-09T10:26:43.392 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5520000000000005e-06 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571726+0000", 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.493e-06 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571730+0000", 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044968710000000004 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.393 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576227+0000", 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023127900000000001 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576458+0000", 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043961500000000003 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.394 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576898+0000", 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048120180000000004 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.581710+0000", 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.96e-06 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.581718+0000", 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.460789267 2023-10-09T10:26:43.395 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042507+0000", 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4877999999999999e-05 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042562+0000", 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5174e-05 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.396 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042587+0000", 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.47087400299999999 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:157308 11.4 11:2062cb3d:::benchmark_data_smithi012_404402_object44525:head [delete] snapc 0=[] ondisk+write+known_if_redirected e537)", 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:40.576237+0000", 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 2.7013448069999999, 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46641645599999998, 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:26:43.397 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 157308 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576237+0000", 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.398 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576237+0000", 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576235+0000", 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6810000000000003e-06 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:26:43.399 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576241+0000", 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9400999999999993e-05 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576330+0000", 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0400000000000004e-06 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576337+0000", 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002802999 2023-10-09T10:26:43.400 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.579140+0000", 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112234 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.579252+0000", 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000386172 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:26:43.401 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.579638+0000", 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021832420000000002 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.581822+0000", 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050634300000000003 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.582328+0000", 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46028158600000002 2023-10-09T10:26:43.402 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042610+0000", 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3995999999999999e-05 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042634+0000", 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9375e-05 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:26:43.403 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:26:43.404 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042653+0000", 2023-10-09T10:26:43.404 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46641645599999998 2023-10-09T10:26:43.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.404 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.404 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:26:43.404 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:26:43.404 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:26:43.468 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:43.697 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:26:43.715 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:43.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:43.825 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[152263]: pgmap v1506: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 33538/205617 objects degraded (16.311%); 17293/205617 objects misplaced (8.410%); 15 MiB/s, 243 objects/s recovering 2023-10-09T10:26:43.825 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.825 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.825 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.826 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.826 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 33538/205617 objects degraded (16.311%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:43.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:43.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[147926]: pgmap v1506: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 33538/205617 objects degraded (16.311%); 17293/205617 objects misplaced (8.410%); 15 MiB/s, 243 objects/s recovering 2023-10-09T10:26:43.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.830 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 33538/205617 objects degraded (16.311%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:43.896 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:43.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:43 smithi110 ceph-mon[136032]: pgmap v1506: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 33538/205617 objects degraded (16.311%); 17293/205617 objects misplaced (8.410%); 15 MiB/s, 243 objects/s recovering 2023-10-09T10:26:43.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:43 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:43.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 33538/205617 objects degraded (16.311%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:44.021 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:44.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:44.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:44.036 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:44.131 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:44.165 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:26:44.173 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:44.206 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:26:44.242 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:26:44.291 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:26:44.296 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:44.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:44.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:26:44.374 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:26:44.405 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:44.426 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:26:44.449 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:44.470 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:26:44.498 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:26:44.498 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:26:44.498 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:26:44.498 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:26:44.498 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:26:44.498 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:26:44.498 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:26:44.498 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:26:44.866 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:44.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:44.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:44.991 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:45.068 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:45.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:44 smithi110 ceph-mon[136032]: pgmap v1507: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 33382/204177 objects degraded (16.350%); 17137/204177 objects misplaced (8.393%); 12 MiB/s, 191 objects/s recovering 2023-10-09T10:26:45.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:45.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:45.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1053101210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:45.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/371306100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:45.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:44 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:45.197 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:45.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:45.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:45.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[152263]: pgmap v1507: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 33382/204177 objects degraded (16.350%); 17137/204177 objects misplaced (8.393%); 12 MiB/s, 191 objects/s recovering 2023-10-09T10:26:45.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:45.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:45.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1053101210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:45.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/371306100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:45.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:45.269 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[147926]: pgmap v1507: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 33382/204177 objects degraded (16.350%); 17137/204177 objects misplaced (8.393%); 12 MiB/s, 191 objects/s recovering 2023-10-09T10:26:45.269 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:45.269 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:45.269 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1053101210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:45.269 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/371306100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:45.269 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:44 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:26:45.316 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:45.405 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:45.585 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:45.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:45.665 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:26:45.690 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:45.754 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:45.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:45 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:45.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:45 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:26:45.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:45 smithi012 ceph-mon[147926]: osdmap e538: 8 total, 8 up, 6 in 2023-10-09T10:26:46.121 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:46.127 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:45 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:46.127 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:45 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:26:46.127 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:45 smithi110 ceph-mon[136032]: osdmap e538: 8 total, 8 up, 6 in 2023-10-09T10:26:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:46.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:46.229 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:46.301 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:45 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:26:46.302 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:45 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:26:46.302 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:45 smithi012 ceph-mon[152263]: osdmap e538: 8 total, 8 up, 6 in 2023-10-09T10:26:46.308 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:46.401 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:46.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:46.525 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:46.636 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:46.735 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:46.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:46.842 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:26:46.842 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:26:46.843 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:46.929 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:46.992 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:46.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:47.057 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:26:47.062 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:26:47.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:47.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:46 smithi012 ceph-mon[147926]: failed to encode map e538 with expected crc 2023-10-09T10:26:47.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:46 smithi012 ceph-mon[147926]: failed to encode map e538 with expected crc 2023-10-09T10:26:47.165 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:46 smithi012 ceph-mon[147926]: pgmap v1509: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 525 op/s; 32815/200610 objects degraded (16.358%); 16606/200610 objects misplaced (8.278%); 16 MiB/s, 263 objects/s recovering 2023-10-09T10:26:47.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:46 smithi012 ceph-mon[152263]: failed to encode map e538 with expected crc 2023-10-09T10:26:47.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:46 smithi012 ceph-mon[152263]: failed to encode map e538 with expected crc 2023-10-09T10:26:47.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:46 smithi012 ceph-mon[152263]: pgmap v1509: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 525 op/s; 32815/200610 objects degraded (16.358%); 16606/200610 objects misplaced (8.278%); 16 MiB/s, 263 objects/s recovering 2023-10-09T10:26:47.175 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:47.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:46 smithi110 ceph-mon[136032]: failed to encode map e538 with expected crc 2023-10-09T10:26:47.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:46 smithi110 ceph-mon[136032]: failed to encode map e538 with expected crc 2023-10-09T10:26:47.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:46 smithi110 ceph-mon[136032]: pgmap v1509: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 525 op/s; 32815/200610 objects degraded (16.358%); 16606/200610 objects misplaced (8.278%); 16 MiB/s, 263 objects/s recovering 2023-10-09T10:26:47.229 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:26:47.252 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:47.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:47.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:47.286 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:26:47.360 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:47.411 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:47.465 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:47.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:47.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:47.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:26:47.705 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:48.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:26:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:26:47] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.33.4" 2023-10-09T10:26:48.135 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:26:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:48.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:26:48.162 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:48.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:26:48.284 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:48 smithi110 ceph-mon[136032]: from='client.57112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:48.288 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:48.343 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:26:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:48.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:26:48.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:48 smithi012 ceph-mon[147926]: from='client.57112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:48.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:48 smithi012 ceph-mon[152263]: from='client.57112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:26:48.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:48.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:26:48.525 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:49.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:49 smithi110 ceph-mon[136032]: pgmap v1510: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 585 op/s; 32473/197283 objects degraded (16.460%); 16302/197283 objects misplaced (8.263%); 16 MiB/s, 255 objects/s recovering 2023-10-09T10:26:49.342 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:49 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 32473/197283 objects degraded (16.460%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:49 smithi012 ceph-mon[147926]: pgmap v1510: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 585 op/s; 32473/197283 objects degraded (16.460%); 16302/197283 objects misplaced (8.263%); 16 MiB/s, 255 objects/s recovering 2023-10-09T10:26:49.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:49 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 32473/197283 objects degraded (16.460%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:49.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:49 smithi012 ceph-mon[152263]: pgmap v1510: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 585 op/s; 32473/197283 objects degraded (16.460%); 16302/197283 objects misplaced (8.263%); 16 MiB/s, 255 objects/s recovering 2023-10-09T10:26:49.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:49 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 32473/197283 objects degraded (16.460%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:49.716 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:26:49.716 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:26:50.566 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1509,"stamp":"2023-10-09T10:26:45.813131+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4381251512,"num_objects":66870,"num_object_clones":0,"num_object_copies":200610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32815,"num_objects_misplaced":16606,"num_objects_unfound":0,"num_objects_dirty":66870,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":144360,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21466,"num_bytes_recovered":1376814844,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147527,"ondisk_log_size":147527,"up":297,"acting":284,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":20738872,"kb_used_data":15063368,"kb_used_omap":163,"kb_used_meta":5675100,"kb_avail":541732040,"statfs":{"total":575970213888,"available":554733608960,"internally_reserved":0,"allocated":15424888832,"data_stored":15415680389,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":5811303411},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,6,4,7,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-252575744,"num_objects":-3854,"num_object_clones":0,"num_object_copies":-11562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1604,"num_objects_misplaced":-1778,"num_objects_unfound":0,"num_objects_dirty":-3854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3854,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1929,"num_bytes_recovered":126418944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.330365"},"pg_stats":[{"pgid":"11.12","version":"537'5932","reported_seq":6003,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:41.543767+0000","last_change":"2023-10-09T10:25:09.709209+0000","last_active":"2023-10-09T10:26:41.543767+0000","last_peered":"2023-10-09T10:26:41.543767+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:26:41.543767+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142835712,"num_objects":2180,"num_object_clones":0,"num_object_copies":6540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3119,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4984,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-939},{"shards":"0,2,5","objects":1717}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":130,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.503552+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:26:38.503552+0000","last_peered":"2023-10-09T10:26:38.503552+0000","last_clean":"2023-10-09T10:26:38.503552+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:26:38.503552+0000","last_undegraded":"2023-10-09T10:26:38.503552+0000","last_fullsized":"2023-10-09T10:26:38.503552+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":491,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504073+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:26:38.504073+0000","last_peered":"2023-10-09T10:26:38.504073+0000","last_clean":"2023-10-09T10:26:38.504073+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:26:38.504073+0000","last_undegraded":"2023-10-09T10:26:38.504073+0000","last_fullsized":"2023-10-09T10:26:38.504073+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"537'5889","reported_seq":10208,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:41.643344+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:26:41.643344+0000","last_peered":"2023-10-09T10:26:41.643344+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:26:41.643344+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5889,"log_dups_size":0,"ondisk_log_size":5889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138805248,"num_objects":2118,"num_object_clones":0,"num_object_copies":6354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2547,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4944,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1413,"num_bytes_recovered":92602368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":851},{"shards":"0,2,6","objects":845},{"shards":"0,6","objects":422}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":568,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.502639+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:26:38.502639+0000","last_peered":"2023-10-09T10:26:38.502639+0000","last_clean":"2023-10-09T10:26:38.502639+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:26:38.502639+0000","last_undegraded":"2023-10-09T10:26:38.502639+0000","last_fullsized":"2023-10-09T10:26:38.502639+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":462,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.516067+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:26:39.516067+0000","last_peered":"2023-10-09T10:26:39.516067+0000","last_clean":"2023-10-09T10:26:39.516067+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:26:39.516067+0000","last_undegraded":"2023-10-09T10:26:39.516067+0000","last_fullsized":"2023-10-09T10:26:39.516067+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"537'5814","reported_seq":10362,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:41.613293+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:26:41.613293+0000","last_peered":"2023-10-09T10:26:41.613293+0000","last_clean":"2023-10-09T10:26:41.613293+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:41.613293+0000","last_undegraded":"2023-10-09T10:26:41.613293+0000","last_fullsized":"2023-10-09T10:26:41.613293+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137920512,"num_objects":2105,"num_object_clones":0,"num_object_copies":6315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4877,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":32,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.484808+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:26:38.484808+0000","last_peered":"2023-10-09T10:26:38.484808+0000","last_clean":"2023-10-09T10:26:38.484808+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:26:38.484808+0000","last_undegraded":"2023-10-09T10:26:38.484808+0000","last_fullsized":"2023-10-09T10:26:38.484808+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":114,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.486239+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:26:38.486239+0000","last_peered":"2023-10-09T10:26:38.486239+0000","last_clean":"2023-10-09T10:26:38.486239+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:26:38.486239+0000","last_undegraded":"2023-10-09T10:26:38.486239+0000","last_fullsized":"2023-10-09T10:26:38.486239+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":496,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.516761+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:26:39.516761+0000","last_peered":"2023-10-09T10:26:39.516761+0000","last_clean":"2023-10-09T10:26:39.516761+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:26:39.516761+0000","last_undegraded":"2023-10-09T10:26:39.516761+0000","last_fullsized":"2023-10-09T10:26:39.516761+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":444,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.480187+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:26:38.480187+0000","last_peered":"2023-10-09T10:26:38.480187+0000","last_clean":"2023-10-09T10:26:38.480187+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:26:38.480187+0000","last_undegraded":"2023-10-09T10:26:38.480187+0000","last_fullsized":"2023-10-09T10:26:38.480187+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":212,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.505611+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:26:38.505611+0000","last_peered":"2023-10-09T10:26:38.505611+0000","last_clean":"2023-10-09T10:26:38.505611+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:26:38.505611+0000","last_undegraded":"2023-10-09T10:26:38.505611+0000","last_fullsized":"2023-10-09T10:26:38.505611+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":444,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.479448+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:26:38.479448+0000","last_peered":"2023-10-09T10:26:38.479448+0000","last_clean":"2023-10-09T10:26:38.479448+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:26:38.479448+0000","last_undegraded":"2023-10-09T10:26:38.479448+0000","last_fullsized":"2023-10-09T10:26:38.479448+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":142,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504208+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:26:38.504208+0000","last_peered":"2023-10-09T10:26:38.504208+0000","last_clean":"2023-10-09T10:26:38.504208+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:26:38.504208+0000","last_undegraded":"2023-10-09T10:26:38.504208+0000","last_fullsized":"2023-10-09T10:26:38.504208+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":32,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.486726+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:26:38.486726+0000","last_peered":"2023-10-09T10:26:38.486726+0000","last_clean":"2023-10-09T10:26:38.486726+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:26:38.486726+0000","last_undegraded":"2023-10-09T10:26:38.486726+0000","last_fullsized":"2023-10-09T10:26:38.486726+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":188,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.503218+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:26:38.503218+0000","last_peered":"2023-10-09T10:26:38.503218+0000","last_clean":"2023-10-09T10:26:38.503218+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:26:38.503218+0000","last_undegraded":"2023-10-09T10:26:38.503218+0000","last_fullsized":"2023-10-09T10:26:38.503218+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":148,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504674+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:26:38.504674+0000","last_peered":"2023-10-09T10:26:38.504674+0000","last_clean":"2023-10-09T10:26:38.504674+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:26:38.504674+0000","last_undegraded":"2023-10-09T10:26:38.504674+0000","last_fullsized":"2023-10-09T10:26:38.504674+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":108,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504507+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:26:38.504507+0000","last_peered":"2023-10-09T10:26:38.504507+0000","last_clean":"2023-10-09T10:26:38.504507+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:26:38.504507+0000","last_undegraded":"2023-10-09T10:26:38.504507+0000","last_fullsized":"2023-10-09T10:26:38.504507+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.508975+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:26:38.508975+0000","last_peered":"2023-10-09T10:26:38.508975+0000","last_clean":"2023-10-09T10:26:38.508975+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:26:38.508975+0000","last_undegraded":"2023-10-09T10:26:38.508975+0000","last_fullsized":"2023-10-09T10:26:38.508975+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":460,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.490599+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:26:38.490599+0000","last_peered":"2023-10-09T10:26:38.490599+0000","last_clean":"2023-10-09T10:26:38.490599+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:26:38.490599+0000","last_undegraded":"2023-10-09T10:26:38.490599+0000","last_fullsized":"2023-10-09T10:26:38.490599+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":203,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.508195+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:26:38.508195+0000","last_peered":"2023-10-09T10:26:38.508195+0000","last_clean":"2023-10-09T10:26:38.508195+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:26:38.508195+0000","last_undegraded":"2023-10-09T10:26:38.508195+0000","last_fullsized":"2023-10-09T10:26:38.508195+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"537'5966","reported_seq":6770,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:44.848376+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:26:44.848376+0000","last_peered":"2023-10-09T10:26:44.848376+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:26:44.848376+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"498'388","ondisk_log_start":"498'388","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135168000,"num_objects":2063,"num_object_clones":0,"num_object_copies":6189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":2063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5043,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-1088},{"shards":"3,5,6","objects":843}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":148,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.502770+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:26:38.502770+0000","last_peered":"2023-10-09T10:26:38.502770+0000","last_clean":"2023-10-09T10:26:38.502770+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:26:38.502770+0000","last_undegraded":"2023-10-09T10:26:38.502770+0000","last_fullsized":"2023-10-09T10:26:38.502770+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":32,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.485842+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:26:38.485842+0000","last_peered":"2023-10-09T10:26:38.485842+0000","last_clean":"2023-10-09T10:26:38.485842+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:26:38.485842+0000","last_undegraded":"2023-10-09T10:26:38.485842+0000","last_fullsized":"2023-10-09T10:26:38.485842+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"537'6021","reported_seq":14531,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:44.918888+0000","last_change":"2023-10-09T10:26:39.522775+0000","last_active":"2023-10-09T10:26:44.918888+0000","last_peered":"2023-10-09T10:26:44.918888+0000","last_clean":"2023-10-09T10:26:44.918888+0000","last_became_active":"2023-10-09T10:26:39.509024+0000","last_became_peered":"2023-10-09T10:26:39.509024+0000","last_unstale":"2023-10-09T10:26:44.918888+0000","last_undegraded":"2023-10-09T10:26:44.918888+0000","last_fullsized":"2023-10-09T10:26:44.918888+0000","mapping_epoch":536,"log_start":"498'417","ondisk_log_start":"498'417","created":501,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5604,"log_dups_size":0,"ondisk_log_size":5604,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133660672,"num_objects":2040,"num_object_clones":0,"num_object_copies":6120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5056,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3072,"num_bytes_recovered":201326592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":42,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.485516+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:26:38.485516+0000","last_peered":"2023-10-09T10:26:38.485516+0000","last_clean":"2023-10-09T10:26:38.485516+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:26:38.485516+0000","last_undegraded":"2023-10-09T10:26:38.485516+0000","last_fullsized":"2023-10-09T10:26:38.485516+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":35,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.482957+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:26:38.482957+0000","last_peered":"2023-10-09T10:26:38.482957+0000","last_clean":"2023-10-09T10:26:38.482957+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:26:38.482957+0000","last_undegraded":"2023-10-09T10:26:38.482957+0000","last_fullsized":"2023-10-09T10:26:38.482957+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"537'6079","reported_seq":6144,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:44.961383+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:26:44.961383+0000","last_peered":"2023-10-09T10:26:44.961383+0000","last_clean":"2023-10-09T10:26:44.961383+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:26:44.961383+0000","last_undegraded":"2023-10-09T10:26:44.961383+0000","last_fullsized":"2023-10-09T10:26:44.961383+0000","mapping_epoch":494,"log_start":"498'588","ondisk_log_start":"498'588","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":5491,"log_dups_size":0,"ondisk_log_size":5491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136118272,"num_objects":2077,"num_object_clones":0,"num_object_copies":6231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2791,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":38,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.503377+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:26:38.503377+0000","last_peered":"2023-10-09T10:26:38.503377+0000","last_clean":"2023-10-09T10:26:38.503377+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:26:38.503377+0000","last_undegraded":"2023-10-09T10:26:38.503377+0000","last_fullsized":"2023-10-09T10:26:38.503377+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":212,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.509014+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:26:38.509014+0000","last_peered":"2023-10-09T10:26:38.509014+0000","last_clean":"2023-10-09T10:26:38.509014+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:26:38.509014+0000","last_undegraded":"2023-10-09T10:26:38.509014+0000","last_fullsized":"2023-10-09T10:26:38.509014+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"537'5634","reported_seq":11738,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:41.652156+0000","last_change":"2023-10-09T10:26:35.543001+0000","last_active":"2023-10-09T10:26:41.652156+0000","last_peered":"2023-10-09T10:26:41.652156+0000","last_clean":"2023-10-09T10:26:41.652156+0000","last_became_active":"2023-10-09T10:26:35.529481+0000","last_became_peered":"2023-10-09T10:26:35.529481+0000","last_unstale":"2023-10-09T10:26:41.652156+0000","last_undegraded":"2023-10-09T10:26:41.652156+0000","last_fullsized":"2023-10-09T10:26:41.652156+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5634,"log_dups_size":0,"ondisk_log_size":5634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:07.598234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132841472,"num_objects":2027,"num_object_clones":0,"num_object_copies":6081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2874,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3012,"num_bytes_recovered":174915584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":164,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.502867+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:26:38.502867+0000","last_peered":"2023-10-09T10:26:38.502867+0000","last_clean":"2023-10-09T10:26:38.502867+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:26:38.502867+0000","last_undegraded":"2023-10-09T10:26:38.502867+0000","last_fullsized":"2023-10-09T10:26:38.502867+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":151,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.502846+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:26:38.502846+0000","last_peered":"2023-10-09T10:26:38.502846+0000","last_clean":"2023-10-09T10:26:38.502846+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:26:38.502846+0000","last_undegraded":"2023-10-09T10:26:38.502846+0000","last_fullsized":"2023-10-09T10:26:38.502846+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"537'5980","reported_seq":10572,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:45.014175+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:26:45.014175+0000","last_peered":"2023-10-09T10:26:45.014175+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:26:45.014175+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"498'421","ondisk_log_start":"498'421","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133005312,"num_objects":2029,"num_object_clones":0,"num_object_copies":6087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2443,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5035,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1504,"num_bytes_recovered":98566144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":815},{"shards":"0,5,6","objects":813},{"shards":"5,6","objects":401}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":228,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.507287+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:26:38.507287+0000","last_peered":"2023-10-09T10:26:38.507287+0000","last_clean":"2023-10-09T10:26:38.507287+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:26:38.507287+0000","last_undegraded":"2023-10-09T10:26:38.507287+0000","last_fullsized":"2023-10-09T10:26:38.507287+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":129,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.505280+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:26:38.505280+0000","last_peered":"2023-10-09T10:26:38.505280+0000","last_clean":"2023-10-09T10:26:38.505280+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:26:38.505280+0000","last_undegraded":"2023-10-09T10:26:38.505280+0000","last_fullsized":"2023-10-09T10:26:38.505280+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"537'5886","reported_seq":10076,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:44.932179+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:26:44.932179+0000","last_peered":"2023-10-09T10:26:44.932179+0000","last_clean":"2023-10-09T10:26:44.932179+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:26:44.932179+0000","last_undegraded":"2023-10-09T10:26:44.932179+0000","last_fullsized":"2023-10-09T10:26:44.932179+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137035776,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2972,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":212,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.508614+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:26:38.508614+0000","last_peered":"2023-10-09T10:26:38.508614+0000","last_clean":"2023-10-09T10:26:38.508614+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:26:38.508614+0000","last_undegraded":"2023-10-09T10:26:38.508614+0000","last_fullsized":"2023-10-09T10:26:38.508614+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":446,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.484837+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:26:38.484837+0000","last_peered":"2023-10-09T10:26:38.484837+0000","last_clean":"2023-10-09T10:26:38.484837+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:26:38.484837+0000","last_undegraded":"2023-10-09T10:26:38.484837+0000","last_fullsized":"2023-10-09T10:26:38.484837+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"537'5859","reported_seq":7153,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:44.985207+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:26:44.985207+0000","last_peered":"2023-10-09T10:26:44.985207+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:26:44.985207+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"498'214","ondisk_log_start":"498'214","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136347648,"num_objects":2080,"num_object_clones":0,"num_object_copies":6240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4938,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":811},{"shards":"2,5","objects":451}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":147,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.484749+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:26:38.484749+0000","last_peered":"2023-10-09T10:26:38.484749+0000","last_clean":"2023-10-09T10:26:38.484749+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:26:38.484749+0000","last_undegraded":"2023-10-09T10:26:38.484749+0000","last_fullsized":"2023-10-09T10:26:38.484749+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":148,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.503138+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:26:38.503138+0000","last_peered":"2023-10-09T10:26:38.503138+0000","last_clean":"2023-10-09T10:26:38.503138+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:26:38.503138+0000","last_undegraded":"2023-10-09T10:26:38.503138+0000","last_fullsized":"2023-10-09T10:26:38.503138+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"537'5866","reported_seq":5990,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:41.596378+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:26:41.596378+0000","last_peered":"2023-10-09T10:26:41.596378+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:26:41.596378+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5866,"log_dups_size":0,"ondisk_log_size":5866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136904704,"num_objects":2089,"num_object_clones":0,"num_object_copies":6267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2978,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-969},{"shards":"0,2,5","objects":1641}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":148,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.503405+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:26:38.503405+0000","last_peered":"2023-10-09T10:26:38.503405+0000","last_clean":"2023-10-09T10:26:38.503405+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:26:38.503405+0000","last_undegraded":"2023-10-09T10:26:38.503405+0000","last_fullsized":"2023-10-09T10:26:38.503405+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":114,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504079+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:26:38.504079+0000","last_peered":"2023-10-09T10:26:38.504079+0000","last_clean":"2023-10-09T10:26:38.504079+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:26:38.504079+0000","last_undegraded":"2023-10-09T10:26:38.504079+0000","last_fullsized":"2023-10-09T10:26:38.504079+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"537'5875","reported_seq":10150,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:41.404665+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:26:41.404665+0000","last_peered":"2023-10-09T10:26:41.404665+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:26:41.404665+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5875,"log_dups_size":0,"ondisk_log_size":5875,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139853824,"num_objects":2134,"num_object_clones":0,"num_object_copies":6402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4930,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1398,"num_bytes_recovered":91619328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":856},{"shards":"0,2,6","objects":806},{"shards":"0,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":130,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.502992+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:26:38.502992+0000","last_peered":"2023-10-09T10:26:38.502992+0000","last_clean":"2023-10-09T10:26:38.502992+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:26:38.502992+0000","last_undegraded":"2023-10-09T10:26:38.502992+0000","last_fullsized":"2023-10-09T10:26:38.502992+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":157,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.502720+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:26:38.502720+0000","last_peered":"2023-10-09T10:26:38.502720+0000","last_clean":"2023-10-09T10:26:38.502720+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:26:38.502720+0000","last_undegraded":"2023-10-09T10:26:38.502720+0000","last_fullsized":"2023-10-09T10:26:38.502720+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"537'5824","reported_seq":6262,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:41.584039+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:26:41.584039+0000","last_peered":"2023-10-09T10:26:41.584039+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:26:41.584039+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136970240,"num_objects":2090,"num_object_clones":0,"num_object_copies":6270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2717,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":836},{"shards":"0,3","objects":9}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":489,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.508958+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:26:38.508958+0000","last_peered":"2023-10-09T10:26:38.508958+0000","last_clean":"2023-10-09T10:26:38.508958+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:26:38.508958+0000","last_undegraded":"2023-10-09T10:26:38.508958+0000","last_fullsized":"2023-10-09T10:26:38.508958+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":551,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.503641+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:26:38.503641+0000","last_peered":"2023-10-09T10:26:38.503641+0000","last_clean":"2023-10-09T10:26:38.503641+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:26:38.503641+0000","last_undegraded":"2023-10-09T10:26:38.503641+0000","last_fullsized":"2023-10-09T10:26:38.503641+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"537'9851","reported_seq":9937,"reported_epoch":537,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:43.071039+0000","last_change":"2023-10-09T10:25:17.929569+0000","last_active":"2023-10-09T10:26:43.071039+0000","last_peered":"2023-10-09T10:26:43.071039+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:26:43.071039+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"504'3900","ondisk_log_start":"504'3900","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5951,"log_dups_size":0,"ondisk_log_size":5951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278134808,"num_objects":4245,"num_object_clones":0,"num_object_copies":12735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4242,"num_objects_misplaced":4245,"num_objects_unfound":0,"num_objects_dirty":4245,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9851,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":4245}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":258,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.486192+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:26:38.486192+0000","last_peered":"2023-10-09T10:26:38.486192+0000","last_clean":"2023-10-09T10:26:38.486192+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:26:38.486192+0000","last_undegraded":"2023-10-09T10:26:38.486192+0000","last_fullsized":"2023-10-09T10:26:38.486192+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":221,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.508110+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:26:38.508110+0000","last_peered":"2023-10-09T10:26:38.508110+0000","last_clean":"2023-10-09T10:26:38.508110+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:26:38.508110+0000","last_undegraded":"2023-10-09T10:26:38.508110+0000","last_fullsized":"2023-10-09T10:26:38.508110+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":501,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.505324+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:26:38.505324+0000","last_peered":"2023-10-09T10:26:38.505324+0000","last_clean":"2023-10-09T10:26:38.505324+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:26:38.505324+0000","last_undegraded":"2023-10-09T10:26:38.505324+0000","last_fullsized":"2023-10-09T10:26:38.505324+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"537'9938","reported_seq":9990,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:41.668640+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:26:41.668640+0000","last_peered":"2023-10-09T10:26:41.668640+0000","last_clean":"2023-10-09T10:26:41.668640+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:26:41.668640+0000","last_undegraded":"2023-10-09T10:26:41.668640+0000","last_fullsized":"2023-10-09T10:26:41.668640+0000","mapping_epoch":494,"log_start":"508'4906","ondisk_log_start":"508'4906","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284426240,"num_objects":4340,"num_object_clones":0,"num_object_copies":13020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9938,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":221,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.505253+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:26:38.505253+0000","last_peered":"2023-10-09T10:26:38.505253+0000","last_clean":"2023-10-09T10:26:38.505253+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:26:38.505253+0000","last_undegraded":"2023-10-09T10:26:38.505253+0000","last_fullsized":"2023-10-09T10:26:38.505253+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":139,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504648+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:26:38.504648+0000","last_peered":"2023-10-09T10:26:38.504648+0000","last_clean":"2023-10-09T10:26:38.504648+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:26:38.504648+0000","last_undegraded":"2023-10-09T10:26:38.504648+0000","last_fullsized":"2023-10-09T10:26:38.504648+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":449,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504122+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:26:38.504122+0000","last_peered":"2023-10-09T10:26:38.504122+0000","last_clean":"2023-10-09T10:26:38.504122+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:26:38.504122+0000","last_undegraded":"2023-10-09T10:26:38.504122+0000","last_fullsized":"2023-10-09T10:26:38.504122+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"537'5924","reported_seq":10363,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:44.974751+0000","last_change":"2023-10-09T10:25:09.703037+0000","last_active":"2023-10-09T10:26:44.974751+0000","last_peered":"2023-10-09T10:26:44.974751+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:25:09.696864+0000","last_became_peered":"2023-10-09T10:25:09.696864+0000","last_unstale":"2023-10-09T10:26:44.974751+0000","last_undegraded":"2023-10-09T10:25:09.545735+0000","last_fullsized":"2023-10-09T10:25:09.430540+0000","mapping_epoch":513,"log_start":"498'251","ondisk_log_start":"498'251","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5673,"log_dups_size":0,"ondisk_log_size":5673,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134381568,"num_objects":2050,"num_object_clones":0,"num_object_copies":6150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4958,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1454,"num_bytes_recovered":95289344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":836},{"shards":"2,5,6","objects":804},{"shards":"5,6","objects":410}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":487,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.491811+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:26:38.491811+0000","last_peered":"2023-10-09T10:26:38.491811+0000","last_clean":"2023-10-09T10:26:38.491811+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:26:38.491811+0000","last_undegraded":"2023-10-09T10:26:38.491811+0000","last_fullsized":"2023-10-09T10:26:38.491811+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.485417+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:26:38.485417+0000","last_peered":"2023-10-09T10:26:38.485417+0000","last_clean":"2023-10-09T10:26:38.485417+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:26:38.485417+0000","last_undegraded":"2023-10-09T10:26:38.485417+0000","last_fullsized":"2023-10-09T10:26:38.485417+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":550,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.505955+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:26:38.505955+0000","last_peered":"2023-10-09T10:26:38.505955+0000","last_clean":"2023-10-09T10:26:38.505955+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:26:38.505955+0000","last_undegraded":"2023-10-09T10:26:38.505955+0000","last_fullsized":"2023-10-09T10:26:38.505955+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"537'9994","reported_seq":10069,"reported_epoch":537,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:44.966406+0000","last_change":"2023-10-09T10:26:37.693448+0000","last_active":"2023-10-09T10:26:44.966406+0000","last_peered":"2023-10-09T10:26:44.966406+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:26:44.966406+0000","last_undegraded":"2023-10-09T10:26:44.966406+0000","last_fullsized":"2023-10-09T10:26:44.966406+0000","mapping_epoch":507,"log_start":"509'5100","ondisk_log_start":"509'5100","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4894,"log_dups_size":0,"ondisk_log_size":4894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269877248,"num_objects":4118,"num_object_clones":0,"num_object_copies":12354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4117,"num_objects_unfound":0,"num_objects_dirty":4118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9994,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":4118}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":191,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.507574+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:26:38.507574+0000","last_peered":"2023-10-09T10:26:38.507574+0000","last_clean":"2023-10-09T10:26:38.507574+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:26:38.507574+0000","last_undegraded":"2023-10-09T10:26:38.507574+0000","last_fullsized":"2023-10-09T10:26:38.507574+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":730,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.484607+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:26:38.484607+0000","last_peered":"2023-10-09T10:26:38.484607+0000","last_clean":"2023-10-09T10:26:38.484607+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:38.484607+0000","last_undegraded":"2023-10-09T10:26:38.484607+0000","last_fullsized":"2023-10-09T10:26:38.484607+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":239,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.505069+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:26:38.505069+0000","last_peered":"2023-10-09T10:26:38.505069+0000","last_clean":"2023-10-09T10:26:38.505069+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:26:38.505069+0000","last_undegraded":"2023-10-09T10:26:38.505069+0000","last_fullsized":"2023-10-09T10:26:38.505069+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":148,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504760+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:26:38.504760+0000","last_peered":"2023-10-09T10:26:38.504760+0000","last_clean":"2023-10-09T10:26:38.504760+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:26:38.504760+0000","last_undegraded":"2023-10-09T10:26:38.504760+0000","last_fullsized":"2023-10-09T10:26:38.504760+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"537'6005","reported_seq":10420,"reported_epoch":537,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:26:45.031930+0000","last_change":"2023-10-09T10:25:13.759251+0000","last_active":"2023-10-09T10:26:45.031930+0000","last_peered":"2023-10-09T10:26:45.031930+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:26:45.031930+0000","last_undegraded":"2023-10-09T10:25:13.707357+0000","last_fullsized":"2023-10-09T10:26:45.031930+0000","mapping_epoch":516,"log_start":"498'525","ondisk_log_start":"498'525","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141393920,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":843,"num_objects_misplaced":1794,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5081,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1463,"num_bytes_recovered":95879168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":843},{"shards":"3,5,6","objects":363}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":222,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.485591+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:26:38.485591+0000","last_peered":"2023-10-09T10:26:38.485591+0000","last_clean":"2023-10-09T10:26:38.485591+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:26:38.485591+0000","last_undegraded":"2023-10-09T10:26:38.485591+0000","last_fullsized":"2023-10-09T10:26:38.485591+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":481,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.491948+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:26:38.491948+0000","last_peered":"2023-10-09T10:26:38.491948+0000","last_clean":"2023-10-09T10:26:38.491948+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:26:38.491948+0000","last_undegraded":"2023-10-09T10:26:38.491948+0000","last_fullsized":"2023-10-09T10:26:38.491948+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":32,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.485627+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:26:38.485627+0000","last_peered":"2023-10-09T10:26:38.485627+0000","last_clean":"2023-10-09T10:26:38.485627+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:26:38.485627+0000","last_undegraded":"2023-10-09T10:26:38.485627+0000","last_fullsized":"2023-10-09T10:26:38.485627+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"537'10011","reported_seq":10062,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:43.116081+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:26:43.116081+0000","last_peered":"2023-10-09T10:26:43.116081+0000","last_clean":"2023-10-09T10:26:43.116081+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:26:43.116081+0000","last_undegraded":"2023-10-09T10:26:43.116081+0000","last_fullsized":"2023-10-09T10:26:43.116081+0000","mapping_epoch":494,"log_start":"504'4100","ondisk_log_start":"504'4100","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275316736,"num_objects":4201,"num_object_clones":0,"num_object_copies":12603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10011,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":32,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.486782+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:26:38.486782+0000","last_peered":"2023-10-09T10:26:38.486782+0000","last_clean":"2023-10-09T10:26:38.486782+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:26:38.486782+0000","last_undegraded":"2023-10-09T10:26:38.486782+0000","last_fullsized":"2023-10-09T10:26:38.486782+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":518,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.508840+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:26:38.508840+0000","last_peered":"2023-10-09T10:26:38.508840+0000","last_clean":"2023-10-09T10:26:38.508840+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:26:38.508840+0000","last_undegraded":"2023-10-09T10:26:38.508840+0000","last_fullsized":"2023-10-09T10:26:38.508840+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":449,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.485126+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:26:38.485126+0000","last_peered":"2023-10-09T10:26:38.485126+0000","last_clean":"2023-10-09T10:26:38.485126+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:26:38.485126+0000","last_undegraded":"2023-10-09T10:26:38.485126+0000","last_fullsized":"2023-10-09T10:26:38.485126+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"537'9590","reported_seq":9642,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:41.567774+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:26:41.567774+0000","last_peered":"2023-10-09T10:26:41.567774+0000","last_clean":"2023-10-09T10:26:41.567774+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:26:41.567774+0000","last_undegraded":"2023-10-09T10:26:41.567774+0000","last_fullsized":"2023-10-09T10:26:41.567774+0000","mapping_epoch":494,"log_start":"508'4576","ondisk_log_start":"508'4576","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5014,"log_dups_size":0,"ondisk_log_size":5014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271712256,"num_objects":4146,"num_object_clones":0,"num_object_copies":12438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9590,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":337,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.486561+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:26:38.486561+0000","last_peered":"2023-10-09T10:26:38.486561+0000","last_clean":"2023-10-09T10:26:38.486561+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:26:38.486561+0000","last_undegraded":"2023-10-09T10:26:38.486561+0000","last_fullsized":"2023-10-09T10:26:38.486561+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":518,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.502646+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:26:38.502646+0000","last_peered":"2023-10-09T10:26:38.502646+0000","last_clean":"2023-10-09T10:26:38.502646+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:26:38.502646+0000","last_undegraded":"2023-10-09T10:26:38.502646+0000","last_fullsized":"2023-10-09T10:26:38.502646+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":32,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.484717+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:26:38.484717+0000","last_peered":"2023-10-09T10:26:38.484717+0000","last_clean":"2023-10-09T10:26:38.484717+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:26:38.484717+0000","last_undegraded":"2023-10-09T10:26:38.484717+0000","last_fullsized":"2023-10-09T10:26:38.484717+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"537'10061","reported_seq":10112,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:43.129031+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:26:43.129031+0000","last_peered":"2023-10-09T10:26:43.129031+0000","last_clean":"2023-10-09T10:26:43.129031+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:26:43.129031+0000","last_undegraded":"2023-10-09T10:26:43.129031+0000","last_fullsized":"2023-10-09T10:26:43.129031+0000","mapping_epoch":494,"log_start":"504'4100","ondisk_log_start":"504'4100","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5961,"log_dups_size":0,"ondisk_log_size":5961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275447808,"num_objects":4203,"num_object_clones":0,"num_object_copies":12609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10061,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":40,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.512615+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:26:38.512615+0000","last_peered":"2023-10-09T10:26:38.512615+0000","last_clean":"2023-10-09T10:26:38.512615+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:26:38.512615+0000","last_undegraded":"2023-10-09T10:26:38.512615+0000","last_fullsized":"2023-10-09T10:26:38.512615+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":36,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.485920+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:26:38.485920+0000","last_peered":"2023-10-09T10:26:38.485920+0000","last_clean":"2023-10-09T10:26:38.485920+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:26:38.485920+0000","last_undegraded":"2023-10-09T10:26:38.485920+0000","last_fullsized":"2023-10-09T10:26:38.485920+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":217,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.505028+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:26:38.505028+0000","last_peered":"2023-10-09T10:26:38.505028+0000","last_clean":"2023-10-09T10:26:38.505028+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:26:38.505028+0000","last_undegraded":"2023-10-09T10:26:38.505028+0000","last_fullsized":"2023-10-09T10:26:38.505028+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"537'5830","reported_seq":5929,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:45.022959+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:26:45.022959+0000","last_peered":"2023-10-09T10:26:45.022959+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:26:45.022959+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"498'145","ondisk_log_start":"498'145","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132481024,"num_objects":2022,"num_object_clones":0,"num_object_copies":6066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4910,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":786},{"shards":"2,5","objects":-194}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":32,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.485056+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:26:38.485056+0000","last_peered":"2023-10-09T10:26:38.485056+0000","last_clean":"2023-10-09T10:26:38.485056+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:26:38.485056+0000","last_undegraded":"2023-10-09T10:26:38.485056+0000","last_fullsized":"2023-10-09T10:26:38.485056+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":41,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.484856+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:26:38.484856+0000","last_peered":"2023-10-09T10:26:38.484856+0000","last_clean":"2023-10-09T10:26:38.484856+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:26:38.484856+0000","last_undegraded":"2023-10-09T10:26:38.484856+0000","last_fullsized":"2023-10-09T10:26:38.484856+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"537'5908","reported_seq":6621,"reported_epoch":537,"state":"active+recovering+remapped","last_fresh":"2023-10-09T10:26:41.683557+0000","last_change":"2023-10-09T10:26:37.693292+0000","last_active":"2023-10-09T10:26:41.683557+0000","last_peered":"2023-10-09T10:26:41.683557+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:26:41.683557+0000","last_undegraded":"2023-10-09T10:26:41.683557+0000","last_fullsized":"2023-10-09T10:26:41.683557+0000","mapping_epoch":507,"log_start":"498'10","ondisk_log_start":"498'10","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135495680,"num_objects":2068,"num_object_clones":0,"num_object_copies":6204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2012,"num_objects_unfound":0,"num_objects_dirty":2068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4984,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":320,"num_bytes_recovered":20971520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":56}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":267,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.508673+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:26:38.508673+0000","last_peered":"2023-10-09T10:26:38.508673+0000","last_clean":"2023-10-09T10:26:38.508673+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:26:38.508673+0000","last_undegraded":"2023-10-09T10:26:38.508673+0000","last_fullsized":"2023-10-09T10:26:38.508673+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":130,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.504748+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:26:38.504748+0000","last_peered":"2023-10-09T10:26:38.504748+0000","last_clean":"2023-10-09T10:26:38.504748+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:26:38.504748+0000","last_undegraded":"2023-10-09T10:26:38.504748+0000","last_fullsized":"2023-10-09T10:26:38.504748+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"537'5936","reported_seq":10348,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:44.994322+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:26:44.994322+0000","last_peered":"2023-10-09T10:26:44.994322+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:26:44.994322+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"498'273","ondisk_log_start":"498'273","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5663,"log_dups_size":0,"ondisk_log_size":5663,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137592832,"num_objects":2100,"num_object_clones":0,"num_object_copies":6300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2422,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4992,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1446,"num_bytes_recovered":94765056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":794},{"shards":"0,5,6","objects":833},{"shards":"5,6","objects":472}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":250,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.508007+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:26:38.508007+0000","last_peered":"2023-10-09T10:26:38.508007+0000","last_clean":"2023-10-09T10:26:38.508007+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:26:38.508007+0000","last_undegraded":"2023-10-09T10:26:38.508007+0000","last_fullsized":"2023-10-09T10:26:38.508007+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":360,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.490462+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:26:38.490462+0000","last_peered":"2023-10-09T10:26:38.490462+0000","last_clean":"2023-10-09T10:26:38.490462+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:26:38.490462+0000","last_undegraded":"2023-10-09T10:26:38.490462+0000","last_fullsized":"2023-10-09T10:26:38.490462+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"537'5861","reported_seq":7081,"reported_epoch":537,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:44.460657+0000","last_change":"2023-10-09T10:26:33.871249+0000","last_active":"2023-10-09T10:26:44.460657+0000","last_peered":"2023-10-09T10:26:44.460657+0000","last_clean":"2023-10-09T10:25:11.551777+0000","last_became_active":"2023-10-09T10:25:13.754797+0000","last_became_peered":"2023-10-09T10:25:13.754797+0000","last_unstale":"2023-10-09T10:26:44.460657+0000","last_undegraded":"2023-10-09T10:25:13.625003+0000","last_fullsized":"2023-10-09T10:25:13.604794+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:58.037975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134676480,"num_objects":2055,"num_object_clones":0,"num_object_copies":6165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1274,"num_objects_misplaced":1287,"num_objects_unfound":0,"num_objects_dirty":2055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2924,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":576,"num_bytes_recovered":37748736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":-506},{"shards":"2,4,5","objects":1287}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":35,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.483055+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:26:38.483055+0000","last_peered":"2023-10-09T10:26:38.483055+0000","last_clean":"2023-10-09T10:26:38.483055+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:26:38.483055+0000","last_undegraded":"2023-10-09T10:26:38.483055+0000","last_fullsized":"2023-10-09T10:26:38.483055+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":589,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.491807+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:26:38.491807+0000","last_peered":"2023-10-09T10:26:38.491807+0000","last_clean":"2023-10-09T10:26:38.491807+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:26:38.491807+0000","last_undegraded":"2023-10-09T10:26:38.491807+0000","last_fullsized":"2023-10-09T10:26:38.491807+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"537'5928","reported_seq":5992,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:44.904130+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:26:44.904130+0000","last_peered":"2023-10-09T10:26:44.904130+0000","last_clean":"2023-10-09T10:26:44.904130+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:26:44.904130+0000","last_undegraded":"2023-10-09T10:26:44.904130+0000","last_fullsized":"2023-10-09T10:26:44.904130+0000","mapping_epoch":501,"log_start":"498'215","ondisk_log_start":"498'215","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":5713,"log_dups_size":0,"ondisk_log_size":5713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132251648,"num_objects":2018,"num_object_clones":0,"num_object_copies":6054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2667,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":53,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.486105+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:26:38.486105+0000","last_peered":"2023-10-09T10:26:38.486105+0000","last_clean":"2023-10-09T10:26:38.486105+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:26:38.486105+0000","last_undegraded":"2023-10-09T10:26:38.486105+0000","last_fullsized":"2023-10-09T10:26:38.486105+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T10:26:38.502817+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:26:38.502817+0000","last_peered":"2023-10-09T10:26:38.502817+0000","last_clean":"2023-10-09T10:26:38.502817+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:26:38.502817+0000","last_undegraded":"2023-10-09T10:26:38.502817+0000","last_fullsized":"2023-10-09T10:26:38.502817+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":4380655640,"num_objects":66846,"num_object_clones":0,"num_object_copies":200538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32815,"num_objects_misplaced":16606,"num_objects_unfound":0,"num_objects_dirty":66846,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144100,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21306,"num_bytes_recovered":1373831168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15347097600,"data_stored":15347089456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147333,"ondisk_log_size":147333,"up":78,"acting":65,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430685,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626913,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2216892,"kb_used_data":1309428,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91528260,"statfs":{"total":95995035648,"available":93724938240,"internally_reserved":0,"allocated":1340854272,"data_stored":1339911842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.4330000000000001,"15min":1.6339999999999999},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":4.7990000000000004,"5min":9.4260000000000002,"15min":44.055},"last":1.0349999999999999},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.591,"15min":1.8600000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":5.6079999999999997,"5min":9.2609999999999992,"15min":43.869999999999997},"last":0.32000000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":1.464,"15min":2.1499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":6.3899999999999997,"5min":6.3899999999999997,"15min":155.01400000000001},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.53,"15min":2.27},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":6.3369999999999997,"5min":7.7320000000000002,"15min":154.71100000000001},"last":0.41799999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":1.821,"15min":1.865},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":8.5069999999999997,"5min":8.6280000000000001,"15min":29.367999999999999},"last":2.956},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.617,"15min":1.752},"min":{"1min":0.32600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.0640000000000001,"5min":7.5629999999999997,"15min":14.542999999999999},"last":0.997}]},{"osd":3,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.76,"15min":1.573},"min":{"1min":0.34599999999999997,"5min":0.32700000000000001,"15min":0.308},"max":{"1min":9.4619999999999997,"5min":9.4619999999999997,"15min":9.4619999999999997},"last":0.95499999999999996},{"interface":"front","average":{"1min":2.3130000000000002,"5min":1.946,"15min":1.6759999999999999},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.31},"max":{"1min":7.9939999999999998,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.34300000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.3610000000000002,"15min":1.9430000000000001},"min":{"1min":0.30299999999999999,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":7.3289999999999997,"5min":10.618,"15min":122.041},"last":0.25800000000000001},{"interface":"front","average":{"1min":1.478,"5min":2.04,"15min":1.7809999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.4729999999999999,"5min":11.266,"15min":122.13200000000001},"last":0.93899999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":2.4100000000000001,"15min":2.0350000000000001},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1980000000000004,"5min":12.894,"15min":121.98999999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.9019999999999999,"5min":1.5960000000000001,"15min":1.6919999999999999},"min":{"1min":0.33300000000000002,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":7.0060000000000002,"5min":15.041,"15min":122.346},"last":0.91700000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":2.327,"15min":1.972},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":5.0149999999999997,"5min":49.991,"15min":130.66999999999999},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.6819999999999999,"5min":2.3999999999999999,"15min":2.0249999999999999},"min":{"1min":0.38900000000000001,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":5.0910000000000002,"5min":50.023000000000003,"15min":130.702},"last":0.97999999999999998}]}]},{"osd":1,"up_from":0,"seq":360777253226,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416750,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3834460,"kb_used_data":2868180,"kb_used_omap":28,"kb_used_meta":966051,"kb_avail":89910692,"statfs":{"total":95995035648,"available":92068548608,"internally_reserved":0,"allocated":2937016320,"data_stored":2934921230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":989237072},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:25:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.6200000000000001,"15min":2.0059999999999998},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":17.161999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.585,"5min":1.506,"15min":2.4689999999999999},"min":{"1min":0.218,"5min":0.214,"15min":0.192},"max":{"1min":10.786,"5min":17.033000000000001,"15min":56.950000000000003},"last":2.7290000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":1.4970000000000001,"15min":2.1190000000000002},"min":{"1min":0.26600000000000001,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":17.093,"5min":17.093,"15min":53.988999999999997},"last":0.69399999999999995},{"interface":"front","average":{"1min":3.3370000000000002,"5min":1.8759999999999999,"15min":2.161},"min":{"1min":0.32100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.053999999999998,"5min":17.053999999999998,"15min":54.119},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 10:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.609,"15min":1.4850000000000001},"min":{"1min":0.28599999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.763,"5min":11.933,"15min":56.984999999999999},"last":1.6830000000000001},{"interface":"front","average":{"1min":2.8660000000000001,"5min":1.9790000000000001,"15min":1.5640000000000001},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.067,"5min":10.067,"15min":56.475999999999999},"last":1.893}]},{"osd":4,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":1.974,"15min":2.254},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":10.941000000000001,"5min":28.289000000000001,"15min":57.719000000000001},"last":1.1259999999999999},{"interface":"front","average":{"1min":2.7709999999999999,"5min":2.3420000000000001,"15min":2.2650000000000001},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.32400000000000001},"max":{"1min":9.9499999999999993,"5min":28.331,"15min":56.378},"last":2.9009999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":8.6020000000000003,"5min":3.4510000000000001,"15min":3.544},"min":{"1min":0.51700000000000002,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":90.412000000000006,"5min":90.412000000000006,"15min":229.589},"last":3.1829999999999998},{"interface":"front","average":{"1min":6.9400000000000004,"5min":2.7509999999999999,"15min":3.1909999999999998},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":90.393000000000001,"5min":90.393000000000001,"15min":228.88900000000001},"last":0.67900000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.919,"15min":2.29},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":54.482999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.806,"5min":2.1240000000000001,"15min":2.5230000000000001},"min":{"1min":0.47899999999999998,"5min":0.39700000000000002,"15min":0.34799999999999998},"max":{"1min":8.8309999999999995,"5min":8.8309999999999995,"15min":57.753},"last":1.0920000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0470000000000002,"5min":2.2679999999999998,"15min":2.4220000000000002},"min":{"1min":0.48299999999999998,"5min":0.40500000000000003,"15min":0.34999999999999998},"max":{"1min":12.029999999999999,"5min":12.029999999999999,"15min":56.411999999999999},"last":1.6240000000000001},{"interface":"front","average":{"1min":3.0350000000000001,"5min":2.1869999999999998,"15min":2.3530000000000002},"min":{"1min":0.44800000000000001,"5min":0.38600000000000001,"15min":0.38300000000000001},"max":{"1min":17.129999999999999,"5min":17.129999999999999,"15min":54.451999999999998},"last":1.758}]}]},{"osd":2,"up_from":217,"seq":932007903486,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3782676,"kb_used_data":2938124,"kb_used_omap":83,"kb_used_meta":844460,"kb_avail":89962476,"statfs":{"total":95995035648,"available":92121575424,"internally_reserved":0,"allocated":3008638976,"data_stored":3006736026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":864727535},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2000000000000002,"5min":2.0579999999999998,"15min":2.0049999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.185},"max":{"1min":11.093999999999999,"5min":48.990000000000002,"15min":48.990000000000002},"last":10.121},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.1160000000000001,"15min":2.3220000000000001},"min":{"1min":0.45600000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.2699999999999996,"5min":49.412999999999997,"15min":49.412999999999997},"last":1.002}]},{"osd":1,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":1.893,"15min":2.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":13.303000000000001,"5min":13.303000000000001,"15min":34.893999999999998},"last":10.045},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.5740000000000001,"15min":2.0339999999999998},"min":{"1min":0.35099999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":9.9239999999999995,"5min":9.9239999999999995,"15min":34.962000000000003},"last":1.2490000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.4139999999999999,"15min":1.202},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.248},"max":{"1min":9.3119999999999994,"5min":9.3119999999999994,"15min":35.043999999999997},"last":15.115},{"interface":"front","average":{"1min":2.7909999999999999,"5min":1.2829999999999999,"15min":1.3},"min":{"1min":0.35699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.206,"5min":11.206,"15min":50.406999999999996},"last":0.46200000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:26:14 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.8149999999999999,"15min":1.6870000000000001},"min":{"1min":0.31900000000000001,"5min":0.29799999999999999,"15min":0.28999999999999998},"max":{"1min":4.4710000000000001,"5min":47.503999999999998,"15min":47.503999999999998},"last":1.0669999999999999},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.0640000000000001,"15min":1.835},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":11.298,"5min":47.704000000000001,"15min":47.704000000000001},"last":0.55000000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.8340000000000001,"15min":2.012},"min":{"1min":0.40500000000000003,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":57.298000000000002},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.8639999999999999,"5min":2.0430000000000001,"15min":2.0169999999999999},"min":{"1min":0.442,"5min":0.31,"15min":0.31},"max":{"1min":11.195,"5min":11.195,"15min":57.828000000000003},"last":10.093}]},{"osd":6,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":2.0590000000000002,"15min":2.0230000000000001},"min":{"1min":0.35399999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":4.5579999999999998,"5min":12.411,"15min":18.943999999999999},"last":1.097},{"interface":"front","average":{"1min":2.4860000000000002,"5min":1.855,"15min":1.97},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":13.975,"5min":13.975,"15min":24.379000000000001},"last":0.496}]},{"osd":7,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.1349999999999998,"15min":2.2949999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.308},"max":{"1min":5.415,"5min":11.704000000000001,"15min":35.008000000000003},"last":1.0329999999999999},{"interface":"front","average":{"1min":2.907,"5min":1.99,"15min":2.1579999999999999},"min":{"1min":0.47099999999999997,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":15.548,"5min":15.548,"15min":24.323},"last":0.57199999999999995}]}]},{"osd":3,"up_from":352,"seq":1511828488325,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2799256,"kb_used_data":2275984,"kb_used_omap":15,"kb_used_meta":523248,"kb_avail":90945896,"statfs":{"total":95995035648,"available":93128597504,"internally_reserved":0,"allocated":2330607616,"data_stored":2328227797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":535806406},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8259999999999996,"5min":3.1200000000000001,"15min":1.696},"min":{"1min":0.34999999999999998,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":70.004000000000005,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.44900000000000001},{"interface":"front","average":{"1min":6.2519999999999998,"5min":2.8159999999999998,"15min":1.6739999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":66.802000000000007,"5min":66.802000000000007,"15min":66.802000000000007},"last":10.313000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7889999999999999,"15min":1.8620000000000001},"min":{"1min":0.311,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.785,"5min":25.370000000000001,"15min":105.401},"last":0.51100000000000001},{"interface":"front","average":{"1min":2.7109999999999999,"5min":2.04,"15min":1.7969999999999999},"min":{"1min":0.30299999999999999,"5min":0.25900000000000001,"15min":0.23300000000000001},"max":{"1min":9.2140000000000004,"5min":14.613,"15min":104.914},"last":10.422000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.2989999999999999,"15min":1.9490000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":11.141999999999999,"5min":11.141999999999999,"15min":11.141999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":3.1400000000000001,"5min":1.956,"15min":1.4279999999999999},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.27600000000000002},"max":{"1min":16.475000000000001,"5min":25.338999999999999,"15min":25.338999999999999},"last":10.045999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.194,"15min":1.415},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":6.9370000000000003,"5min":21.196000000000002,"15min":21.831},"last":10.268000000000001},{"interface":"front","average":{"1min":3.3980000000000001,"5min":2.8820000000000001,"15min":1.958},"min":{"1min":0.36399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":11.087,"5min":19.323,"15min":22.097999999999999},"last":3.4020000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":2.0910000000000002,"15min":1.294},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":11.363,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.46,"5min":2.411,"15min":1.5429999999999999},"min":{"1min":0.36599999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":6.9900000000000002,"5min":13.394,"15min":13.394},"last":10.388}]},{"osd":6,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.4780000000000002,"15min":1.7250000000000001},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.35599999999999998},"max":{"1min":11.114000000000001,"5min":19.114000000000001,"15min":19.114000000000001},"last":0.375},{"interface":"front","average":{"1min":2.3279999999999998,"5min":2.1230000000000002,"15min":1.468},"min":{"1min":0.47699999999999998,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":11.32,"5min":25.306999999999999,"15min":25.306999999999999},"last":10.117000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":2.0779999999999998,"15min":1.514},"min":{"1min":0.42299999999999999,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":6.9640000000000004,"5min":9.7880000000000003,"15min":10.459},"last":10.345000000000001},{"interface":"front","average":{"1min":2.327,"5min":2.3220000000000001,"15min":1.762},"min":{"1min":0.432,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":11.045999999999999,"5min":19.280000000000001,"15min":19.280000000000001},"last":0.41799999999999998}]}]},{"osd":4,"up_from":266,"seq":1142461300955,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3683968,"kb_used_data":2754936,"kb_used_omap":35,"kb_used_meta":928988,"kb_avail":90061184,"statfs":{"total":95995035648,"available":92222652416,"internally_reserved":0,"allocated":2821054464,"data_stored":2820106360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":951283950},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.893,"15min":2.1469999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":6.5019999999999998,"5min":14.548,"15min":97.721999999999994},"last":0.80200000000000005},{"interface":"front","average":{"1min":2.8940000000000001,"5min":2.694,"15min":2.8740000000000001},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.224},"max":{"1min":11.022,"5min":11.257,"15min":97.915999999999997},"last":1.734}]},{"osd":1,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4449999999999998,"15min":2.3700000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.24299999999999999},"max":{"1min":12.209,"5min":12.209,"15min":71.457999999999998},"last":4.4470000000000001},{"interface":"front","average":{"1min":2.992,"5min":2.8999999999999999,"15min":2.2269999999999999},"min":{"1min":0.376,"5min":0.248,"15min":0.248},"max":{"1min":8.327,"5min":49.781999999999996,"15min":71.658000000000001},"last":4.5330000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3809999999999998,"15min":1.845},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":7.3230000000000004,"5min":15.944000000000001,"15min":15.944000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.9970000000000001,"15min":1.7989999999999999},"min":{"1min":0.36199999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.102,"5min":14.473000000000001,"15min":14.473000000000001},"last":0.48699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.7410000000000001,"5min":2.9199999999999999,"15min":2.024},"min":{"1min":0.39800000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.347,"5min":12.074,"15min":12.074},"last":0.73399999999999999},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.907,"15min":1.7390000000000001},"min":{"1min":0.40899999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.369999999999999,"5min":16.067,"15min":16.067},"last":0.81399999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":2.5939999999999999,"15min":2.069},"min":{"1min":0.314,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":3.3279999999999998,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.054,"5min":2.4969999999999999,"15min":1.831},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.464,"5min":49.915999999999997,"15min":99.384},"last":0.58599999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.881,"15min":1.3740000000000001},"min":{"1min":0.33800000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.5389999999999997,"5min":14.499000000000001,"15min":14.499000000000001},"last":3.5030000000000001},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.8220000000000001,"15min":1.7949999999999999},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":12.242000000000001,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.44800000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":3.0409999999999999,"15min":1.9239999999999999},"min":{"1min":0.39500000000000002,"5min":0.316,"15min":0.307},"max":{"1min":12.391999999999999,"5min":17.369,"15min":17.369},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.3320000000000001,"5min":2.7829999999999999,"15min":1.778},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.30399999999999999},"max":{"1min":4.6689999999999996,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.52300000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823140,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4421620,"kb_used_data":2916716,"kb_used_omap":0,"kb_used_meta":1504896,"kb_avail":89323532,"statfs":{"total":95995035648,"available":91467296768,"internally_reserved":0,"allocated":2986717184,"data_stored":2985777134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541013504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.754,"15min":1.9370000000000001},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":16.536999999999999,"5min":22.469999999999999,"15min":22.469999999999999},"last":7.0529999999999999},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.1309999999999998,"15min":1.7270000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":8.9339999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.70799999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:26:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":2.3260000000000001,"15min":2.1549999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.23999999999999999},"max":{"1min":4.375,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.6499999999999999,"15min":1.8089999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":4.593,"5min":13.143000000000001,"15min":29.065000000000001},"last":1.54}]},{"osd":2,"last update":"Mon Oct 9 10:26:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":2.5990000000000002,"15min":2.1309999999999998},"min":{"1min":0.33000000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":12.247999999999999,"5min":16.629999999999999,"15min":16.629999999999999},"last":0.311},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.907,"15min":1.6919999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":3.1389999999999998,"5min":13.095000000000001,"15min":15.929},"last":2.835}]},{"osd":3,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.2170000000000001,"15min":1.7889999999999999},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":10.362,"5min":10.561999999999999,"15min":10.561999999999999},"last":8.7330000000000005},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.0830000000000002,"15min":1.7270000000000001},"min":{"1min":0.41599999999999998,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9130000000000003,"5min":13.624000000000001,"15min":13.624000000000001},"last":8.7479999999999993}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":2.242,"15min":1.5760000000000001},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":10.826000000000001,"5min":12.023,"15min":12.023},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.6579999999999999,"5min":2.0369999999999999,"15min":1.399},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":8.8620000000000001,"5min":11.956,"15min":11.956},"last":0.78000000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.621,"15min":1.794},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":10.664999999999999,"5min":11.686,"15min":12.772},"last":0.434},{"interface":"front","average":{"1min":2.4500000000000002,"5min":2.1640000000000001,"15min":1.665},"min":{"1min":0.29699999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.34499999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:26:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.3719999999999999,"15min":1.7709999999999999},"min":{"1min":0.378,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":4.25,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.1990000000000001,"15min":1.1319999999999999},"min":{"1min":0.31900000000000001,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":2.0299999999999998,"5min":11.994,"15min":11.994},"last":0.745}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2920742912,"data_stored":2920742912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2987393024,"data_stored":2987393024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2323513344,"data_stored":2323513344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2805272576,"data_stored":2805268504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2975010816,"data_stored":2975006744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1335164928,"data_stored":1335164928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:26:50.610 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:26:51.082 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:51.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:51.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:51.256 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:51.331 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:51.336 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:51 smithi012 ceph-mon[152263]: pgmap v1511: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 32316/195351 objects degraded (16.543%); 15867/195351 objects misplaced (8.122%); 15 MiB/s, 239 objects/s recovering 2023-10-09T10:26:51.336 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:51 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:51.336 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:51 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:51.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:51 smithi012 ceph-mon[147926]: pgmap v1511: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 32316/195351 objects degraded (16.543%); 15867/195351 objects misplaced (8.122%); 15 MiB/s, 239 objects/s recovering 2023-10-09T10:26:51.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:51 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:51.337 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:51 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:51.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:51.389 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:26:51.438 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:51.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:51 smithi110 ceph-mon[136032]: pgmap v1511: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 32316/195351 objects degraded (16.543%); 15867/195351 objects misplaced (8.122%); 15 MiB/s, 239 objects/s recovering 2023-10-09T10:26:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:51 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:51 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:51.476 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:51.660 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:51.669 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:52.746 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:52.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:52.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:52.851 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:52.928 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:53.045 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:53.046 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:53 smithi110 ceph-mon[136032]: pgmap v1512: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 31163/187692 objects degraded (16.603%); 15223/187692 objects misplaced (8.111%); 14 MiB/s, 233 objects/s recovering 2023-10-09T10:26:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:53.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:53.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:53 smithi012 ceph-mon[147926]: pgmap v1512: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 31163/187692 objects degraded (16.603%); 15223/187692 objects misplaced (8.111%); 14 MiB/s, 233 objects/s recovering 2023-10-09T10:26:53.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:53 smithi012 ceph-mon[152263]: pgmap v1512: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 31163/187692 objects degraded (16.603%); 15223/187692 objects misplaced (8.111%); 14 MiB/s, 233 objects/s recovering 2023-10-09T10:26:53.230 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:53.437 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:53.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:53.543 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:53.575 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:26:53.646 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:53.842 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:53.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:53.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:54.057 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:54 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 31163/187692 objects degraded (16.603%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:54.084 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:54.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:54 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 31163/187692 objects degraded (16.603%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:54.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:54 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 31163/187692 objects degraded (16.603%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:54.358 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:54.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:54.413 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:26:54.458 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:26:54.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:54.511 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:26:54.549 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:26:54.562 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:54.585 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:26:54.618 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:26:54.618 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:26:54.635 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:54.646 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:26:54.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:54.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:54.677 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:26:54.707 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:26:54.733 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:26:54.748 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:26:54.768 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:26:54.768 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:26:54.768 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:26:54.768 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:26:54.768 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:26:54.768 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:26:54.768 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:26:54.769 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:26:54.808 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:54.985 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:26:54.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:54.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:26:55.073 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:26:55.088 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:26:55.090 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:55.145 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:55.198 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:55.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:55.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:55.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:55.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[147926]: pgmap v1513: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 30944/185754 objects degraded (16.659%); 15004/185754 objects misplaced (8.077%); 14 MiB/s, 232 objects/s recovering 2023-10-09T10:26:55.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1970179188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:55.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:55.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3319102555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:55.342 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:55.343 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[152263]: pgmap v1513: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 30944/185754 objects degraded (16.659%); 15004/185754 objects misplaced (8.077%); 14 MiB/s, 232 objects/s recovering 2023-10-09T10:26:55.343 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1970179188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:55.343 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:55.343 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3319102555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:55.343 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:55 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:55.345 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:55 smithi110 ceph-mon[136032]: pgmap v1513: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 30944/185754 objects degraded (16.659%); 15004/185754 objects misplaced (8.077%); 14 MiB/s, 232 objects/s recovering 2023-10-09T10:26:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1970179188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:26:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:55 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:55 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3319102555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:26:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:55 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:56.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:26:56.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[147926]: failed to encode map e539 with expected crc 2023-10-09T10:26:56.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1970179188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:56.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[147926]: osdmap e539: 8 total, 8 up, 6 in 2023-10-09T10:26:56.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:26:56.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:26:56.430 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:56.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:26:56.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[152263]: failed to encode map e539 with expected crc 2023-10-09T10:26:56.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1970179188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:56.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[152263]: osdmap e539: 8 total, 8 up, 6 in 2023-10-09T10:26:56.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:26:56.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:26:56.433 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:56 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:56 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:26:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:56 smithi110 ceph-mon[136032]: failed to encode map e539 with expected crc 2023-10-09T10:26:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:56 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1970179188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:26:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:56 smithi110 ceph-mon[136032]: osdmap e539: 8 total, 8 up, 6 in 2023-10-09T10:26:56.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:56 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:26:56.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:56 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:26:56.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:56 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:56.624 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:26:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:56.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:26:56.738 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:56.797 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:56.861 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:56.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:56.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:56.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:57.021 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:57.090 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:57.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:57.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:57.196 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:26:57.278 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:57.355 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:26:57.358 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:57 smithi110 ceph-mon[136032]: pgmap v1515: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 632 op/s; 30239/181614 objects degraded (16.650%); 14433/181614 objects misplaced (7.947%); 15 MiB/s, 242 objects/s recovering 2023-10-09T10:26:57.358 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:57 smithi110 ceph-mon[136032]: failed to encode map e539 with expected crc 2023-10-09T10:26:57.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:57.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:26:57.460 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:57.486 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:57.544 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:57.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:57.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:57.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:26:57.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:57 smithi012 ceph-mon[147926]: pgmap v1515: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 632 op/s; 30239/181614 objects degraded (16.650%); 14433/181614 objects misplaced (7.947%); 15 MiB/s, 242 objects/s recovering 2023-10-09T10:26:57.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:57 smithi012 ceph-mon[147926]: failed to encode map e539 with expected crc 2023-10-09T10:26:57.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:57 smithi012 ceph-mon[152263]: pgmap v1515: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 632 op/s; 30239/181614 objects degraded (16.650%); 14433/181614 objects misplaced (7.947%); 15 MiB/s, 242 objects/s recovering 2023-10-09T10:26:57.681 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:57 smithi012 ceph-mon[152263]: failed to encode map e539 with expected crc 2023-10-09T10:26:57.719 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:57.771 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:26:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:57.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:26:57.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:26:57.969 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:58.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:26:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:26:57] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.33.4" 2023-10-09T10:26:58.432 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:26:58.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:58.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:26:58.455 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:26:58.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:26:58.582 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:58.672 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:26:58.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:26:58.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:26:58.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:26:58.862 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:26:59.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:59 smithi012 ceph-mon[147926]: pgmap v1516: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 29857/178146 objects degraded (16.760%); 14021/178146 objects misplaced (7.871%); 13 MiB/s, 255 objects/s recovering 2023-10-09T10:26:59.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:59 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 29857/178146 objects degraded (16.760%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:59.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:26:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:59 smithi012 ceph-mon[152263]: pgmap v1516: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 29857/178146 objects degraded (16.760%); 14021/178146 objects misplaced (7.871%); 13 MiB/s, 255 objects/s recovering 2023-10-09T10:26:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:59 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 29857/178146 objects degraded (16.760%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:59.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:26:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:59 smithi110 ceph-mon[136032]: pgmap v1516: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 29857/178146 objects degraded (16.760%); 14021/178146 objects misplaced (7.871%); 13 MiB/s, 255 objects/s recovering 2023-10-09T10:26:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:59 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 29857/178146 objects degraded (16.760%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:26:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:26:59.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:26:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:26:59.996 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:00.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:00.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:00.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:00.137 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:00.221 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:00.230 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:00.356 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:00.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:00.466 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:00.502 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:00.546 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:00.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:00.650 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:00.688 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:00.721 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:27:00.729 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:00.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:00.837 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:00.873 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:00.923 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:00.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:00.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:01.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:01.110 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:01.204 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:27:01.207 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:01.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:01.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:01.226 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:27:01.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:01.405 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:01 smithi012 ceph-mon[152263]: pgmap v1517: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 29661/175986 objects degraded (16.854%); 13530/175986 objects misplaced (7.688%); 13 MiB/s, 257 objects/s recovering 2023-10-09T10:27:01.406 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:01 smithi012 ceph-mon[147926]: pgmap v1517: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 29661/175986 objects degraded (16.854%); 13530/175986 objects misplaced (7.688%); 13 MiB/s, 257 objects/s recovering 2023-10-09T10:27:01.409 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:01.477 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:01.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:01.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:01.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:01.672 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:01.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:01 smithi110 ceph-mon[136032]: pgmap v1517: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 29661/175986 objects degraded (16.854%); 13530/175986 objects misplaced (7.688%); 13 MiB/s, 257 objects/s recovering 2023-10-09T10:27:02.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:02 smithi012 ceph-mon[152263]: from='client.37164 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:02.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:02 smithi012 ceph-mon[147926]: from='client.37164 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:02.696 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:02 smithi110 ceph-mon[136032]: from='client.37164 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:03.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:03 smithi012 ceph-mon[147926]: pgmap v1518: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 28534/168030 objects degraded (16.981%); 12740/168030 objects misplaced (7.582%); 11 MiB/s, 269 objects/s recovering 2023-10-09T10:27:03.627 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:03 smithi012 ceph-mon[152263]: pgmap v1518: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 28534/168030 objects degraded (16.981%); 12740/168030 objects misplaced (7.582%); 11 MiB/s, 269 objects/s recovering 2023-10-09T10:27:03.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:03 smithi110 ceph-mon[136032]: pgmap v1518: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 28534/168030 objects degraded (16.981%); 12740/168030 objects misplaced (7.582%); 11 MiB/s, 269 objects/s recovering 2023-10-09T10:27:04.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:04 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 28534/168030 objects degraded (16.981%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:04.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:04 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 28534/168030 objects degraded (16.981%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:04.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:04 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 28534/168030 objects degraded (16.981%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:05.142 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:05.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:05.248 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:05.298 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:05.349 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:05.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:05.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:05.432 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1517,"stamp":"2023-10-09T10:26:59.831914+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3843332024,"num_objects":58662,"num_object_clones":0,"num_object_copies":175986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29661,"num_objects_misplaced":13530,"num_objects_unfound":0,"num_objects_dirty":58662,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":152568,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24798,"num_bytes_recovered":1566607100,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":145176,"ondisk_log_size":145176,"up":297,"acting":284,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":19445172,"kb_used_data":13777796,"kb_used_omap":163,"kb_used_meta":5666908,"kb_avail":543025740,"statfs":{"total":575970213888,"available":556058357760,"internally_reserved":0,"allocated":14108463104,"data_stored":14099211417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":5802914803},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-423034880,"num_objects":-6455,"num_object_clones":0,"num_object_copies":-19365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2655,"num_objects_misplaced":-2337,"num_objects_unfound":0,"num_objects_dirty":-6455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6455,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2579,"num_bytes_recovered":140443648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.012357"},"pg_stats":[{"pgid":"11.12","version":"539'6259","reported_seq":6332,"reported_epoch":539,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:56.755724+0000","last_change":"2023-10-09T10:25:09.709209+0000","last_active":"2023-10-09T10:26:56.755724+0000","last_peered":"2023-10-09T10:26:56.755724+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:26:56.755724+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"498'611","ondisk_log_start":"498'611","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5648,"log_dups_size":0,"ondisk_log_size":5648,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121405440,"num_objects":1853,"num_object_clones":0,"num_object_copies":5559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3119,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5311,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-1266},{"shards":"0,2,5","objects":1390}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":132,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.655688+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:26:45.655688+0000","last_peered":"2023-10-09T10:26:45.655688+0000","last_clean":"2023-10-09T10:26:45.655688+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:26:45.655688+0000","last_undegraded":"2023-10-09T10:26:45.655688+0000","last_fullsized":"2023-10-09T10:26:45.655688+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":493,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.656470+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:26:45.656470+0000","last_peered":"2023-10-09T10:26:45.656470+0000","last_clean":"2023-10-09T10:26:45.656470+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:26:45.656470+0000","last_undegraded":"2023-10-09T10:26:45.656470+0000","last_fullsized":"2023-10-09T10:26:45.656470+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"539'6175","reported_seq":11353,"reported_epoch":539,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:56.767147+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:26:56.767147+0000","last_peered":"2023-10-09T10:26:56.767147+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:26:56.767147+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"498'457","ondisk_log_start":"498'457","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120061952,"num_objects":1832,"num_object_clones":0,"num_object_copies":5496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1975,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5230,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1699,"num_bytes_recovered":111345664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":565},{"shards":"0,2,6","objects":845},{"shards":"0,6","objects":422}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":570,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.655311+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:26:45.655311+0000","last_peered":"2023-10-09T10:26:45.655311+0000","last_clean":"2023-10-09T10:26:45.655311+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:26:45.655311+0000","last_undegraded":"2023-10-09T10:26:45.655311+0000","last_fullsized":"2023-10-09T10:26:45.655311+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":464,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.671594+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:26:45.671594+0000","last_peered":"2023-10-09T10:26:45.671594+0000","last_clean":"2023-10-09T10:26:45.671594+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:26:45.671594+0000","last_undegraded":"2023-10-09T10:26:45.671594+0000","last_fullsized":"2023-10-09T10:26:45.671594+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"539'6095","reported_seq":10645,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T10:26:56.572345+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:26:56.572345+0000","last_peered":"2023-10-09T10:26:56.572345+0000","last_clean":"2023-10-09T10:26:56.572345+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:56.572345+0000","last_undegraded":"2023-10-09T10:26:56.572345+0000","last_fullsized":"2023-10-09T10:26:56.572345+0000","mapping_epoch":527,"log_start":"498'293","ondisk_log_start":"498'293","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119504896,"num_objects":1824,"num_object_clones":0,"num_object_copies":5472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5158,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":34,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669862+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:26:45.669862+0000","last_peered":"2023-10-09T10:26:45.669862+0000","last_clean":"2023-10-09T10:26:45.669862+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:26:45.669862+0000","last_undegraded":"2023-10-09T10:26:45.669862+0000","last_fullsized":"2023-10-09T10:26:45.669862+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":116,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669143+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:26:45.669143+0000","last_peered":"2023-10-09T10:26:45.669143+0000","last_clean":"2023-10-09T10:26:45.669143+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:26:45.669143+0000","last_undegraded":"2023-10-09T10:26:45.669143+0000","last_fullsized":"2023-10-09T10:26:45.669143+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":498,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669734+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:26:45.669734+0000","last_peered":"2023-10-09T10:26:45.669734+0000","last_clean":"2023-10-09T10:26:45.669734+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:26:45.669734+0000","last_undegraded":"2023-10-09T10:26:45.669734+0000","last_fullsized":"2023-10-09T10:26:45.669734+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":446,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.683585+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:26:45.683585+0000","last_peered":"2023-10-09T10:26:45.683585+0000","last_clean":"2023-10-09T10:26:45.683585+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:26:45.683585+0000","last_undegraded":"2023-10-09T10:26:45.683585+0000","last_fullsized":"2023-10-09T10:26:45.683585+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501201+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:26:39.501201+0000","last_peered":"2023-10-09T10:26:39.501201+0000","last_clean":"2023-10-09T10:26:39.501201+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:26:39.501201+0000","last_undegraded":"2023-10-09T10:26:39.501201+0000","last_fullsized":"2023-10-09T10:26:39.501201+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":446,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.679484+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:26:45.679484+0000","last_peered":"2023-10-09T10:26:45.679484+0000","last_clean":"2023-10-09T10:26:45.679484+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:26:45.679484+0000","last_undegraded":"2023-10-09T10:26:45.679484+0000","last_fullsized":"2023-10-09T10:26:45.679484+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":144,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.656703+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:26:45.656703+0000","last_peered":"2023-10-09T10:26:45.656703+0000","last_clean":"2023-10-09T10:26:45.656703+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:26:45.656703+0000","last_undegraded":"2023-10-09T10:26:45.656703+0000","last_fullsized":"2023-10-09T10:26:45.656703+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":34,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.668980+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:26:45.668980+0000","last_peered":"2023-10-09T10:26:45.668980+0000","last_clean":"2023-10-09T10:26:45.668980+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:26:45.668980+0000","last_undegraded":"2023-10-09T10:26:45.668980+0000","last_fullsized":"2023-10-09T10:26:45.668980+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":190,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.658917+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:26:45.658917+0000","last_peered":"2023-10-09T10:26:45.658917+0000","last_clean":"2023-10-09T10:26:45.658917+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:26:45.658917+0000","last_undegraded":"2023-10-09T10:26:45.658917+0000","last_fullsized":"2023-10-09T10:26:45.658917+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":150,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.657327+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:26:45.657327+0000","last_peered":"2023-10-09T10:26:45.657327+0000","last_clean":"2023-10-09T10:26:45.657327+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:26:45.657327+0000","last_undegraded":"2023-10-09T10:26:45.657327+0000","last_fullsized":"2023-10-09T10:26:45.657327+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":110,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.655977+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:26:45.655977+0000","last_peered":"2023-10-09T10:26:45.655977+0000","last_clean":"2023-10-09T10:26:45.655977+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:26:45.655977+0000","last_undegraded":"2023-10-09T10:26:45.655977+0000","last_fullsized":"2023-10-09T10:26:45.655977+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":63,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501687+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:26:39.501687+0000","last_peered":"2023-10-09T10:26:39.501687+0000","last_clean":"2023-10-09T10:26:39.501687+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:26:39.501687+0000","last_undegraded":"2023-10-09T10:26:39.501687+0000","last_fullsized":"2023-10-09T10:26:39.501687+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":462,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.676299+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:26:45.676299+0000","last_peered":"2023-10-09T10:26:45.676299+0000","last_clean":"2023-10-09T10:26:45.676299+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:26:45.676299+0000","last_undegraded":"2023-10-09T10:26:45.676299+0000","last_fullsized":"2023-10-09T10:26:45.676299+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":204,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.504761+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:26:39.504761+0000","last_peered":"2023-10-09T10:26:39.504761+0000","last_clean":"2023-10-09T10:26:39.504761+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:26:39.504761+0000","last_undegraded":"2023-10-09T10:26:39.504761+0000","last_fullsized":"2023-10-09T10:26:39.504761+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"538'6153","reported_seq":6958,"reported_epoch":538,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:54.949763+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:26:54.949763+0000","last_peered":"2023-10-09T10:26:54.949763+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:26:54.949763+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"498'734","ondisk_log_start":"498'734","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5419,"log_dups_size":0,"ondisk_log_size":5419,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122912768,"num_objects":1876,"num_object_clones":0,"num_object_copies":5628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":1876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5230,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-1275},{"shards":"3,5,6","objects":843}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":150,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.655443+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:26:45.655443+0000","last_peered":"2023-10-09T10:26:45.655443+0000","last_clean":"2023-10-09T10:26:45.655443+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:26:45.655443+0000","last_undegraded":"2023-10-09T10:26:45.655443+0000","last_fullsized":"2023-10-09T10:26:45.655443+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":34,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.671229+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:26:45.671229+0000","last_peered":"2023-10-09T10:26:45.671229+0000","last_clean":"2023-10-09T10:26:45.671229+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:26:45.671229+0000","last_undegraded":"2023-10-09T10:26:45.671229+0000","last_fullsized":"2023-10-09T10:26:45.671229+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"538'6214","reported_seq":14725,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:54.976840+0000","last_change":"2023-10-09T10:26:39.522775+0000","last_active":"2023-10-09T10:26:54.976840+0000","last_peered":"2023-10-09T10:26:54.976840+0000","last_clean":"2023-10-09T10:26:54.976840+0000","last_became_active":"2023-10-09T10:26:39.509024+0000","last_became_peered":"2023-10-09T10:26:39.509024+0000","last_unstale":"2023-10-09T10:26:54.976840+0000","last_undegraded":"2023-10-09T10:26:54.976840+0000","last_fullsized":"2023-10-09T10:26:54.976840+0000","mapping_epoch":536,"log_start":"498'809","ondisk_log_start":"498'809","created":501,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5405,"log_dups_size":0,"ondisk_log_size":5405,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121012224,"num_objects":1847,"num_object_clones":0,"num_object_copies":5541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5249,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3072,"num_bytes_recovered":201326592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":44,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.672501+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:26:45.672501+0000","last_peered":"2023-10-09T10:26:45.672501+0000","last_clean":"2023-10-09T10:26:45.672501+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:26:45.672501+0000","last_undegraded":"2023-10-09T10:26:45.672501+0000","last_fullsized":"2023-10-09T10:26:45.672501+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":37,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.679699+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:26:45.679699+0000","last_peered":"2023-10-09T10:26:45.679699+0000","last_clean":"2023-10-09T10:26:45.679699+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:26:45.679699+0000","last_undegraded":"2023-10-09T10:26:45.679699+0000","last_fullsized":"2023-10-09T10:26:45.679699+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"538'6272","reported_seq":6338,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:54.853167+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:26:54.853167+0000","last_peered":"2023-10-09T10:26:54.853167+0000","last_clean":"2023-10-09T10:26:54.853167+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:26:54.853167+0000","last_undegraded":"2023-10-09T10:26:54.853167+0000","last_fullsized":"2023-10-09T10:26:54.853167+0000","mapping_epoch":494,"log_start":"498'976","ondisk_log_start":"498'976","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":5296,"log_dups_size":0,"ondisk_log_size":5296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123469824,"num_objects":1884,"num_object_clones":0,"num_object_copies":5652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2984,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":40,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.658302+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:26:45.658302+0000","last_peered":"2023-10-09T10:26:45.658302+0000","last_clean":"2023-10-09T10:26:45.658302+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:26:45.658302+0000","last_undegraded":"2023-10-09T10:26:45.658302+0000","last_fullsized":"2023-10-09T10:26:45.658302+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":213,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501131+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:26:39.501131+0000","last_peered":"2023-10-09T10:26:39.501131+0000","last_clean":"2023-10-09T10:26:39.501131+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:26:39.501131+0000","last_undegraded":"2023-10-09T10:26:39.501131+0000","last_fullsized":"2023-10-09T10:26:39.501131+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"539'5915","reported_seq":12021,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T10:26:56.763958+0000","last_change":"2023-10-09T10:26:35.543001+0000","last_active":"2023-10-09T10:26:56.763958+0000","last_peered":"2023-10-09T10:26:56.763958+0000","last_clean":"2023-10-09T10:26:56.763958+0000","last_became_active":"2023-10-09T10:26:35.529481+0000","last_became_peered":"2023-10-09T10:26:35.529481+0000","last_unstale":"2023-10-09T10:26:56.763958+0000","last_undegraded":"2023-10-09T10:26:56.763958+0000","last_fullsized":"2023-10-09T10:26:56.763958+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5915,"log_dups_size":0,"ondisk_log_size":5915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:07.598234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114425856,"num_objects":1746,"num_object_clones":0,"num_object_copies":5238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3155,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3012,"num_bytes_recovered":174915584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":166,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.656670+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:26:45.656670+0000","last_peered":"2023-10-09T10:26:45.656670+0000","last_clean":"2023-10-09T10:26:45.656670+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:26:45.656670+0000","last_undegraded":"2023-10-09T10:26:45.656670+0000","last_fullsized":"2023-10-09T10:26:45.656670+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.655554+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:26:45.655554+0000","last_peered":"2023-10-09T10:26:45.655554+0000","last_clean":"2023-10-09T10:26:45.655554+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:26:45.655554+0000","last_undegraded":"2023-10-09T10:26:45.655554+0000","last_fullsized":"2023-10-09T10:26:45.655554+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"538'6180","reported_seq":11373,"reported_epoch":538,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:55.021649+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:26:55.021649+0000","last_peered":"2023-10-09T10:26:55.021649+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:26:55.021649+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"498'815","ondisk_log_start":"498'815","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5365,"log_dups_size":0,"ondisk_log_size":5365,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119898112,"num_objects":1829,"num_object_clones":0,"num_object_copies":5487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2043,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5235,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1704,"num_bytes_recovered":111673344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":615},{"shards":"0,5,6","objects":813},{"shards":"5,6","objects":401}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":229,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.502497+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:26:39.502497+0000","last_peered":"2023-10-09T10:26:39.502497+0000","last_clean":"2023-10-09T10:26:39.502497+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:26:39.502497+0000","last_undegraded":"2023-10-09T10:26:39.502497+0000","last_fullsized":"2023-10-09T10:26:39.502497+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":130,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.502063+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:26:39.502063+0000","last_peered":"2023-10-09T10:26:39.502063+0000","last_clean":"2023-10-09T10:26:39.502063+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:26:39.502063+0000","last_undegraded":"2023-10-09T10:26:39.502063+0000","last_fullsized":"2023-10-09T10:26:39.502063+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"538'6086","reported_seq":10277,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:54.922825+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:26:54.922825+0000","last_peered":"2023-10-09T10:26:54.922825+0000","last_clean":"2023-10-09T10:26:54.922825+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:26:54.922825+0000","last_undegraded":"2023-10-09T10:26:54.922825+0000","last_fullsized":"2023-10-09T10:26:54.922825+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":6086,"log_dups_size":0,"ondisk_log_size":6086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123928576,"num_objects":1891,"num_object_clones":0,"num_object_copies":5673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3172,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":213,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501296+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:26:39.501296+0000","last_peered":"2023-10-09T10:26:39.501296+0000","last_clean":"2023-10-09T10:26:39.501296+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:26:39.501296+0000","last_undegraded":"2023-10-09T10:26:39.501296+0000","last_fullsized":"2023-10-09T10:26:39.501296+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":448,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669204+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:26:45.669204+0000","last_peered":"2023-10-09T10:26:45.669204+0000","last_clean":"2023-10-09T10:26:45.669204+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:26:45.669204+0000","last_undegraded":"2023-10-09T10:26:45.669204+0000","last_fullsized":"2023-10-09T10:26:45.669204+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"538'6089","reported_seq":7384,"reported_epoch":538,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:55.005289+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:26:55.005289+0000","last_peered":"2023-10-09T10:26:55.005289+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:26:55.005289+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"498'652","ondisk_log_start":"498'652","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5437,"log_dups_size":0,"ondisk_log_size":5437,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121274368,"num_objects":1850,"num_object_clones":0,"num_object_copies":5550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5168,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":581},{"shards":"2,5","objects":221}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":149,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669102+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:26:45.669102+0000","last_peered":"2023-10-09T10:26:45.669102+0000","last_clean":"2023-10-09T10:26:45.669102+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:26:45.669102+0000","last_undegraded":"2023-10-09T10:26:45.669102+0000","last_fullsized":"2023-10-09T10:26:45.669102+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":150,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.658815+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:26:45.658815+0000","last_peered":"2023-10-09T10:26:45.658815+0000","last_clean":"2023-10-09T10:26:45.658815+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:26:45.658815+0000","last_undegraded":"2023-10-09T10:26:45.658815+0000","last_fullsized":"2023-10-09T10:26:45.658815+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"539'6150","reported_seq":6276,"reported_epoch":539,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:56.608678+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:26:56.608678+0000","last_peered":"2023-10-09T10:26:56.608678+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:26:56.608678+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"498'420","ondisk_log_start":"498'420","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118292480,"num_objects":1805,"num_object_clones":0,"num_object_copies":5415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3262,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-1253},{"shards":"0,2,5","objects":1357}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":150,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.655547+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:26:45.655547+0000","last_peered":"2023-10-09T10:26:45.655547+0000","last_clean":"2023-10-09T10:26:45.655547+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:26:45.655547+0000","last_undegraded":"2023-10-09T10:26:45.655547+0000","last_fullsized":"2023-10-09T10:26:45.655547+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.656554+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:26:45.656554+0000","last_peered":"2023-10-09T10:26:45.656554+0000","last_clean":"2023-10-09T10:26:45.656554+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:26:45.656554+0000","last_undegraded":"2023-10-09T10:26:45.656554+0000","last_fullsized":"2023-10-09T10:26:45.656554+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"539'6168","reported_seq":11323,"reported_epoch":539,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:56.646123+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:26:56.646123+0000","last_peered":"2023-10-09T10:26:56.646123+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:26:56.646123+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"498'443","ondisk_log_start":"498'443","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120651776,"num_objects":1841,"num_object_clones":0,"num_object_copies":5523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1932,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5223,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1691,"num_bytes_recovered":110821376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":563},{"shards":"0,2,6","objects":806},{"shards":"0,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":132,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.655716+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:26:45.655716+0000","last_peered":"2023-10-09T10:26:45.655716+0000","last_clean":"2023-10-09T10:26:45.655716+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:26:45.655716+0000","last_undegraded":"2023-10-09T10:26:45.655716+0000","last_fullsized":"2023-10-09T10:26:45.655716+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.656527+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:26:45.656527+0000","last_peered":"2023-10-09T10:26:45.656527+0000","last_clean":"2023-10-09T10:26:45.656527+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:26:45.656527+0000","last_undegraded":"2023-10-09T10:26:45.656527+0000","last_fullsized":"2023-10-09T10:26:45.656527+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"539'6092","reported_seq":6532,"reported_epoch":539,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:56.727904+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:26:56.727904+0000","last_peered":"2023-10-09T10:26:56.727904+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:26:56.727904+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"498'331","ondisk_log_start":"498'331","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5761,"log_dups_size":0,"ondisk_log_size":5761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119406592,"num_objects":1822,"num_object_clones":0,"num_object_copies":5466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2985,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":568},{"shards":"0,3","objects":-259}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":490,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501049+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:26:39.501049+0000","last_peered":"2023-10-09T10:26:39.501049+0000","last_clean":"2023-10-09T10:26:39.501049+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:26:39.501049+0000","last_undegraded":"2023-10-09T10:26:39.501049+0000","last_fullsized":"2023-10-09T10:26:39.501049+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":553,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.658570+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:26:45.658570+0000","last_peered":"2023-10-09T10:26:45.658570+0000","last_clean":"2023-10-09T10:26:45.658570+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:26:45.658570+0000","last_undegraded":"2023-10-09T10:26:45.658570+0000","last_fullsized":"2023-10-09T10:26:45.658570+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"539'10423","reported_seq":10511,"reported_epoch":539,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:58.094701+0000","last_change":"2023-10-09T10:25:17.929569+0000","last_active":"2023-10-09T10:26:58.094701+0000","last_peered":"2023-10-09T10:26:58.094701+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:26:58.094701+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"508'4500","ondisk_log_start":"508'4500","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5923,"log_dups_size":0,"ondisk_log_size":5923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240648216,"num_objects":3673,"num_object_clones":0,"num_object_copies":11019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3670,"num_objects_misplaced":3673,"num_objects_unfound":0,"num_objects_dirty":3673,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10423,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":3673}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":260,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.676756+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:26:45.676756+0000","last_peered":"2023-10-09T10:26:45.676756+0000","last_clean":"2023-10-09T10:26:45.676756+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:26:45.676756+0000","last_undegraded":"2023-10-09T10:26:45.676756+0000","last_fullsized":"2023-10-09T10:26:45.676756+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":222,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.504681+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:26:39.504681+0000","last_peered":"2023-10-09T10:26:39.504681+0000","last_clean":"2023-10-09T10:26:39.504681+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:26:39.504681+0000","last_undegraded":"2023-10-09T10:26:39.504681+0000","last_fullsized":"2023-10-09T10:26:39.504681+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":502,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.502140+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:26:39.502140+0000","last_peered":"2023-10-09T10:26:39.502140+0000","last_clean":"2023-10-09T10:26:39.502140+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:26:39.502140+0000","last_undegraded":"2023-10-09T10:26:39.502140+0000","last_fullsized":"2023-10-09T10:26:39.502140+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"539'10563","reported_seq":10618,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T10:26:56.710146+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:26:56.710146+0000","last_peered":"2023-10-09T10:26:56.710146+0000","last_clean":"2023-10-09T10:26:56.710146+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:26:56.710146+0000","last_undegraded":"2023-10-09T10:26:56.710146+0000","last_fullsized":"2023-10-09T10:26:56.710146+0000","mapping_epoch":494,"log_start":"509'5506","ondisk_log_start":"509'5506","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5057,"log_dups_size":0,"ondisk_log_size":5057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243466240,"num_objects":3715,"num_object_clones":0,"num_object_copies":11145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10563,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":222,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501986+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:26:39.501986+0000","last_peered":"2023-10-09T10:26:39.501986+0000","last_clean":"2023-10-09T10:26:39.501986+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:26:39.501986+0000","last_undegraded":"2023-10-09T10:26:39.501986+0000","last_fullsized":"2023-10-09T10:26:39.501986+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":141,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.657450+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:26:45.657450+0000","last_peered":"2023-10-09T10:26:45.657450+0000","last_clean":"2023-10-09T10:26:45.657450+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:26:45.657450+0000","last_undegraded":"2023-10-09T10:26:45.657450+0000","last_fullsized":"2023-10-09T10:26:45.657450+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":451,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.661819+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:26:45.661819+0000","last_peered":"2023-10-09T10:26:45.661819+0000","last_clean":"2023-10-09T10:26:45.661819+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:26:45.661819+0000","last_undegraded":"2023-10-09T10:26:45.661819+0000","last_fullsized":"2023-10-09T10:26:45.661819+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"538'6130","reported_seq":11188,"reported_epoch":538,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:54.978129+0000","last_change":"2023-10-09T10:25:09.703037+0000","last_active":"2023-10-09T10:26:54.978129+0000","last_peered":"2023-10-09T10:26:54.978129+0000","last_clean":"2023-10-09T10:24:39.576381+0000","last_became_active":"2023-10-09T10:25:09.696864+0000","last_became_peered":"2023-10-09T10:25:09.696864+0000","last_unstale":"2023-10-09T10:26:54.978129+0000","last_undegraded":"2023-10-09T10:25:09.545735+0000","last_fullsized":"2023-10-09T10:25:09.430540+0000","mapping_epoch":513,"log_start":"498'661","ondisk_log_start":"498'661","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5469,"log_dups_size":0,"ondisk_log_size":5469,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120881152,"num_objects":1844,"num_object_clones":0,"num_object_copies":5532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2064,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5164,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1660,"num_bytes_recovered":108789760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,6],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":630},{"shards":"2,5,6","objects":804},{"shards":"5,6","objects":410}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":489,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.682930+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:26:45.682930+0000","last_peered":"2023-10-09T10:26:45.682930+0000","last_clean":"2023-10-09T10:26:45.682930+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:26:45.682930+0000","last_undegraded":"2023-10-09T10:26:45.682930+0000","last_fullsized":"2023-10-09T10:26:45.682930+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":107,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.672418+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:26:45.672418+0000","last_peered":"2023-10-09T10:26:45.672418+0000","last_clean":"2023-10-09T10:26:45.672418+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:26:45.672418+0000","last_undegraded":"2023-10-09T10:26:45.672418+0000","last_fullsized":"2023-10-09T10:26:45.672418+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":551,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.502384+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:26:39.502384+0000","last_peered":"2023-10-09T10:26:39.502384+0000","last_clean":"2023-10-09T10:26:39.502384+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:26:39.502384+0000","last_undegraded":"2023-10-09T10:26:39.502384+0000","last_fullsized":"2023-10-09T10:26:39.502384+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"538'10351","reported_seq":10428,"reported_epoch":538,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:26:55.029204+0000","last_change":"2023-10-09T10:26:37.693448+0000","last_active":"2023-10-09T10:26:55.029204+0000","last_peered":"2023-10-09T10:26:55.029204+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:26:55.029204+0000","last_undegraded":"2023-10-09T10:26:55.029204+0000","last_fullsized":"2023-10-09T10:26:55.029204+0000","mapping_epoch":507,"log_start":"509'5500","ondisk_log_start":"509'5500","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4851,"log_dups_size":0,"ondisk_log_size":4851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246480896,"num_objects":3761,"num_object_clones":0,"num_object_copies":11283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3761,"num_objects_unfound":0,"num_objects_dirty":3761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10351,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":3761}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":192,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.503006+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:26:39.503006+0000","last_peered":"2023-10-09T10:26:39.503006+0000","last_clean":"2023-10-09T10:26:39.503006+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:26:39.503006+0000","last_undegraded":"2023-10-09T10:26:39.503006+0000","last_fullsized":"2023-10-09T10:26:39.503006+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":732,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669720+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:26:45.669720+0000","last_peered":"2023-10-09T10:26:45.669720+0000","last_clean":"2023-10-09T10:26:45.669720+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:26:45.669720+0000","last_undegraded":"2023-10-09T10:26:45.669720+0000","last_fullsized":"2023-10-09T10:26:45.669720+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":240,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501775+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:26:39.501775+0000","last_peered":"2023-10-09T10:26:39.501775+0000","last_clean":"2023-10-09T10:26:39.501775+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:26:39.501775+0000","last_undegraded":"2023-10-09T10:26:39.501775+0000","last_fullsized":"2023-10-09T10:26:39.501775+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":150,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.656622+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:26:45.656622+0000","last_peered":"2023-10-09T10:26:45.656622+0000","last_clean":"2023-10-09T10:26:45.656622+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:26:45.656622+0000","last_undegraded":"2023-10-09T10:26:45.656622+0000","last_fullsized":"2023-10-09T10:26:45.656622+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"538'6220","reported_seq":11279,"reported_epoch":538,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:26:55.028821+0000","last_change":"2023-10-09T10:25:13.759251+0000","last_active":"2023-10-09T10:26:55.028821+0000","last_peered":"2023-10-09T10:26:55.028821+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:26:55.028821+0000","last_undegraded":"2023-10-09T10:25:13.707357+0000","last_fullsized":"2023-10-09T10:26:55.028821+0000","mapping_epoch":516,"log_start":"498'978","ondisk_log_start":"498'978","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5242,"log_dups_size":0,"ondisk_log_size":5242,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127303680,"num_objects":1942,"num_object_clones":0,"num_object_copies":5826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":629,"num_objects_misplaced":1580,"num_objects_unfound":0,"num_objects_dirty":1942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5296,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1677,"num_bytes_recovered":109903872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":629},{"shards":"3,5,6","objects":362}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":224,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.672577+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:26:45.672577+0000","last_peered":"2023-10-09T10:26:45.672577+0000","last_clean":"2023-10-09T10:26:45.672577+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:26:45.672577+0000","last_undegraded":"2023-10-09T10:26:45.672577+0000","last_fullsized":"2023-10-09T10:26:45.672577+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":483,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.683091+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:26:45.683091+0000","last_peered":"2023-10-09T10:26:45.683091+0000","last_clean":"2023-10-09T10:26:45.683091+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:26:45.683091+0000","last_undegraded":"2023-10-09T10:26:45.683091+0000","last_fullsized":"2023-10-09T10:26:45.683091+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":34,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.670998+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:26:45.670998+0000","last_peered":"2023-10-09T10:26:45.670998+0000","last_clean":"2023-10-09T10:26:45.670998+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:26:45.670998+0000","last_undegraded":"2023-10-09T10:26:45.670998+0000","last_fullsized":"2023-10-09T10:26:45.670998+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"539'10651","reported_seq":10704,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T10:26:58.104991+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:26:58.104991+0000","last_peered":"2023-10-09T10:26:58.104991+0000","last_clean":"2023-10-09T10:26:58.104991+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:26:58.104991+0000","last_undegraded":"2023-10-09T10:26:58.104991+0000","last_fullsized":"2023-10-09T10:26:58.104991+0000","mapping_epoch":494,"log_start":"508'4700","ondisk_log_start":"508'4700","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5951,"log_dups_size":0,"ondisk_log_size":5951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233373696,"num_objects":3561,"num_object_clones":0,"num_object_copies":10683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10651,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":34,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.673424+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:26:45.673424+0000","last_peered":"2023-10-09T10:26:45.673424+0000","last_clean":"2023-10-09T10:26:45.673424+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:26:45.673424+0000","last_undegraded":"2023-10-09T10:26:45.673424+0000","last_fullsized":"2023-10-09T10:26:45.673424+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":519,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501554+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:26:39.501554+0000","last_peered":"2023-10-09T10:26:39.501554+0000","last_clean":"2023-10-09T10:26:39.501554+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:26:39.501554+0000","last_undegraded":"2023-10-09T10:26:39.501554+0000","last_fullsized":"2023-10-09T10:26:39.501554+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":451,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669631+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:26:45.669631+0000","last_peered":"2023-10-09T10:26:45.669631+0000","last_clean":"2023-10-09T10:26:45.669631+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:26:45.669631+0000","last_undegraded":"2023-10-09T10:26:45.669631+0000","last_fullsized":"2023-10-09T10:26:45.669631+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"539'10187","reported_seq":10242,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T10:26:56.738736+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:26:56.738736+0000","last_peered":"2023-10-09T10:26:56.738736+0000","last_clean":"2023-10-09T10:26:56.738736+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:26:56.738736+0000","last_undegraded":"2023-10-09T10:26:56.738736+0000","last_fullsized":"2023-10-09T10:26:56.738736+0000","mapping_epoch":494,"log_start":"509'5176","ondisk_log_start":"509'5176","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5011,"log_dups_size":0,"ondisk_log_size":5011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232587264,"num_objects":3549,"num_object_clones":0,"num_object_copies":10647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10187,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":339,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669562+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:26:45.669562+0000","last_peered":"2023-10-09T10:26:45.669562+0000","last_clean":"2023-10-09T10:26:45.669562+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:26:45.669562+0000","last_undegraded":"2023-10-09T10:26:45.669562+0000","last_fullsized":"2023-10-09T10:26:45.669562+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":520,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.656439+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:26:45.656439+0000","last_peered":"2023-10-09T10:26:45.656439+0000","last_clean":"2023-10-09T10:26:45.656439+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:26:45.656439+0000","last_undegraded":"2023-10-09T10:26:45.656439+0000","last_fullsized":"2023-10-09T10:26:45.656439+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":34,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669794+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:26:45.669794+0000","last_peered":"2023-10-09T10:26:45.669794+0000","last_clean":"2023-10-09T10:26:45.669794+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:26:45.669794+0000","last_undegraded":"2023-10-09T10:26:45.669794+0000","last_fullsized":"2023-10-09T10:26:45.669794+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"539'10670","reported_seq":10723,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T10:26:58.121610+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:26:58.121610+0000","last_peered":"2023-10-09T10:26:58.121610+0000","last_clean":"2023-10-09T10:26:58.121610+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:26:58.121610+0000","last_undegraded":"2023-10-09T10:26:58.121610+0000","last_fullsized":"2023-10-09T10:26:58.121610+0000","mapping_epoch":494,"log_start":"508'4700","ondisk_log_start":"508'4700","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235536384,"num_objects":3594,"num_object_clones":0,"num_object_copies":10782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10670,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":41,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.507881+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:26:39.507881+0000","last_peered":"2023-10-09T10:26:39.507881+0000","last_clean":"2023-10-09T10:26:39.507881+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:26:39.507881+0000","last_undegraded":"2023-10-09T10:26:39.507881+0000","last_fullsized":"2023-10-09T10:26:39.507881+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":38,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.671309+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:26:45.671309+0000","last_peered":"2023-10-09T10:26:45.671309+0000","last_clean":"2023-10-09T10:26:45.671309+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:26:45.671309+0000","last_undegraded":"2023-10-09T10:26:45.671309+0000","last_fullsized":"2023-10-09T10:26:45.671309+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":218,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.502340+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:26:39.502340+0000","last_peered":"2023-10-09T10:26:39.502340+0000","last_clean":"2023-10-09T10:26:39.502340+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:26:39.502340+0000","last_undegraded":"2023-10-09T10:26:39.502340+0000","last_fullsized":"2023-10-09T10:26:39.502340+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"538'6037","reported_seq":6137,"reported_epoch":538,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:55.003981+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:26:55.003981+0000","last_peered":"2023-10-09T10:26:55.003981+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:26:55.003981+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"498'565","ondisk_log_start":"498'565","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118915072,"num_objects":1815,"num_object_clones":0,"num_object_copies":5445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5117,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":579},{"shards":"2,5","objects":-401}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":34,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.669559+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:26:45.669559+0000","last_peered":"2023-10-09T10:26:45.669559+0000","last_clean":"2023-10-09T10:26:45.669559+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:26:45.669559+0000","last_undegraded":"2023-10-09T10:26:45.669559+0000","last_fullsized":"2023-10-09T10:26:45.669559+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":43,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.676619+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:26:45.676619+0000","last_peered":"2023-10-09T10:26:45.676619+0000","last_clean":"2023-10-09T10:26:45.676619+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:26:45.676619+0000","last_undegraded":"2023-10-09T10:26:45.676619+0000","last_fullsized":"2023-10-09T10:26:45.676619+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"539'6183","reported_seq":9042,"reported_epoch":539,"state":"active+recovering+remapped","last_fresh":"2023-10-09T10:26:56.778020+0000","last_change":"2023-10-09T10:26:37.693292+0000","last_active":"2023-10-09T10:26:56.778020+0000","last_peered":"2023-10-09T10:26:56.778020+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:24:41.890721+0000","last_became_peered":"2023-10-09T10:24:41.890721+0000","last_unstale":"2023-10-09T10:26:56.778020+0000","last_undegraded":"2023-10-09T10:26:56.778020+0000","last_fullsized":"2023-10-09T10:26:56.778020+0000","mapping_epoch":507,"log_start":"498'559","ondisk_log_start":"498'559","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5624,"log_dups_size":0,"ondisk_log_size":5624,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117473280,"num_objects":1793,"num_object_clones":0,"num_object_copies":5379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":940,"num_objects_unfound":0,"num_objects_dirty":1793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5259,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1392,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":853}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":268,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.501367+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:26:39.501367+0000","last_peered":"2023-10-09T10:26:39.501367+0000","last_clean":"2023-10-09T10:26:39.501367+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:26:39.501367+0000","last_undegraded":"2023-10-09T10:26:39.501367+0000","last_fullsized":"2023-10-09T10:26:39.501367+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":132,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.657390+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:26:45.657390+0000","last_peered":"2023-10-09T10:26:45.657390+0000","last_clean":"2023-10-09T10:26:45.657390+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:26:45.657390+0000","last_undegraded":"2023-10-09T10:26:45.657390+0000","last_fullsized":"2023-10-09T10:26:45.657390+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"538'6135","reported_seq":11145,"reported_epoch":538,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:55.025840+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:26:55.025840+0000","last_peered":"2023-10-09T10:26:55.025840+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:26:55.025840+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"498'669","ondisk_log_start":"498'669","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124551168,"num_objects":1901,"num_object_clones":0,"num_object_copies":5703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2024,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5191,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1645,"num_bytes_recovered":107806720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":595},{"shards":"0,5,6","objects":833},{"shards":"5,6","objects":472}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":251,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T10:26:39.504504+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:26:39.504504+0000","last_peered":"2023-10-09T10:26:39.504504+0000","last_clean":"2023-10-09T10:26:39.504504+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:26:39.504504+0000","last_undegraded":"2023-10-09T10:26:39.504504+0000","last_fullsized":"2023-10-09T10:26:39.504504+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":362,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.676195+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:26:45.676195+0000","last_peered":"2023-10-09T10:26:45.676195+0000","last_clean":"2023-10-09T10:26:45.676195+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:26:45.676195+0000","last_undegraded":"2023-10-09T10:26:45.676195+0000","last_fullsized":"2023-10-09T10:26:45.676195+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"539'6146","reported_seq":9093,"reported_epoch":539,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:26:59.482529+0000","last_change":"2023-10-09T10:26:33.871249+0000","last_active":"2023-10-09T10:26:59.482529+0000","last_peered":"2023-10-09T10:26:59.482529+0000","last_clean":"2023-10-09T10:25:11.551777+0000","last_became_active":"2023-10-09T10:25:13.754797+0000","last_became_peered":"2023-10-09T10:25:13.754797+0000","last_unstale":"2023-10-09T10:26:59.482529+0000","last_undegraded":"2023-10-09T10:25:13.625003+0000","last_fullsized":"2023-10-09T10:25:13.604794+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":6146,"log_dups_size":0,"ondisk_log_size":6146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:58.037975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115998720,"num_objects":1770,"num_object_clones":0,"num_object_copies":5310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1274,"num_objects_misplaced":425,"num_objects_unfound":0,"num_objects_dirty":1770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3209,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1438,"num_bytes_recovered":94240768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4","objects":71},{"shards":"2,4,5","objects":425}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":37,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.685088+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:26:45.685088+0000","last_peered":"2023-10-09T10:26:45.685088+0000","last_clean":"2023-10-09T10:26:45.685088+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:26:45.685088+0000","last_undegraded":"2023-10-09T10:26:45.685088+0000","last_fullsized":"2023-10-09T10:26:45.685088+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":591,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.682748+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:26:45.682748+0000","last_peered":"2023-10-09T10:26:45.682748+0000","last_clean":"2023-10-09T10:26:45.682748+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:26:45.682748+0000","last_undegraded":"2023-10-09T10:26:45.682748+0000","last_fullsized":"2023-10-09T10:26:45.682748+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"538'6126","reported_seq":6191,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:54.984526+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:26:54.984526+0000","last_peered":"2023-10-09T10:26:54.984526+0000","last_clean":"2023-10-09T10:26:54.984526+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:26:54.984526+0000","last_undegraded":"2023-10-09T10:26:54.984526+0000","last_fullsized":"2023-10-09T10:26:54.984526+0000","mapping_epoch":501,"log_start":"498'633","ondisk_log_start":"498'633","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":5493,"log_dups_size":0,"ondisk_log_size":5493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119275520,"num_objects":1820,"num_object_clones":0,"num_object_copies":5460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2865,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":55,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.668971+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:26:45.668971+0000","last_peered":"2023-10-09T10:26:45.668971+0000","last_clean":"2023-10-09T10:26:45.668971+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:26:45.668971+0000","last_undegraded":"2023-10-09T10:26:45.668971+0000","last_fullsized":"2023-10-09T10:26:45.668971+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":159,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T10:26:45.654902+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:26:45.654902+0000","last_peered":"2023-10-09T10:26:45.654902+0000","last_clean":"2023-10-09T10:26:45.654902+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:26:45.654902+0000","last_undegraded":"2023-10-09T10:26:45.654902+0000","last_fullsized":"2023-10-09T10:26:45.654902+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":3842736152,"num_objects":58638,"num_object_clones":0,"num_object_copies":175914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29661,"num_objects_misplaced":13530,"num_objects_unfound":0,"num_objects_dirty":58638,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152308,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24638,"num_bytes_recovered":1563623424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14010294272,"data_stored":14010286128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":144982,"ondisk_log_size":144982,"up":78,"acting":65,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430687,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626915,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2203272,"kb_used_data":1291712,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":91541880,"statfs":{"total":95995035648,"available":93738885120,"internally_reserved":0,"allocated":1322713088,"data_stored":1321766713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.4330000000000001,"15min":1.6339999999999999},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":4.7990000000000004,"5min":9.4260000000000002,"15min":44.055},"last":1.871},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.591,"15min":1.8600000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":5.6079999999999997,"5min":9.2609999999999992,"15min":43.869999999999997},"last":4.4720000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":1.464,"15min":2.1499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":6.3899999999999997,"5min":6.3899999999999997,"15min":155.01400000000001},"last":1.2849999999999999},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.53,"15min":2.27},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":6.3369999999999997,"5min":7.7320000000000002,"15min":154.71100000000001},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":1.821,"15min":1.865},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":8.5069999999999997,"5min":8.6280000000000001,"15min":29.367999999999999},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.617,"15min":1.752},"min":{"1min":0.32600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.0640000000000001,"5min":7.5629999999999997,"15min":14.542999999999999},"last":1.4470000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.76,"15min":1.573},"min":{"1min":0.34599999999999997,"5min":0.32700000000000001,"15min":0.308},"max":{"1min":9.4619999999999997,"5min":9.4619999999999997,"15min":9.4619999999999997},"last":0.96599999999999997},{"interface":"front","average":{"1min":2.3130000000000002,"5min":1.946,"15min":1.6759999999999999},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.31},"max":{"1min":7.9939999999999998,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.97399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.3610000000000002,"15min":1.9430000000000001},"min":{"1min":0.30299999999999999,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":7.3289999999999997,"5min":10.618,"15min":122.041},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.478,"5min":2.04,"15min":1.7809999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.4729999999999999,"5min":11.266,"15min":122.13200000000001},"last":0.55000000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":2.4100000000000001,"15min":2.0350000000000001},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1980000000000004,"5min":12.894,"15min":121.98999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.9019999999999999,"5min":1.5960000000000001,"15min":1.6919999999999999},"min":{"1min":0.33300000000000002,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":7.0060000000000002,"5min":15.041,"15min":122.346},"last":0.57899999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":2.327,"15min":1.972},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":5.0149999999999997,"5min":49.991,"15min":130.66999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.6819999999999999,"5min":2.3999999999999999,"15min":2.0249999999999999},"min":{"1min":0.38900000000000001,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":5.0910000000000002,"5min":50.023000000000003,"15min":130.702},"last":0.63500000000000001}]}]},{"osd":1,"up_from":0,"seq":360777253229,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416753,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3552500,"kb_used_data":2598444,"kb_used_omap":28,"kb_used_meta":953763,"kb_avail":90192652,"statfs":{"total":95995035648,"available":92357275648,"internally_reserved":0,"allocated":2660806656,"data_stored":2658703676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":976654160},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:26:49 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.659,"15min":1.9350000000000001},"min":{"1min":0.26600000000000001,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":7.3609999999999998,"5min":17.161999999999999,"15min":54.518000000000001},"last":2.8780000000000001},{"interface":"front","average":{"1min":2.4390000000000001,"5min":1.48,"15min":2.2400000000000002},"min":{"1min":0.249,"5min":0.214,"15min":0.192},"max":{"1min":11.808,"5min":11.808,"15min":56.950000000000003},"last":1.1619999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:26:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":1.4970000000000001,"15min":2.1190000000000002},"min":{"1min":0.26600000000000001,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":17.093,"5min":17.093,"15min":53.988999999999997},"last":2.9209999999999998},{"interface":"front","average":{"1min":3.3370000000000002,"5min":1.8759999999999999,"15min":2.161},"min":{"1min":0.32100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.053999999999998,"5min":17.053999999999998,"15min":54.119},"last":4.1440000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.609,"15min":1.4850000000000001},"min":{"1min":0.28599999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.763,"5min":11.933,"15min":56.984999999999999},"last":0.38},{"interface":"front","average":{"1min":2.8660000000000001,"5min":1.9790000000000001,"15min":1.5640000000000001},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.067,"5min":10.067,"15min":56.475999999999999},"last":3.8730000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":1.974,"15min":2.254},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":10.941000000000001,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.7709999999999999,"5min":2.3420000000000001,"15min":2.2650000000000001},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.32400000000000001},"max":{"1min":9.9499999999999993,"5min":28.331,"15min":56.378},"last":3.7050000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":8.6020000000000003,"5min":3.4510000000000001,"15min":3.544},"min":{"1min":0.51700000000000002,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":90.412000000000006,"5min":90.412000000000006,"15min":229.589},"last":3.7480000000000002},{"interface":"front","average":{"1min":6.9400000000000004,"5min":2.7509999999999999,"15min":3.1909999999999998},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":90.393000000000001,"5min":90.393000000000001,"15min":228.88900000000001},"last":2.9449999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.919,"15min":2.29},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":54.482999999999997},"last":8.5719999999999992},{"interface":"front","average":{"1min":2.806,"5min":2.1240000000000001,"15min":2.5230000000000001},"min":{"1min":0.47899999999999998,"5min":0.39700000000000002,"15min":0.34799999999999998},"max":{"1min":8.8309999999999995,"5min":8.8309999999999995,"15min":57.753},"last":3.6309999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0470000000000002,"5min":2.2679999999999998,"15min":2.4220000000000002},"min":{"1min":0.48299999999999998,"5min":0.40500000000000003,"15min":0.34999999999999998},"max":{"1min":12.029999999999999,"5min":12.029999999999999,"15min":56.411999999999999},"last":3.8220000000000001},{"interface":"front","average":{"1min":3.0350000000000001,"5min":2.1869999999999998,"15min":2.3530000000000002},"min":{"1min":0.44800000000000001,"5min":0.38600000000000001,"15min":0.38300000000000001},"max":{"1min":17.129999999999999,"5min":17.129999999999999,"15min":54.451999999999998},"last":3}]}]},{"osd":2,"up_from":217,"seq":932007903489,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3509552,"kb_used_data":2665000,"kb_used_omap":83,"kb_used_meta":844460,"kb_avail":90235600,"statfs":{"total":95995035648,"available":92401254400,"internally_reserved":0,"allocated":2728960000,"data_stored":2727045366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":864727535},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2000000000000002,"5min":2.0579999999999998,"15min":2.0049999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.185},"max":{"1min":11.093999999999999,"5min":48.990000000000002,"15min":48.990000000000002},"last":8.4280000000000008},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.1160000000000001,"15min":2.3220000000000001},"min":{"1min":0.45600000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.2699999999999996,"5min":49.412999999999997,"15min":49.412999999999997},"last":2.1440000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":1.893,"15min":2.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":13.303000000000001,"5min":13.303000000000001,"15min":34.893999999999998},"last":5.0039999999999996},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.5740000000000001,"15min":2.0339999999999998},"min":{"1min":0.35099999999999998,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":9.9239999999999995,"5min":9.9239999999999995,"15min":34.962000000000003},"last":1.6160000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:26:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4460000000000002,"5min":1.8700000000000001,"15min":1.373},"min":{"1min":0.32000000000000001,"5min":0.25700000000000001,"15min":0.248},"max":{"1min":15.115,"5min":15.115,"15min":35.043999999999997},"last":1.9199999999999999},{"interface":"front","average":{"1min":1.923,"5min":1.482,"15min":1.361},"min":{"1min":0.33500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.7880000000000003,"5min":11.206,"15min":50.406999999999996},"last":3.0619999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:26:14 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.8149999999999999,"15min":1.6870000000000001},"min":{"1min":0.31900000000000001,"5min":0.29799999999999999,"15min":0.28999999999999998},"max":{"1min":4.4710000000000001,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.0640000000000001,"15min":1.835},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":11.298,"5min":47.704000000000001,"15min":47.704000000000001},"last":3.2200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.8340000000000001,"15min":2.012},"min":{"1min":0.40500000000000003,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":57.298000000000002},"last":3.1400000000000001},{"interface":"front","average":{"1min":2.8639999999999999,"5min":2.0430000000000001,"15min":2.0169999999999999},"min":{"1min":0.442,"5min":0.31,"15min":0.31},"max":{"1min":11.195,"5min":11.195,"15min":57.828000000000003},"last":5.0490000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":2.0590000000000002,"15min":2.0230000000000001},"min":{"1min":0.35399999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":4.5579999999999998,"5min":12.411,"15min":18.943999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.4860000000000002,"5min":1.855,"15min":1.97},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":13.975,"5min":13.975,"15min":24.379000000000001},"last":3.1829999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.1349999999999998,"15min":2.2949999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.308},"max":{"1min":5.415,"5min":11.704000000000001,"15min":35.008000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.907,"5min":1.99,"15min":2.1579999999999999},"min":{"1min":0.47099999999999997,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":15.548,"5min":15.548,"15min":24.323},"last":3.2559999999999998}]}]},{"osd":3,"up_from":352,"seq":1511828488328,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2577904,"kb_used_data":2050536,"kb_used_omap":15,"kb_used_meta":527344,"kb_avail":91167248,"statfs":{"total":95995035648,"available":93355261952,"internally_reserved":0,"allocated":2099748864,"data_stored":2097361155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":540000710},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8259999999999996,"5min":3.1200000000000001,"15min":1.696},"min":{"1min":0.34999999999999998,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":70.004000000000005,"5min":70.004000000000005,"15min":70.004000000000005},"last":4.0960000000000001},{"interface":"front","average":{"1min":6.2519999999999998,"5min":2.8159999999999998,"15min":1.6739999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":66.802000000000007,"5min":66.802000000000007,"15min":66.802000000000007},"last":3.7629999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7889999999999999,"15min":1.8620000000000001},"min":{"1min":0.311,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.785,"5min":25.370000000000001,"15min":105.401},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.7109999999999999,"5min":2.04,"15min":1.7969999999999999},"min":{"1min":0.30299999999999999,"5min":0.25900000000000001,"15min":0.23300000000000001},"max":{"1min":9.2140000000000004,"5min":14.613,"15min":104.914},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.2989999999999999,"15min":1.9490000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":11.141999999999999,"5min":11.141999999999999,"15min":11.141999999999999},"last":1.825},{"interface":"front","average":{"1min":3.1400000000000001,"5min":1.956,"15min":1.4279999999999999},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.27600000000000002},"max":{"1min":16.475000000000001,"5min":25.338999999999999,"15min":25.338999999999999},"last":3.681}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.194,"15min":1.415},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":6.9370000000000003,"5min":21.196000000000002,"15min":21.831},"last":0.59599999999999997},{"interface":"front","average":{"1min":3.3980000000000001,"5min":2.8820000000000001,"15min":1.958},"min":{"1min":0.36399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":11.087,"5min":19.323,"15min":22.097999999999999},"last":5.5259999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":2.0910000000000002,"15min":1.294},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":11.363,"5min":25.251999999999999,"15min":25.251999999999999},"last":3.6269999999999998},{"interface":"front","average":{"1min":2.46,"5min":2.411,"15min":1.5429999999999999},"min":{"1min":0.36599999999999999,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":6.9900000000000002,"5min":13.394,"15min":13.394},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.4780000000000002,"15min":1.7250000000000001},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.35599999999999998},"max":{"1min":11.114000000000001,"5min":19.114000000000001,"15min":19.114000000000001},"last":5.5640000000000001},{"interface":"front","average":{"1min":2.3279999999999998,"5min":2.1230000000000002,"15min":1.468},"min":{"1min":0.47699999999999998,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":11.32,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.46700000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":2.0779999999999998,"15min":1.514},"min":{"1min":0.42299999999999999,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":6.9640000000000004,"5min":9.7880000000000003,"15min":10.459},"last":3.7309999999999999},{"interface":"front","average":{"1min":2.327,"5min":2.3220000000000001,"15min":1.762},"min":{"1min":0.432,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":11.045999999999999,"5min":19.280000000000001,"15min":19.280000000000001},"last":1.859}]}]},{"osd":4,"up_from":266,"seq":1142461300958,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3389592,"kb_used_data":2460560,"kb_used_omap":35,"kb_used_meta":928988,"kb_avail":90355560,"statfs":{"total":95995035648,"available":92524093440,"internally_reserved":0,"allocated":2519613440,"data_stored":2518657446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":951283950},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.893,"15min":2.1469999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":6.5019999999999998,"5min":14.548,"15min":97.721999999999994},"last":4.726},{"interface":"front","average":{"1min":2.8940000000000001,"5min":2.694,"15min":2.8740000000000001},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.224},"max":{"1min":11.022,"5min":11.257,"15min":97.915999999999997},"last":11.169}]},{"osd":1,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4449999999999998,"15min":2.3700000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.24299999999999999},"max":{"1min":12.209,"5min":12.209,"15min":71.457999999999998},"last":0.499},{"interface":"front","average":{"1min":2.992,"5min":2.8999999999999999,"15min":2.2269999999999999},"min":{"1min":0.376,"5min":0.248,"15min":0.248},"max":{"1min":8.327,"5min":49.781999999999996,"15min":71.658000000000001},"last":5.133}]},{"osd":2,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3809999999999998,"15min":1.845},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":7.3230000000000004,"5min":15.944000000000001,"15min":15.944000000000001},"last":1.9099999999999999},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.9970000000000001,"15min":1.7989999999999999},"min":{"1min":0.36199999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.102,"5min":14.473000000000001,"15min":14.473000000000001},"last":1.851}]},{"osd":3,"last update":"Mon Oct 9 10:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.7410000000000001,"5min":2.9199999999999999,"15min":2.024},"min":{"1min":0.39800000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.347,"5min":12.074,"15min":12.074},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.907,"15min":1.7390000000000001},"min":{"1min":0.40899999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.369999999999999,"5min":16.067,"15min":16.067},"last":0.50900000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":2.5939999999999999,"15min":2.069},"min":{"1min":0.314,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":3.3279999999999998,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.054,"5min":2.4969999999999999,"15min":1.831},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.464,"5min":49.915999999999997,"15min":99.384},"last":0.34100000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.881,"15min":1.3740000000000001},"min":{"1min":0.33800000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.5389999999999997,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.372},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.8220000000000001,"15min":1.7949999999999999},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":12.242000000000001,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:26:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":3.0409999999999999,"15min":1.9239999999999999},"min":{"1min":0.39500000000000002,"5min":0.316,"15min":0.307},"max":{"1min":12.391999999999999,"5min":17.369,"15min":17.369},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.3320000000000001,"5min":2.7829999999999999,"15min":1.778},"min":{"1min":0.40200000000000002,"5min":0.379,"15min":0.30399999999999999},"max":{"1min":4.6689999999999996,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.45900000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823142,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4212352,"kb_used_data":2711544,"kb_used_omap":0,"kb_used_meta":1500800,"kb_avail":89532800,"statfs":{"total":95995035648,"available":91681587200,"internally_reserved":0,"allocated":2776621056,"data_stored":2775677061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536819200},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.754,"15min":1.9370000000000001},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":16.536999999999999,"5min":22.469999999999999,"15min":22.469999999999999},"last":3.589},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.1309999999999998,"15min":1.7270000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":8.9339999999999993,"5min":22.437000000000001,"15min":22.437000000000001},"last":0.501}]},{"osd":1,"last update":"Mon Oct 9 10:26:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":2.3260000000000001,"15min":2.1549999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.23999999999999999},"max":{"1min":4.375,"5min":16.562999999999999,"15min":29.370000000000001},"last":3.524},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.6499999999999999,"15min":1.8089999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":4.593,"5min":13.143000000000001,"15min":29.065000000000001},"last":0.48099999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:26:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":2.5990000000000002,"15min":2.1309999999999998},"min":{"1min":0.33000000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":12.247999999999999,"5min":16.629999999999999,"15min":16.629999999999999},"last":3.556},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.907,"15min":1.6919999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":3.1389999999999998,"5min":13.095000000000001,"15min":15.929},"last":0.97799999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.2170000000000001,"15min":1.7889999999999999},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":10.362,"5min":10.561999999999999,"15min":10.561999999999999},"last":13.614000000000001},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.0830000000000002,"15min":1.7270000000000001},"min":{"1min":0.41599999999999998,"5min":0.34200000000000003,"15min":0.315},"max":{"1min":8.9130000000000003,"5min":13.624000000000001,"15min":13.624000000000001},"last":1.0289999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":2.242,"15min":1.5760000000000001},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":10.826000000000001,"5min":12.023,"15min":12.023},"last":13.523999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":2.0369999999999999,"15min":1.399},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":8.8620000000000001,"5min":11.956,"15min":11.956},"last":0.45200000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.621,"15min":1.794},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":10.664999999999999,"5min":11.686,"15min":12.772},"last":13.567},{"interface":"front","average":{"1min":2.4500000000000002,"5min":2.1640000000000001,"15min":1.665},"min":{"1min":0.29699999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":3.4249999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:26:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.3719999999999999,"15min":1.7709999999999999},"min":{"1min":0.378,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":4.25,"5min":21.109000000000002,"15min":21.109000000000002},"last":3.371},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.1990000000000001,"15min":1.1319999999999999},"min":{"1min":0.31900000000000001,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":2.0299999999999998,"5min":11.994,"15min":11.994},"last":1.0049999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2645229568,"data_stored":2645229568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2709454848,"data_stored":2709454848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2078408704,"data_stored":2078408704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2497646592,"data_stored":2497642520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2765754368,"data_stored":2765750296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1313800192,"data_stored":1313800192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:27:05.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:27:05.457 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:05.497 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:05.558 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:05.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:05.564 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:05 smithi110 ceph-mon[136032]: pgmap v1519: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 28314/166068 objects degraded (17.050%); 12520/166068 objects misplaced (7.539%); 11 MiB/s, 269 objects/s recovering 2023-10-09T10:27:05.665 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:05.725 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:05 smithi012 ceph-mon[147926]: pgmap v1519: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 28314/166068 objects degraded (17.050%); 12520/166068 objects misplaced (7.539%); 11 MiB/s, 269 objects/s recovering 2023-10-09T10:27:05.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:05 smithi012 ceph-mon[152263]: pgmap v1519: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 28314/166068 objects degraded (17.050%); 12520/166068 objects misplaced (7.539%); 11 MiB/s, 269 objects/s recovering 2023-10-09T10:27:05.741 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:07.205 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:07.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:07.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:07.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:07.347 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:07.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[147926]: pgmap v1520: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 627 op/s; 27550/161340 objects degraded (17.076%); 11864/161340 objects misplaced (7.353%); 11 MiB/s, 254 objects/s recovering 2023-10-09T10:27:07.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[147926]: osdmap e540: 8 total, 8 up, 6 in 2023-10-09T10:27:07.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[147926]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.529 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[147926]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[152263]: pgmap v1520: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 627 op/s; 27550/161340 objects degraded (17.076%); 11864/161340 objects misplaced (7.353%); 11 MiB/s, 254 objects/s recovering 2023-10-09T10:27:07.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[152263]: osdmap e540: 8 total, 8 up, 6 in 2023-10-09T10:27:07.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[152263]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[152263]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[152263]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[152263]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[147926]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.921 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-mon[147926]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:07 smithi110 ceph-mon[136032]: pgmap v1520: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 627 op/s; 27550/161340 objects degraded (17.076%); 11864/161340 objects misplaced (7.353%); 11 MiB/s, 254 objects/s recovering 2023-10-09T10:27:07.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:07 smithi110 ceph-mon[136032]: osdmap e540: 8 total, 8 up, 6 in 2023-10-09T10:27:07.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:07 smithi110 ceph-mon[136032]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:07 smithi110 ceph-mon[136032]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:07 smithi110 ceph-mon[136032]: failed to encode map e540 with expected crc 2023-10-09T10:27:07.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:07 smithi110 ceph-mon[136032]: failed to encode map e540 with expected crc 2023-10-09T10:27:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:27:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:27:07] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.33.4" 2023-10-09T10:27:08.472 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:08.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:08.577 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:08.623 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:08.679 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:08.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:08.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:08.785 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:08.849 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:08.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:08 smithi110 ceph-mon[136032]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:08 smithi110 ceph-mon[136032]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:08 smithi110 ceph-mon[136032]: osdmap e541: 8 total, 8 up, 6 in 2023-10-09T10:27:08.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:08 smithi110 ceph-mon[136032]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:08 smithi110 ceph-mon[136032]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:08 smithi110 ceph-mon[136032]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 27550/161340 objects degraded (17.076%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:08.914 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:08.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:08.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[147926]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[147926]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[147926]: osdmap e541: 8 total, 8 up, 6 in 2023-10-09T10:27:08.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[147926]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[147926]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[147926]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 27550/161340 objects degraded (17.076%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:08.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[152263]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[152263]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[152263]: osdmap e541: 8 total, 8 up, 6 in 2023-10-09T10:27:08.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[152263]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[152263]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[152263]: failed to encode map e541 with expected crc 2023-10-09T10:27:08.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 27550/161340 objects degraded (17.076%), 14 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:08.993 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:09.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:09.110 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:09.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:09 smithi012 ceph-mon[147926]: pgmap v1523: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 27104/157482 objects degraded (17.211%); 11864/157482 objects misplaced (7.534%); 12 MiB/s, 257 objects/s recovering 2023-10-09T10:27:09.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:09 smithi012 ceph-mon[152263]: pgmap v1523: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 27104/157482 objects degraded (17.211%); 11864/157482 objects misplaced (7.534%); 12 MiB/s, 257 objects/s recovering 2023-10-09T10:27:09.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:09 smithi110 ceph-mon[136032]: pgmap v1523: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 770 op/s; 27104/157482 objects degraded (17.211%); 11864/157482 objects misplaced (7.534%); 12 MiB/s, 257 objects/s recovering 2023-10-09T10:27:09.954 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:27:09.954 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:27:09.954 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:176332 11.16 11:6b8c1f7e:::benchmark_data_smithi012_404402_object63549:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:09.880351+0000", 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.025130511000000001, 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.025159552000000002, 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 176332 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880351+0000", 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880351+0000", 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:09.956 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880350+0000", 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880357+0000", 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.464e-06 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880370+0000", 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2904e-05 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.957 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880380+0000", 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5109999999999992e-06 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880440+0000", 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0568e-05 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880606+0000", 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016582899999999999 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880750+0000", 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014387100000000001 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.883818+0000", 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003067817 2023-10-09T10:27:09.959 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:09.960 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:09.960 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:09.960 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.891052+0000", 2023-10-09T10:27:09.960 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007234674 2023-10-09T10:27:09.960 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:09.960 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:09.960 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:09.961 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:09.961 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:27:09.961 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 1 2023-10-09T10:27:09.961 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:27:10.048 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:10.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:10.064 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:27:10.105 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:27:10.144 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:27:10.154 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:10.158 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:10.178 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:27:10.206 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:10.224 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:27:10.260 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:10.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:10.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:10.269 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:27:10.317 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:27:10.358 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:27:10.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:10.378 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:27:10.378 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:27:10.378 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:27:10.378 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:27:10.378 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:27:10.378 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:27:10.379 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:27:10.379 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:27:10.446 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:10.566 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4080831142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:10.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4080831142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:10.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4080831142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:10.757 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:10.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:10.866 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:10.940 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:11.056 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:11.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:11.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:11.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:11.284 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:11.408 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:11.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:11.515 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:11.582 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:11 smithi012 ceph-mon[147926]: pgmap v1524: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 537 op/s; 26505/155136 objects degraded (17.085%); 11493/155136 objects misplaced (7.408%); 8.4 MiB/s, 178 objects/s recovering 2023-10-09T10:27:11.582 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:11 smithi012 ceph-mon[152263]: pgmap v1524: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 537 op/s; 26505/155136 objects degraded (17.085%); 11493/155136 objects misplaced (7.408%); 8.4 MiB/s, 178 objects/s recovering 2023-10-09T10:27:11.586 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:11.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:11 smithi110 ceph-mon[136032]: pgmap v1524: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 537 op/s; 26505/155136 objects degraded (17.085%); 11493/155136 objects misplaced (7.408%); 8.4 MiB/s, 178 objects/s recovering 2023-10-09T10:27:13.305 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:13.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:13.411 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:13.451 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:13.508 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:13.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:13.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:13.614 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:13.681 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:13.749 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:13.753 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:13 smithi110 ceph-mon[136032]: pgmap v1525: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 783 op/s; 23941/147249 objects degraded (16.259%); 11581/147249 objects misplaced (7.865%); 15 MiB/s, 347 objects/s recovering 2023-10-09T10:27:13.753 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 23941/147249 objects degraded (16.259%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:13.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:13.855 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:13.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:13 smithi012 ceph-mon[147926]: pgmap v1525: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 783 op/s; 23941/147249 objects degraded (16.259%); 11581/147249 objects misplaced (7.865%); 15 MiB/s, 347 objects/s recovering 2023-10-09T10:27:13.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 23941/147249 objects degraded (16.259%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:13.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:13 smithi012 ceph-mon[152263]: pgmap v1525: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 783 op/s; 23941/147249 objects degraded (16.259%); 11581/147249 objects misplaced (7.865%); 15 MiB/s, 347 objects/s recovering 2023-10-09T10:27:13.877 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 23941/147249 objects degraded (16.259%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:13.878 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:27:13.886 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:13.945 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:13.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:14.051 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:14.084 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:14.135 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:14.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:14.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:14.321 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:14.593 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:14.696 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:14.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:14.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:14.801 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:14.837 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:14.892 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:14.903 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:14.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:15.005 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:15.060 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:15.124 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:15.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:15.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:15.231 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:15.286 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:15.348 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:15.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:15.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:15.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:27:15.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:15.535 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:15.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:15 smithi012 ceph-mon[147926]: pgmap v1526: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 667 op/s; 23744/145317 objects degraded (16.339%); 11384/145317 objects misplaced (7.834%); 9.6 MiB/s, 254 objects/s recovering 2023-10-09T10:27:15.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:15 smithi012 ceph-mon[152263]: pgmap v1526: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 667 op/s; 23744/145317 objects degraded (16.339%); 11384/145317 objects misplaced (7.834%); 9.6 MiB/s, 254 objects/s recovering 2023-10-09T10:27:15.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:15 smithi110 ceph-mon[136032]: pgmap v1526: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 667 op/s; 23744/145317 objects degraded (16.339%); 11384/145317 objects misplaced (7.834%); 9.6 MiB/s, 254 objects/s recovering 2023-10-09T10:27:16.830 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:16.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:16.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:16.990 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:17.040 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:17.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:17.144 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:17.187 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:17.237 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:17.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:17.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:17.341 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:17.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:17.457 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:17.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:17.563 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:17.618 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:17.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:17 smithi012 ceph-mon[147926]: pgmap v1527: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 22717/139911 objects degraded (16.237%); 10592/139911 objects misplaced (7.571%); 13 MiB/s, 344 objects/s recovering 2023-10-09T10:27:17.918 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:27:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:27:17] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.33.4" 2023-10-09T10:27:17.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:17 smithi012 ceph-mon[152263]: pgmap v1527: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 22717/139911 objects degraded (16.237%); 10592/139911 objects misplaced (7.571%); 13 MiB/s, 344 objects/s recovering 2023-10-09T10:27:17.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:17 smithi110 ceph-mon[136032]: pgmap v1527: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 767 op/s; 22717/139911 objects degraded (16.237%); 10592/139911 objects misplaced (7.571%); 13 MiB/s, 344 objects/s recovering 2023-10-09T10:27:18.200 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:18.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:18.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:18.320 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:18.384 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:18.415 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:18.441 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:18.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:18.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:18.547 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:18.634 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:18.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 22717/139911 objects degraded (16.237%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:18.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 22717/139911 objects degraded (16.237%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:18.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 22717/139911 objects degraded (16.237%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:19.260 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:19.556 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:19.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:19.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:19.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:19.737 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:19.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:19 smithi012 ceph-mon[147926]: pgmap v1528: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 696 op/s; 22209/135921 objects degraded (16.340%); 10592/135921 objects misplaced (7.793%); 12 MiB/s, 314 objects/s recovering 2023-10-09T10:27:19.900 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:19 smithi012 ceph-mon[152263]: pgmap v1528: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 696 op/s; 22209/135921 objects degraded (16.340%); 10592/135921 objects misplaced (7.793%); 12 MiB/s, 314 objects/s recovering 2023-10-09T10:27:19.948 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:19 smithi110 ceph-mon[136032]: pgmap v1528: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 696 op/s; 22209/135921 objects degraded (16.340%); 10592/135921 objects misplaced (7.793%); 12 MiB/s, 314 objects/s recovering 2023-10-09T10:27:20.031 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:20.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:20.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:20.142 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:20.193 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:20.248 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:20.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:20.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:20.355 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:20.429 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:21.063 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:27:21.063 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 21499/133245 objects degraded (16.135%), 13 pgs degraded, 13 pgs undersized","count":26},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1958,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":542,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846541,"num_in_osds":6,"osd_in_since":1696847079,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":10},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":44415,"data_bytes":2909640632,"bytes_used":18062688256,"bytes_avail":557907525632,"bytes_total":575970213888,"degraded_objects":21499,"degraded_total":133245,"degraded_ratio":0.16134939397350745,"misplaced_objects":10355,"misplaced_total":133245,"misplaced_ratio":0.077713985515403955,"recovering_objects_per_sec":309,"recovering_bytes_per_sec":11005875,"recovering_keys_per_sec":0,"num_objects_recovered":3713,"num_bytes_recovered":132120576,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":672},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":33,"modified":"2023-10-09T10:27:11.837030+0000","services":{}},"progress_events":{"4aaa0d12-13e6-4ea3-ae4b-c27ebe383851":{"message":"Global Recovery Event (6m)\n [=======================.....] (remaining: 72s)","progress":0.83838385343551636,"add_to_ceph_s":true},"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T10:27:21.393 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:21.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:21.499 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:21.525 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:21.556 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:21.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:21.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:21.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[147926]: pgmap v1529: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 21499/133245 objects degraded (16.135%); 10355/133245 objects misplaced (7.771%); 10 MiB/s, 309 objects/s recovering 2023-10-09T10:27:21.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[147926]: osdmap e542: 8 total, 8 up, 6 in 2023-10-09T10:27:21.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[147926]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[147926]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[147926]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1014939835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2584778479' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[152263]: pgmap v1529: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 21499/133245 objects degraded (16.135%); 10355/133245 objects misplaced (7.771%); 10 MiB/s, 309 objects/s recovering 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[152263]: osdmap e542: 8 total, 8 up, 6 in 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[152263]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[152263]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[152263]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1014939835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:27:21.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:27:21.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2584778479' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:27:21.697 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:27:21.704 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:21.825 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:21.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:21.932 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:21.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:21 smithi110 ceph-mon[136032]: pgmap v1529: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 21499/133245 objects degraded (16.135%); 10355/133245 objects misplaced (7.771%); 10 MiB/s, 309 objects/s recovering 2023-10-09T10:27:21.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:21 smithi110 ceph-mon[136032]: osdmap e542: 8 total, 8 up, 6 in 2023-10-09T10:27:21.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:21 smithi110 ceph-mon[136032]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:21 smithi110 ceph-mon[136032]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:21 smithi110 ceph-mon[136032]: failed to encode map e542 with expected crc 2023-10-09T10:27:21.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1014939835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:27:21.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:21 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:27:21.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2584778479' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:27:21.995 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:22.516 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:27:22.516 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:27:22.573 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:22.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:22.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:22.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:22.739 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:22.787 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:22.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:22.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:22.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[147926]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[147926]: osdmap e543: 8 total, 8 up, 6 in 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[147926]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[147926]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[147926]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[152263]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.866 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:27:22.867 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[152263]: osdmap e543: 8 total, 8 up, 6 in 2023-10-09T10:27:22.867 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[152263]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.867 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[152263]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.867 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:22 smithi012 ceph-mon[152263]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.890 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:22.936 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:22.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:22 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:22.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:22 smithi110 ceph-mon[136032]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:27:22.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:22 smithi110 ceph-mon[136032]: osdmap e543: 8 total, 8 up, 6 in 2023-10-09T10:27:22.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:22 smithi110 ceph-mon[136032]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:22 smithi110 ceph-mon[136032]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:22 smithi110 ceph-mon[136032]: failed to encode map e543 with expected crc 2023-10-09T10:27:22.989 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:22.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:23.099 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:23.143 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:23.205 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:23.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:23.309 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:23.379 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:23.464 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:23.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:23.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:23.568 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:23.650 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:23.719 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:23 smithi012 ceph-mon[152263]: pgmap v1532: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 19549/124467 objects degraded (15.706%); 10242/124467 objects misplaced (8.229%); 17 MiB/s, 388 objects/s recovering 2023-10-09T10:27:23.719 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:23 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 19549/124467 objects degraded (15.706%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:23.720 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:23 smithi012 ceph-mon[147926]: pgmap v1532: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 19549/124467 objects degraded (15.706%); 10242/124467 objects misplaced (8.229%); 17 MiB/s, 388 objects/s recovering 2023-10-09T10:27:23.720 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:23 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 19549/124467 objects degraded (15.706%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:23.889 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:23.912 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:23.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:23.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:24.022 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:24.061 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:23 smithi110 ceph-mon[136032]: pgmap v1532: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 19549/124467 objects degraded (15.706%); 10242/124467 objects misplaced (8.229%); 17 MiB/s, 388 objects/s recovering 2023-10-09T10:27:24.061 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:23 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 19549/124467 objects degraded (15.706%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:24.064 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:24.115 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:24.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:24.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:24.156 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:27:24.156 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:27:24.220 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:24.259 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:24.303 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:24.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:24.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:24.311 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:27:24.353 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:27:24.385 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:27:24.407 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:24.434 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:27:24.451 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:24.473 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:27:24.498 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:24.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:24.510 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:27:24.565 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:27:24.602 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:27:24.602 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:24.606 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:27:24.606 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:27:24.606 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:27:24.606 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:27:24.606 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:27:24.606 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:27:24.606 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:27:24.606 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:27:24.631 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:24.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:27:24.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:27:24.692 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:24.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:24.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3281320205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:24.783 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3281320205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:24.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:24.967 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:25.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3281320205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:25.436 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:25.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:25.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:25.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:25.600 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:26.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:25 smithi012 ceph-mon[147926]: pgmap v1533: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 726 op/s; 19306/122463 objects degraded (15.765%); 9999/122463 objects misplaced (8.165%); 11 MiB/s, 242 objects/s recovering 2023-10-09T10:27:26.185 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:25 smithi012 ceph-mon[152263]: pgmap v1533: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 726 op/s; 19306/122463 objects degraded (15.765%); 9999/122463 objects misplaced (8.165%); 11 MiB/s, 242 objects/s recovering 2023-10-09T10:27:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:25 smithi110 ceph-mon[136032]: pgmap v1533: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 726 op/s; 19306/122463 objects degraded (15.765%); 9999/122463 objects misplaced (8.165%); 11 MiB/s, 242 objects/s recovering 2023-10-09T10:27:27.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:27:27.160 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:26 smithi012 ceph-mon[147926]: pgmap v1534: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 19106/118020 objects degraded (16.189%); 9746/118020 objects misplaced (8.258%); 10 MiB/s, 226 objects/s recovering 2023-10-09T10:27:27.161 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:26 smithi012 ceph-mon[152263]: pgmap v1534: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 19106/118020 objects degraded (16.189%); 9746/118020 objects misplaced (8.258%); 10 MiB/s, 226 objects/s recovering 2023-10-09T10:27:27.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:26 smithi110 ceph-mon[136032]: pgmap v1534: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 19106/118020 objects degraded (16.189%); 9746/118020 objects misplaced (8.258%); 10 MiB/s, 226 objects/s recovering 2023-10-09T10:27:27.473 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:27.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:27.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:27.579 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:27.623 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:27.656 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:27.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:27.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:27.764 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:27.809 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:27.860 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:27.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:27.916 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:27:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:27:27] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.33.4" 2023-10-09T10:27:27.963 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:28.061 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:28.388 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:28.395 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:28.431 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:28.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:28.540 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:28.626 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:29.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:28 smithi012 ceph-mon[147926]: pgmap v1535: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 827 op/s; 19000/113370 objects degraded (16.759%); 9523/113370 objects misplaced (8.400%); 12 MiB/s, 202 objects/s recovering 2023-10-09T10:27:29.181 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:28 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 19000/113370 objects degraded (16.759%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:29.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:28 smithi012 ceph-mon[152263]: pgmap v1535: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 827 op/s; 19000/113370 objects degraded (16.759%); 9523/113370 objects misplaced (8.400%); 12 MiB/s, 202 objects/s recovering 2023-10-09T10:27:29.182 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:28 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 19000/113370 objects degraded (16.759%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:29.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:28 smithi110 ceph-mon[136032]: pgmap v1535: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 827 op/s; 19000/113370 objects degraded (16.759%); 9523/113370 objects misplaced (8.400%); 12 MiB/s, 202 objects/s recovering 2023-10-09T10:27:29.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:28 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 19000/113370 objects degraded (16.759%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:29.576 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:29.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:29.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:29.684 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:29.687 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:29.791 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:29.877 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:29.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:29.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:29.993 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:30.042 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:30.096 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:30.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:30.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:30.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:27:30.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:30.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:27:30.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:30.207 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:30.252 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:27:30.264 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:27:30.288 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:30.374 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:30.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:30.445 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:27:30.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:30.479 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:30.527 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:30.581 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:30.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:30.686 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:30.733 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:30.785 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:30.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:30.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:30.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:31.013 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:31.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:31.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:31.118 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:31.189 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:31.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:30 smithi110 ceph-mon[136032]: pgmap v1536: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 18665/110820 objects degraded (16.843%); 9188/110820 objects misplaced (8.291%); 5.1 MiB/s, 85 objects/s recovering 2023-10-09T10:27:31.196 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:30 smithi110 ceph-mon[136032]: from='client.47150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:31.355 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:31.365 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:30 smithi012 ceph-mon[147926]: pgmap v1536: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 18665/110820 objects degraded (16.843%); 9188/110820 objects misplaced (8.291%); 5.1 MiB/s, 85 objects/s recovering 2023-10-09T10:27:31.366 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:30 smithi012 ceph-mon[147926]: from='client.47150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:31.366 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:30 smithi012 ceph-mon[152263]: pgmap v1536: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 18665/110820 objects degraded (16.843%); 9188/110820 objects misplaced (8.291%); 5.1 MiB/s, 85 objects/s recovering 2023-10-09T10:27:31.366 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:30 smithi012 ceph-mon[152263]: from='client.47150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:31.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:31.467 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:31.508 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:31.571 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:31.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:31.675 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:31.738 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:33.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:33 smithi012 ceph-mon[147926]: pgmap v1537: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 18665/105114 objects degraded (17.757%); 8888/105114 objects misplaced (8.456%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T10:27:33.009 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:33 smithi012 ceph-mon[152263]: pgmap v1537: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 18665/105114 objects degraded (17.757%); 8888/105114 objects misplaced (8.456%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T10:27:33.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:33 smithi110 ceph-mon[136032]: pgmap v1537: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 18665/105114 objects degraded (17.757%); 8888/105114 objects misplaced (8.456%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T10:27:34.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:34 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 18665/105114 objects degraded (17.757%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:34.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:34 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 18665/105114 objects degraded (17.757%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:34.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:34 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 18665/105114 objects degraded (17.757%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:34.511 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:34.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:34.626 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:34.666 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:34.719 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:34.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:34.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:34.902 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:35.628 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:35.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:35.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:35.680 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:35 smithi012 ceph-mon[147926]: pgmap v1538: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 18392/97479 objects degraded (18.868%); 8397/97479 objects misplaced (8.614%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:27:35.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:35 smithi012 ceph-mon[152263]: pgmap v1538: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 18392/97479 objects degraded (18.868%); 8397/97479 objects misplaced (8.614%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:27:35.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:35.788 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:35 smithi110 ceph-mon[136032]: pgmap v1538: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 18392/97479 objects degraded (18.868%); 8397/97479 objects misplaced (8.614%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:27:35.790 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:35.844 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:35.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:35.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:35.953 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:36.031 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:36.234 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:36.235 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1536,"stamp":"2023-10-09T10:27:29.846266+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2419759032,"num_objects":36940,"num_object_clones":0,"num_object_copies":110820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18665,"num_objects_misplaced":9188,"num_objects_unfound":0,"num_objects_dirty":36940,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":174290,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32191,"num_bytes_recovered":1905952508,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":134940,"ondisk_log_size":134940,"up":297,"acting":286,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16351960,"kb_used_data":10307560,"kb_used_omap":163,"kb_used_meta":6043868,"kb_avail":546118952,"statfs":{"total":575970213888,"available":559225806848,"internally_reserved":0,"allocated":10554941440,"data_stored":10545544265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":6188921843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,1,5,6,4,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-377094144,"num_objects":-5754,"num_object_clones":0,"num_object_copies":-17262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2218,"num_objects_misplaced":-315,"num_objects_unfound":0,"num_objects_dirty":-5754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5754,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":781,"num_bytes_recovered":48824320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.158247"},"pg_stats":[{"pgid":"11.12","version":"543'6925","reported_seq":7707,"reported_epoch":543,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:26.837319+0000","last_change":"2023-10-09T10:27:19.878100+0000","last_active":"2023-10-09T10:27:26.837319+0000","last_peered":"2023-10-09T10:27:26.837319+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:27:26.837319+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"502'1906","ondisk_log_start":"502'1906","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5019,"log_dups_size":0,"ondisk_log_size":5019,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77758464,"num_objects":1187,"num_object_clones":0,"num_object_copies":3561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2251,"num_objects_misplaced":516,"num_objects_unfound":0,"num_objects_dirty":1187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5977,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":352,"num_bytes_recovered":23068672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-1580},{"shards":"0,2,5","objects":516}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":136,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698720+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:27:20.698720+0000","last_peered":"2023-10-09T10:27:20.698720+0000","last_clean":"2023-10-09T10:27:20.698720+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:27:20.698720+0000","last_undegraded":"2023-10-09T10:27:20.698720+0000","last_fullsized":"2023-10-09T10:27:20.698720+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":497,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710030+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:27:20.710030+0000","last_peered":"2023-10-09T10:27:20.710030+0000","last_clean":"2023-10-09T10:27:20.710030+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:27:20.710030+0000","last_undegraded":"2023-10-09T10:27:20.710030+0000","last_fullsized":"2023-10-09T10:27:20.710030+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"543'6847","reported_seq":13725,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:26.619960+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:27:26.619960+0000","last_peered":"2023-10-09T10:27:26.619960+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:27:26.619960+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"500'1749","ondisk_log_start":"500'1749","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5098,"log_dups_size":0,"ondisk_log_size":5098,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76021760,"num_objects":1160,"num_object_clones":0,"num_object_copies":3480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":845,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5902,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,2,6","objects":738},{"shards":"0,6","objects":315}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":574,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698172+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:27:20.698172+0000","last_peered":"2023-10-09T10:27:20.698172+0000","last_clean":"2023-10-09T10:27:20.698172+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:27:20.698172+0000","last_undegraded":"2023-10-09T10:27:20.698172+0000","last_fullsized":"2023-10-09T10:27:20.698172+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":468,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696418+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:27:20.696418+0000","last_peered":"2023-10-09T10:27:20.696418+0000","last_clean":"2023-10-09T10:27:20.696418+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:27:20.696418+0000","last_undegraded":"2023-10-09T10:27:20.696418+0000","last_fullsized":"2023-10-09T10:27:20.696418+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"543'6777","reported_seq":11331,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:26.784416+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:27:26.784416+0000","last_peered":"2023-10-09T10:27:26.784416+0000","last_clean":"2023-10-09T10:27:26.784416+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:27:26.784416+0000","last_undegraded":"2023-10-09T10:27:26.784416+0000","last_fullsized":"2023-10-09T10:27:26.784416+0000","mapping_epoch":527,"log_start":"500'1681","ondisk_log_start":"500'1681","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74809344,"num_objects":1142,"num_object_clones":0,"num_object_copies":3426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5840,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":38,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696591+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:27:20.696591+0000","last_peered":"2023-10-09T10:27:20.696591+0000","last_clean":"2023-10-09T10:27:20.696591+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:27:20.696591+0000","last_undegraded":"2023-10-09T10:27:20.696591+0000","last_fullsized":"2023-10-09T10:27:20.696591+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":120,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.695872+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:27:20.695872+0000","last_peered":"2023-10-09T10:27:20.695872+0000","last_clean":"2023-10-09T10:27:20.695872+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:27:20.695872+0000","last_undegraded":"2023-10-09T10:27:20.695872+0000","last_fullsized":"2023-10-09T10:27:20.695872+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":502,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696355+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:27:20.696355+0000","last_peered":"2023-10-09T10:27:20.696355+0000","last_clean":"2023-10-09T10:27:20.696355+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:27:20.696355+0000","last_undegraded":"2023-10-09T10:27:20.696355+0000","last_fullsized":"2023-10-09T10:27:20.696355+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":451,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.689575+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:27:20.689575+0000","last_peered":"2023-10-09T10:27:20.689575+0000","last_clean":"2023-10-09T10:27:20.689575+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:27:20.689575+0000","last_undegraded":"2023-10-09T10:27:20.689575+0000","last_fullsized":"2023-10-09T10:27:20.689575+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.691731+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:27:20.691731+0000","last_peered":"2023-10-09T10:27:20.691731+0000","last_clean":"2023-10-09T10:27:20.691731+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:27:20.691731+0000","last_undegraded":"2023-10-09T10:27:20.691731+0000","last_fullsized":"2023-10-09T10:27:20.691731+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":451,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.089451+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:27:21.089451+0000","last_peered":"2023-10-09T10:27:21.089451+0000","last_clean":"2023-10-09T10:27:21.089451+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:27:21.089451+0000","last_undegraded":"2023-10-09T10:27:21.089451+0000","last_fullsized":"2023-10-09T10:27:21.089451+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":148,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710275+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:27:20.710275+0000","last_peered":"2023-10-09T10:27:20.710275+0000","last_clean":"2023-10-09T10:27:20.710275+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:27:20.710275+0000","last_undegraded":"2023-10-09T10:27:20.710275+0000","last_fullsized":"2023-10-09T10:27:20.710275+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":38,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.695486+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:27:20.695486+0000","last_peered":"2023-10-09T10:27:20.695486+0000","last_clean":"2023-10-09T10:27:20.695486+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:27:20.695486+0000","last_undegraded":"2023-10-09T10:27:20.695486+0000","last_fullsized":"2023-10-09T10:27:20.695486+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":194,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.699259+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:27:20.699259+0000","last_peered":"2023-10-09T10:27:20.699259+0000","last_clean":"2023-10-09T10:27:20.699259+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:27:20.699259+0000","last_undegraded":"2023-10-09T10:27:20.699259+0000","last_fullsized":"2023-10-09T10:27:20.699259+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":154,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.699142+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:27:20.699142+0000","last_peered":"2023-10-09T10:27:20.699142+0000","last_clean":"2023-10-09T10:27:20.699142+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:27:20.699142+0000","last_undegraded":"2023-10-09T10:27:20.699142+0000","last_fullsized":"2023-10-09T10:27:20.699142+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":114,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698991+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:27:20.698991+0000","last_peered":"2023-10-09T10:27:20.698991+0000","last_clean":"2023-10-09T10:27:20.698991+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:27:20.698991+0000","last_undegraded":"2023-10-09T10:27:20.698991+0000","last_fullsized":"2023-10-09T10:27:20.698991+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":68,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.694383+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:27:20.694383+0000","last_peered":"2023-10-09T10:27:20.694383+0000","last_clean":"2023-10-09T10:27:20.694383+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:27:20.694383+0000","last_undegraded":"2023-10-09T10:27:20.694383+0000","last_fullsized":"2023-10-09T10:27:20.694383+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":466,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.700968+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:27:20.700968+0000","last_peered":"2023-10-09T10:27:20.700968+0000","last_clean":"2023-10-09T10:27:20.700968+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:27:20.700968+0000","last_undegraded":"2023-10-09T10:27:20.700968+0000","last_fullsized":"2023-10-09T10:27:20.700968+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":209,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710373+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:27:20.710373+0000","last_peered":"2023-10-09T10:27:20.710373+0000","last_clean":"2023-10-09T10:27:20.710373+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:27:20.710373+0000","last_undegraded":"2023-10-09T10:27:20.710373+0000","last_fullsized":"2023-10-09T10:27:20.710373+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"543'6844","reported_seq":7654,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:24.997125+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:27:24.997125+0000","last_peered":"2023-10-09T10:27:24.997125+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:27:24.997125+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"503'1996","ondisk_log_start":"503'1996","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77627392,"num_objects":1185,"num_object_clones":0,"num_object_copies":3555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":1185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5921,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-1966},{"shards":"3,5,6","objects":784}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698293+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:27:20.698293+0000","last_peered":"2023-10-09T10:27:20.698293+0000","last_clean":"2023-10-09T10:27:20.698293+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:27:20.698293+0000","last_undegraded":"2023-10-09T10:27:20.698293+0000","last_fullsized":"2023-10-09T10:27:20.698293+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":38,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696036+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:27:20.696036+0000","last_peered":"2023-10-09T10:27:20.696036+0000","last_clean":"2023-10-09T10:27:20.696036+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:27:20.696036+0000","last_undegraded":"2023-10-09T10:27:20.696036+0000","last_fullsized":"2023-10-09T10:27:20.696036+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"543'6864","reported_seq":15380,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:24.965779+0000","last_change":"2023-10-09T10:26:39.522775+0000","last_active":"2023-10-09T10:27:24.965779+0000","last_peered":"2023-10-09T10:27:24.965779+0000","last_clean":"2023-10-09T10:27:24.965779+0000","last_became_active":"2023-10-09T10:26:39.509024+0000","last_became_peered":"2023-10-09T10:26:39.509024+0000","last_unstale":"2023-10-09T10:27:24.965779+0000","last_undegraded":"2023-10-09T10:27:24.965779+0000","last_fullsized":"2023-10-09T10:27:24.965779+0000","mapping_epoch":536,"log_start":"502'1997","ondisk_log_start":"502'1997","created":501,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4867,"log_dups_size":0,"ondisk_log_size":4867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78413824,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5899,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3072,"num_bytes_recovered":201326592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":48,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698189+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:27:20.698189+0000","last_peered":"2023-10-09T10:27:20.698189+0000","last_clean":"2023-10-09T10:27:20.698189+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:27:20.698189+0000","last_undegraded":"2023-10-09T10:27:20.698189+0000","last_fullsized":"2023-10-09T10:27:20.698189+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":41,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.683887+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:27:20.683887+0000","last_peered":"2023-10-09T10:27:20.683887+0000","last_clean":"2023-10-09T10:27:20.683887+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:27:20.683887+0000","last_undegraded":"2023-10-09T10:27:20.683887+0000","last_fullsized":"2023-10-09T10:27:20.683887+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"543'6966","reported_seq":7037,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:25.017313+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:27:25.017313+0000","last_peered":"2023-10-09T10:27:25.017313+0000","last_clean":"2023-10-09T10:27:25.017313+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:27:25.017313+0000","last_undegraded":"2023-10-09T10:27:25.017313+0000","last_fullsized":"2023-10-09T10:27:25.017313+0000","mapping_epoch":494,"log_start":"503'2046","ondisk_log_start":"503'2046","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":4920,"log_dups_size":0,"ondisk_log_size":4920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77987840,"num_objects":1190,"num_object_clones":0,"num_object_copies":3570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3678,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":44,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710910+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:27:20.710910+0000","last_peered":"2023-10-09T10:27:20.710910+0000","last_clean":"2023-10-09T10:27:20.710910+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:27:20.710910+0000","last_undegraded":"2023-10-09T10:27:20.710910+0000","last_fullsized":"2023-10-09T10:27:20.710910+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710368+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:27:20.710368+0000","last_peered":"2023-10-09T10:27:20.710368+0000","last_clean":"2023-10-09T10:27:20.710368+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:27:20.710368+0000","last_undegraded":"2023-10-09T10:27:20.710368+0000","last_fullsized":"2023-10-09T10:27:20.710368+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"543'6563","reported_seq":12673,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:26.829820+0000","last_change":"2023-10-09T10:26:35.543001+0000","last_active":"2023-10-09T10:27:26.829820+0000","last_peered":"2023-10-09T10:27:26.829820+0000","last_clean":"2023-10-09T10:27:26.829820+0000","last_became_active":"2023-10-09T10:26:35.529481+0000","last_became_peered":"2023-10-09T10:26:35.529481+0000","last_unstale":"2023-10-09T10:27:26.829820+0000","last_undegraded":"2023-10-09T10:27:26.829820+0000","last_fullsized":"2023-10-09T10:27:26.829820+0000","mapping_epoch":534,"log_start":"499'1265","ondisk_log_start":"499'1265","created":494,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5298,"log_dups_size":0,"ondisk_log_size":5298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:07.598234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3803,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3012,"num_bytes_recovered":174915584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":170,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.711575+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:27:20.711575+0000","last_peered":"2023-10-09T10:27:20.711575+0000","last_clean":"2023-10-09T10:27:20.711575+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:27:20.711575+0000","last_undegraded":"2023-10-09T10:27:20.711575+0000","last_fullsized":"2023-10-09T10:27:20.711575+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698359+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:27:20.698359+0000","last_peered":"2023-10-09T10:27:20.698359+0000","last_clean":"2023-10-09T10:27:20.698359+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:27:20.698359+0000","last_undegraded":"2023-10-09T10:27:20.698359+0000","last_fullsized":"2023-10-09T10:27:20.698359+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"543'6857","reported_seq":13900,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:24.965735+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:27:24.965735+0000","last_peered":"2023-10-09T10:27:24.965735+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:27:24.965735+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"502'1962","ondisk_log_start":"502'1962","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4895,"log_dups_size":0,"ondisk_log_size":4895,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75530240,"num_objects":1152,"num_object_clones":0,"num_object_copies":3456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5912,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2319,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"0,5,6","objects":752},{"shards":"5,6","objects":339}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":234,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698048+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:27:20.698048+0000","last_peered":"2023-10-09T10:27:20.698048+0000","last_clean":"2023-10-09T10:27:20.698048+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:27:20.698048+0000","last_undegraded":"2023-10-09T10:27:20.698048+0000","last_fullsized":"2023-10-09T10:27:20.698048+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":135,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.691429+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:27:20.691429+0000","last_peered":"2023-10-09T10:27:20.691429+0000","last_clean":"2023-10-09T10:27:20.691429+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:27:20.691429+0000","last_undegraded":"2023-10-09T10:27:20.691429+0000","last_fullsized":"2023-10-09T10:27:20.691429+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"543'6772","reported_seq":10968,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:24.971537+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:27:24.971537+0000","last_peered":"2023-10-09T10:27:24.971537+0000","last_clean":"2023-10-09T10:27:24.971537+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:27:24.971537+0000","last_undegraded":"2023-10-09T10:27:24.971537+0000","last_fullsized":"2023-10-09T10:27:24.971537+0000","mapping_epoch":513,"log_start":"498'326","ondisk_log_start":"498'326","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":6446,"log_dups_size":0,"ondisk_log_size":6446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78970880,"num_objects":1205,"num_object_clones":0,"num_object_copies":3615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3858,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":218,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.693764+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:27:20.693764+0000","last_peered":"2023-10-09T10:27:20.693764+0000","last_clean":"2023-10-09T10:27:20.693764+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:27:20.693764+0000","last_undegraded":"2023-10-09T10:27:20.693764+0000","last_fullsized":"2023-10-09T10:27:20.693764+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":453,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696615+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:27:20.696615+0000","last_peered":"2023-10-09T10:27:20.696615+0000","last_clean":"2023-10-09T10:27:20.696615+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:27:20.696615+0000","last_undegraded":"2023-10-09T10:27:20.696615+0000","last_fullsized":"2023-10-09T10:27:20.696615+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"543'6729","reported_seq":8029,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:25.001676+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:27:25.001676+0000","last_peered":"2023-10-09T10:27:25.001676+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:27:25.001676+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"502'1851","ondisk_log_start":"502'1851","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4878,"log_dups_size":0,"ondisk_log_size":4878,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79331328,"num_objects":1210,"num_object_clones":0,"num_object_copies":3630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5808,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,5","objects":-419}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":153,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696529+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:27:20.696529+0000","last_peered":"2023-10-09T10:27:20.696529+0000","last_clean":"2023-10-09T10:27:20.696529+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:27:20.696529+0000","last_undegraded":"2023-10-09T10:27:20.696529+0000","last_fullsized":"2023-10-09T10:27:20.696529+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":154,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698858+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:27:20.698858+0000","last_peered":"2023-10-09T10:27:20.698858+0000","last_clean":"2023-10-09T10:27:20.698858+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:27:20.698858+0000","last_undegraded":"2023-10-09T10:27:20.698858+0000","last_fullsized":"2023-10-09T10:27:20.698858+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"543'6815","reported_seq":6944,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:26.816071+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:27:26.816071+0000","last_peered":"2023-10-09T10:27:26.816071+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:27:26.816071+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"500'1723","ondisk_log_start":"500'1723","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5092,"log_dups_size":0,"ondisk_log_size":5092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74711040,"num_objects":1140,"num_object_clones":0,"num_object_copies":3420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3927,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-1918},{"shards":"0,2,5","objects":692}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":154,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698588+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:27:20.698588+0000","last_peered":"2023-10-09T10:27:20.698588+0000","last_clean":"2023-10-09T10:27:20.698588+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:27:20.698588+0000","last_undegraded":"2023-10-09T10:27:20.698588+0000","last_fullsized":"2023-10-09T10:27:20.698588+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710105+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:27:20.710105+0000","last_peered":"2023-10-09T10:27:20.710105+0000","last_clean":"2023-10-09T10:27:20.710105+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:27:20.710105+0000","last_undegraded":"2023-10-09T10:27:20.710105+0000","last_fullsized":"2023-10-09T10:27:20.710105+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"543'6840","reported_seq":13688,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:26.784929+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:27:26.784929+0000","last_peered":"2023-10-09T10:27:26.784929+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:27:26.784929+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"500'1777","ondisk_log_start":"500'1777","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5063,"log_dups_size":0,"ondisk_log_size":5063,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76611584,"num_objects":1169,"num_object_clones":0,"num_object_copies":3507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":806,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5895,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2254,"num_bytes_recovered":147718144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,2,6","objects":697},{"shards":"0,6","objects":363}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":136,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698493+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:27:20.698493+0000","last_peered":"2023-10-09T10:27:20.698493+0000","last_clean":"2023-10-09T10:27:20.698493+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:27:20.698493+0000","last_undegraded":"2023-10-09T10:27:20.698493+0000","last_fullsized":"2023-10-09T10:27:20.698493+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698931+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:27:20.698931+0000","last_peered":"2023-10-09T10:27:20.698931+0000","last_clean":"2023-10-09T10:27:20.698931+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:27:20.698931+0000","last_undegraded":"2023-10-09T10:27:20.698931+0000","last_fullsized":"2023-10-09T10:27:20.698931+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"543'6782","reported_seq":7226,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:26.796493+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:27:26.796493+0000","last_peered":"2023-10-09T10:27:26.796493+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:27:26.796493+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"500'1671","ondisk_log_start":"500'1671","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5111,"log_dups_size":0,"ondisk_log_size":5111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74186752,"num_objects":1132,"num_object_clones":0,"num_object_copies":3396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3675,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,3","objects":-949}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":495,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710297+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:27:20.710297+0000","last_peered":"2023-10-09T10:27:20.710297+0000","last_clean":"2023-10-09T10:27:20.710297+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:27:20.710297+0000","last_undegraded":"2023-10-09T10:27:20.710297+0000","last_fullsized":"2023-10-09T10:27:20.710297+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":557,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.711072+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:27:20.711072+0000","last_peered":"2023-10-09T10:27:20.711072+0000","last_clean":"2023-10-09T10:27:20.711072+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:27:20.711072+0000","last_undegraded":"2023-10-09T10:27:20.711072+0000","last_fullsized":"2023-10-09T10:27:20.711072+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"543'11908","reported_seq":12005,"reported_epoch":543,"state":"active+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:28.108385+0000","last_change":"2023-10-09T10:27:05.872079+0000","last_active":"2023-10-09T10:27:28.108385+0000","last_peered":"2023-10-09T10:27:28.108385+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:27:28.108385+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"510'6000","ondisk_log_start":"510'6000","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327256,"num_objects":2188,"num_object_clones":0,"num_object_copies":6564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2185,"num_objects_misplaced":2188,"num_objects_unfound":0,"num_objects_dirty":2188,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11908,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":2188}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":264,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696296+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:27:20.696296+0000","last_peered":"2023-10-09T10:27:20.696296+0000","last_clean":"2023-10-09T10:27:20.696296+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:27:20.696296+0000","last_undegraded":"2023-10-09T10:27:20.696296+0000","last_fullsized":"2023-10-09T10:27:20.696296+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":227,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710282+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:27:20.710282+0000","last_peered":"2023-10-09T10:27:20.710282+0000","last_clean":"2023-10-09T10:27:20.710282+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:27:20.710282+0000","last_undegraded":"2023-10-09T10:27:20.710282+0000","last_fullsized":"2023-10-09T10:27:20.710282+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":507,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.691502+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:27:20.691502+0000","last_peered":"2023-10-09T10:27:20.691502+0000","last_clean":"2023-10-09T10:27:20.691502+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:27:20.691502+0000","last_undegraded":"2023-10-09T10:27:20.691502+0000","last_fullsized":"2023-10-09T10:27:20.691502+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"543'11946","reported_seq":12005,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:26.830431+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:27:26.830431+0000","last_peered":"2023-10-09T10:27:26.830431+0000","last_clean":"2023-10-09T10:27:26.830431+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:27:26.830431+0000","last_undegraded":"2023-10-09T10:27:26.830431+0000","last_fullsized":"2023-10-09T10:27:26.830431+0000","mapping_epoch":494,"log_start":"521'6906","ondisk_log_start":"521'6906","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5040,"log_dups_size":0,"ondisk_log_size":5040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152829952,"num_objects":2332,"num_object_clones":0,"num_object_copies":6996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11946,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":227,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.691356+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:27:20.691356+0000","last_peered":"2023-10-09T10:27:20.691356+0000","last_clean":"2023-10-09T10:27:20.691356+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:27:20.691356+0000","last_undegraded":"2023-10-09T10:27:20.691356+0000","last_fullsized":"2023-10-09T10:27:20.691356+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":145,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.699141+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:27:20.699141+0000","last_peered":"2023-10-09T10:27:20.699141+0000","last_clean":"2023-10-09T10:27:20.699141+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:27:20.699141+0000","last_undegraded":"2023-10-09T10:27:20.699141+0000","last_fullsized":"2023-10-09T10:27:20.699141+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":456,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.709172+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:27:20.709172+0000","last_peered":"2023-10-09T10:27:20.709172+0000","last_clean":"2023-10-09T10:27:20.709172+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:27:20.709172+0000","last_undegraded":"2023-10-09T10:27:20.709172+0000","last_fullsized":"2023-10-09T10:27:20.709172+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"543'6822","reported_seq":15394,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:24.916026+0000","last_change":"2023-10-09T10:27:21.711478+0000","last_active":"2023-10-09T10:27:24.916026+0000","last_peered":"2023-10-09T10:27:24.916026+0000","last_clean":"2023-10-09T10:27:24.916026+0000","last_became_active":"2023-10-09T10:27:21.694370+0000","last_became_peered":"2023-10-09T10:27:21.694370+0000","last_unstale":"2023-10-09T10:27:24.916026+0000","last_undegraded":"2023-10-09T10:27:24.916026+0000","last_fullsized":"2023-10-09T10:27:24.916026+0000","mapping_epoch":542,"log_start":"502'1933","ondisk_log_start":"502'1933","created":494,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4889,"log_dups_size":0,"ondisk_log_size":4889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75530240,"num_objects":1152,"num_object_clones":0,"num_object_copies":3456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5856,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3094,"num_bytes_recovered":202768384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":493,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698400+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:27:20.698400+0000","last_peered":"2023-10-09T10:27:20.698400+0000","last_clean":"2023-10-09T10:27:20.698400+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:27:20.698400+0000","last_undegraded":"2023-10-09T10:27:20.698400+0000","last_fullsized":"2023-10-09T10:27:20.698400+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":111,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698101+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:27:20.698101+0000","last_peered":"2023-10-09T10:27:20.698101+0000","last_clean":"2023-10-09T10:27:20.698101+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:27:20.698101+0000","last_undegraded":"2023-10-09T10:27:20.698101+0000","last_fullsized":"2023-10-09T10:27:20.698101+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":556,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.692979+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:27:20.692979+0000","last_peered":"2023-10-09T10:27:20.692979+0000","last_clean":"2023-10-09T10:27:20.692979+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:27:20.692979+0000","last_undegraded":"2023-10-09T10:27:20.692979+0000","last_fullsized":"2023-10-09T10:27:20.692979+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"543'11730","reported_seq":11812,"reported_epoch":543,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:27:24.873376+0000","last_change":"2023-10-09T10:26:37.693448+0000","last_active":"2023-10-09T10:27:24.873376+0000","last_peered":"2023-10-09T10:27:24.873376+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:27:24.873376+0000","last_undegraded":"2023-10-09T10:27:24.873376+0000","last_fullsized":"2023-10-09T10:27:24.873376+0000","mapping_epoch":507,"log_start":"521'6800","ondisk_log_start":"521'6800","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4930,"log_dups_size":0,"ondisk_log_size":4930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156106752,"num_objects":2382,"num_object_clones":0,"num_object_copies":7146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2382,"num_objects_unfound":0,"num_objects_dirty":2382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11730,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2382}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":197,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.692723+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:27:20.692723+0000","last_peered":"2023-10-09T10:27:20.692723+0000","last_clean":"2023-10-09T10:27:20.692723+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:27:20.692723+0000","last_undegraded":"2023-10-09T10:27:20.692723+0000","last_fullsized":"2023-10-09T10:27:20.692723+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":736,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696411+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:27:20.696411+0000","last_peered":"2023-10-09T10:27:20.696411+0000","last_clean":"2023-10-09T10:27:20.696411+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:27:20.696411+0000","last_undegraded":"2023-10-09T10:27:20.696411+0000","last_fullsized":"2023-10-09T10:27:20.696411+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":245,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.691020+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:27:20.691020+0000","last_peered":"2023-10-09T10:27:20.691020+0000","last_clean":"2023-10-09T10:27:20.691020+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:27:20.691020+0000","last_undegraded":"2023-10-09T10:27:20.691020+0000","last_fullsized":"2023-10-09T10:27:20.691020+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":154,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710181+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:27:20.710181+0000","last_peered":"2023-10-09T10:27:20.710181+0000","last_clean":"2023-10-09T10:27:20.710181+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:27:20.710181+0000","last_undegraded":"2023-10-09T10:27:20.710181+0000","last_fullsized":"2023-10-09T10:27:20.710181+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"543'6910","reported_seq":13860,"reported_epoch":543,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:27:24.997129+0000","last_change":"2023-10-09T10:27:22.518901+0000","last_active":"2023-10-09T10:27:24.997129+0000","last_peered":"2023-10-09T10:27:24.997129+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:27:24.997129+0000","last_undegraded":"2023-10-09T10:27:24.997129+0000","last_fullsized":"2023-10-09T10:27:24.997129+0000","mapping_epoch":516,"log_start":"502'1988","ondisk_log_start":"502'1988","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4922,"log_dups_size":0,"ondisk_log_size":4922,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82083840,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":951,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5986,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2306,"num_bytes_recovered":151126016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","6"],"object_location_counts":[{"shards":"3,5,6","objects":301}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":228,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698272+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:27:20.698272+0000","last_peered":"2023-10-09T10:27:20.698272+0000","last_clean":"2023-10-09T10:27:20.698272+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:27:20.698272+0000","last_undegraded":"2023-10-09T10:27:20.698272+0000","last_fullsized":"2023-10-09T10:27:20.698272+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":487,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698584+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:27:20.698584+0000","last_peered":"2023-10-09T10:27:20.698584+0000","last_clean":"2023-10-09T10:27:20.698584+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:27:20.698584+0000","last_undegraded":"2023-10-09T10:27:20.698584+0000","last_fullsized":"2023-10-09T10:27:20.698584+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":39,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.694348+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:27:21.694348+0000","last_peered":"2023-10-09T10:27:21.694348+0000","last_clean":"2023-10-09T10:27:21.694348+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:27:21.694348+0000","last_undegraded":"2023-10-09T10:27:21.694348+0000","last_fullsized":"2023-10-09T10:27:21.694348+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"543'12022","reported_seq":12079,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:28.099939+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:27:28.099939+0000","last_peered":"2023-10-09T10:27:28.099939+0000","last_clean":"2023-10-09T10:27:28.099939+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:27:28.099939+0000","last_undegraded":"2023-10-09T10:27:28.099939+0000","last_fullsized":"2023-10-09T10:27:28.099939+0000","mapping_epoch":494,"log_start":"510'6100","ondisk_log_start":"510'6100","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143523840,"num_objects":2190,"num_object_clones":0,"num_object_copies":6570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12022,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":39,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.688849+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:27:21.688849+0000","last_peered":"2023-10-09T10:27:21.688849+0000","last_clean":"2023-10-09T10:27:21.688849+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:27:21.688849+0000","last_undegraded":"2023-10-09T10:27:21.688849+0000","last_fullsized":"2023-10-09T10:27:21.688849+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":524,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.694220+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:27:20.694220+0000","last_peered":"2023-10-09T10:27:20.694220+0000","last_clean":"2023-10-09T10:27:20.694220+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:27:20.694220+0000","last_undegraded":"2023-10-09T10:27:20.694220+0000","last_fullsized":"2023-10-09T10:27:20.694220+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":455,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696949+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:27:20.696949+0000","last_peered":"2023-10-09T10:27:20.696949+0000","last_clean":"2023-10-09T10:27:20.696949+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:27:20.696949+0000","last_undegraded":"2023-10-09T10:27:20.696949+0000","last_fullsized":"2023-10-09T10:27:20.696949+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"543'11496","reported_seq":11555,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:26.819614+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:27:26.819614+0000","last_peered":"2023-10-09T10:27:26.819614+0000","last_clean":"2023-10-09T10:27:26.819614+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:27:26.819614+0000","last_undegraded":"2023-10-09T10:27:26.819614+0000","last_fullsized":"2023-10-09T10:27:26.819614+0000","mapping_epoch":494,"log_start":"520'6476","ondisk_log_start":"520'6476","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5020,"log_dups_size":0,"ondisk_log_size":5020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146800640,"num_objects":2240,"num_object_clones":0,"num_object_copies":6720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11496,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":344,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692457+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:27:21.692457+0000","last_peered":"2023-10-09T10:27:21.692457+0000","last_clean":"2023-10-09T10:27:21.692457+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:27:21.692457+0000","last_undegraded":"2023-10-09T10:27:21.692457+0000","last_fullsized":"2023-10-09T10:27:21.692457+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":524,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.709994+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:27:20.709994+0000","last_peered":"2023-10-09T10:27:20.709994+0000","last_clean":"2023-10-09T10:27:20.709994+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:27:20.709994+0000","last_undegraded":"2023-10-09T10:27:20.709994+0000","last_fullsized":"2023-10-09T10:27:20.709994+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":39,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696497+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:27:20.696497+0000","last_peered":"2023-10-09T10:27:20.696497+0000","last_clean":"2023-10-09T10:27:20.696497+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:27:20.696497+0000","last_undegraded":"2023-10-09T10:27:20.696497+0000","last_fullsized":"2023-10-09T10:27:20.696497+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"543'12063","reported_seq":12120,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:28.088779+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:27:28.088779+0000","last_peered":"2023-10-09T10:27:28.088779+0000","last_clean":"2023-10-09T10:27:28.088779+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:27:28.088779+0000","last_undegraded":"2023-10-09T10:27:28.088779+0000","last_fullsized":"2023-10-09T10:27:28.088779+0000","mapping_epoch":494,"log_start":"510'6100","ondisk_log_start":"510'6100","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5963,"log_dups_size":0,"ondisk_log_size":5963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144244736,"num_objects":2201,"num_object_clones":0,"num_object_copies":6603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12063,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":46,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.709918+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:27:20.709918+0000","last_peered":"2023-10-09T10:27:20.709918+0000","last_clean":"2023-10-09T10:27:20.709918+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:27:20.709918+0000","last_undegraded":"2023-10-09T10:27:20.709918+0000","last_fullsized":"2023-10-09T10:27:20.709918+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":43,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.694575+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:27:21.694575+0000","last_peered":"2023-10-09T10:27:21.694575+0000","last_clean":"2023-10-09T10:27:21.694575+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:27:21.694575+0000","last_undegraded":"2023-10-09T10:27:21.694575+0000","last_fullsized":"2023-10-09T10:27:21.694575+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":223,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.694360+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:27:20.694360+0000","last_peered":"2023-10-09T10:27:20.694360+0000","last_clean":"2023-10-09T10:27:20.694360+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:27:20.694360+0000","last_undegraded":"2023-10-09T10:27:20.694360+0000","last_fullsized":"2023-10-09T10:27:20.694360+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"543'6674","reported_seq":6779,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:24.920609+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:27:24.920609+0000","last_peered":"2023-10-09T10:27:24.920609+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:27:24.920609+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"500'1785","ondisk_log_start":"500'1785","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4889,"log_dups_size":0,"ondisk_log_size":4889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77168640,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5754,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,5","objects":-1038}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":38,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.696862+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:27:20.696862+0000","last_peered":"2023-10-09T10:27:20.696862+0000","last_clean":"2023-10-09T10:27:20.696862+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:27:20.696862+0000","last_undegraded":"2023-10-09T10:27:20.696862+0000","last_fullsized":"2023-10-09T10:27:20.696862+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":47,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.695981+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:27:20.695981+0000","last_peered":"2023-10-09T10:27:20.695981+0000","last_clean":"2023-10-09T10:27:20.695981+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:27:20.695981+0000","last_undegraded":"2023-10-09T10:27:20.695981+0000","last_fullsized":"2023-10-09T10:27:20.695981+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"543'6846","reported_seq":11601,"reported_epoch":543,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:27:24.944107+0000","last_change":"2023-10-09T10:27:07.588772+0000","last_active":"2023-10-09T10:27:24.944107+0000","last_peered":"2023-10-09T10:27:24.944107+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:27:07.549958+0000","last_became_peered":"2023-10-09T10:27:07.549958+0000","last_unstale":"2023-10-09T10:27:24.944107+0000","last_undegraded":"2023-10-09T10:27:24.944107+0000","last_fullsized":"2023-10-09T10:27:24.944107+0000","mapping_epoch":540,"log_start":"498'986","ondisk_log_start":"498'986","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5860,"log_dups_size":0,"ondisk_log_size":5860,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74022912,"num_objects":1130,"num_object_clones":0,"num_object_copies":3390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5922,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2333,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":["6","0","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":1130}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":273,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.693877+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:27:20.693877+0000","last_peered":"2023-10-09T10:27:20.693877+0000","last_clean":"2023-10-09T10:27:20.693877+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:27:20.693877+0000","last_undegraded":"2023-10-09T10:27:20.693877+0000","last_fullsized":"2023-10-09T10:27:20.693877+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":136,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.699235+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:27:20.699235+0000","last_peered":"2023-10-09T10:27:20.699235+0000","last_clean":"2023-10-09T10:27:20.699235+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:27:20.699235+0000","last_undegraded":"2023-10-09T10:27:20.699235+0000","last_fullsized":"2023-10-09T10:27:20.699235+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"543'6798","reported_seq":13598,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:24.885669+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:27:24.885669+0000","last_peered":"2023-10-09T10:27:24.885669+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:27:24.885669+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"502'1950","ondisk_log_start":"502'1950","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81100800,"num_objects":1238,"num_object_clones":0,"num_object_copies":3714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":834,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5854,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2240,"num_bytes_recovered":146800640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"0,5,6","objects":765},{"shards":"5,6","objects":404}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":256,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.710115+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:27:20.710115+0000","last_peered":"2023-10-09T10:27:20.710115+0000","last_clean":"2023-10-09T10:27:20.710115+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:27:20.710115+0000","last_undegraded":"2023-10-09T10:27:20.710115+0000","last_fullsized":"2023-10-09T10:27:20.710115+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":366,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.700837+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:27:20.700837+0000","last_peered":"2023-10-09T10:27:20.700837+0000","last_clean":"2023-10-09T10:27:20.700837+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:27:20.700837+0000","last_undegraded":"2023-10-09T10:27:20.700837+0000","last_fullsized":"2023-10-09T10:27:20.700837+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"543'6799","reported_seq":13160,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:26.829701+0000","last_change":"2023-10-09T10:27:21.732594+0000","last_active":"2023-10-09T10:27:26.829701+0000","last_peered":"2023-10-09T10:27:26.829701+0000","last_clean":"2023-10-09T10:27:26.829701+0000","last_became_active":"2023-10-09T10:27:21.699893+0000","last_became_peered":"2023-10-09T10:27:21.699893+0000","last_unstale":"2023-10-09T10:27:26.829701+0000","last_undegraded":"2023-10-09T10:27:26.829701+0000","last_fullsized":"2023-10-09T10:27:26.829701+0000","mapping_epoch":542,"log_start":"500'1757","ondisk_log_start":"500'1757","created":501,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":5042,"log_dups_size":0,"ondisk_log_size":5042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:06.239031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73203712,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3862,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3137,"num_bytes_recovered":122093568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":42,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.688984+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:27:20.688984+0000","last_peered":"2023-10-09T10:27:20.688984+0000","last_clean":"2023-10-09T10:27:20.688984+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:27:20.688984+0000","last_undegraded":"2023-10-09T10:27:20.688984+0000","last_fullsized":"2023-10-09T10:27:20.688984+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":595,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698289+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:27:20.698289+0000","last_peered":"2023-10-09T10:27:20.698289+0000","last_clean":"2023-10-09T10:27:20.698289+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:27:20.698289+0000","last_undegraded":"2023-10-09T10:27:20.698289+0000","last_fullsized":"2023-10-09T10:27:20.698289+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"543'6797","reported_seq":6867,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:24.885636+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:27:24.885636+0000","last_peered":"2023-10-09T10:27:24.885636+0000","last_clean":"2023-10-09T10:27:24.885636+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:27:24.885636+0000","last_undegraded":"2023-10-09T10:27:24.885636+0000","last_fullsized":"2023-10-09T10:27:24.885636+0000","mapping_epoch":501,"log_start":"500'1915","ondisk_log_start":"500'1915","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75300864,"num_objects":1149,"num_object_clones":0,"num_object_copies":3447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3536,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":60,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.695721+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:27:20.695721+0000","last_peered":"2023-10-09T10:27:20.695721+0000","last_clean":"2023-10-09T10:27:20.695721+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:27:20.695721+0000","last_undegraded":"2023-10-09T10:27:20.695721+0000","last_fullsized":"2023-10-09T10:27:20.695721+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":163,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T10:27:20.698433+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:27:20.698433+0000","last_peered":"2023-10-09T10:27:20.698433+0000","last_clean":"2023-10-09T10:27:20.698433+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:27:20.698433+0000","last_undegraded":"2023-10-09T10:27:20.698433+0000","last_fullsized":"2023-10-09T10:27:20.698433+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":2419163160,"num_objects":36916,"num_object_clones":0,"num_object_copies":110748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18665,"num_objects_misplaced":9188,"num_objects_unfound":0,"num_objects_dirty":36916,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174030,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32031,"num_bytes_recovered":1902968832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10464141312,"data_stored":10464133168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":134746,"ondisk_log_size":134746,"up":78,"acting":67,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430693,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626921,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2286788,"kb_used_data":1090940,"kb_used_omap":0,"kb_used_meta":1195840,"kb_avail":91458364,"statfs":{"total":95995035648,"available":93653364736,"internally_reserved":0,"allocated":1117122560,"data_stored":1116156212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1224540160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.625,"15min":1.5669999999999999},"min":{"1min":0.223,"5min":0.216,"15min":0.216},"max":{"1min":6.1849999999999996,"5min":9.4260000000000002,"15min":44.055},"last":1.452},{"interface":"front","average":{"1min":2.9740000000000002,"5min":1.7130000000000001,"15min":1.9079999999999999},"min":{"1min":0.28799999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":9.3499999999999996,"5min":9.3499999999999996,"15min":43.869999999999997},"last":2.3759999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":1.464,"15min":2.1499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":6.3899999999999997,"5min":6.3899999999999997,"15min":155.01400000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.53,"15min":2.27},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":6.3369999999999997,"5min":7.7320000000000002,"15min":154.71100000000001},"last":2.3460000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.887,"15min":1.833},"min":{"1min":0.315,"5min":0.309,"15min":0.24099999999999999},"max":{"1min":5.8300000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":1.262},{"interface":"front","average":{"1min":1.853,"5min":1.667,"15min":1.643},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4379999999999997,"5min":6.4379999999999997,"15min":11.720000000000001},"last":1.353}]},{"osd":3,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":2.9470000000000001,"5min":2.0459999999999998,"15min":1.6579999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":43.393000000000001,"5min":43.393000000000001,"15min":43.393000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":3.4500000000000002,"5min":2.1829999999999998,"15min":1.806},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":43.466000000000001,"5min":43.466000000000001,"15min":43.466000000000001},"last":2.3100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.0680000000000001,"15min":1.974},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":10.772,"5min":10.772,"15min":122.041},"last":0.752},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.73,"15min":1.7769999999999999},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.1059999999999999,"5min":11.266,"15min":122.13200000000001},"last":0.49299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.27,"15min":2.1200000000000001},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.5670000000000002,"5min":9.6890000000000001,"15min":121.98999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.405,"5min":1.4950000000000001,"15min":1.673},"min":{"1min":0.40500000000000003,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":4.835,"5min":7.0060000000000002,"15min":122.346},"last":0.34599999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":2.327,"15min":1.972},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":5.0149999999999997,"5min":49.991,"15min":130.66999999999999},"last":0.38},{"interface":"front","average":{"1min":1.6819999999999999,"5min":2.3999999999999999,"15min":2.0249999999999999},"min":{"1min":0.38900000000000001,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":5.0910000000000002,"5min":50.023000000000003,"15min":130.702},"last":0.40799999999999997}]}]},{"osd":1,"up_from":0,"seq":360777253235,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416759,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2907316,"kb_used_data":1931372,"kb_used_omap":28,"kb_used_meta":975907,"kb_avail":90837836,"statfs":{"total":95995035648,"available":93017944064,"internally_reserved":0,"allocated":1977724928,"data_stored":1975590855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":999329616},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3,1,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:26:49 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.659,"15min":1.9350000000000001},"min":{"1min":0.26600000000000001,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":7.3609999999999998,"5min":17.161999999999999,"15min":54.518000000000001},"last":13.077999999999999},{"interface":"front","average":{"1min":2.4390000000000001,"5min":1.48,"15min":2.2400000000000002},"min":{"1min":0.249,"5min":0.214,"15min":0.192},"max":{"1min":11.808,"5min":11.808,"15min":56.950000000000003},"last":3.7160000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:27:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":1.845,"15min":2.0230000000000001},"min":{"1min":0.36099999999999999,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":8.859,"5min":17.093,"15min":53.988999999999997},"last":5.3230000000000004},{"interface":"front","average":{"1min":2.286,"5min":2.081,"15min":2.1150000000000002},"min":{"1min":0.29999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.3739999999999997,"5min":17.053999999999998,"15min":54.119},"last":13.021000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:27:05 2023","interfaces":[{"interface":"back","average":{"1min":3.7349999999999999,"5min":2.0649999999999999,"15min":1.655},"min":{"1min":0.248,"5min":0.217,"15min":0.217},"max":{"1min":29.835999999999999,"5min":29.835999999999999,"15min":56.984999999999999},"last":3.766},{"interface":"front","average":{"1min":4.242,"5min":2.585,"15min":1.7749999999999999},"min":{"1min":0.32900000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":29.776,"5min":29.776,"15min":56.475999999999999},"last":4.9020000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":2.077,"15min":2.1579999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":9.2859999999999996,"5min":28.289000000000001,"15min":57.719000000000001},"last":2.3860000000000001},{"interface":"front","average":{"1min":2.5920000000000001,"5min":2.4710000000000001,"15min":2.1920000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.896000000000001,"5min":28.331,"15min":56.378},"last":4.7000000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":3.5830000000000002,"15min":3.4689999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":11.867000000000001,"5min":90.412000000000006,"15min":229.589},"last":4.8280000000000003},{"interface":"front","average":{"1min":2.625,"5min":3.0270000000000001,"15min":3.194},"min":{"1min":0.35799999999999998,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":6.9539999999999997,"5min":90.393000000000001,"15min":228.88900000000001},"last":1.694}]},{"osd":6,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":2.0640000000000001,"15min":2.2690000000000001},"min":{"1min":0.41399999999999998,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":8.5719999999999992,"5min":9.7840000000000007,"15min":54.482999999999997},"last":1.7310000000000001},{"interface":"front","average":{"1min":2.7989999999999999,"5min":2.2450000000000001,"15min":2.4049999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.34799999999999998},"max":{"1min":9.3439999999999994,"5min":9.3439999999999994,"15min":57.753},"last":2.4169999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.2749999999999999,"15min":2.2959999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.34999999999999998},"max":{"1min":5.6959999999999997,"5min":12.029999999999999,"15min":56.411999999999999},"last":4.8650000000000002},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.335,"15min":2.3410000000000002},"min":{"1min":0.47199999999999998,"5min":0.39900000000000002,"15min":0.38300000000000001},"max":{"1min":7.0490000000000004,"5min":17.129999999999999,"15min":54.451999999999998},"last":1.7529999999999999}]}]},{"osd":2,"up_from":217,"seq":932007903495,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2859244,"kb_used_data":1992292,"kb_used_omap":83,"kb_used_meta":866604,"kb_avail":90885908,"statfs":{"total":95995035648,"available":93067169792,"internally_reserved":0,"allocated":2040107008,"data_stored":2038165046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":887402991},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.5939999999999999,"15min":2.008},"min":{"1min":0.22900000000000001,"5min":0.192,"15min":0.185},"max":{"1min":14.666,"5min":48.990000000000002,"15min":48.990000000000002},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.8340000000000001,"5min":2.363,"15min":2.1379999999999999},"min":{"1min":0.23599999999999999,"5min":0.22600000000000001,"15min":0.20899999999999999},"max":{"1min":13.773,"5min":49.412999999999997,"15min":49.412999999999997},"last":0.57699999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":2.343,"15min":2.0230000000000001},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":14.632999999999999,"5min":14.632999999999999,"15min":34.893999999999998},"last":2.1240000000000001},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.7689999999999999,"15min":1.823},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":13.356999999999999,"5min":13.356999999999999,"15min":34.962000000000003},"last":2.254}]},{"osd":3,"last update":"Mon Oct 9 10:26:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4460000000000002,"5min":1.8700000000000001,"15min":1.373},"min":{"1min":0.32000000000000001,"5min":0.25700000000000001,"15min":0.248},"max":{"1min":15.115,"5min":15.115,"15min":35.043999999999997},"last":0.441},{"interface":"front","average":{"1min":1.923,"5min":1.482,"15min":1.361},"min":{"1min":0.33500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.7880000000000003,"5min":11.206,"15min":50.406999999999996},"last":2.1659999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:27:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":1.899,"15min":1.6220000000000001},"min":{"1min":0.35899999999999999,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":8.1859999999999999,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.7639999999999998,"5min":2.2309999999999999,"15min":1.913},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":8.7219999999999995,"5min":47.704000000000001,"15min":47.704000000000001},"last":0.38700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":1.8260000000000001,"15min":2.0169999999999999},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":8.6839999999999993,"5min":11.266999999999999,"15min":57.298000000000002},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.8860000000000001,"5min":2.4199999999999999,"15min":2.1400000000000001},"min":{"1min":0.375,"5min":0.31,"15min":0.31},"max":{"1min":14.31,"5min":14.31,"15min":57.828000000000003},"last":0.64900000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":2.0310000000000001,"15min":1.9299999999999999},"min":{"1min":0.40300000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":7.9390000000000001,"5min":12.411,"15min":18.943999999999999},"last":0.503},{"interface":"front","average":{"1min":2.556,"5min":1.95,"15min":1.948},"min":{"1min":0.39300000000000002,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":8.7590000000000003,"5min":13.975,"15min":24.379000000000001},"last":0.42499999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.95,"5min":2.1139999999999999,"15min":2.1920000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":9.5199999999999996,"5min":11.704000000000001,"15min":35.008000000000003},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.1230000000000002,"15min":2.121},"min":{"1min":0.51400000000000001,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":8.8149999999999995,"5min":15.548,"15min":24.323},"last":0.46200000000000002}]}]},{"osd":3,"up_from":352,"seq":1511828488334,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2091568,"kb_used_data":1524008,"kb_used_omap":15,"kb_used_meta":567536,"kb_avail":91653584,"statfs":{"total":95995035648,"available":93853270016,"internally_reserved":0,"allocated":1560584192,"data_stored":1558165390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":581157318},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.9740000000000002,"5min":3.4220000000000002,"15min":1.8859999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":13.661,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.56100000000000005},{"interface":"front","average":{"1min":3.9849999999999999,"5min":3.1499999999999999,"15min":1.8779999999999999},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":10.313000000000001,"5min":66.802000000000007,"15min":66.802000000000007},"last":6.1589999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":1.8160000000000001,"15min":1.9490000000000001},"min":{"1min":0.42499999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.3100000000000005,"5min":25.370000000000001,"15min":105.401},"last":1.9810000000000001},{"interface":"front","average":{"1min":2.6800000000000002,"5min":2.0430000000000001,"15min":1.9299999999999999},"min":{"1min":0.28399999999999997,"5min":0.25900000000000001,"15min":0.23300000000000001},"max":{"1min":10.422000000000001,"5min":14.613,"15min":104.914},"last":6.2460000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2250000000000001,"5min":2.266,"15min":2.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":13.497999999999999,"5min":13.497999999999999,"15min":13.497999999999999},"last":2.226},{"interface":"front","average":{"1min":3.4689999999999999,"5min":2.2549999999999999,"15min":1.5780000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27600000000000002},"max":{"1min":10.339,"5min":25.338999999999999,"15min":25.338999999999999},"last":7.2409999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":2.016,"15min":1.5169999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.268000000000001,"5min":21.196000000000002,"15min":21.831},"last":5.9580000000000002},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.927,"15min":2.1179999999999999},"min":{"1min":0.34499999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":13.577999999999999,"5min":19.323,"15min":22.097999999999999},"last":2.2570000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5259999999999998,"5min":2.0609999999999999,"15min":1.4159999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":8.2710000000000008,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.9319999999999999,"5min":2.3639999999999999,"15min":1.6910000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.388,"5min":10.388,"15min":13.394},"last":6.0430000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":2.262,"15min":1.845},"min":{"1min":0.375,"5min":0.36199999999999999,"15min":0.35599999999999998},"max":{"1min":13.545999999999999,"5min":19.114000000000001,"15min":19.114000000000001},"last":2.3199999999999998},{"interface":"front","average":{"1min":2.758,"5min":2.0979999999999999,"15min":1.593},"min":{"1min":0.37,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":10.117000000000001,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.371}]},{"osd":7,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.1589999999999998,"15min":1.643},"min":{"1min":0.41299999999999998,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":10.345000000000001,"5min":10.345000000000001,"15min":10.459},"last":6.0030000000000001},{"interface":"front","average":{"1min":2.9060000000000001,"5min":2.3239999999999998,"15min":1.8660000000000001},"min":{"1min":0.41799999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.234,"5min":19.280000000000001,"15min":19.280000000000001},"last":2.3010000000000002}]}]},{"osd":4,"up_from":266,"seq":1142461300964,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2683400,"kb_used_data":1750208,"kb_used_omap":35,"kb_used_meta":933084,"kb_avail":91061752,"statfs":{"total":95995035648,"available":93247234048,"internally_reserved":0,"allocated":1792212992,"data_stored":1791240970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":955478254},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":8.9870000000000001,"5min":3.1680000000000001,"15min":2.6859999999999999},"min":{"1min":0.245,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":109.193,"5min":109.193,"15min":109.193},"last":0.91400000000000003},{"interface":"front","average":{"1min":8.6099999999999994,"5min":3.548,"15min":3.2469999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.224},"max":{"1min":110.438,"5min":110.438,"15min":110.438},"last":0.32400000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":2.1040000000000001,"15min":2.2709999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.24299999999999999},"max":{"1min":10.696,"5min":12.209,"15min":71.457999999999998},"last":3.2909999999999999},{"interface":"front","average":{"1min":2.2799999999999998,"5min":2.7090000000000001,"15min":2.2330000000000001},"min":{"1min":0.32700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":9.2040000000000006,"5min":49.781999999999996,"15min":71.658000000000001},"last":3.3450000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.141,"15min":1.8160000000000001},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":5.4710000000000001,"5min":15.944000000000001,"15min":15.944000000000001},"last":4.8179999999999996},{"interface":"front","average":{"1min":1.419,"5min":1.7430000000000001,"15min":1.766},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.0309999999999997,"5min":14.473000000000001,"15min":14.473000000000001},"last":3.1629999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:26:59 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":2.552,"15min":2.0659999999999998},"min":{"1min":0.40999999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.257,"5min":11.347,"15min":12.074},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.5600000000000001,"5min":2.786,"15min":1.839},"min":{"1min":0.50900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.1270000000000007,"5min":16.067,"15min":16.067},"last":2.4420000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.0369999999999999,"15min":2.0510000000000002},"min":{"1min":0.28000000000000003,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":3.8290000000000002,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.629},{"interface":"front","average":{"1min":1.0409999999999999,"5min":2.0619999999999998,"15min":1.871},"min":{"1min":0.317,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":5.9509999999999996,"5min":49.915999999999997,"15min":99.384},"last":2.3140000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.526,"15min":1.4079999999999999},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.0579999999999998,"5min":14.499000000000001,"15min":14.499000000000001},"last":3.379},{"interface":"front","average":{"1min":1.8600000000000001,"5min":2.2930000000000001,"15min":1.804},"min":{"1min":0.34000000000000002,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":10.824,"5min":18.952999999999999,"15min":18.952999999999999},"last":3.7810000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":2.4569999999999999,"15min":1.913},"min":{"1min":0.39400000000000002,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":9.016,"5min":12.391999999999999,"15min":17.369},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.2350000000000001,"5min":2.431,"15min":1.778},"min":{"1min":0.37,"5min":0.37,"15min":0.314},"max":{"1min":3.9750000000000001,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.60099999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823148,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3523644,"kb_used_data":2018740,"kb_used_omap":0,"kb_used_meta":1504896,"kb_avail":90221508,"statfs":{"total":95995035648,"available":92386824192,"internally_reserved":0,"allocated":2067189760,"data_stored":2066225792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541013504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":2.9260000000000002,"15min":2.0979999999999999},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":19.102,"5min":19.102,"15min":22.469999999999999},"last":3.782},{"interface":"front","average":{"1min":2.7850000000000001,"5min":1.784,"15min":1.7270000000000001},"min":{"1min":0.253,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":17.887,"5min":17.887,"15min":22.437000000000001},"last":2.069}]},{"osd":1,"last update":"Mon Oct 9 10:26:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":2.3260000000000001,"15min":2.1549999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.23999999999999999},"max":{"1min":4.375,"5min":16.562999999999999,"15min":29.370000000000001},"last":10.425000000000001},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.6499999999999999,"15min":1.8089999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":4.593,"5min":13.143000000000001,"15min":29.065000000000001},"last":0.35499999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:27:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":2.5369999999999999,"15min":2.0710000000000002},"min":{"1min":0.311,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":7.8849999999999998,"5min":16.629999999999999,"15min":16.629999999999999},"last":1.0509999999999999},{"interface":"front","average":{"1min":2.7999999999999998,"5min":1.9399999999999999,"15min":1.774},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":7.1479999999999997,"5min":9.6929999999999996,"15min":15.929},"last":0.96999999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.8109999999999999,"5min":2.3940000000000001,"15min":1.9450000000000001},"min":{"1min":0.377,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":13.614000000000001,"5min":13.614000000000001,"15min":13.614000000000001},"last":2.5910000000000002},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.8660000000000001,"15min":1.7869999999999999},"min":{"1min":0.38400000000000001,"5min":0.35099999999999998,"15min":0.315},"max":{"1min":11.124000000000001,"5min":11.124000000000001,"15min":13.624000000000001},"last":2.5830000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.3679999999999999,"15min":1.6890000000000001},"min":{"1min":0.378,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":13.523999999999999,"5min":13.523999999999999,"15min":13.523999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.7130000000000001,"15min":1.4419999999999999},"min":{"1min":0.29399999999999998,"5min":0.28199999999999997,"15min":0.23599999999999999},"max":{"1min":4.1219999999999999,"5min":10.153,"15min":11.956},"last":0.32000000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.4430000000000001,"15min":1.8799999999999999},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":13.567,"5min":13.567,"15min":13.567},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.02,"15min":1.724},"min":{"1min":0.34499999999999997,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":7.2999999999999998,"5min":16.497,"15min":16.497},"last":0.34300000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:26:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.3719999999999999,"15min":1.7709999999999999},"min":{"1min":0.378,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":4.25,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.1990000000000001,"15min":1.1319999999999999},"min":{"1min":0.31900000000000001,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":2.0299999999999998,"5min":11.994,"15min":11.994},"last":0.371}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1956708352,"data_stored":1956708352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2017263616,"data_stored":2017263616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1549533184,"data_stored":1549533184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1786318848,"data_stored":1786314776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2056916992,"data_stored":2056912920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1097400320,"data_stored":1097400320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:27:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:36.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:36.343 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:36.385 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:36.452 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:36.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:36.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:36.561 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:36.594 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:36.648 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:36.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:36.759 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:36.804 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:36.850 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:36.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:36.961 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:36.996 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:37.048 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:37.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:37.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(9.1f 398/377 [PushOp(9:ff99a8f0:::benchmark_data_smithi012_313365_object80195:head, version: 333'5053, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:ff99a8f0:::benchmark_data_smithi012_313365_object80195:head@333'5053, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 548.04495921399996, 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071748132000000006, 2023-10-09T10:27:37.100 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050421+0000", 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.101 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050420+0000", 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050484+0000", 2023-10-09T10:27:37.102 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3660999999999999e-05 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050499+0000", 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4959e-05 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050510+0000", 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1165999999999999e-05 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.103 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050573+0000", 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2452000000000004e-05 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.050587+0000", 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4083000000000001e-05 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:18:29.122169+0000", 2023-10-09T10:27:37.104 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071582693000000003 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53350 10.5 e433/424 10:a3334060:::benchmark_data_smithi012_366312_object53349:head v 433'3357, mlcod=433'3355)", 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.338586+0000", 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 440.75679413500001, 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.104754867, 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.105 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338586+0000", 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338586+0000", 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.106 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338585+0000", 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338614+0000", 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8943999999999999e-05 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.108 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338629+0000", 2023-10-09T10:27:37.108 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4309e-05 2023-10-09T10:27:37.108 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.108 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.108 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338638+0000", 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4679999999999998e-06 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338684+0000", 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6298000000000003e-05 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.109 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.338704+0000", 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9813000000000002e-05 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.342706+0000", 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0040014919999999997 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443341+0000", 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10063549400000001 2023-10-09T10:27:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53352 10.e e433/424 10:732e479f:::benchmark_data_smithi012_366312_object53351:head v 433'3315, mlcod=433'3312)", 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.340487+0000", 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 440.75489343300001, 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.10286616599999999, 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.111 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340487+0000", 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340487+0000", 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.112 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340486+0000", 2023-10-09T10:27:37.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:27:37.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.113 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.113 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340529+0000", 2023-10-09T10:27:37.113 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.3291000000000003e-05 2023-10-09T10:27:37.113 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.113 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340540+0000", 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0995999999999999e-05 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340548+0000", 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4379999999999998e-06 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.114 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340591+0000", 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2740000000000001e-05 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340612+0000", 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0996e-05 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.342863+0000", 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0022512690000000002 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443353+0000", 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.100490447 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.116 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.117 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.117 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53356 10.5 e433/424 10:aeaac023:::benchmark_data_smithi012_366312_object53355:head v 433'3358, mlcod=433'3355)", 2023-10-09T10:27:37.117 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.340839+0000", 2023-10-09T10:27:37.117 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 440.75454185799998, 2023-10-09T10:27:37.117 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.107630108, 2023-10-09T10:27:37.117 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.117 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.118 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.118 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.118 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.118 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340839+0000", 2023-10-09T10:27:37.118 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.119 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.119 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.119 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.119 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340839+0000", 2023-10-09T10:27:37.120 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.120 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.120 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.120 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.120 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340838+0000", 2023-10-09T10:27:37.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:27:37.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.121 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.121 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340862+0000", 2023-10-09T10:27:37.121 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3927e-05 2023-10-09T10:27:37.121 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.121 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340872+0000", 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0509999999999999e-05 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340879+0000", 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.4110000000000001e-06 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.122 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340939+0000", 2023-10-09T10:27:37.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0615000000000002e-05 2023-10-09T10:27:37.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.123 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.123 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.123 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.340958+0000", 2023-10-09T10:27:37.123 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8502999999999999e-05 2023-10-09T10:27:37.123 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.448424+0000", 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.107465777 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.448469+0000", 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5042e-05 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.124 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53363 10.5 e433/424 10:a71fae04:::benchmark_data_smithi012_366312_object53362:head v 433'3359, mlcod=433'3355)", 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.343610+0000", 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 440.75177037600002, 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.104896063, 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.125 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.343610+0000", 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.343610+0000", 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.126 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.127 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.127 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.343609+0000", 2023-10-09T10:27:37.127 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:27:37.127 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.127 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.128 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.343634+0000", 2023-10-09T10:27:37.128 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5088000000000001e-05 2023-10-09T10:27:37.128 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.128 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.128 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.129 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443297+0000", 2023-10-09T10:27:37.129 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.099662744999999997 2023-10-09T10:27:37.129 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.129 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.129 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443322+0000", 2023-10-09T10:27:37.130 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5011000000000002e-05 2023-10-09T10:27:37.130 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.130 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.130 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.130 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443361+0000", 2023-10-09T10:27:37.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9066999999999997e-05 2023-10-09T10:27:37.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.131 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.131 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.131 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443378+0000", 2023-10-09T10:27:37.131 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6393e-05 2023-10-09T10:27:37.131 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.132 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.132 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.448478+0000", 2023-10-09T10:27:37.132 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051004739999999998 2023-10-09T10:27:37.132 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.132 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.133 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.133 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.448506+0000", 2023-10-09T10:27:37.133 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8220000000000001e-05 2023-10-09T10:27:37.133 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.133 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.133 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.134 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.134 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.134 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:53361 10.e e433/424 10:7ba25ff8:::benchmark_data_smithi012_366312_object53360:head v 433'3316, mlcod=433'3313)", 2023-10-09T10:27:37.134 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:20:16.344867+0000", 2023-10-09T10:27:37.134 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 440.75051386400003, 2023-10-09T10:27:37.134 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.102124111, 2023-10-09T10:27:37.135 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.135 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.135 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.135 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.135 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.135 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.344867+0000", 2023-10-09T10:27:37.135 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.135 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.136 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.344867+0000", 2023-10-09T10:27:37.136 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.136 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.136 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.136 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.344866+0000", 2023-10-09T10:27:37.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:27:37.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.137 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.137 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.137 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.344918+0000", 2023-10-09T10:27:37.137 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2333000000000002e-05 2023-10-09T10:27:37.137 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.138 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443328+0000", 2023-10-09T10:27:37.138 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.098409900999999994 2023-10-09T10:27:37.138 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.138 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.138 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443338+0000", 2023-10-09T10:27:37.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0236e-05 2023-10-09T10:27:37.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.139 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.139 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.139 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443375+0000", 2023-10-09T10:27:37.139 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6925000000000003e-05 2023-10-09T10:27:37.139 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.142 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.142 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.142 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.443390+0000", 2023-10-09T10:27:37.142 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4415e-05 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.446946+0000", 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0035561339999999999 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:20:16.446991+0000", 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.4795000000000001e-05 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.143 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:182365 10.14 e474/463 10:29b20442:::benchmark_data_smithi012_366312_object71036:head v 474'9809, mlcod=474'9807)", 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:22:47.945625+0000", 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 289.14975543899999, 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.084676611999999998, 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.144 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.145 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.145 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.145 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945625+0000", 2023-10-09T10:27:37.145 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.145 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945625+0000", 2023-10-09T10:27:37.146 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.146 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.146 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.146 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.146 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945625+0000", 2023-10-09T10:27:37.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:27:37.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.147 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.147 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945629+0000", 2023-10-09T10:27:37.147 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8299999999999998e-06 2023-10-09T10:27:37.147 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.147 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945635+0000", 2023-10-09T10:27:37.148 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8930000000000002e-06 2023-10-09T10:27:37.148 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.148 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.148 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.148 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945639+0000", 2023-10-09T10:27:37.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2610000000000004e-06 2023-10-09T10:27:37.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.149 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.149 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945677+0000", 2023-10-09T10:27:37.149 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.8506e-05 2023-10-09T10:27:37.149 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.149 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.945692+0000", 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4191e-05 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.030267+0000", 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.084575633999999997 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.150 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.030302+0000", 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4527999999999999e-05 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:182367 10.a e474/467 10:5ed41c5c:::benchmark_data_smithi012_366312_object71038:head v 474'11310, mlcod=474'11307)", 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:22:47.946616+0000", 2023-10-09T10:27:37.151 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 289.14876477500002, 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.091614227000000006, 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946616+0000", 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946616+0000", 2023-10-09T10:27:37.152 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946615+0000", 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946621+0000", 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4580000000000001e-06 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.153 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.154 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946629+0000", 2023-10-09T10:27:37.154 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.4889999999999995e-06 2023-10-09T10:27:37.154 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.154 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.154 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.946635+0000", 2023-10-09T10:27:37.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4609999999999999e-06 2023-10-09T10:27:37.155 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.155 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.155 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.155 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.032469+0000", 2023-10-09T10:27:37.155 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.085834405000000003 2023-10-09T10:27:37.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.156 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.156 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.032515+0000", 2023-10-09T10:27:37.156 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6402000000000001e-05 2023-10-09T10:27:37.156 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.156 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.038199+0000", 2023-10-09T10:27:37.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0056834420000000004 2023-10-09T10:27:37.157 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.157 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.157 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.157 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.038230+0000", 2023-10-09T10:27:37.157 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1109000000000003e-05 2023-10-09T10:27:37.158 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.158 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.158 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.158 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.158 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.158 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:182369 10.e e474/467 10:7c2770ab:::benchmark_data_smithi012_366312_object71040:head v 474'11217, mlcod=474'11214)", 2023-10-09T10:27:37.158 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:22:47.949829+0000", 2023-10-09T10:27:37.159 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 289.14555152100002, 2023-10-09T10:27:37.159 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.089488281000000003, 2023-10-09T10:27:37.159 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.159 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.159 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.159 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.159 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.159 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.949829+0000", 2023-10-09T10:27:37.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.160 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.160 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.160 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.949829+0000", 2023-10-09T10:27:37.160 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.160 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.949828+0000", 2023-10-09T10:27:37.161 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:27:37.161 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.161 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.161 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.161 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:47.949834+0000", 2023-10-09T10:27:37.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0109999999999997e-06 2023-10-09T10:27:37.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.162 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.162 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.162 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.029649+0000", 2023-10-09T10:27:37.162 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.079814211999999995 2023-10-09T10:27:37.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.029665+0000", 2023-10-09T10:27:37.163 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6007000000000001e-05 2023-10-09T10:27:37.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.163 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.163 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.163 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.032286+0000", 2023-10-09T10:27:37.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026212729999999999 2023-10-09T10:27:37.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.164 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.164 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.164 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.032318+0000", 2023-10-09T10:27:37.164 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2382000000000003e-05 2023-10-09T10:27:37.164 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.164 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.039276+0000", 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0069573680000000002 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:22:48.039317+0000", 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1610000000000003e-05 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.165 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:107693 11.16 11:6cd9ccbf:::benchmark_data_smithi012_404402_object107692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e521)", 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:23.668146+0000", 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 133.42723456100001, 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071013134000000006, 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.166 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 107693 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668146+0000", 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.167 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668146+0000", 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668144+0000", 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999976 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668170+0000", 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6064999999999999e-05 2023-10-09T10:27:37.168 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668188+0000", 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8023999999999999e-05 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668197+0000", 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6929999999999996e-06 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.169 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.170 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.670231+0000", 2023-10-09T10:27:37.170 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0020343610000000002 2023-10-09T10:27:37.170 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.170 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.170 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.670428+0000", 2023-10-09T10:27:37.171 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000196603 2023-10-09T10:27:37.171 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.171 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.171 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.171 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.670610+0000", 2023-10-09T10:27:37.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018279400000000001 2023-10-09T10:27:37.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.172 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.172 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.172 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.675440+0000", 2023-10-09T10:27:37.172 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0048299859999999997 2023-10-09T10:27:37.172 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.678163+0000", 2023-10-09T10:27:37.173 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027225050000000001 2023-10-09T10:27:37.173 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.173 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.173 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.173 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.737783+0000", 2023-10-09T10:27:37.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.059619802999999999 2023-10-09T10:27:37.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.174 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.174 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.737832+0000", 2023-10-09T10:27:37.174 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9431999999999997e-05 2023-10-09T10:27:37.174 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.174 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.175 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.175 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.739159+0000", 2023-10-09T10:27:37.175 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0013270199999999999 2023-10-09T10:27:37.175 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.175 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.175 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.175 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.175 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.176 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:107694 11.16 11:6816a903:::benchmark_data_smithi012_404402_object107693:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e521)", 2023-10-09T10:27:37.176 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:23.668425+0000", 2023-10-09T10:27:37.176 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 133.42695572, 2023-10-09T10:27:37.176 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.072028426000000007, 2023-10-09T10:27:37.176 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.177 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.177 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.177 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.177 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.177 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 107694 2023-10-09T10:27:37.177 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.177 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.178 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668425+0000", 2023-10-09T10:27:37.178 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.178 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.178 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.178 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668425+0000", 2023-10-09T10:27:37.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.179 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.179 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.179 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668424+0000", 2023-10-09T10:27:37.179 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:27:37.179 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.180 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668447+0000", 2023-10-09T10:27:37.180 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.3204999999999999e-05 2023-10-09T10:27:37.180 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.180 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.180 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668460+0000", 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3256e-05 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.668469+0000", 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.8419999999999994e-06 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.181 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.672670+0000", 2023-10-09T10:27:37.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0042014319999999997 2023-10-09T10:27:37.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.182 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.182 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.182 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.672837+0000", 2023-10-09T10:27:37.182 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000166708 2023-10-09T10:27:37.182 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.673023+0000", 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00018611999999999999 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.675591+0000", 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025681319999999999 2023-10-09T10:27:37.183 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.678195+0000", 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026036570000000001 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.739301+0000", 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.061105916000000003 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.184 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.739377+0000", 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6205999999999995e-05 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:23.740453+0000", 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001076037 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.185 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:125509 11.6 11:62aa1ea7:::benchmark_data_smithi012_404402_object12726:head [delete] snapc 0=[] ondisk+write+known_if_redirected e528)", 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:50.500876+0000", 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 106.594505011, 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.097691501, 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 125509 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.186 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.500876+0000", 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.500876+0000", 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.187 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.500874+0000", 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.500880+0000", 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9619999999999998e-06 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.500891+0000", 2023-10-09T10:27:37.188 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0798999999999999e-05 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.500898+0000", 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.418e-06 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.506485+0000", 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055863179999999998 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.189 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.506628+0000", 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014316900000000001 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.506770+0000", 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014204699999999999 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.190 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.509725+0000", 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029547419999999998 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.581564+0000", 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071839175000000005 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.596778+0000", 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.015214442999999999 2023-10-09T10:27:37.191 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.596841+0000", 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2446999999999994e-05 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.598567+0000", 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001726432 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.192 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:125513 11.6 11:671a12e2:::benchmark_data_smithi012_404402_object12730:head [delete] snapc 0=[] ondisk+write+known_if_redirected e528)", 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:50.505553+0000", 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 106.589827837, 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.095736897000000001, 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 125513 2023-10-09T10:27:37.193 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.505553+0000", 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.505553+0000", 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.194 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.505551+0000", 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.505558+0000", 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.2140000000000001e-06 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.195 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.505570+0000", 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2077e-05 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.505578+0000", 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.738e-06 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.508559+0000", 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029807729999999999 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.196 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.508665+0000", 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000106051 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.508752+0000", 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6627999999999998e-05 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.197 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.509822+0000", 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001069696 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.581605+0000", 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071783817 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.599567+0000", 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.017961261999999999 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.198 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.599618+0000", 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1496000000000003e-05 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.601290+0000", 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0016713590000000001 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.199 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:125514 11.16 11:6dacbd51:::benchmark_data_smithi012_404402_object12731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e528)", 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:50.510689+0000", 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 106.584691467, 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.089446647000000004, 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 125514 2023-10-09T10:27:37.200 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.510689+0000", 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.510689+0000", 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.201 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.510688+0000", 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.510694+0000", 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5809999999999996e-06 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.510703+0000", 2023-10-09T10:27:37.202 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2490000000000007e-06 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.510709+0000", 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.9279999999999998e-06 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.510761+0000", 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1916000000000002e-05 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.203 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.510898+0000", 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000137083 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.511022+0000", 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00012440199999999999 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.513532+0000", 2023-10-09T10:27:37.204 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0025101849999999998 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.581647+0000", 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.068114479000000006 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.598623+0000", 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.016976602 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.205 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.598682+0000", 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8593e-05 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.600136+0000", 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.001453724 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.206 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:125516 11.a e528/507 11:5d281d60:::benchmark_data_smithi012_404402_object12733:head v 528'7868, mlcod=528'7865)", 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:25:50.514301+0000", 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 106.581079769, 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071105642999999996, 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.514301+0000", 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.207 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.514301+0000", 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.514300+0000", 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.514308+0000", 2023-10-09T10:27:37.208 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.4569999999999999e-06 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.514326+0000", 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8261999999999999e-05 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.514331+0000", 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3569999999999997e-06 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.209 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.514364+0000", 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3141e-05 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.514379+0000", 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5112e-05 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.585360+0000", 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.070980469000000004 2023-10-09T10:27:37.210 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:25:50.585406+0000", 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6536000000000003e-05 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:137460 11.16 11:690cf47c:::benchmark_data_smithi012_404402_object24677:head [delete] snapc 0=[] ondisk+write+known_if_redirected e530)", 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:09.092619+0000", 2023-10-09T10:27:37.211 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 88.002761371000005, 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.100663435, 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 137460 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.212 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.092619+0000", 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.092619+0000", 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.092617+0000", 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999971 2023-10-09T10:27:37.213 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.092630+0000", 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.3467e-05 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.092657+0000", 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6999999999999999e-05 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.214 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.092672+0000", 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5503e-05 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.092780+0000", 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010794300000000001 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.093013+0000", 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00023251200000000001 2023-10-09T10:27:37.215 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.093119+0000", 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000105956 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.094998+0000", 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0018793919999999999 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.216 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.096706+0000", 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0017077500000000001 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.191762+0000", 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.095056404999999997 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.191818+0000", 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5461999999999999e-05 2023-10-09T10:27:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:09.193283+0000", 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0014646920000000001 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:166112 11.16 11:6e646d2f:::benchmark_data_smithi012_404402_object53329:head [delete] snapc 0=[] ondisk+write+known_if_redirected e539)", 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:55.323850+0000", 2023-10-09T10:27:37.218 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 41.771531011, 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.080601053000000006, 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 166112 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.219 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.323850+0000", 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.323850+0000", 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.323848+0000", 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:27:37.220 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.323857+0000", 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.431e-06 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.323878+0000", 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1163e-05 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.221 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.323888+0000", 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.2599999999999994e-06 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.323942+0000", 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.4082000000000002e-05 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.324064+0000", 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.000121922 2023-10-09T10:27:37.222 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.324159+0000", 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5174999999999996e-05 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.326891+0000", 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0027324049999999998 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.223 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.326963+0000", 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2033000000000006e-05 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.403474+0000", 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.076510992 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.403545+0000", 2023-10-09T10:27:37.224 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.0727000000000002e-05 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:55.404451+0000", 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00090557300000000001 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:168212 11.16 11:6c7d9752:::benchmark_data_smithi012_404402_object55429:head [delete] snapc 0=[] ondisk+write+known_if_redirected e539)", 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:58.261100+0000", 2023-10-09T10:27:37.225 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 38.834280620000001, 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.095123202000000004, 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 168212 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.226 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261100+0000", 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261100+0000", 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261099+0000", 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:27:37.227 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261104+0000", 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.631e-06 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261114+0000", 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7110000000000007e-06 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.228 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261122+0000", 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.6820000000000003e-06 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261171+0000", 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9384e-05 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261281+0000", 2023-10-09T10:27:37.229 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00010980300000000001 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.261380+0000", 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9102000000000001e-05 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.263578+0000", 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021981520000000001 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.230 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.264097+0000", 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00051894599999999995 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.355264+0000", 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.091167309000000002 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.355330+0000", 2023-10-09T10:27:37.231 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5983999999999997e-05 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:58.356223+0000", 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00089281099999999997 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:176180 11.4 e541/513 11:22df2737:::benchmark_data_smithi012_404402_object63397:head v 541'6412, mlcod=541'6410)", 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:09.582216+0000", 2023-10-09T10:27:37.232 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.513164956000001, 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.105806393, 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.582216+0000", 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.233 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.582216+0000", 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.582215+0000", 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.582219+0000", 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1699999999999999e-06 2023-10-09T10:27:37.234 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.582227+0000", 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7870000000000002e-06 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.582232+0000", 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5889999999999996e-06 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.235 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.582260+0000", 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8309999999999998e-05 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.582273+0000", 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2911999999999999e-05 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.584419+0000", 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021465849999999999 2023-10-09T10:27:37.236 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.688022+0000", 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.103602509 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_op(client.36870.0:176332 11.16 11:6b8c1f7e:::benchmark_data_smithi012_404402_object63549:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:09.880351+0000", 2023-10-09T10:27:37.237 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 27.215029172000001, 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.15106297899999999, 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "client_info": { 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "client": "client.36870", 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "tid": 176332 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880351+0000", 2023-10-09T10:27:37.238 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880351+0000", 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880350+0000", 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999981 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.239 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880357+0000", 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.464e-06 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880370+0000", 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2904e-05 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880380+0000", 2023-10-09T10:27:37.240 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.5109999999999992e-06 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880440+0000", 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.0568e-05 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880606+0000", 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00016582899999999999 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.241 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.880750+0000", 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00014387100000000001 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "op_commit", 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.883818+0000", 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.003067817 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.891052+0000", 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007234674 2023-10-09T10:27:37.242 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:10.030929+0000", 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.13987674 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:10.030988+0000", 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8872000000000001e-05 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:37.243 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:27:37.244 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:10.031414+0000", 2023-10-09T10:27:37.244 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00042640899999999999 2023-10-09T10:27:37.244 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.244 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.244 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.244 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:37.244 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:37.244 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:27:37.244 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:37.282 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:37.329 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:37.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:37.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:37.434 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:37.472 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:37.474 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:27:37.516 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:27:37.536 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:37.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:37 smithi110 ceph-mon[136032]: pgmap v1539: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 18392/91236 objects degraded (20.159%); 8111/91236 objects misplaced (8.890%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:27:37.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3858201254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:37.540 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:37 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3425321617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:27:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:37.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:37.554 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:27:37.566 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:27:37.589 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:27:37.624 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:27:37.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:37.666 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:27:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:37 smithi012 ceph-mon[152263]: pgmap v1539: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 18392/91236 objects degraded (20.159%); 8111/91236 objects misplaced (8.890%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:27:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3858201254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:37.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:37 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3425321617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:27:37.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:37 smithi012 ceph-mon[147926]: pgmap v1539: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 18392/91236 objects degraded (20.159%); 8111/91236 objects misplaced (8.890%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T10:27:37.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3858201254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:37.679 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:37 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3425321617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:27:37.708 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:37.744 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:27:37.744 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:27:37.745 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:27:37.745 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:27:37.745 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:27:37.745 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:27:37.745 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:27:37.745 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:27:37.746 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:27:37.746 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:27:38.179 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:27:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:27:37] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.33.4" 2023-10-09T10:27:38.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3425321617' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[147926]: osdmap e544: 8 total, 8 up, 6 in 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[147926]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[147926]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[147926]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 18392/91236 objects degraded (20.159%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3425321617' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[152263]: osdmap e544: 8 total, 8 up, 6 in 2023-10-09T10:27:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[152263]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[152263]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[152263]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:38 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 18392/91236 objects degraded (20.159%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:38 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3425321617' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:27:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:38 smithi110 ceph-mon[136032]: osdmap e544: 8 total, 8 up, 6 in 2023-10-09T10:27:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:38 smithi110 ceph-mon[136032]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:38 smithi110 ceph-mon[136032]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:38 smithi110 ceph-mon[136032]: failed to encode map e544 with expected crc 2023-10-09T10:27:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:38 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 18392/91236 objects degraded (20.159%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:39.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:27:39.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:39 smithi012 ceph-mon[147926]: pgmap v1541: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 911 op/s; 18392/86007 objects degraded (21.384%); 7873/86007 objects misplaced (9.154%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:39.664 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:39 smithi012 ceph-mon[152263]: pgmap v1541: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 911 op/s; 18392/86007 objects degraded (21.384%); 7873/86007 objects misplaced (9.154%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:39.706 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:27:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:39 smithi110 ceph-mon[136032]: pgmap v1541: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 911 op/s; 18392/86007 objects degraded (21.384%); 7873/86007 objects misplaced (9.154%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:40.484 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:27:41.649 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:41 smithi110 ceph-mon[136032]: pgmap v1542: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 907 op/s; 18116/83577 objects degraded (21.676%); 7596/83577 objects misplaced (9.089%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:41 smithi012 ceph-mon[147926]: pgmap v1542: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 907 op/s; 18116/83577 objects degraded (21.676%); 7596/83577 objects misplaced (9.089%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:41.680 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:41 smithi012 ceph-mon[152263]: pgmap v1542: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 907 op/s; 18116/83577 objects degraded (21.676%); 7596/83577 objects misplaced (9.089%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:42.072 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:27:42.072 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:27:42.241 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:42.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:42.353 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:42.411 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:42.479 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:42.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:42.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:42.489 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:27:42.590 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:42.671 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:42.944 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:42.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:42.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:43.055 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:43.105 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:43.172 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:43.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:43.250 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:27:43.254 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:43.280 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:43.333 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:43.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:43.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:43.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:27:43.506 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:43.560 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:43.620 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:43.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:43.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:43 smithi110 ceph-mon[136032]: pgmap v1543: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 902 op/s; 18116/78003 objects degraded (23.225%); 7328/78003 objects misplaced (9.395%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:43.695 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:43 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 18116/78003 objects degraded (23.225%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:43.734 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:43.805 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:43 smithi012 ceph-mon[152263]: pgmap v1543: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 902 op/s; 18116/78003 objects degraded (23.225%); 7328/78003 objects misplaced (9.395%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:43.805 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:43 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 18116/78003 objects degraded (23.225%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:43.809 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:43.809 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:43 smithi012 ceph-mon[147926]: pgmap v1543: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 902 op/s; 18116/78003 objects degraded (23.225%); 7328/78003 objects misplaced (9.395%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:27:43.810 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:43 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 18116/78003 objects degraded (23.225%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:44.163 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:44.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:44.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:44.270 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:44.316 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:44.370 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:44.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:44.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:44.481 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:44.526 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:44.577 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:44.646 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:44.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:44.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:44.751 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:44.762 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:27:44.762 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:27:44.824 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:44.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:44.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:44.999 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:45.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:45.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:45.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:27:45.112 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:45.195 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:45.251 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:27:45.281 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:45.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:45.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:45.394 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:45.450 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:45.521 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:45.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:45.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:45.576 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:45 smithi012 ceph-mon[147926]: pgmap v1544: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 906 op/s; 17632/70266 objects degraded (25.093%); 6983/70266 objects misplaced (9.938%); 1.9 MiB/s, 49 objects/s recovering 2023-10-09T10:27:45.627 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:45.673 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:45.725 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:45.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:45.732 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:45.833 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:45.903 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:45 smithi012 ceph-mon[152263]: pgmap v1544: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 906 op/s; 17632/70266 objects degraded (25.093%); 6983/70266 objects misplaced (9.938%); 1.9 MiB/s, 49 objects/s recovering 2023-10-09T10:27:45.916 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:45.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:45 smithi110 ceph-mon[136032]: pgmap v1544: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 906 op/s; 17632/70266 objects degraded (25.093%); 6983/70266 objects misplaced (9.938%); 1.9 MiB/s, 49 objects/s recovering 2023-10-09T10:27:46.011 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:27:46.837 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:46.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:46.951 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:47.016 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:47.137 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:47.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:47.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:47.170 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:47.246 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:47.327 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:47.499 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:47.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:47.585 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:47 smithi110 ceph-mon[136032]: pgmap v1545: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 898 op/s; 17632/64248 objects degraded (27.444%); 6689/64248 objects misplaced (10.411%); 1.9 MiB/s, 49 objects/s recovering 2023-10-09T10:27:47.604 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:47.690 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:47.776 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:47.778 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:27:47.778 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:27:47.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:47.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:47.834 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:47 smithi012 ceph-mon[152263]: pgmap v1545: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 898 op/s; 17632/64248 objects degraded (27.444%); 6689/64248 objects misplaced (10.411%); 1.9 MiB/s, 49 objects/s recovering 2023-10-09T10:27:47.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:47 smithi012 ceph-mon[147926]: pgmap v1545: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 898 op/s; 17632/64248 objects degraded (27.444%); 6689/64248 objects misplaced (10.411%); 1.9 MiB/s, 49 objects/s recovering 2023-10-09T10:27:47.884 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:47.966 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:48.155 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:48.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:48.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:48.173 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:27:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:27:47] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.33.4" 2023-10-09T10:27:48.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:27:48.272 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:48.385 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:48.751 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:48 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17632/64248 objects degraded (27.444%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:48.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:48 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17632/64248 objects degraded (27.444%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:48.852 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:48.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:48.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:48.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:48 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17632/64248 objects degraded (27.444%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:48.959 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:49.069 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:49.256 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:49.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:49.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:49.366 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:49.408 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:49.451 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:49.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:49.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:49.555 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:49.559 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:27:49.576 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:27:49.601 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:49.646 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:49.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:49.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:49.651 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:49 smithi012 ceph-mon[147926]: pgmap v1546: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 859 op/s; 17340/59346 objects degraded (29.218%); 6689/59346 objects misplaced (11.271%); 372 KiB/s, 53 objects/s recovering 2023-10-09T10:27:49.651 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:49 smithi012 ceph-mon[152263]: pgmap v1546: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 859 op/s; 17340/59346 objects degraded (29.218%); 6689/59346 objects misplaced (11.271%); 372 KiB/s, 53 objects/s recovering 2023-10-09T10:27:49.658 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:49.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:49 smithi110 ceph-mon[136032]: pgmap v1546: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 859 op/s; 17340/59346 objects degraded (29.218%); 6689/59346 objects misplaced (11.271%); 372 KiB/s, 53 objects/s recovering 2023-10-09T10:27:49.750 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:49.837 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:49.966 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:27:49.992 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:49.999 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:27:50.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:50.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:50.046 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:27:50.081 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:27:50.109 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:50.118 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:27:50.146 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:50.165 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:27:50.196 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:50.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:50.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:50.220 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:27:50.277 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:27:50.285 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:27:50.286 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:27:50.286 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:27:50.286 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:27:50.286 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:27:50.286 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:27:50.286 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:27:50.286 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:27:50.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:50.396 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:50.494 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:50.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:50.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:50.553 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1546,"stamp":"2023-10-09T10:27:47.861027+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1295292344,"num_objects":19782,"num_object_clones":0,"num_object_copies":59346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17340,"num_objects_misplaced":6689,"num_objects_unfound":0,"num_objects_dirty":19782,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":191448,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33200,"num_bytes_recovered":1939769084,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133741,"ondisk_log_size":133741,"up":297,"acting":286,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13660264,"kb_used_data":7239864,"kb_used_omap":163,"kb_used_meta":6420060,"kb_avail":548810648,"statfs":{"total":575970213888,"available":561982103552,"internally_reserved":0,"allocated":7413620736,"data_stored":7404191757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":6574142451},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-582418432,"num_objects":-8887,"num_object_clones":0,"num_object_copies":-26661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1052,"num_objects_misplaced":-1184,"num_objects_unfound":0,"num_objects_dirty":-8887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8887,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":553,"num_bytes_recovered":3932160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.335361"},"pg_stats":[{"pgid":"11.12","version":"544'7514","reported_seq":10314,"reported_epoch":544,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:46.874005+0000","last_change":"2023-10-09T10:27:19.878100+0000","last_active":"2023-10-09T10:27:46.874005+0000","last_peered":"2023-10-09T10:27:46.874005+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:27:46.874005+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"503'2506","ondisk_log_start":"503'2506","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5008,"log_dups_size":0,"ondisk_log_size":5008,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39157760,"num_objects":598,"num_object_clones":0,"num_object_copies":1794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1758,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6566,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1361,"num_bytes_recovered":56885248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-1160}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":137,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.698595+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:27:21.698595+0000","last_peered":"2023-10-09T10:27:21.698595+0000","last_clean":"2023-10-09T10:27:21.698595+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:27:21.698595+0000","last_undegraded":"2023-10-09T10:27:21.698595+0000","last_fullsized":"2023-10-09T10:27:21.698595+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":498,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699423+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:27:21.699423+0000","last_peered":"2023-10-09T10:27:21.699423+0000","last_clean":"2023-10-09T10:27:21.699423+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:27:21.699423+0000","last_undegraded":"2023-10-09T10:27:21.699423+0000","last_fullsized":"2023-10-09T10:27:21.699423+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"544'7425","reported_seq":14304,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:46.740367+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:27:46.740367+0000","last_peered":"2023-10-09T10:27:46.740367+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:27:46.740367+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"503'2399","ondisk_log_start":"503'2399","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5026,"log_dups_size":0,"ondisk_log_size":5026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38141952,"num_objects":582,"num_object_clones":0,"num_object_copies":1746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":845,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6480,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,2,6","objects":160},{"shards":"0,6","objects":-263}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":575,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.698619+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:27:21.698619+0000","last_peered":"2023-10-09T10:27:21.698619+0000","last_clean":"2023-10-09T10:27:21.698619+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:27:21.698619+0000","last_undegraded":"2023-10-09T10:27:21.698619+0000","last_fullsized":"2023-10-09T10:27:21.698619+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":469,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.694730+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:27:21.694730+0000","last_peered":"2023-10-09T10:27:21.694730+0000","last_clean":"2023-10-09T10:27:21.694730+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:27:21.694730+0000","last_undegraded":"2023-10-09T10:27:21.694730+0000","last_fullsized":"2023-10-09T10:27:21.694730+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"544'7322","reported_seq":11877,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:46.843065+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:27:46.843065+0000","last_peered":"2023-10-09T10:27:46.843065+0000","last_clean":"2023-10-09T10:27:46.843065+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:27:46.843065+0000","last_undegraded":"2023-10-09T10:27:46.843065+0000","last_fullsized":"2023-10-09T10:27:46.843065+0000","mapping_epoch":527,"log_start":"503'2305","ondisk_log_start":"503'2305","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5017,"log_dups_size":0,"ondisk_log_size":5017,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39092224,"num_objects":597,"num_object_clones":0,"num_object_copies":1791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6385,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":40,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:36.875783+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:27:36.875783+0000","last_peered":"2023-10-09T10:27:36.875783+0000","last_clean":"2023-10-09T10:27:36.875783+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:27:36.875783+0000","last_undegraded":"2023-10-09T10:27:36.875783+0000","last_fullsized":"2023-10-09T10:27:36.875783+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":121,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.693869+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:27:21.693869+0000","last_peered":"2023-10-09T10:27:21.693869+0000","last_clean":"2023-10-09T10:27:21.693869+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:27:21.693869+0000","last_undegraded":"2023-10-09T10:27:21.693869+0000","last_fullsized":"2023-10-09T10:27:21.693869+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":503,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692602+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:27:21.692602+0000","last_peered":"2023-10-09T10:27:21.692602+0000","last_clean":"2023-10-09T10:27:21.692602+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:27:21.692602+0000","last_undegraded":"2023-10-09T10:27:21.692602+0000","last_fullsized":"2023-10-09T10:27:21.692602+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":452,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.685737+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:27:21.685737+0000","last_peered":"2023-10-09T10:27:21.685737+0000","last_clean":"2023-10-09T10:27:21.685737+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:27:21.685737+0000","last_undegraded":"2023-10-09T10:27:21.685737+0000","last_fullsized":"2023-10-09T10:27:21.685737+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692205+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:27:21.692205+0000","last_peered":"2023-10-09T10:27:21.692205+0000","last_clean":"2023-10-09T10:27:21.692205+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:27:21.692205+0000","last_undegraded":"2023-10-09T10:27:21.692205+0000","last_fullsized":"2023-10-09T10:27:21.692205+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":452,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.686065+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:27:21.686065+0000","last_peered":"2023-10-09T10:27:21.686065+0000","last_clean":"2023-10-09T10:27:21.686065+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:27:21.686065+0000","last_undegraded":"2023-10-09T10:27:21.686065+0000","last_fullsized":"2023-10-09T10:27:21.686065+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":149,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699539+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:27:21.699539+0000","last_peered":"2023-10-09T10:27:21.699539+0000","last_clean":"2023-10-09T10:27:21.699539+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:27:21.699539+0000","last_undegraded":"2023-10-09T10:27:21.699539+0000","last_fullsized":"2023-10-09T10:27:21.699539+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":40,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:31.866658+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:27:31.866658+0000","last_peered":"2023-10-09T10:27:31.866658+0000","last_clean":"2023-10-09T10:27:31.866658+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:27:31.866658+0000","last_undegraded":"2023-10-09T10:27:31.866658+0000","last_fullsized":"2023-10-09T10:27:31.866658+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":195,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699107+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:27:21.699107+0000","last_peered":"2023-10-09T10:27:21.699107+0000","last_clean":"2023-10-09T10:27:21.699107+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:27:21.699107+0000","last_undegraded":"2023-10-09T10:27:21.699107+0000","last_fullsized":"2023-10-09T10:27:21.699107+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":155,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699015+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:27:21.699015+0000","last_peered":"2023-10-09T10:27:21.699015+0000","last_clean":"2023-10-09T10:27:21.699015+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:27:21.699015+0000","last_undegraded":"2023-10-09T10:27:21.699015+0000","last_fullsized":"2023-10-09T10:27:21.699015+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":115,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.698928+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:27:21.698928+0000","last_peered":"2023-10-09T10:27:21.698928+0000","last_clean":"2023-10-09T10:27:21.698928+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:27:21.698928+0000","last_undegraded":"2023-10-09T10:27:21.698928+0000","last_fullsized":"2023-10-09T10:27:21.698928+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692278+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:27:21.692278+0000","last_peered":"2023-10-09T10:27:21.692278+0000","last_clean":"2023-10-09T10:27:21.692278+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:27:21.692278+0000","last_undegraded":"2023-10-09T10:27:21.692278+0000","last_fullsized":"2023-10-09T10:27:21.692278+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":467,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.695966+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:27:21.695966+0000","last_peered":"2023-10-09T10:27:21.695966+0000","last_clean":"2023-10-09T10:27:21.695966+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:27:21.695966+0000","last_undegraded":"2023-10-09T10:27:21.695966+0000","last_fullsized":"2023-10-09T10:27:21.695966+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":210,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.693462+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:27:21.693462+0000","last_peered":"2023-10-09T10:27:21.693462+0000","last_clean":"2023-10-09T10:27:21.693462+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:27:21.693462+0000","last_undegraded":"2023-10-09T10:27:21.693462+0000","last_fullsized":"2023-10-09T10:27:21.693462+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"544'7378","reported_seq":8189,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:45.045351+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:27:45.045351+0000","last_peered":"2023-10-09T10:27:45.045351+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:27:45.045351+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"503'2496","ondisk_log_start":"503'2496","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42631168,"num_objects":651,"num_object_clones":0,"num_object_copies":1953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6455,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-2500},{"shards":"3,5,6","objects":250}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":155,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.698763+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:27:21.698763+0000","last_peered":"2023-10-09T10:27:21.698763+0000","last_clean":"2023-10-09T10:27:21.698763+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:27:21.698763+0000","last_undegraded":"2023-10-09T10:27:21.698763+0000","last_fullsized":"2023-10-09T10:27:21.698763+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":40,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:36.879247+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:27:36.879247+0000","last_peered":"2023-10-09T10:27:36.879247+0000","last_clean":"2023-10-09T10:27:36.879247+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:27:36.879247+0000","last_undegraded":"2023-10-09T10:27:36.879247+0000","last_fullsized":"2023-10-09T10:27:36.879247+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"544'7396","reported_seq":15913,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:45.019624+0000","last_change":"2023-10-09T10:26:39.522775+0000","last_active":"2023-10-09T10:27:45.019624+0000","last_peered":"2023-10-09T10:27:45.019624+0000","last_clean":"2023-10-09T10:27:45.019624+0000","last_became_active":"2023-10-09T10:26:39.509024+0000","last_became_peered":"2023-10-09T10:26:39.509024+0000","last_unstale":"2023-10-09T10:27:45.019624+0000","last_undegraded":"2023-10-09T10:27:45.019624+0000","last_fullsized":"2023-10-09T10:27:45.019624+0000","mapping_epoch":536,"log_start":"503'2497","ondisk_log_start":"503'2497","created":501,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4899,"log_dups_size":0,"ondisk_log_size":4899,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43548672,"num_objects":665,"num_object_clones":0,"num_object_copies":1995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6431,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3072,"num_bytes_recovered":201326592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":50,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:31.866549+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:27:31.866549+0000","last_peered":"2023-10-09T10:27:31.866549+0000","last_clean":"2023-10-09T10:27:31.866549+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:27:31.866549+0000","last_undegraded":"2023-10-09T10:27:31.866549+0000","last_fullsized":"2023-10-09T10:27:31.866549+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":43,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:36.109459+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:27:36.109459+0000","last_peered":"2023-10-09T10:27:36.109459+0000","last_clean":"2023-10-09T10:27:36.109459+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:27:36.109459+0000","last_undegraded":"2023-10-09T10:27:36.109459+0000","last_fullsized":"2023-10-09T10:27:36.109459+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"544'7545","reported_seq":7617,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:45.059318+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:27:45.059318+0000","last_peered":"2023-10-09T10:27:45.059318+0000","last_clean":"2023-10-09T10:27:45.059318+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:27:45.059318+0000","last_undegraded":"2023-10-09T10:27:45.059318+0000","last_fullsized":"2023-10-09T10:27:45.059318+0000","mapping_epoch":494,"log_start":"503'2646","ondisk_log_start":"503'2646","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":4899,"log_dups_size":0,"ondisk_log_size":4899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40042496,"num_objects":611,"num_object_clones":0,"num_object_copies":1833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4257,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":45,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699110+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:27:21.699110+0000","last_peered":"2023-10-09T10:27:21.699110+0000","last_clean":"2023-10-09T10:27:21.699110+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:27:21.699110+0000","last_undegraded":"2023-10-09T10:27:21.699110+0000","last_fullsized":"2023-10-09T10:27:21.699110+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":219,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691756+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:27:21.691756+0000","last_peered":"2023-10-09T10:27:21.691756+0000","last_clean":"2023-10-09T10:27:21.691756+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:27:21.691756+0000","last_undegraded":"2023-10-09T10:27:21.691756+0000","last_fullsized":"2023-10-09T10:27:21.691756+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"544'7128","reported_seq":13239,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:46.871215+0000","last_change":"2023-10-09T10:26:35.543001+0000","last_active":"2023-10-09T10:27:46.871215+0000","last_peered":"2023-10-09T10:27:46.871215+0000","last_clean":"2023-10-09T10:27:46.871215+0000","last_became_active":"2023-10-09T10:26:35.529481+0000","last_became_peered":"2023-10-09T10:26:35.529481+0000","last_unstale":"2023-10-09T10:27:46.871215+0000","last_undegraded":"2023-10-09T10:27:46.871215+0000","last_fullsized":"2023-10-09T10:27:46.871215+0000","mapping_epoch":534,"log_start":"503'2041","ondisk_log_start":"503'2041","created":494,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:07.598234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34930688,"num_objects":533,"num_object_clones":0,"num_object_copies":1599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4368,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3012,"num_bytes_recovered":174915584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":171,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699328+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:27:21.699328+0000","last_peered":"2023-10-09T10:27:21.699328+0000","last_clean":"2023-10-09T10:27:21.699328+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:27:21.699328+0000","last_undegraded":"2023-10-09T10:27:21.699328+0000","last_fullsized":"2023-10-09T10:27:21.699328+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":158,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.698861+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:27:21.698861+0000","last_peered":"2023-10-09T10:27:21.698861+0000","last_clean":"2023-10-09T10:27:21.698861+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:27:21.698861+0000","last_undegraded":"2023-10-09T10:27:21.698861+0000","last_fullsized":"2023-10-09T10:27:21.698861+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"544'7422","reported_seq":14466,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:44.956263+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:27:44.956263+0000","last_peered":"2023-10-09T10:27:44.956263+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:27:44.956263+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"503'2562","ondisk_log_start":"503'2562","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4860,"log_dups_size":0,"ondisk_log_size":4860,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38502400,"num_objects":587,"num_object_clones":0,"num_object_copies":1761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6477,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2319,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"0,5,6","objects":187},{"shards":"5,6","objects":-226}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":235,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692743+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:27:21.692743+0000","last_peered":"2023-10-09T10:27:21.692743+0000","last_clean":"2023-10-09T10:27:21.692743+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:27:21.692743+0000","last_undegraded":"2023-10-09T10:27:21.692743+0000","last_fullsized":"2023-10-09T10:27:21.692743+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":136,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691622+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:27:21.691622+0000","last_peered":"2023-10-09T10:27:21.691622+0000","last_clean":"2023-10-09T10:27:21.691622+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:27:21.691622+0000","last_undegraded":"2023-10-09T10:27:21.691622+0000","last_fullsized":"2023-10-09T10:27:21.691622+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"544'7320","reported_seq":11517,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:44.954590+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:27:44.954590+0000","last_peered":"2023-10-09T10:27:44.954590+0000","last_clean":"2023-10-09T10:27:44.954590+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:27:44.954590+0000","last_undegraded":"2023-10-09T10:27:44.954590+0000","last_fullsized":"2023-10-09T10:27:44.954590+0000","mapping_epoch":513,"log_start":"499'1481","ondisk_log_start":"499'1481","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":5839,"log_dups_size":0,"ondisk_log_size":5839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43057152,"num_objects":657,"num_object_clones":0,"num_object_copies":1971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4406,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":219,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691404+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:27:21.691404+0000","last_peered":"2023-10-09T10:27:21.691404+0000","last_clean":"2023-10-09T10:27:21.691404+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:27:21.691404+0000","last_undegraded":"2023-10-09T10:27:21.691404+0000","last_fullsized":"2023-10-09T10:27:21.691404+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":454,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.688838+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:27:21.688838+0000","last_peered":"2023-10-09T10:27:21.688838+0000","last_clean":"2023-10-09T10:27:21.688838+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:27:21.688838+0000","last_undegraded":"2023-10-09T10:27:21.688838+0000","last_fullsized":"2023-10-09T10:27:21.688838+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"544'7312","reported_seq":8613,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:44.940489+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:27:44.940489+0000","last_peered":"2023-10-09T10:27:44.940489+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:27:44.940489+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"503'2451","ondisk_log_start":"503'2451","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4861,"log_dups_size":0,"ondisk_log_size":4861,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41123840,"num_objects":627,"num_object_clones":0,"num_object_copies":1881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6391,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,5","objects":-1002}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":155,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552145+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:27:37.552145+0000","last_peered":"2023-10-09T10:27:37.552145+0000","last_clean":"2023-10-09T10:27:37.552145+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:27:37.552145+0000","last_undegraded":"2023-10-09T10:27:37.552145+0000","last_fullsized":"2023-10-09T10:27:37.552145+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699019+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:27:21.699019+0000","last_peered":"2023-10-09T10:27:21.699019+0000","last_clean":"2023-10-09T10:27:21.699019+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:27:21.699019+0000","last_undegraded":"2023-10-09T10:27:21.699019+0000","last_fullsized":"2023-10-09T10:27:21.699019+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"544'7372","reported_seq":7503,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:46.871196+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:27:46.871196+0000","last_peered":"2023-10-09T10:27:46.871196+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:27:46.871196+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"503'2348","ondisk_log_start":"503'2348","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5024,"log_dups_size":0,"ondisk_log_size":5024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38207488,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4484,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-2475},{"shards":"0,2,5","objects":135}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":155,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.698524+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:27:21.698524+0000","last_peered":"2023-10-09T10:27:21.698524+0000","last_clean":"2023-10-09T10:27:21.698524+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:27:21.698524+0000","last_undegraded":"2023-10-09T10:27:21.698524+0000","last_fullsized":"2023-10-09T10:27:21.698524+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":121,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699428+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:27:21.699428+0000","last_peered":"2023-10-09T10:27:21.699428+0000","last_clean":"2023-10-09T10:27:21.699428+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:27:21.699428+0000","last_undegraded":"2023-10-09T10:27:21.699428+0000","last_fullsized":"2023-10-09T10:27:21.699428+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"544'7381","reported_seq":14230,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:46.694765+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:27:46.694765+0000","last_peered":"2023-10-09T10:27:46.694765+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:27:46.694765+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"503'2365","ondisk_log_start":"503'2365","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5016,"log_dups_size":0,"ondisk_log_size":5016,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41156608,"num_objects":628,"num_object_clones":0,"num_object_copies":1884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":806,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6436,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2254,"num_bytes_recovered":147718144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,2,6","objects":156},{"shards":"0,6","objects":-178}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":137,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699042+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:27:21.699042+0000","last_peered":"2023-10-09T10:27:21.699042+0000","last_clean":"2023-10-09T10:27:21.699042+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:27:21.699042+0000","last_undegraded":"2023-10-09T10:27:21.699042+0000","last_fullsized":"2023-10-09T10:27:21.699042+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":164,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699169+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:27:21.699169+0000","last_peered":"2023-10-09T10:27:21.699169+0000","last_clean":"2023-10-09T10:27:21.699169+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:27:21.699169+0000","last_undegraded":"2023-10-09T10:27:21.699169+0000","last_fullsized":"2023-10-09T10:27:21.699169+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"544'7349","reported_seq":7794,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:46.904306+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:27:46.904306+0000","last_peered":"2023-10-09T10:27:46.904306+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:27:46.904306+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"503'2273","ondisk_log_start":"503'2273","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5076,"log_dups_size":0,"ondisk_log_size":5076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37027840,"num_objects":565,"num_object_clones":0,"num_object_copies":1695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4242,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,3","objects":-1516}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":496,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691746+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:27:21.691746+0000","last_peered":"2023-10-09T10:27:21.691746+0000","last_clean":"2023-10-09T10:27:21.691746+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:27:21.691746+0000","last_undegraded":"2023-10-09T10:27:21.691746+0000","last_fullsized":"2023-10-09T10:27:21.691746+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":558,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699375+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:27:21.699375+0000","last_peered":"2023-10-09T10:27:21.699375+0000","last_clean":"2023-10-09T10:27:21.699375+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:27:21.699375+0000","last_undegraded":"2023-10-09T10:27:21.699375+0000","last_fullsized":"2023-10-09T10:27:21.699375+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"544'12743","reported_seq":12843,"reported_epoch":544,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:27:43.119085+0000","last_change":"2023-10-09T10:27:35.872423+0000","last_active":"2023-10-09T10:27:43.119085+0000","last_peered":"2023-10-09T10:27:43.119085+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:27:43.119085+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"521'6800","ondisk_log_start":"521'6800","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88604696,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1353,"num_objects_misplaced":1353,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12743,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":1353}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":265,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692187+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:27:21.692187+0000","last_peered":"2023-10-09T10:27:21.692187+0000","last_clean":"2023-10-09T10:27:21.692187+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:27:21.692187+0000","last_undegraded":"2023-10-09T10:27:21.692187+0000","last_fullsized":"2023-10-09T10:27:21.692187+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":228,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.693419+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:27:21.693419+0000","last_peered":"2023-10-09T10:27:21.693419+0000","last_clean":"2023-10-09T10:27:21.693419+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:27:21.693419+0000","last_undegraded":"2023-10-09T10:27:21.693419+0000","last_fullsized":"2023-10-09T10:27:21.693419+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":508,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691698+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:27:21.691698+0000","last_peered":"2023-10-09T10:27:21.691698+0000","last_clean":"2023-10-09T10:27:21.691698+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:27:21.691698+0000","last_undegraded":"2023-10-09T10:27:21.691698+0000","last_fullsized":"2023-10-09T10:27:21.691698+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"544'13155","reported_seq":13215,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:46.883160+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:27:46.883160+0000","last_peered":"2023-10-09T10:27:46.883160+0000","last_clean":"2023-10-09T10:27:46.883160+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:27:46.883160+0000","last_undegraded":"2023-10-09T10:27:46.883160+0000","last_fullsized":"2023-10-09T10:27:46.883160+0000","mapping_epoch":494,"log_start":"528'8106","ondisk_log_start":"528'8106","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5049,"log_dups_size":0,"ondisk_log_size":5049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73596928,"num_objects":1123,"num_object_clones":0,"num_object_copies":3369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13155,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":228,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691545+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:27:21.691545+0000","last_peered":"2023-10-09T10:27:21.691545+0000","last_clean":"2023-10-09T10:27:21.691545+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:27:21.691545+0000","last_undegraded":"2023-10-09T10:27:21.691545+0000","last_fullsized":"2023-10-09T10:27:21.691545+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":146,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.698994+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:27:21.698994+0000","last_peered":"2023-10-09T10:27:21.698994+0000","last_clean":"2023-10-09T10:27:21.698994+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:27:21.698994+0000","last_undegraded":"2023-10-09T10:27:21.698994+0000","last_fullsized":"2023-10-09T10:27:21.698994+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":457,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.697832+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:27:21.697832+0000","last_peered":"2023-10-09T10:27:21.697832+0000","last_clean":"2023-10-09T10:27:21.697832+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:27:21.697832+0000","last_undegraded":"2023-10-09T10:27:21.697832+0000","last_fullsized":"2023-10-09T10:27:21.697832+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"544'7377","reported_seq":15950,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:44.983708+0000","last_change":"2023-10-09T10:27:21.711478+0000","last_active":"2023-10-09T10:27:44.983708+0000","last_peered":"2023-10-09T10:27:44.983708+0000","last_clean":"2023-10-09T10:27:44.983708+0000","last_became_active":"2023-10-09T10:27:21.694370+0000","last_became_peered":"2023-10-09T10:27:21.694370+0000","last_unstale":"2023-10-09T10:27:44.983708+0000","last_undegraded":"2023-10-09T10:27:44.983708+0000","last_fullsized":"2023-10-09T10:27:44.983708+0000","mapping_epoch":542,"log_start":"503'2533","ondisk_log_start":"503'2533","created":494,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4844,"log_dups_size":0,"ondisk_log_size":4844,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39157760,"num_objects":597,"num_object_clones":0,"num_object_copies":1791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6411,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3094,"num_bytes_recovered":202768384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":495,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.554831+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:27:37.554831+0000","last_peered":"2023-10-09T10:27:37.554831+0000","last_clean":"2023-10-09T10:27:37.554831+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:27:37.554831+0000","last_undegraded":"2023-10-09T10:27:37.554831+0000","last_fullsized":"2023-10-09T10:27:37.554831+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":112,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.688584+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:27:21.688584+0000","last_peered":"2023-10-09T10:27:21.688584+0000","last_clean":"2023-10-09T10:27:21.688584+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:27:21.688584+0000","last_undegraded":"2023-10-09T10:27:21.688584+0000","last_fullsized":"2023-10-09T10:27:21.688584+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":557,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692467+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:27:21.692467+0000","last_peered":"2023-10-09T10:27:21.692467+0000","last_clean":"2023-10-09T10:27:21.692467+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:27:21.692467+0000","last_undegraded":"2023-10-09T10:27:21.692467+0000","last_fullsized":"2023-10-09T10:27:21.692467+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"544'12878","reported_seq":12961,"reported_epoch":544,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:27:45.019791+0000","last_change":"2023-10-09T10:26:37.693448+0000","last_active":"2023-10-09T10:27:45.019791+0000","last_peered":"2023-10-09T10:27:45.019791+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:27:45.019791+0000","last_undegraded":"2023-10-09T10:27:45.019791+0000","last_fullsized":"2023-10-09T10:27:45.019791+0000","mapping_epoch":507,"log_start":"528'8000","ondisk_log_start":"528'8000","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4878,"log_dups_size":0,"ondisk_log_size":4878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80871424,"num_objects":1234,"num_object_clones":0,"num_object_copies":3702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1234,"num_objects_unfound":0,"num_objects_dirty":1234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12878,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1234}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":198,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692600+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:27:21.692600+0000","last_peered":"2023-10-09T10:27:21.692600+0000","last_clean":"2023-10-09T10:27:21.692600+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:27:21.692600+0000","last_undegraded":"2023-10-09T10:27:21.692600+0000","last_fullsized":"2023-10-09T10:27:21.692600+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":737,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.688364+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:27:21.688364+0000","last_peered":"2023-10-09T10:27:21.688364+0000","last_clean":"2023-10-09T10:27:21.688364+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:27:21.688364+0000","last_undegraded":"2023-10-09T10:27:21.688364+0000","last_fullsized":"2023-10-09T10:27:21.688364+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":246,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691320+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:27:21.691320+0000","last_peered":"2023-10-09T10:27:21.691320+0000","last_clean":"2023-10-09T10:27:21.691320+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:27:21.691320+0000","last_undegraded":"2023-10-09T10:27:21.691320+0000","last_fullsized":"2023-10-09T10:27:21.691320+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":155,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699505+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:27:21.699505+0000","last_peered":"2023-10-09T10:27:21.699505+0000","last_clean":"2023-10-09T10:27:21.699505+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:27:21.699505+0000","last_undegraded":"2023-10-09T10:27:21.699505+0000","last_fullsized":"2023-10-09T10:27:21.699505+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"544'7487","reported_seq":14438,"reported_epoch":544,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:27:45.027524+0000","last_change":"2023-10-09T10:27:22.518901+0000","last_active":"2023-10-09T10:27:45.027524+0000","last_peered":"2023-10-09T10:27:45.027524+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:27:45.027524+0000","last_undegraded":"2023-10-09T10:27:45.027524+0000","last_fullsized":"2023-10-09T10:27:45.027524+0000","mapping_epoch":516,"log_start":"503'2588","ondisk_log_start":"503'2588","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4899,"log_dups_size":0,"ondisk_log_size":4899,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44269568,"num_objects":675,"num_object_clones":0,"num_object_copies":2025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":951,"num_objects_unfound":0,"num_objects_dirty":675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6563,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2306,"num_bytes_recovered":151126016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","6"],"object_location_counts":[{"shards":"3,5,6","objects":-276}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":229,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.688758+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:27:21.688758+0000","last_peered":"2023-10-09T10:27:21.688758+0000","last_clean":"2023-10-09T10:27:21.688758+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:27:21.688758+0000","last_undegraded":"2023-10-09T10:27:21.688758+0000","last_fullsized":"2023-10-09T10:27:21.688758+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":489,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:31.866642+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:27:31.866642+0000","last_peered":"2023-10-09T10:27:31.866642+0000","last_clean":"2023-10-09T10:27:31.866642+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:27:31.866642+0000","last_undegraded":"2023-10-09T10:27:31.866642+0000","last_fullsized":"2023-10-09T10:27:31.866642+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":40,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.836352+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:27:21.836352+0000","last_peered":"2023-10-09T10:27:21.836352+0000","last_clean":"2023-10-09T10:27:21.836352+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:27:21.836352+0000","last_undegraded":"2023-10-09T10:27:21.836352+0000","last_fullsized":"2023-10-09T10:27:21.836352+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"544'12856","reported_seq":12914,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:43.113271+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:27:43.113271+0000","last_peered":"2023-10-09T10:27:43.113271+0000","last_clean":"2023-10-09T10:27:43.113271+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:27:43.113271+0000","last_undegraded":"2023-10-09T10:27:43.113271+0000","last_fullsized":"2023-10-09T10:27:43.113271+0000","mapping_epoch":494,"log_start":"521'6900","ondisk_log_start":"521'6900","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5956,"log_dups_size":0,"ondisk_log_size":5956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88866816,"num_objects":1356,"num_object_clones":0,"num_object_copies":4068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12856,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":40,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.836330+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:27:21.836330+0000","last_peered":"2023-10-09T10:27:21.836330+0000","last_clean":"2023-10-09T10:27:21.836330+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:27:21.836330+0000","last_undegraded":"2023-10-09T10:27:21.836330+0000","last_fullsized":"2023-10-09T10:27:21.836330+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":525,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692158+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:27:21.692158+0000","last_peered":"2023-10-09T10:27:21.692158+0000","last_clean":"2023-10-09T10:27:21.692158+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:27:21.692158+0000","last_undegraded":"2023-10-09T10:27:21.692158+0000","last_fullsized":"2023-10-09T10:27:21.692158+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":457,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:31.866744+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:27:31.866744+0000","last_peered":"2023-10-09T10:27:31.866744+0000","last_clean":"2023-10-09T10:27:31.866744+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:27:31.866744+0000","last_undegraded":"2023-10-09T10:27:31.866744+0000","last_fullsized":"2023-10-09T10:27:31.866744+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"544'12563","reported_seq":12623,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:46.883530+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:27:46.883530+0000","last_peered":"2023-10-09T10:27:46.883530+0000","last_clean":"2023-10-09T10:27:46.883530+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:27:46.883530+0000","last_undegraded":"2023-10-09T10:27:46.883530+0000","last_fullsized":"2023-10-09T10:27:46.883530+0000","mapping_epoch":494,"log_start":"526'7476","ondisk_log_start":"526'7476","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76873728,"num_objects":1173,"num_object_clones":0,"num_object_copies":3519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12563,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":345,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:26.860182+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:27:26.860182+0000","last_peered":"2023-10-09T10:27:26.860182+0000","last_clean":"2023-10-09T10:27:26.860182+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:27:26.860182+0000","last_undegraded":"2023-10-09T10:27:26.860182+0000","last_fullsized":"2023-10-09T10:27:26.860182+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":525,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699089+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:27:21.699089+0000","last_peered":"2023-10-09T10:27:21.699089+0000","last_clean":"2023-10-09T10:27:21.699089+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:27:21.699089+0000","last_undegraded":"2023-10-09T10:27:21.699089+0000","last_fullsized":"2023-10-09T10:27:21.699089+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":40,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.688388+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:27:21.688388+0000","last_peered":"2023-10-09T10:27:21.688388+0000","last_clean":"2023-10-09T10:27:21.688388+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:27:21.688388+0000","last_undegraded":"2023-10-09T10:27:21.688388+0000","last_fullsized":"2023-10-09T10:27:21.688388+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"544'12921","reported_seq":12979,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:43.148195+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:27:43.148195+0000","last_peered":"2023-10-09T10:27:43.148195+0000","last_clean":"2023-10-09T10:27:43.148195+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:27:43.148195+0000","last_undegraded":"2023-10-09T10:27:43.148195+0000","last_fullsized":"2023-10-09T10:27:43.148195+0000","mapping_epoch":494,"log_start":"521'7000","ondisk_log_start":"521'7000","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5921,"log_dups_size":0,"ondisk_log_size":5921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88014848,"num_objects":1343,"num_object_clones":0,"num_object_copies":4029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12921,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":47,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.689076+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:27:21.689076+0000","last_peered":"2023-10-09T10:27:21.689076+0000","last_clean":"2023-10-09T10:27:21.689076+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:27:21.689076+0000","last_undegraded":"2023-10-09T10:27:21.689076+0000","last_fullsized":"2023-10-09T10:27:21.689076+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":44,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.836399+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:27:21.836399+0000","last_peered":"2023-10-09T10:27:21.836399+0000","last_clean":"2023-10-09T10:27:21.836399+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:27:21.836399+0000","last_undegraded":"2023-10-09T10:27:21.836399+0000","last_fullsized":"2023-10-09T10:27:21.836399+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":224,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691282+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:27:21.691282+0000","last_peered":"2023-10-09T10:27:21.691282+0000","last_clean":"2023-10-09T10:27:21.691282+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:27:21.691282+0000","last_undegraded":"2023-10-09T10:27:21.691282+0000","last_fullsized":"2023-10-09T10:27:21.691282+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"544'7235","reported_seq":7341,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:45.054777+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:27:45.054777+0000","last_peered":"2023-10-09T10:27:45.054777+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:27:45.054777+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"503'2385","ondisk_log_start":"503'2385","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40402944,"num_objects":617,"num_object_clones":0,"num_object_copies":1851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6315,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,5","objects":-1599}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":40,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552445+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:27:37.552445+0000","last_peered":"2023-10-09T10:27:37.552445+0000","last_clean":"2023-10-09T10:27:37.552445+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:27:37.552445+0000","last_undegraded":"2023-10-09T10:27:37.552445+0000","last_fullsized":"2023-10-09T10:27:37.552445+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":49,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550009+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:27:37.550009+0000","last_peered":"2023-10-09T10:27:37.550009+0000","last_clean":"2023-10-09T10:27:37.550009+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:27:37.550009+0000","last_undegraded":"2023-10-09T10:27:37.550009+0000","last_fullsized":"2023-10-09T10:27:37.550009+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"544'7397","reported_seq":12153,"reported_epoch":544,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:27:44.933710+0000","last_change":"2023-10-09T10:27:07.588772+0000","last_active":"2023-10-09T10:27:44.933710+0000","last_peered":"2023-10-09T10:27:44.933710+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:27:07.549958+0000","last_became_peered":"2023-10-09T10:27:07.549958+0000","last_unstale":"2023-10-09T10:27:44.933710+0000","last_undegraded":"2023-10-09T10:27:44.933710+0000","last_fullsized":"2023-10-09T10:27:44.933710+0000","mapping_epoch":540,"log_start":"500'1623","ondisk_log_start":"500'1623","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37912576,"num_objects":579,"num_object_clones":0,"num_object_copies":1737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6473,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2333,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":["6","0","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":579}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":274,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.691497+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:27:21.691497+0000","last_peered":"2023-10-09T10:27:21.691497+0000","last_clean":"2023-10-09T10:27:21.691497+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:27:21.691497+0000","last_undegraded":"2023-10-09T10:27:21.691497+0000","last_fullsized":"2023-10-09T10:27:21.691497+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":137,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.699076+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:27:21.699076+0000","last_peered":"2023-10-09T10:27:21.699076+0000","last_clean":"2023-10-09T10:27:21.699076+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:27:21.699076+0000","last_undegraded":"2023-10-09T10:27:21.699076+0000","last_fullsized":"2023-10-09T10:27:21.699076+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"544'7364","reported_seq":14165,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:45.006166+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:27:45.006166+0000","last_peered":"2023-10-09T10:27:45.006166+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:27:45.006166+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"503'2450","ondisk_log_start":"503'2450","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4914,"log_dups_size":0,"ondisk_log_size":4914,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44007424,"num_objects":672,"num_object_clones":0,"num_object_copies":2016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":834,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6420,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2240,"num_bytes_recovered":146800640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"0,5,6","objects":199},{"shards":"5,6","objects":-162}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":257,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.693334+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:27:21.693334+0000","last_peered":"2023-10-09T10:27:21.693334+0000","last_clean":"2023-10-09T10:27:21.693334+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:27:21.693334+0000","last_undegraded":"2023-10-09T10:27:21.693334+0000","last_fullsized":"2023-10-09T10:27:21.693334+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":367,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.695819+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:27:21.695819+0000","last_peered":"2023-10-09T10:27:21.695819+0000","last_clean":"2023-10-09T10:27:21.695819+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:27:21.695819+0000","last_undegraded":"2023-10-09T10:27:21.695819+0000","last_fullsized":"2023-10-09T10:27:21.695819+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"544'7365","reported_seq":13727,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:46.858720+0000","last_change":"2023-10-09T10:27:21.732594+0000","last_active":"2023-10-09T10:27:46.858720+0000","last_peered":"2023-10-09T10:27:46.858720+0000","last_clean":"2023-10-09T10:27:46.858720+0000","last_became_active":"2023-10-09T10:27:21.699893+0000","last_became_peered":"2023-10-09T10:27:21.699893+0000","last_unstale":"2023-10-09T10:27:46.858720+0000","last_undegraded":"2023-10-09T10:27:46.858720+0000","last_fullsized":"2023-10-09T10:27:46.858720+0000","mapping_epoch":542,"log_start":"503'2357","ondisk_log_start":"503'2357","created":501,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":5008,"log_dups_size":0,"ondisk_log_size":5008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:06.239031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36110336,"num_objects":551,"num_object_clones":0,"num_object_copies":1653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4428,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3137,"num_bytes_recovered":122093568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":43,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.692327+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:27:21.692327+0000","last_peered":"2023-10-09T10:27:21.692327+0000","last_clean":"2023-10-09T10:27:21.692327+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:27:21.692327+0000","last_undegraded":"2023-10-09T10:27:21.692327+0000","last_fullsized":"2023-10-09T10:27:21.692327+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":597,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:31.866610+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:27:31.866610+0000","last_peered":"2023-10-09T10:27:31.866610+0000","last_clean":"2023-10-09T10:27:31.866610+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:27:31.866610+0000","last_undegraded":"2023-10-09T10:27:31.866610+0000","last_fullsized":"2023-10-09T10:27:31.866610+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"544'7345","reported_seq":7416,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:45.009524+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:27:45.009524+0000","last_peered":"2023-10-09T10:27:45.009524+0000","last_clean":"2023-10-09T10:27:45.009524+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:27:45.009524+0000","last_undegraded":"2023-10-09T10:27:45.009524+0000","last_fullsized":"2023-10-09T10:27:45.009524+0000","mapping_epoch":501,"log_start":"503'2415","ondisk_log_start":"503'2415","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":4930,"log_dups_size":0,"ondisk_log_size":4930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39387136,"num_objects":601,"num_object_clones":0,"num_object_copies":1803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4084,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":61,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.689416+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:27:21.689416+0000","last_peered":"2023-10-09T10:27:21.689416+0000","last_clean":"2023-10-09T10:27:21.689416+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:27:21.689416+0000","last_undegraded":"2023-10-09T10:27:21.689416+0000","last_fullsized":"2023-10-09T10:27:21.689416+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":164,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-09T10:27:21.698727+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:27:21.698727+0000","last_peered":"2023-10-09T10:27:21.698727+0000","last_clean":"2023-10-09T10:27:21.698727+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:27:21.698727+0000","last_undegraded":"2023-10-09T10:27:21.698727+0000","last_fullsized":"2023-10-09T10:27:21.698727+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":1294696472,"num_objects":19758,"num_object_clones":0,"num_object_copies":59274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17340,"num_objects_misplaced":6689,"num_objects_unfound":0,"num_objects_dirty":19758,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191188,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33040,"num_bytes_recovered":1936785408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7320903680,"data_stored":7320895536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133547,"ondisk_log_size":133547,"up":78,"acting":67,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430697,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626925,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2111824,"kb_used_data":594504,"kb_used_omap":0,"kb_used_meta":1517312,"kb_avail":91633328,"statfs":{"total":95995035648,"available":93832527872,"internally_reserved":0,"allocated":608772096,"data_stored":607801712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553727488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.625,"15min":1.5669999999999999},"min":{"1min":0.223,"5min":0.216,"15min":0.216},"max":{"1min":6.1849999999999996,"5min":9.4260000000000002,"15min":44.055},"last":6.016},{"interface":"front","average":{"1min":2.9740000000000002,"5min":1.7130000000000001,"15min":1.9079999999999999},"min":{"1min":0.28799999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":9.3499999999999996,"5min":9.3499999999999996,"15min":43.869999999999997},"last":0.29399999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.3400000000000001,"15min":1.998},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":2.4049999999999998,"5min":6.3899999999999997,"15min":155.01400000000001},"last":3.0910000000000002},{"interface":"front","average":{"1min":1.482,"5min":1.3580000000000001,"15min":2.1739999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.6050000000000004,"5min":6.3369999999999997,"15min":154.71100000000001},"last":2.7090000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.887,"15min":1.833},"min":{"1min":0.315,"5min":0.309,"15min":0.24099999999999999},"max":{"1min":5.8300000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":2.7730000000000001},{"interface":"front","average":{"1min":1.853,"5min":1.667,"15min":1.643},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4379999999999997,"5min":6.4379999999999997,"15min":11.720000000000001},"last":3.2749999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":2.9470000000000001,"5min":2.0459999999999998,"15min":1.6579999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":43.393000000000001,"5min":43.393000000000001,"15min":43.393000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":3.4500000000000002,"5min":2.1829999999999998,"15min":1.806},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":43.466000000000001,"5min":43.466000000000001,"15min":43.466000000000001},"last":0.45600000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.0680000000000001,"15min":1.974},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":10.772,"5min":10.772,"15min":122.041},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.73,"15min":1.7769999999999999},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.1059999999999999,"5min":11.266,"15min":122.13200000000001},"last":0.32300000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.27,"15min":2.1200000000000001},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.5670000000000002,"5min":9.6890000000000001,"15min":121.98999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.405,"5min":1.4950000000000001,"15min":1.673},"min":{"1min":0.40500000000000003,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":4.835,"5min":7.0060000000000002,"15min":122.346},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.0219999999999998,"15min":1.966},"min":{"1min":0.38,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":12.612,"5min":49.991,"15min":130.66999999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.5880000000000001,"5min":2.097,"15min":2.02},"min":{"1min":0.40799999999999997,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":12.659000000000001,"5min":50.023000000000003,"15min":130.702},"last":0.874}]}]},{"osd":1,"up_from":0,"seq":360777253238,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416763,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2358340,"kb_used_data":1345724,"kb_used_omap":28,"kb_used_meta":1012579,"kb_avail":91386812,"statfs":{"total":95995035648,"available":93580095488,"internally_reserved":0,"allocated":1378021376,"data_stored":1375879473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1036881744},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:26:49 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.659,"15min":1.9350000000000001},"min":{"1min":0.26600000000000001,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":7.3609999999999998,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.26000000000000001},{"interface":"front","average":{"1min":2.4390000000000001,"5min":1.48,"15min":2.2400000000000002},"min":{"1min":0.249,"5min":0.214,"15min":0.192},"max":{"1min":11.808,"5min":11.808,"15min":56.950000000000003},"last":3.8319999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:27:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":1.845,"15min":2.0230000000000001},"min":{"1min":0.36099999999999999,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":8.859,"5min":17.093,"15min":53.988999999999997},"last":3.484},{"interface":"front","average":{"1min":2.286,"5min":2.081,"15min":2.1150000000000002},"min":{"1min":0.29999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.3739999999999997,"5min":17.053999999999998,"15min":54.119},"last":0.30399999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:27:05 2023","interfaces":[{"interface":"back","average":{"1min":3.7349999999999999,"5min":2.0649999999999999,"15min":1.655},"min":{"1min":0.248,"5min":0.217,"15min":0.217},"max":{"1min":29.835999999999999,"5min":29.835999999999999,"15min":56.984999999999999},"last":4.7489999999999997},{"interface":"front","average":{"1min":4.242,"5min":2.585,"15min":1.7749999999999999},"min":{"1min":0.32900000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":29.776,"5min":29.776,"15min":56.475999999999999},"last":0.40000000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":2.077,"15min":2.1579999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":9.2859999999999996,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.69799999999999995},{"interface":"front","average":{"1min":2.5920000000000001,"5min":2.4710000000000001,"15min":2.1920000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.896000000000001,"5min":28.331,"15min":56.378},"last":0.495}]},{"osd":5,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":3.5830000000000002,"15min":3.4689999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":11.867000000000001,"5min":90.412000000000006,"15min":229.589},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.625,"5min":3.0270000000000001,"15min":3.194},"min":{"1min":0.35799999999999998,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":6.9539999999999997,"5min":90.393000000000001,"15min":228.88900000000001},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":2.0640000000000001,"15min":2.2690000000000001},"min":{"1min":0.41399999999999998,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":8.5719999999999992,"5min":9.7840000000000007,"15min":54.482999999999997},"last":0.505},{"interface":"front","average":{"1min":2.7989999999999999,"5min":2.2450000000000001,"15min":2.4049999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.34799999999999998},"max":{"1min":9.3439999999999994,"5min":9.3439999999999994,"15min":57.753},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.2749999999999999,"15min":2.2959999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.34999999999999998},"max":{"1min":5.6959999999999997,"5min":12.029999999999999,"15min":56.411999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.335,"15min":2.3410000000000002},"min":{"1min":0.47199999999999998,"5min":0.39900000000000002,"15min":0.38300000000000001},"max":{"1min":7.0490000000000004,"5min":17.129999999999999,"15min":54.451999999999998},"last":3.4430000000000001}]}]},{"osd":2,"up_from":217,"seq":932007903498,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2400120,"kb_used_data":1525232,"kb_used_omap":83,"kb_used_meta":874796,"kb_avail":91345032,"statfs":{"total":95995035648,"available":93537312768,"internally_reserved":0,"allocated":1561837568,"data_stored":1559891570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":895791599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.5939999999999999,"15min":2.008},"min":{"1min":0.22900000000000001,"5min":0.192,"15min":0.185},"max":{"1min":14.666,"5min":48.990000000000002,"15min":48.990000000000002},"last":2.5920000000000001},{"interface":"front","average":{"1min":2.8340000000000001,"5min":2.363,"15min":2.1379999999999999},"min":{"1min":0.23599999999999999,"5min":0.22600000000000001,"15min":0.20899999999999999},"max":{"1min":13.773,"5min":49.412999999999997,"15min":49.412999999999997},"last":3.4260000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":2.343,"15min":2.0230000000000001},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":14.632999999999999,"5min":14.632999999999999,"15min":34.893999999999998},"last":0.317},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.7689999999999999,"15min":1.823},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":13.356999999999999,"5min":13.356999999999999,"15min":34.962000000000003},"last":0.54200000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:26:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4460000000000002,"5min":1.8700000000000001,"15min":1.373},"min":{"1min":0.32000000000000001,"5min":0.25700000000000001,"15min":0.248},"max":{"1min":15.115,"5min":15.115,"15min":35.043999999999997},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.923,"5min":1.482,"15min":1.361},"min":{"1min":0.33500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.7880000000000003,"5min":11.206,"15min":50.406999999999996},"last":6.6820000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:27:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":1.899,"15min":1.6220000000000001},"min":{"1min":0.35899999999999999,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":8.1859999999999999,"5min":47.503999999999998,"15min":47.503999999999998},"last":3.391},{"interface":"front","average":{"1min":2.7639999999999998,"5min":2.2309999999999999,"15min":1.913},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":8.7219999999999995,"5min":47.704000000000001,"15min":47.704000000000001},"last":6.7869999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":1.8260000000000001,"15min":2.0169999999999999},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":8.6839999999999993,"5min":11.266999999999999,"15min":57.298000000000002},"last":6.7300000000000004},{"interface":"front","average":{"1min":3.8860000000000001,"5min":2.4199999999999999,"15min":2.1400000000000001},"min":{"1min":0.375,"5min":0.31,"15min":0.31},"max":{"1min":14.31,"5min":14.31,"15min":57.828000000000003},"last":4.1749999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":2.0310000000000001,"15min":1.9299999999999999},"min":{"1min":0.40300000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":7.9390000000000001,"5min":12.411,"15min":18.943999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":2.556,"5min":1.95,"15min":1.948},"min":{"1min":0.39300000000000002,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":8.7590000000000003,"5min":13.975,"15min":24.379000000000001},"last":6.8120000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.95,"5min":2.1139999999999999,"15min":2.1920000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":9.5199999999999996,"5min":11.704000000000001,"15min":35.008000000000003},"last":4.0449999999999999},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.1230000000000002,"15min":2.121},"min":{"1min":0.51400000000000001,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":8.8149999999999995,"5min":15.548,"15min":24.323},"last":6.7610000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488338,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595392,"kb_used_data":1017912,"kb_used_omap":15,"kb_used_meta":577392,"kb_avail":92149760,"statfs":{"total":95995035648,"available":94361354240,"internally_reserved":0,"allocated":1042341888,"data_stored":1039915256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":591249862},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.9740000000000002,"5min":3.4220000000000002,"15min":1.8859999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":13.661,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.40400000000000003},{"interface":"front","average":{"1min":3.9849999999999999,"5min":3.1499999999999999,"15min":1.8779999999999999},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":10.313000000000001,"5min":66.802000000000007,"15min":66.802000000000007},"last":0.27200000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":1.8160000000000001,"15min":1.9490000000000001},"min":{"1min":0.42499999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.3100000000000005,"5min":25.370000000000001,"15min":105.401},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.6800000000000002,"5min":2.0430000000000001,"15min":1.9299999999999999},"min":{"1min":0.28399999999999997,"5min":0.25900000000000001,"15min":0.23300000000000001},"max":{"1min":10.422000000000001,"5min":14.613,"15min":104.914},"last":0.875}]},{"osd":2,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2250000000000001,"5min":2.266,"15min":2.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":13.497999999999999,"5min":13.497999999999999,"15min":13.497999999999999},"last":8.2919999999999998},{"interface":"front","average":{"1min":3.4689999999999999,"5min":2.2549999999999999,"15min":1.5780000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27600000000000002},"max":{"1min":10.339,"5min":25.338999999999999,"15min":25.338999999999999},"last":0.68600000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":2.016,"15min":1.5169999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.268000000000001,"5min":21.196000000000002,"15min":21.831},"last":0.84499999999999997},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.927,"15min":2.1179999999999999},"min":{"1min":0.34499999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":13.577999999999999,"5min":19.323,"15min":22.097999999999999},"last":5.8479999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5259999999999998,"5min":2.0609999999999999,"15min":1.4159999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":8.2710000000000008,"5min":25.251999999999999,"15min":25.251999999999999},"last":1.7190000000000001},{"interface":"front","average":{"1min":2.9319999999999999,"5min":2.3639999999999999,"15min":1.6910000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.388,"5min":10.388,"15min":13.394},"last":1.423}]},{"osd":6,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":2.262,"15min":1.845},"min":{"1min":0.375,"5min":0.36199999999999999,"15min":0.35599999999999998},"max":{"1min":13.545999999999999,"5min":19.114000000000001,"15min":19.114000000000001},"last":5.891},{"interface":"front","average":{"1min":2.758,"5min":2.0979999999999999,"15min":1.593},"min":{"1min":0.37,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":10.117000000000001,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.57599999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.1589999999999998,"15min":1.643},"min":{"1min":0.41299999999999998,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":10.345000000000001,"5min":10.345000000000001,"15min":10.459},"last":1.8280000000000001},{"interface":"front","average":{"1min":2.9060000000000001,"5min":2.3239999999999998,"15min":1.8660000000000001},"min":{"1min":0.41799999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.234,"5min":19.280000000000001,"15min":19.280000000000001},"last":12.4}]}]},{"osd":4,"up_from":266,"seq":1142461300967,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2251476,"kb_used_data":1314252,"kb_used_omap":35,"kb_used_meta":937180,"kb_avail":91493676,"statfs":{"total":95995035648,"available":93689524224,"internally_reserved":0,"allocated":1345794048,"data_stored":1344817990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":959672558},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":8.9870000000000001,"5min":3.1680000000000001,"15min":2.6859999999999999},"min":{"1min":0.245,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":109.193,"5min":109.193,"15min":109.193},"last":2.944},{"interface":"front","average":{"1min":8.6099999999999994,"5min":3.548,"15min":3.2469999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.224},"max":{"1min":110.438,"5min":110.438,"15min":110.438},"last":0.30599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":2.1040000000000001,"15min":2.2709999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.24299999999999999},"max":{"1min":10.696,"5min":12.209,"15min":71.457999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.2799999999999998,"5min":2.7090000000000001,"15min":2.2330000000000001},"min":{"1min":0.32700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":9.2040000000000006,"5min":49.781999999999996,"15min":71.658000000000001},"last":0.46800000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.141,"15min":1.8160000000000001},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":5.4710000000000001,"5min":15.944000000000001,"15min":15.944000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.419,"5min":1.7430000000000001,"15min":1.766},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.0309999999999997,"5min":14.473000000000001,"15min":14.473000000000001},"last":0.39300000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:26:59 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":2.552,"15min":2.0659999999999998},"min":{"1min":0.40999999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.257,"5min":11.347,"15min":12.074},"last":3.1869999999999998},{"interface":"front","average":{"1min":2.5600000000000001,"5min":2.786,"15min":1.839},"min":{"1min":0.50900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.1270000000000007,"5min":16.067,"15min":16.067},"last":0.56599999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.0369999999999999,"15min":2.0510000000000002},"min":{"1min":0.28000000000000003,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":3.8290000000000002,"5min":16.094000000000001,"15min":99.585999999999999},"last":1.75},{"interface":"front","average":{"1min":1.0409999999999999,"5min":2.0619999999999998,"15min":1.871},"min":{"1min":0.317,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":5.9509999999999996,"5min":49.915999999999997,"15min":99.384},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.526,"15min":1.4079999999999999},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.0579999999999998,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.8600000000000001,"5min":2.2930000000000001,"15min":1.804},"min":{"1min":0.34000000000000002,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":10.824,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.58299999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":2.4569999999999999,"15min":1.913},"min":{"1min":0.39400000000000002,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":9.016,"5min":12.391999999999999,"15min":17.369},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.2350000000000001,"5min":2.431,"15min":1.778},"min":{"1min":0.37,"5min":0.37,"15min":0.314},"max":{"1min":3.9750000000000001,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.5}]}]},{"osd":5,"up_from":36,"seq":154618823152,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2943112,"kb_used_data":1442240,"kb_used_omap":0,"kb_used_meta":1500800,"kb_avail":90802040,"statfs":{"total":95995035648,"available":92981288960,"internally_reserved":0,"allocated":1476853760,"data_stored":1475885756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536819200},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":2.9260000000000002,"15min":2.0979999999999999},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":19.102,"5min":19.102,"15min":22.469999999999999},"last":3.581},{"interface":"front","average":{"1min":2.7850000000000001,"5min":1.784,"15min":1.7270000000000001},"min":{"1min":0.253,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":17.887,"5min":17.887,"15min":22.437000000000001},"last":0.35499999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:27:43 2023","interfaces":[{"interface":"back","average":{"1min":3.3319999999999999,"5min":2.387,"15min":2.1680000000000001},"min":{"1min":0.36799999999999999,"5min":0.27900000000000003,"15min":0.23999999999999999},"max":{"1min":10.425000000000001,"5min":16.562999999999999,"15min":29.370000000000001},"last":2.109},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.6299999999999999,"15min":1.788},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23499999999999999},"max":{"1min":5.2329999999999997,"5min":8.9529999999999994,"15min":29.065000000000001},"last":0.38600000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:27:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":2.5369999999999999,"15min":2.0710000000000002},"min":{"1min":0.311,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":7.8849999999999998,"5min":16.629999999999999,"15min":16.629999999999999},"last":2.1640000000000001},{"interface":"front","average":{"1min":2.7999999999999998,"5min":1.9399999999999999,"15min":1.774},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":7.1479999999999997,"5min":9.6929999999999996,"15min":15.929},"last":0.436}]},{"osd":3,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.8109999999999999,"5min":2.3940000000000001,"15min":1.9450000000000001},"min":{"1min":0.377,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":13.614000000000001,"5min":13.614000000000001,"15min":13.614000000000001},"last":3.54},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.8660000000000001,"15min":1.7869999999999999},"min":{"1min":0.38400000000000001,"5min":0.35099999999999998,"15min":0.315},"max":{"1min":11.124000000000001,"5min":11.124000000000001,"15min":13.624000000000001},"last":3.5299999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.3679999999999999,"15min":1.6890000000000001},"min":{"1min":0.378,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":13.523999999999999,"5min":13.523999999999999,"15min":13.523999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.7130000000000001,"15min":1.4419999999999999},"min":{"1min":0.29399999999999998,"5min":0.28199999999999997,"15min":0.23599999999999999},"max":{"1min":4.1219999999999999,"5min":10.153,"15min":11.956},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.4430000000000001,"15min":1.8799999999999999},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":13.567,"5min":13.567,"15min":13.567},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.02,"15min":1.724},"min":{"1min":0.34499999999999997,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":7.2999999999999998,"5min":16.497,"15min":16.497},"last":2.6760000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.9390000000000001,"15min":1.7450000000000001},"min":{"1min":0.38600000000000001,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":5.4160000000000004,"5min":21.109000000000002,"15min":21.109000000000002},"last":2.1400000000000001},{"interface":"front","average":{"1min":1.089,"5min":1.0580000000000001,"15min":1.127},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":5.5519999999999996,"5min":9.0739999999999998,"15min":11.994},"last":0.58299999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1364656128,"data_stored":1364656128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1541210112,"data_stored":1541210112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1027014656,"data_stored":1027014656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1337462784,"data_stored":1337458712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1465454592,"data_stored":1465450520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":585105408,"data_stored":585105408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:27:50.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:27:50.606 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:50.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:50 smithi012 ceph-mon[147926]: from='client.37203 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:50.661 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2758790465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:50.661 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:50 smithi012 ceph-mon[152263]: from='client.37203 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:50.661 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2758790465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:50.692 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:50.779 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:50.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:50.888 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:50.921 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:50 smithi110 ceph-mon[136032]: from='client.37203 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:27:50.921 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2758790465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:50.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:50.979 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:50.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:50.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:51.084 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:51.122 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:51.170 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:51.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:51.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:51.280 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:51.327 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:51.378 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:51.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:51.486 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:51.535 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:51.586 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:51.661 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:27:51.702 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:51.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:51.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:51.812 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:51.851 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:51.898 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:51.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:51.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:51 smithi110 ceph-mon[136032]: pgmap v1547: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 17081/56868 objects degraded (30.036%); 6430/56868 objects misplaced (11.307%); 320 KiB/s, 46 objects/s recovering 2023-10-09T10:27:51.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:51 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4200256549' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:27:51.905 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:51 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:27:51.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:51 smithi012 ceph-mon[147926]: pgmap v1547: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 17081/56868 objects degraded (30.036%); 6430/56868 objects misplaced (11.307%); 320 KiB/s, 46 objects/s recovering 2023-10-09T10:27:51.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:51 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4200256549' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:27:51.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:51 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:27:51.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:51 smithi012 ceph-mon[152263]: pgmap v1547: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 17081/56868 objects degraded (30.036%); 6430/56868 objects misplaced (11.307%); 320 KiB/s, 46 objects/s recovering 2023-10-09T10:27:51.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:51 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4200256549' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:27:51.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:51 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:27:51.982 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:27:51.982 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:27:51.982 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.982 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:211167 11.19 e545/515 11:99cfc045:::benchmark_data_smithi012_404402_object98384:head v 545'7560, mlcod=0'0)", 2023-10-09T10:27:51.982 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:51.973710+0000", 2023-10-09T10:27:51.982 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0072163610000000001, 2023-10-09T10:27:51.982 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072513710000000004, 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973710+0000", 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.983 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.984 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:51.984 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973710+0000", 2023-10-09T10:27:51.984 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:51.984 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.984 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.985 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:51.985 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973709+0000", 2023-10-09T10:27:51.985 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:27:51.985 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.985 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.986 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:51.986 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973717+0000", 2023-10-09T10:27:51.986 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.14e-06 2023-10-09T10:27:51.986 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.987 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.987 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:51.987 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973728+0000", 2023-10-09T10:27:51.987 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0947e-05 2023-10-09T10:27:51.987 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.988 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.988 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:51.988 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973736+0000", 2023-10-09T10:27:51.988 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1450000000000008e-06 2023-10-09T10:27:51.988 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.989 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.989 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:51.989 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.974004+0000", 2023-10-09T10:27:51.989 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00026758400000000002 2023-10-09T10:27:51.989 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.990 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.990 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:51.990 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.974037+0000", 2023-10-09T10:27:51.990 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.3200000000000001e-05 2023-10-09T10:27:51.990 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:51.991 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:51.991 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:51.991 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.991 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.991 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:211164 11.0 e545/507 11:01c4c8a4:::benchmark_data_smithi012_404402_object98381:head v 545'7472, mlcod=498'332)", 2023-10-09T10:27:51.991 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:51.973762+0000", 2023-10-09T10:27:51.991 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0071642750000000003, 2023-10-09T10:27:51.992 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0072696949999999996, 2023-10-09T10:27:51.992 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:51.992 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:27:51.992 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:51.992 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:51.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973762+0000", 2023-10-09T10:27:51.993 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:51.993 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.993 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.993 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:51.993 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973762+0000", 2023-10-09T10:27:51.994 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:51.994 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.994 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:51.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973762+0000", 2023-10-09T10:27:51.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:27:51.995 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.995 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.995 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:51.995 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973767+0000", 2023-10-09T10:27:51.995 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9130000000000002e-06 2023-10-09T10:27:51.996 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.996 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.996 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:51.996 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973773+0000", 2023-10-09T10:27:51.997 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.618e-06 2023-10-09T10:27:51.997 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.997 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.997 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:51.997 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973778+0000", 2023-10-09T10:27:51.998 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7119999999999999e-06 2023-10-09T10:27:51.998 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.998 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.998 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:51.998 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.975956+0000", 2023-10-09T10:27:51.999 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021780599999999999 2023-10-09T10:27:51.999 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:51.999 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:51.999 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:51.999 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.975973+0000", 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6586999999999999e-05 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:211159 11.12 e545/513 11:4fd968b7:::benchmark_data_smithi012_404402_object98376:head v 545'7639, mlcod=530'5270)", 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:51.973784+0000", 2023-10-09T10:27:52.000 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.007142135, 2023-10-09T10:27:52.001 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073099380000000002, 2023-10-09T10:27:52.001 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:52.001 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:27:52.001 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:52.001 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.001 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:52.001 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973784+0000", 2023-10-09T10:27:52.002 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.002 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.002 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.002 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:52.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973784+0000", 2023-10-09T10:27:52.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.003 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.003 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.003 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:52.003 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973784+0000", 2023-10-09T10:27:52.003 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973788+0000", 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1250000000000003e-06 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973793+0000", 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9010000000000002e-06 2023-10-09T10:27:52.004 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973797+0000", 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.7840000000000001e-06 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973833+0000", 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5734e-05 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.005 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973850+0000", 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.747e-05 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:211158 11.11 e545/513 11:8e3fa6a3:::benchmark_data_smithi012_404402_object98375:head v 545'7574, mlcod=505'3191)", 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:51.973807+0000", 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0071197229999999997, 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073483460000000004, 2023-10-09T10:27:52.006 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973807+0000", 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973807+0000", 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.007 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973806+0000", 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973814+0000", 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8499999999999994e-06 2023-10-09T10:27:52.008 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:52.009 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973820+0000", 2023-10-09T10:27:52.009 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.2809999999999999e-06 2023-10-09T10:27:52.009 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.009 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.009 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:52.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973824+0000", 2023-10-09T10:27:52.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1219999999999997e-06 2023-10-09T10:27:52.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.010 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.010 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:52.010 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.976694+0000", 2023-10-09T10:27:52.010 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00287061 2023-10-09T10:27:52.010 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.011 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.011 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:52.011 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.976719+0000", 2023-10-09T10:27:52.011 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4709999999999999e-05 2023-10-09T10:27:52.011 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.011 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:211170 11.0 e545/507)", 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:51.973827+0000", 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0070988930000000002, 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073853039999999997, 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:52.012 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:27:52.013 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:52.013 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.013 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:52.013 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973827+0000", 2023-10-09T10:27:52.013 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.013 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.014 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:52.014 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973827+0000", 2023-10-09T10:27:52.014 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.014 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.014 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:52.015 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973827+0000", 2023-10-09T10:27:52.015 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:27:52.015 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.015 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.015 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:52.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973830+0000", 2023-10-09T10:27:52.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0869999999999998e-06 2023-10-09T10:27:52.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.016 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.016 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:52.016 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973835+0000", 2023-10-09T10:27:52.016 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.7509999999999999e-06 2023-10-09T10:27:52.016 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.017 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.017 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:52.017 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973839+0000", 2023-10-09T10:27:52.017 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6890000000000002e-06 2023-10-09T10:27:52.017 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.017 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:52.017 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.018 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.018 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.018 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:211165 11.11 e545/513)", 2023-10-09T10:27:52.018 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:51.973842+0000", 2023-10-09T10:27:52.018 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.0070839520000000001, 2023-10-09T10:27:52.018 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074158999999999996, 2023-10-09T10:27:52.018 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:52.019 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "queued for pg", 2023-10-09T10:27:52.019 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:52.019 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.019 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:52.019 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973842+0000", 2023-10-09T10:27:52.019 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.019 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:52.020 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973842+0000", 2023-10-09T10:27:52.020 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.020 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.020 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.020 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:52.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973842+0000", 2023-10-09T10:27:52.021 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:27:52.021 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.021 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.021 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:52.021 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973846+0000", 2023-10-09T10:27:52.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.5149999999999998e-06 2023-10-09T10:27:52.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.022 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.022 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:52.022 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973851+0000", 2023-10-09T10:27:52.022 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.6990000000000002e-06 2023-10-09T10:27:52.022 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.973854+0000", 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.698e-06 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.023 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:211173 11.14 e545/540 11:2d6d9873:::benchmark_data_smithi012_404402_object98390:head v 545'7563, mlcod=545'7561)", 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:51.978310+0000", 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 0.002616569, 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029980549999999999, 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.978310+0000", 2023-10-09T10:27:52.024 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.978310+0000", 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.978309+0000", 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.025 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:27:52.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.978325+0000", 2023-10-09T10:27:52.026 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6583e-05 2023-10-09T10:27:52.026 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.026 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.026 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:27:52.026 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.978335+0000", 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9150000000000008e-06 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.978342+0000", 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9580000000000001e-06 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.978405+0000", 2023-10-09T10:27:52.027 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3083000000000006e-05 2023-10-09T10:27:52.028 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:27:52.028 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:27:52.028 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:27:52.028 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:51.978427+0000", 2023-10-09T10:27:52.028 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1679e-05 2023-10-09T10:27:52.028 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.028 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:27:52.028 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.029 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:27:52.029 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2023-10-09T10:27:52.029 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 7 2023-10-09T10:27:52.029 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:27:52.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:27:52.101 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:52.358 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:27:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:52.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:27:52.471 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:52.519 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:52.577 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:52.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:52.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:52.620 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:52.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:27:52.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[147926]: osdmap e545: 8 total, 8 up, 6 in 2023-10-09T10:27:52.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[147926]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[147926]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[147926]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.685 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:52.744 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:52.794 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:27:52.796 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:52.801 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:52 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:52.801 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:52 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:27:52.801 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:52 smithi110 ceph-mon[136032]: osdmap e545: 8 total, 8 up, 6 in 2023-10-09T10:27:52.801 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:52 smithi110 ceph-mon[136032]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.801 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:52 smithi110 ceph-mon[136032]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.801 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:52 smithi110 ceph-mon[136032]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:52.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:52.870 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:27:52.870 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:27:52.870 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[152263]: osdmap e545: 8 total, 8 up, 6 in 2023-10-09T10:27:52.870 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[152263]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.870 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[152263]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.870 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:52 smithi012 ceph-mon[152263]: failed to encode map e545 with expected crc 2023-10-09T10:27:52.904 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:52.940 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:52.994 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:53.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:53.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:53.075 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:27:53.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:53.274 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:53.410 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:53.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:53.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:53.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:53.639 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:53.745 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:53.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:53.754 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:53 smithi012 ceph-mon[147926]: pgmap v1549: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 17081/51498 objects degraded (33.168%); 6159/51498 objects misplaced (11.960%); 384 KiB/s, 55 objects/s recovering 2023-10-09T10:27:53.754 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:53 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 17081/51498 objects degraded (33.168%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:53.754 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:53 smithi012 ceph-mon[152263]: pgmap v1549: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 17081/51498 objects degraded (33.168%); 6159/51498 objects misplaced (11.960%); 384 KiB/s, 55 objects/s recovering 2023-10-09T10:27:53.754 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:53 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 17081/51498 objects degraded (33.168%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:53.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:53.891 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:53.941 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:53 smithi110 ceph-mon[136032]: pgmap v1549: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 17081/51498 objects degraded (33.168%); 6159/51498 objects misplaced (11.960%); 384 KiB/s, 55 objects/s recovering 2023-10-09T10:27:53.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:53 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 17081/51498 objects degraded (33.168%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:53.959 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:54.073 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:54.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:54.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:54.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:54.232 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:27:54.232 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 17081/51498 objects degraded (33.168%), 11 pgs degraded, 11 pgs undersized","count":22},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1991,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":545,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846541,"num_in_osds":6,"osd_in_since":1696847079,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":9},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":17166,"data_bytes":1123850168,"bytes_used":13303783424,"bytes_avail":562666430464,"bytes_total":575970213888,"degraded_objects":17081,"degraded_total":51498,"degraded_ratio":0.3316827837974291,"misplaced_objects":6159,"misplaced_total":51498,"misplaced_ratio":0.11959687754864266,"recovering_objects_per_sec":55,"recovering_bytes_per_sec":392958,"recovering_keys_per_sec":0,"num_objects_recovered":553,"num_bytes_recovered":3932160,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":882},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":33,"modified":"2023-10-09T10:27:11.837030+0000","services":{}},"progress_events":{"4aaa0d12-13e6-4ea3-ae4b-c27ebe383851":{"message":"Global Recovery Event (6m)\n [========================....] (remaining: 66s)","progress":0.85858583450317383,"add_to_ceph_s":true},"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T10:27:54.236 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:54.281 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:54.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:54.390 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:54.463 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:54.511 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:27:54.513 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:54.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:54.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:54.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:54.674 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:54 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2145829305' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:27:54.751 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:54.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:54 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2145829305' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:27:54.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:54 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2145829305' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:27:55.060 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:55.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:55.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:55.130 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:27:55.130 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:27:55.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:55.269 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:55.617 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:55.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:55.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:55.726 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:27:55.765 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:55.813 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:27:55.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:55.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:27:55.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:55.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:55 smithi012 ceph-mon[147926]: pgmap v1550: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 931 MiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 850 op/s; 16565/44739 objects degraded (37.026%); 5896/44739 objects misplaced (13.179%); 0 B/s, 54 objects/s recovering 2023-10-09T10:27:55.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:55 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:27:55.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:55 smithi012 ceph-mon[152263]: pgmap v1550: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 931 MiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 850 op/s; 16565/44739 objects degraded (37.026%); 5896/44739 objects misplaced (13.179%); 0 B/s, 54 objects/s recovering 2023-10-09T10:27:55.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:55 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:27:55.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:55 smithi110 ceph-mon[136032]: pgmap v1550: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 931 MiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 850 op/s; 16565/44739 objects degraded (37.026%); 5896/44739 objects misplaced (13.179%); 0 B/s, 54 objects/s recovering 2023-10-09T10:27:55.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:55 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:27:56.012 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:56.078 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:56.097 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:56.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:56.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:56.206 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:56.271 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:56.340 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:56.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:56.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:56.387 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:56.446 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:27:56.487 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:56.545 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:27:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:56.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:27:56.651 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:56.703 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:56.842 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:56.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:56.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:56.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:56.998 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:27:57.029 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:57.033 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:27:57.081 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:27:57.114 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:27:57.114 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:27:57.114 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:27:57.115 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:27:57.115 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:27:57.127 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:27:57.170 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:27:57.211 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:27:57.246 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:27:57.275 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:27:57.302 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:27:57.302 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:27:57.302 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:27:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:27:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:27:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:27:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:27:57.303 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:27:57.721 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:57.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:57.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:57.866 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:57.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:57 smithi012 ceph-mon[147926]: pgmap v1551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 817 MiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 832 op/s; 16565/39261 objects degraded (42.192%); 5646/39261 objects misplaced (14.381%); 0 B/s, 54 objects/s recovering 2023-10-09T10:27:57.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4009566536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:57.917 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:57 smithi012 ceph-mon[152263]: pgmap v1551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 817 MiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 832 op/s; 16565/39261 objects degraded (42.192%); 5646/39261 objects misplaced (14.381%); 0 B/s, 54 objects/s recovering 2023-10-09T10:27:57.917 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4009566536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:58.181 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:27:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:27:57] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.33.4" 2023-10-09T10:27:58.184 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:58.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:57 smithi110 ceph-mon[136032]: pgmap v1551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 817 MiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 832 op/s; 16565/39261 objects degraded (42.192%); 5646/39261 objects misplaced (14.381%); 0 B/s, 54 objects/s recovering 2023-10-09T10:27:58.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4009566536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:27:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:58.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:27:58.295 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:27:58.324 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:58.443 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:27:58.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:58.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:27:58.550 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:27:58.623 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:58.740 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:27:58.915 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:27:58.918 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:58 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 16565/39261 objects degraded (42.192%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:58.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:58 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 16565/39261 objects degraded (42.192%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:58.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:58.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:27:59.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:27:59.155 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:59.203 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:58 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 16565/39261 objects degraded (42.192%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:27:59.302 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:27:59.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:59.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:27:59.412 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:27:59.466 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:59.637 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:27:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:27:59.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:27:59.745 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:27:59.847 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:27:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:59 smithi110 ceph-mon[136032]: pgmap v1552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 725 MiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 16311/34839 objects degraded (46.818%); 5646/34839 objects misplaced (16.206%); 0 B/s, 50 objects/s recovering 2023-10-09T10:27:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:27:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:27:59.945 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:27:59.988 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:27:59.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:00.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:59 smithi012 ceph-mon[147926]: pgmap v1552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 725 MiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 16311/34839 objects degraded (46.818%); 5646/34839 objects misplaced (16.206%); 0 B/s, 50 objects/s recovering 2023-10-09T10:28:00.000 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:27:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:00.000 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:59 smithi012 ceph-mon[152263]: pgmap v1552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 725 MiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 16311/34839 objects degraded (46.818%); 5646/34839 objects misplaced (16.206%); 0 B/s, 50 objects/s recovering 2023-10-09T10:28:00.000 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:27:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:00.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:00.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:00.146 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:00.205 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:00.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:00.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:00.303 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:28:00.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:00.431 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:00.529 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:00.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:00.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:00.647 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:00.694 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:00.755 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:28:00.783 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:28:00.790 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:28:00.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1311161307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:00.946 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:00 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:01.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1311161307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:01.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:00 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:01.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1311161307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:01.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:00 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:01.559 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:01.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:01.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:01.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:01.740 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:01.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:01 smithi110 ceph-mon[136032]: pgmap v1553: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 678 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 809 op/s; 16048/32580 objects degraded (49.257%); 5383/32580 objects misplaced (16.522%); 0 B/s, 50 objects/s recovering 2023-10-09T10:28:01.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:01 smithi110 ceph-mon[136032]: from='client.37230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:01.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:01 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:28:01.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:01 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:01.947 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:01 smithi110 ceph-mon[136032]: osdmap e546: 8 total, 8 up, 6 in 2023-10-09T10:28:01.948 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:01 smithi110 ceph-mon[136032]: failed to encode map e546 with expected crc 2023-10-09T10:28:01.948 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:01 smithi110 ceph-mon[136032]: failed to encode map e546 with expected crc 2023-10-09T10:28:01.948 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:01 smithi110 ceph-mon[136032]: failed to encode map e546 with expected crc 2023-10-09T10:28:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[147926]: pgmap v1553: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 678 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 809 op/s; 16048/32580 objects degraded (49.257%); 5383/32580 objects misplaced (16.522%); 0 B/s, 50 objects/s recovering 2023-10-09T10:28:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[147926]: from='client.37230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:28:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[147926]: osdmap e546: 8 total, 8 up, 6 in 2023-10-09T10:28:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[147926]: failed to encode map e546 with expected crc 2023-10-09T10:28:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[147926]: failed to encode map e546 with expected crc 2023-10-09T10:28:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[147926]: failed to encode map e546 with expected crc 2023-10-09T10:28:02.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[152263]: pgmap v1553: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 678 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 809 op/s; 16048/32580 objects degraded (49.257%); 5383/32580 objects misplaced (16.522%); 0 B/s, 50 objects/s recovering 2023-10-09T10:28:02.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[152263]: from='client.37230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:02.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:28:02.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:02.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[152263]: osdmap e546: 8 total, 8 up, 6 in 2023-10-09T10:28:02.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[152263]: failed to encode map e546 with expected crc 2023-10-09T10:28:02.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[152263]: failed to encode map e546 with expected crc 2023-10-09T10:28:02.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:01 smithi012 ceph-mon[152263]: failed to encode map e546 with expected crc 2023-10-09T10:28:03.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:02 smithi012 ceph-mon[147926]: pgmap v1555: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 575 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 16048/27630 objects degraded (58.082%); 5151/27630 objects misplaced (18.643%); 0 B/s, 50 objects/s recovering 2023-10-09T10:28:03.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:02 smithi012 ceph-mon[152263]: pgmap v1555: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 575 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 16048/27630 objects degraded (58.082%); 5151/27630 objects misplaced (18.643%); 0 B/s, 50 objects/s recovering 2023-10-09T10:28:03.195 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:02 smithi110 ceph-mon[136032]: pgmap v1555: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 575 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 16048/27630 objects degraded (58.082%); 5151/27630 objects misplaced (18.643%); 0 B/s, 50 objects/s recovering 2023-10-09T10:28:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:04 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:04 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:04 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 16048/27630 objects degraded (58.082%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:04 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:04 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:04 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 16048/27630 objects degraded (58.082%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:04.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:04 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:04.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:04 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:04.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:04 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 16048/27630 objects degraded (58.082%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:04.976 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1553,"stamp":"2023-10-09T10:27:59.865632+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":710580152,"num_objects":10860,"num_object_clones":0,"num_object_copies":32580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16048,"num_objects_misplaced":5383,"num_objects_unfound":0,"num_objects_dirty":10860,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":200370,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33707,"num_bytes_recovered":1939769084,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133779,"ondisk_log_size":133779,"up":297,"acting":286,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11834424,"kb_used_data":5395784,"kb_used_omap":163,"kb_used_meta":6438108,"kb_avail":550636488,"statfs":{"total":575970213888,"available":563851763712,"internally_reserved":0,"allocated":5525282816,"data_stored":5515822033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":6592623603},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,2,9,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-530579456,"num_objects":-8096,"num_object_clones":0,"num_object_copies":-24288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1033,"num_objects_misplaced":-1047,"num_objects_unfound":0,"num_objects_dirty":-8096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8096,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":507,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003756"},"pg_stats":[{"pgid":"11.12","version":"545'7754","reported_seq":11570,"reported_epoch":545,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:56.896410+0000","last_change":"2023-10-09T10:27:19.878100+0000","last_active":"2023-10-09T10:27:56.896410+0000","last_peered":"2023-10-09T10:27:56.896410+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:27:56.896410+0000","last_undegraded":"2023-10-09T10:25:09.456831+0000","last_fullsized":"2023-10-09T10:25:09.428007+0000","mapping_epoch":513,"log_start":"503'2706","ondisk_log_start":"503'2706","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5048,"log_dups_size":0,"ondisk_log_size":5048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23429120,"num_objects":358,"num_object_clones":0,"num_object_copies":1074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1251,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6806,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1868,"num_bytes_recovered":56885248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-893}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":138,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.544007+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:27:37.544007+0000","last_peered":"2023-10-09T10:27:37.544007+0000","last_clean":"2023-10-09T10:27:37.544007+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:27:37.544007+0000","last_undegraded":"2023-10-09T10:27:37.544007+0000","last_fullsized":"2023-10-09T10:27:37.544007+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":499,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549406+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:27:37.549406+0000","last_peered":"2023-10-09T10:27:37.549406+0000","last_clean":"2023-10-09T10:27:37.549406+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:27:37.549406+0000","last_undegraded":"2023-10-09T10:27:37.549406+0000","last_fullsized":"2023-10-09T10:27:37.549406+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"545'7703","reported_seq":14583,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:56.886054+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:27:56.886054+0000","last_peered":"2023-10-09T10:27:56.886054+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:27:56.886054+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"503'2699","ondisk_log_start":"503'2699","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5004,"log_dups_size":0,"ondisk_log_size":5004,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19922944,"num_objects":304,"num_object_clones":0,"num_object_copies":912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":845,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6758,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,6","objects":-541}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":576,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.543408+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:27:37.543408+0000","last_peered":"2023-10-09T10:27:37.543408+0000","last_clean":"2023-10-09T10:27:37.543408+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:27:37.543408+0000","last_undegraded":"2023-10-09T10:27:37.543408+0000","last_fullsized":"2023-10-09T10:27:37.543408+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":470,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551140+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:27:37.551140+0000","last_peered":"2023-10-09T10:27:37.551140+0000","last_clean":"2023-10-09T10:27:37.551140+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:27:37.551140+0000","last_undegraded":"2023-10-09T10:27:37.551140+0000","last_fullsized":"2023-10-09T10:27:37.551140+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"545'7572","reported_seq":12128,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:56.906837+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:27:56.906837+0000","last_peered":"2023-10-09T10:27:56.906837+0000","last_clean":"2023-10-09T10:27:56.906837+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:27:56.906837+0000","last_undegraded":"2023-10-09T10:27:56.906837+0000","last_fullsized":"2023-10-09T10:27:56.906837+0000","mapping_epoch":527,"log_start":"503'2505","ondisk_log_start":"503'2505","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5067,"log_dups_size":0,"ondisk_log_size":5067,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22708224,"num_objects":347,"num_object_clones":0,"num_object_copies":1041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6635,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":41,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550267+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:27:37.550267+0000","last_peered":"2023-10-09T10:27:37.550267+0000","last_clean":"2023-10-09T10:27:37.550267+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:27:37.550267+0000","last_undegraded":"2023-10-09T10:27:37.550267+0000","last_fullsized":"2023-10-09T10:27:37.550267+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550743+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:27:37.550743+0000","last_peered":"2023-10-09T10:27:37.550743+0000","last_clean":"2023-10-09T10:27:37.550743+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:27:37.550743+0000","last_undegraded":"2023-10-09T10:27:37.550743+0000","last_fullsized":"2023-10-09T10:27:37.550743+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":504,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551225+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:27:37.551225+0000","last_peered":"2023-10-09T10:27:37.551225+0000","last_clean":"2023-10-09T10:27:37.551225+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:27:37.551225+0000","last_undegraded":"2023-10-09T10:27:37.551225+0000","last_fullsized":"2023-10-09T10:27:37.551225+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":453,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.533571+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:27:37.533571+0000","last_peered":"2023-10-09T10:27:37.533571+0000","last_clean":"2023-10-09T10:27:37.533571+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:27:37.533571+0000","last_undegraded":"2023-10-09T10:27:37.533571+0000","last_fullsized":"2023-10-09T10:27:37.533571+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551403+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:27:37.551403+0000","last_peered":"2023-10-09T10:27:37.551403+0000","last_clean":"2023-10-09T10:27:37.551403+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:27:37.551403+0000","last_undegraded":"2023-10-09T10:27:37.551403+0000","last_fullsized":"2023-10-09T10:27:37.551403+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":453,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.534989+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:27:37.534989+0000","last_peered":"2023-10-09T10:27:37.534989+0000","last_clean":"2023-10-09T10:27:37.534989+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:27:37.534989+0000","last_undegraded":"2023-10-09T10:27:37.534989+0000","last_fullsized":"2023-10-09T10:27:37.534989+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":150,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549543+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:27:37.549543+0000","last_peered":"2023-10-09T10:27:37.549543+0000","last_clean":"2023-10-09T10:27:37.549543+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:27:37.549543+0000","last_undegraded":"2023-10-09T10:27:37.549543+0000","last_fullsized":"2023-10-09T10:27:37.549543+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":41,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550091+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:27:37.550091+0000","last_peered":"2023-10-09T10:27:37.550091+0000","last_clean":"2023-10-09T10:27:37.550091+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:27:37.550091+0000","last_undegraded":"2023-10-09T10:27:37.550091+0000","last_fullsized":"2023-10-09T10:27:37.550091+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":196,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.544246+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:27:37.544246+0000","last_peered":"2023-10-09T10:27:37.544246+0000","last_clean":"2023-10-09T10:27:37.544246+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:27:37.544246+0000","last_undegraded":"2023-10-09T10:27:37.544246+0000","last_fullsized":"2023-10-09T10:27:37.544246+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":156,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.544484+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:27:37.544484+0000","last_peered":"2023-10-09T10:27:37.544484+0000","last_clean":"2023-10-09T10:27:37.544484+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:27:37.544484+0000","last_undegraded":"2023-10-09T10:27:37.544484+0000","last_fullsized":"2023-10-09T10:27:37.544484+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":116,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.544286+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:27:37.544286+0000","last_peered":"2023-10-09T10:27:37.544286+0000","last_clean":"2023-10-09T10:27:37.544286+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:27:37.544286+0000","last_undegraded":"2023-10-09T10:27:37.544286+0000","last_fullsized":"2023-10-09T10:27:37.544286+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":70,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.553655+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:27:37.553655+0000","last_peered":"2023-10-09T10:27:37.553655+0000","last_clean":"2023-10-09T10:27:37.553655+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:27:37.553655+0000","last_undegraded":"2023-10-09T10:27:37.553655+0000","last_fullsized":"2023-10-09T10:27:37.553655+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":468,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552966+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:27:37.552966+0000","last_peered":"2023-10-09T10:27:37.552966+0000","last_clean":"2023-10-09T10:27:37.552966+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:27:37.552966+0000","last_undegraded":"2023-10-09T10:27:37.552966+0000","last_fullsized":"2023-10-09T10:27:37.552966+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":211,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.554660+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:27:37.554660+0000","last_peered":"2023-10-09T10:27:37.554660+0000","last_clean":"2023-10-09T10:27:37.554660+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:27:37.554660+0000","last_undegraded":"2023-10-09T10:27:37.554660+0000","last_fullsized":"2023-10-09T10:27:37.554660+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"545'7644","reported_seq":8456,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:55.027631+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:27:55.027631+0000","last_peered":"2023-10-09T10:27:55.027631+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:27:55.027631+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"504'2796","ondisk_log_start":"504'2796","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25198592,"num_objects":385,"num_object_clones":0,"num_object_copies":1155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6721,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-2766}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":156,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.543558+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:27:37.543558+0000","last_peered":"2023-10-09T10:27:37.543558+0000","last_clean":"2023-10-09T10:27:37.543558+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:27:37.543558+0000","last_undegraded":"2023-10-09T10:27:37.543558+0000","last_fullsized":"2023-10-09T10:27:37.543558+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":41,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550712+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:27:37.550712+0000","last_peered":"2023-10-09T10:27:37.550712+0000","last_clean":"2023-10-09T10:27:37.550712+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:27:37.550712+0000","last_undegraded":"2023-10-09T10:27:37.550712+0000","last_fullsized":"2023-10-09T10:27:37.550712+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"545'7654","reported_seq":16172,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:55.051377+0000","last_change":"2023-10-09T10:26:39.522775+0000","last_active":"2023-10-09T10:27:55.051377+0000","last_peered":"2023-10-09T10:27:55.051377+0000","last_clean":"2023-10-09T10:27:55.051377+0000","last_became_active":"2023-10-09T10:26:39.509024+0000","last_became_peered":"2023-10-09T10:26:39.509024+0000","last_unstale":"2023-10-09T10:27:55.051377+0000","last_undegraded":"2023-10-09T10:27:55.051377+0000","last_fullsized":"2023-10-09T10:27:55.051377+0000","mapping_epoch":536,"log_start":"503'2797","ondisk_log_start":"503'2797","created":501,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4857,"log_dups_size":0,"ondisk_log_size":4857,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26640384,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6689,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3072,"num_bytes_recovered":201326592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":51,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551577+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:27:37.551577+0000","last_peered":"2023-10-09T10:27:37.551577+0000","last_clean":"2023-10-09T10:27:37.551577+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:27:37.551577+0000","last_undegraded":"2023-10-09T10:27:37.551577+0000","last_fullsized":"2023-10-09T10:27:37.551577+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":44,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.533365+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:27:37.533365+0000","last_peered":"2023-10-09T10:27:37.533365+0000","last_clean":"2023-10-09T10:27:37.533365+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:27:37.533365+0000","last_undegraded":"2023-10-09T10:27:37.533365+0000","last_fullsized":"2023-10-09T10:27:37.533365+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"545'7815","reported_seq":7888,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:55.031795+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:27:55.031795+0000","last_peered":"2023-10-09T10:27:55.031795+0000","last_clean":"2023-10-09T10:27:55.031795+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:27:55.031795+0000","last_undegraded":"2023-10-09T10:27:55.031795+0000","last_fullsized":"2023-10-09T10:27:55.031795+0000","mapping_epoch":494,"log_start":"504'2946","ondisk_log_start":"504'2946","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":4869,"log_dups_size":0,"ondisk_log_size":4869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22347776,"num_objects":341,"num_object_clones":0,"num_object_copies":1023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4527,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":46,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.547701+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:27:37.547701+0000","last_peered":"2023-10-09T10:27:37.547701+0000","last_clean":"2023-10-09T10:27:37.547701+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:27:37.547701+0000","last_undegraded":"2023-10-09T10:27:37.547701+0000","last_fullsized":"2023-10-09T10:27:37.547701+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":220,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550879+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:27:37.550879+0000","last_peered":"2023-10-09T10:27:37.550879+0000","last_clean":"2023-10-09T10:27:37.550879+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:27:37.550879+0000","last_undegraded":"2023-10-09T10:27:37.550879+0000","last_fullsized":"2023-10-09T10:27:37.550879+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"545'7356","reported_seq":13468,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:56.867730+0000","last_change":"2023-10-09T10:26:35.543001+0000","last_active":"2023-10-09T10:27:56.867730+0000","last_peered":"2023-10-09T10:27:56.867730+0000","last_clean":"2023-10-09T10:27:56.867730+0000","last_became_active":"2023-10-09T10:26:35.529481+0000","last_became_peered":"2023-10-09T10:26:35.529481+0000","last_unstale":"2023-10-09T10:27:56.867730+0000","last_undegraded":"2023-10-09T10:27:56.867730+0000","last_fullsized":"2023-10-09T10:27:56.867730+0000","mapping_epoch":534,"log_start":"503'2341","ondisk_log_start":"503'2341","created":494,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5015,"log_dups_size":0,"ondisk_log_size":5015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:07.598234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19988480,"num_objects":305,"num_object_clones":0,"num_object_copies":915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4596,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3012,"num_bytes_recovered":174915584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":172,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549761+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:27:37.549761+0000","last_peered":"2023-10-09T10:27:37.549761+0000","last_clean":"2023-10-09T10:27:37.549761+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:27:37.549761+0000","last_undegraded":"2023-10-09T10:27:37.549761+0000","last_fullsized":"2023-10-09T10:27:37.549761+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":159,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.543632+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:27:37.543632+0000","last_peered":"2023-10-09T10:27:37.543632+0000","last_clean":"2023-10-09T10:27:37.543632+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:27:37.543632+0000","last_undegraded":"2023-10-09T10:27:37.543632+0000","last_fullsized":"2023-10-09T10:27:37.543632+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"545'7679","reported_seq":14724,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:55.032249+0000","last_change":"2023-10-09T10:25:09.748560+0000","last_active":"2023-10-09T10:27:55.032249+0000","last_peered":"2023-10-09T10:27:55.032249+0000","last_clean":"2023-10-09T10:24:39.578400+0000","last_became_active":"2023-10-09T10:25:09.739403+0000","last_became_peered":"2023-10-09T10:25:09.739403+0000","last_unstale":"2023-10-09T10:27:55.032249+0000","last_undegraded":"2023-10-09T10:25:09.539535+0000","last_fullsized":"2023-10-09T10:25:09.429517+0000","mapping_epoch":513,"log_start":"503'2762","ondisk_log_start":"503'2762","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4917,"log_dups_size":0,"ondisk_log_size":4917,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21659648,"num_objects":330,"num_object_clones":0,"num_object_copies":990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6734,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2319,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"5,6","objects":-483}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":236,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551550+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:27:37.551550+0000","last_peered":"2023-10-09T10:27:37.551550+0000","last_clean":"2023-10-09T10:27:37.551550+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:27:37.551550+0000","last_undegraded":"2023-10-09T10:27:37.551550+0000","last_fullsized":"2023-10-09T10:27:37.551550+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":137,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552878+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:27:37.552878+0000","last_peered":"2023-10-09T10:27:37.552878+0000","last_clean":"2023-10-09T10:27:37.552878+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:27:37.552878+0000","last_undegraded":"2023-10-09T10:27:37.552878+0000","last_fullsized":"2023-10-09T10:27:37.552878+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"545'7586","reported_seq":11784,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:54.962506+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:27:54.962506+0000","last_peered":"2023-10-09T10:27:54.962506+0000","last_clean":"2023-10-09T10:27:54.962506+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:27:54.962506+0000","last_undegraded":"2023-10-09T10:27:54.962506+0000","last_fullsized":"2023-10-09T10:27:54.962506+0000","mapping_epoch":513,"log_start":"502'1912","ondisk_log_start":"502'1912","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25624576,"num_objects":391,"num_object_clones":0,"num_object_copies":1173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4672,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":220,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.553188+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:27:37.553188+0000","last_peered":"2023-10-09T10:27:37.553188+0000","last_clean":"2023-10-09T10:27:37.553188+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:27:37.553188+0000","last_undegraded":"2023-10-09T10:27:37.553188+0000","last_fullsized":"2023-10-09T10:27:37.553188+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":455,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552222+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:27:37.552222+0000","last_peered":"2023-10-09T10:27:37.552222+0000","last_clean":"2023-10-09T10:27:37.552222+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:27:37.552222+0000","last_undegraded":"2023-10-09T10:27:37.552222+0000","last_fullsized":"2023-10-09T10:27:37.552222+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"545'7571","reported_seq":8873,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:55.042908+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:27:55.042908+0000","last_peered":"2023-10-09T10:27:55.042908+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:27:55.042908+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"503'2651","ondisk_log_start":"503'2651","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4920,"log_dups_size":0,"ondisk_log_size":4920,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24150016,"num_objects":368,"num_object_clones":0,"num_object_copies":1104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6650,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,5","objects":-1261}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":156,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:46.907718+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:27:46.907718+0000","last_peered":"2023-10-09T10:27:46.907718+0000","last_clean":"2023-10-09T10:27:46.907718+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:27:46.907718+0000","last_undegraded":"2023-10-09T10:27:46.907718+0000","last_fullsized":"2023-10-09T10:27:46.907718+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.544182+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:27:37.544182+0000","last_peered":"2023-10-09T10:27:37.544182+0000","last_clean":"2023-10-09T10:27:37.544182+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:27:37.544182+0000","last_undegraded":"2023-10-09T10:27:37.544182+0000","last_fullsized":"2023-10-09T10:27:37.544182+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"545'7609","reported_seq":7741,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:56.818714+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:27:56.818714+0000","last_peered":"2023-10-09T10:27:56.818714+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:27:56.818714+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"503'2548","ondisk_log_start":"503'2548","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5061,"log_dups_size":0,"ondisk_log_size":5061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22675456,"num_objects":346,"num_object_clones":0,"num_object_copies":1038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4721,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-2712}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":156,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.543260+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:27:37.543260+0000","last_peered":"2023-10-09T10:27:37.543260+0000","last_clean":"2023-10-09T10:27:37.543260+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:27:37.543260+0000","last_undegraded":"2023-10-09T10:27:37.543260+0000","last_fullsized":"2023-10-09T10:27:37.543260+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":122,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549412+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:27:37.549412+0000","last_peered":"2023-10-09T10:27:37.549412+0000","last_clean":"2023-10-09T10:27:37.549412+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:27:37.549412+0000","last_undegraded":"2023-10-09T10:27:37.549412+0000","last_fullsized":"2023-10-09T10:27:37.549412+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"545'7651","reported_seq":14501,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:56.824905+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:27:56.824905+0000","last_peered":"2023-10-09T10:27:56.824905+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:27:56.824905+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"503'2565","ondisk_log_start":"503'2565","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23461888,"num_objects":358,"num_object_clones":0,"num_object_copies":1074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":806,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6706,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2254,"num_bytes_recovered":147718144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,6","objects":-448}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":138,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.543778+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:27:37.543778+0000","last_peered":"2023-10-09T10:27:37.543778+0000","last_clean":"2023-10-09T10:27:37.543778+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:27:37.543778+0000","last_undegraded":"2023-10-09T10:27:37.543778+0000","last_fullsized":"2023-10-09T10:27:37.543778+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":165,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549603+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:27:37.549603+0000","last_peered":"2023-10-09T10:27:37.549603+0000","last_clean":"2023-10-09T10:27:37.549603+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:27:37.549603+0000","last_undegraded":"2023-10-09T10:27:37.549603+0000","last_fullsized":"2023-10-09T10:27:37.549603+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"545'7601","reported_seq":8047,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:56.849592+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:27:56.849592+0000","last_peered":"2023-10-09T10:27:56.849592+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:27:56.849592+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"503'2573","ondisk_log_start":"503'2573","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5028,"log_dups_size":0,"ondisk_log_size":5028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20512768,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4494,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,3","objects":-1768}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":497,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550814+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:27:37.550814+0000","last_peered":"2023-10-09T10:27:37.550814+0000","last_clean":"2023-10-09T10:27:37.550814+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:27:37.550814+0000","last_undegraded":"2023-10-09T10:27:37.550814+0000","last_fullsized":"2023-10-09T10:27:37.550814+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":559,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549585+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:27:37.549585+0000","last_peered":"2023-10-09T10:27:37.549585+0000","last_clean":"2023-10-09T10:27:37.549585+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:27:37.549585+0000","last_undegraded":"2023-10-09T10:27:37.549585+0000","last_fullsized":"2023-10-09T10:27:37.549585+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"545'13528","reported_seq":13629,"reported_epoch":545,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:27:58.139790+0000","last_change":"2023-10-09T10:27:35.872423+0000","last_active":"2023-10-09T10:27:58.139790+0000","last_peered":"2023-10-09T10:27:58.139790+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:27:58.139790+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"525'7500","ondisk_log_start":"525'7500","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":6028,"log_dups_size":0,"ondisk_log_size":6028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158936,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":568,"num_objects_misplaced":568,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13528,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":568}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":266,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550375+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:27:37.550375+0000","last_peered":"2023-10-09T10:27:37.550375+0000","last_clean":"2023-10-09T10:27:37.550375+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:27:37.550375+0000","last_undegraded":"2023-10-09T10:27:37.550375+0000","last_fullsized":"2023-10-09T10:27:37.550375+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":229,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.555584+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:27:37.555584+0000","last_peered":"2023-10-09T10:27:37.555584+0000","last_clean":"2023-10-09T10:27:37.555584+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:27:37.555584+0000","last_undegraded":"2023-10-09T10:27:37.555584+0000","last_fullsized":"2023-10-09T10:27:37.555584+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":509,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552948+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:27:37.552948+0000","last_peered":"2023-10-09T10:27:37.552948+0000","last_clean":"2023-10-09T10:27:37.552948+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:27:37.552948+0000","last_undegraded":"2023-10-09T10:27:37.552948+0000","last_fullsized":"2023-10-09T10:27:37.552948+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"545'13645","reported_seq":13706,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:56.893531+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:27:56.893531+0000","last_peered":"2023-10-09T10:27:56.893531+0000","last_clean":"2023-10-09T10:27:56.893531+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:27:56.893531+0000","last_undegraded":"2023-10-09T10:27:56.893531+0000","last_fullsized":"2023-10-09T10:27:56.893531+0000","mapping_epoch":494,"log_start":"530'8606","ondisk_log_start":"530'8606","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5039,"log_dups_size":0,"ondisk_log_size":5039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41484288,"num_objects":633,"num_object_clones":0,"num_object_copies":1899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13645,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":229,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552802+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:27:37.552802+0000","last_peered":"2023-10-09T10:27:37.552802+0000","last_clean":"2023-10-09T10:27:37.552802+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:27:37.552802+0000","last_undegraded":"2023-10-09T10:27:37.552802+0000","last_fullsized":"2023-10-09T10:27:37.552802+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":147,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.544417+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:27:37.544417+0000","last_peered":"2023-10-09T10:27:37.544417+0000","last_clean":"2023-10-09T10:27:37.544417+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:27:37.544417+0000","last_undegraded":"2023-10-09T10:27:37.544417+0000","last_fullsized":"2023-10-09T10:27:37.544417+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":458,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551358+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:27:37.551358+0000","last_peered":"2023-10-09T10:27:37.551358+0000","last_clean":"2023-10-09T10:27:37.551358+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:27:37.551358+0000","last_undegraded":"2023-10-09T10:27:37.551358+0000","last_fullsized":"2023-10-09T10:27:37.551358+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"545'7621","reported_seq":16195,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:54.902595+0000","last_change":"2023-10-09T10:27:21.711478+0000","last_active":"2023-10-09T10:27:54.902595+0000","last_peered":"2023-10-09T10:27:54.902595+0000","last_clean":"2023-10-09T10:27:54.902595+0000","last_became_active":"2023-10-09T10:27:21.694370+0000","last_became_peered":"2023-10-09T10:27:21.694370+0000","last_unstale":"2023-10-09T10:27:54.902595+0000","last_undegraded":"2023-10-09T10:27:54.902595+0000","last_fullsized":"2023-10-09T10:27:54.902595+0000","mapping_epoch":542,"log_start":"503'2733","ondisk_log_start":"503'2733","created":494,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4888,"log_dups_size":0,"ondisk_log_size":4888,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23166976,"num_objects":353,"num_object_clones":0,"num_object_copies":1059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6655,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3094,"num_bytes_recovered":202768384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":496,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:46.906978+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:27:46.906978+0000","last_peered":"2023-10-09T10:27:46.906978+0000","last_clean":"2023-10-09T10:27:46.906978+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:27:46.906978+0000","last_undegraded":"2023-10-09T10:27:46.906978+0000","last_fullsized":"2023-10-09T10:27:46.906978+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":113,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551490+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:27:37.551490+0000","last_peered":"2023-10-09T10:27:37.551490+0000","last_clean":"2023-10-09T10:27:37.551490+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:27:37.551490+0000","last_undegraded":"2023-10-09T10:27:37.551490+0000","last_fullsized":"2023-10-09T10:27:37.551490+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":558,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551756+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:27:37.551756+0000","last_peered":"2023-10-09T10:27:37.551756+0000","last_clean":"2023-10-09T10:27:37.551756+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:27:37.551756+0000","last_undegraded":"2023-10-09T10:27:37.551756+0000","last_fullsized":"2023-10-09T10:27:37.551756+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"545'13399","reported_seq":13483,"reported_epoch":545,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:27:55.052652+0000","last_change":"2023-10-09T10:26:37.693448+0000","last_active":"2023-10-09T10:27:55.052652+0000","last_peered":"2023-10-09T10:27:55.052652+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:27:55.052652+0000","last_undegraded":"2023-10-09T10:27:55.052652+0000","last_fullsized":"2023-10-09T10:27:55.052652+0000","mapping_epoch":507,"log_start":"530'8500","ondisk_log_start":"530'8500","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4899,"log_dups_size":0,"ondisk_log_size":4899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46727168,"num_objects":713,"num_object_clones":0,"num_object_copies":2139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":713,"num_objects_unfound":0,"num_objects_dirty":713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13399,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":713}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":199,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552245+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:27:37.552245+0000","last_peered":"2023-10-09T10:27:37.552245+0000","last_clean":"2023-10-09T10:27:37.552245+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:27:37.552245+0000","last_undegraded":"2023-10-09T10:27:37.552245+0000","last_fullsized":"2023-10-09T10:27:37.552245+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":739,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:51.652520+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:27:51.652520+0000","last_peered":"2023-10-09T10:27:51.652520+0000","last_clean":"2023-10-09T10:27:51.652520+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:27:51.652520+0000","last_undegraded":"2023-10-09T10:27:51.652520+0000","last_fullsized":"2023-10-09T10:27:51.652520+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":247,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552606+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:27:37.552606+0000","last_peered":"2023-10-09T10:27:37.552606+0000","last_clean":"2023-10-09T10:27:37.552606+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:27:37.552606+0000","last_undegraded":"2023-10-09T10:27:37.552606+0000","last_fullsized":"2023-10-09T10:27:37.552606+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":156,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549504+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:27:37.549504+0000","last_peered":"2023-10-09T10:27:37.549504+0000","last_clean":"2023-10-09T10:27:37.549504+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:27:37.549504+0000","last_undegraded":"2023-10-09T10:27:37.549504+0000","last_fullsized":"2023-10-09T10:27:37.549504+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"545'7759","reported_seq":14711,"reported_epoch":545,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:27:55.019084+0000","last_change":"2023-10-09T10:27:22.518901+0000","last_active":"2023-10-09T10:27:55.019084+0000","last_peered":"2023-10-09T10:27:55.019084+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:27:55.019084+0000","last_undegraded":"2023-10-09T10:27:55.019084+0000","last_fullsized":"2023-10-09T10:27:55.019084+0000","mapping_epoch":516,"log_start":"504'2888","ondisk_log_start":"504'2888","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4871,"log_dups_size":0,"ondisk_log_size":4871,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26443776,"num_objects":403,"num_object_clones":0,"num_object_copies":1209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":951,"num_objects_unfound":0,"num_objects_dirty":403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6835,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2306,"num_bytes_recovered":151126016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","6"],"object_location_counts":[{"shards":"3,5,6","objects":-548}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":230,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551662+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:27:37.551662+0000","last_peered":"2023-10-09T10:27:37.551662+0000","last_clean":"2023-10-09T10:27:37.551662+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:27:37.551662+0000","last_undegraded":"2023-10-09T10:27:37.551662+0000","last_fullsized":"2023-10-09T10:27:37.551662+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":490,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.554966+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:27:37.554966+0000","last_peered":"2023-10-09T10:27:37.554966+0000","last_clean":"2023-10-09T10:27:37.554966+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:27:37.554966+0000","last_undegraded":"2023-10-09T10:27:37.554966+0000","last_fullsized":"2023-10-09T10:27:37.554966+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":41,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550475+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:27:37.550475+0000","last_peered":"2023-10-09T10:27:37.550475+0000","last_clean":"2023-10-09T10:27:37.550475+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:27:37.550475+0000","last_undegraded":"2023-10-09T10:27:37.550475+0000","last_fullsized":"2023-10-09T10:27:37.550475+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"545'13607","reported_seq":13666,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:58.112117+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:27:58.112117+0000","last_peered":"2023-10-09T10:27:58.112117+0000","last_clean":"2023-10-09T10:27:58.112117+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:27:58.112117+0000","last_undegraded":"2023-10-09T10:27:58.112117+0000","last_fullsized":"2023-10-09T10:27:58.112117+0000","mapping_epoch":494,"log_start":"525'7600","ondisk_log_start":"525'7600","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":6007,"log_dups_size":0,"ondisk_log_size":6007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39649280,"num_objects":605,"num_object_clones":0,"num_object_copies":1815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13607,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":41,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549946+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:27:37.549946+0000","last_peered":"2023-10-09T10:27:37.549946+0000","last_clean":"2023-10-09T10:27:37.549946+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:27:37.549946+0000","last_undegraded":"2023-10-09T10:27:37.549946+0000","last_fullsized":"2023-10-09T10:27:37.549946+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":526,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.553496+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:27:37.553496+0000","last_peered":"2023-10-09T10:27:37.553496+0000","last_clean":"2023-10-09T10:27:37.553496+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:27:37.553496+0000","last_undegraded":"2023-10-09T10:27:37.553496+0000","last_fullsized":"2023-10-09T10:27:37.553496+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":458,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552517+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:27:37.552517+0000","last_peered":"2023-10-09T10:27:37.552517+0000","last_clean":"2023-10-09T10:27:37.552517+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:27:37.552517+0000","last_undegraded":"2023-10-09T10:27:37.552517+0000","last_fullsized":"2023-10-09T10:27:37.552517+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"545'13054","reported_seq":13115,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:56.893665+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:27:56.893665+0000","last_peered":"2023-10-09T10:27:56.893665+0000","last_clean":"2023-10-09T10:27:56.893665+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:27:56.893665+0000","last_undegraded":"2023-10-09T10:27:56.893665+0000","last_fullsized":"2023-10-09T10:27:56.893665+0000","mapping_epoch":494,"log_start":"529'7976","ondisk_log_start":"529'7976","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5078,"log_dups_size":0,"ondisk_log_size":5078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44695552,"num_objects":682,"num_object_clones":0,"num_object_copies":2046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13054,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":346,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551084+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:27:37.551084+0000","last_peered":"2023-10-09T10:27:37.551084+0000","last_clean":"2023-10-09T10:27:37.551084+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:27:37.551084+0000","last_undegraded":"2023-10-09T10:27:37.551084+0000","last_fullsized":"2023-10-09T10:27:37.551084+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":526,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.549519+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:27:37.549519+0000","last_peered":"2023-10-09T10:27:37.549519+0000","last_clean":"2023-10-09T10:27:37.549519+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:27:37.549519+0000","last_undegraded":"2023-10-09T10:27:37.549519+0000","last_fullsized":"2023-10-09T10:27:37.549519+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":41,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550177+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:27:37.550177+0000","last_peered":"2023-10-09T10:27:37.550177+0000","last_clean":"2023-10-09T10:27:37.550177+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:27:37.550177+0000","last_undegraded":"2023-10-09T10:27:37.550177+0000","last_fullsized":"2023-10-09T10:27:37.550177+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"545'13713","reported_seq":13772,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:58.137836+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:27:58.137836+0000","last_peered":"2023-10-09T10:27:58.137836+0000","last_clean":"2023-10-09T10:27:58.137836+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:27:58.137836+0000","last_undegraded":"2023-10-09T10:27:58.137836+0000","last_fullsized":"2023-10-09T10:27:58.137836+0000","mapping_epoch":494,"log_start":"525'7700","ondisk_log_start":"525'7700","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":6013,"log_dups_size":0,"ondisk_log_size":6013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36110336,"num_objects":551,"num_object_clones":0,"num_object_copies":1653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13713,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":48,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.551642+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:27:37.551642+0000","last_peered":"2023-10-09T10:27:37.551642+0000","last_clean":"2023-10-09T10:27:37.551642+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:27:37.551642+0000","last_undegraded":"2023-10-09T10:27:37.551642+0000","last_fullsized":"2023-10-09T10:27:37.551642+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":45,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550800+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:27:37.550800+0000","last_peered":"2023-10-09T10:27:37.550800+0000","last_clean":"2023-10-09T10:27:37.550800+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:27:37.550800+0000","last_undegraded":"2023-10-09T10:27:37.550800+0000","last_fullsized":"2023-10-09T10:27:37.550800+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":225,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.553017+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:27:37.553017+0000","last_peered":"2023-10-09T10:27:37.553017+0000","last_clean":"2023-10-09T10:27:37.553017+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:27:37.553017+0000","last_undegraded":"2023-10-09T10:27:37.553017+0000","last_fullsized":"2023-10-09T10:27:37.553017+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"545'7481","reported_seq":7588,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:55.029652+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:27:55.029652+0000","last_peered":"2023-10-09T10:27:55.029652+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:27:55.029652+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"503'2585","ondisk_log_start":"503'2585","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4896,"log_dups_size":0,"ondisk_log_size":4896,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24281088,"num_objects":371,"num_object_clones":0,"num_object_copies":1113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6561,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,5","objects":-1845}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:41.886488+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:27:41.886488+0000","last_peered":"2023-10-09T10:27:41.886488+0000","last_clean":"2023-10-09T10:27:41.886488+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:27:41.886488+0000","last_undegraded":"2023-10-09T10:27:41.886488+0000","last_fullsized":"2023-10-09T10:27:41.886488+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":50,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:41.886519+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:27:41.886519+0000","last_peered":"2023-10-09T10:27:41.886519+0000","last_clean":"2023-10-09T10:27:41.886519+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:27:41.886519+0000","last_undegraded":"2023-10-09T10:27:41.886519+0000","last_fullsized":"2023-10-09T10:27:41.886519+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"545'7637","reported_seq":12394,"reported_epoch":545,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:27:54.978533+0000","last_change":"2023-10-09T10:27:07.588772+0000","last_active":"2023-10-09T10:27:54.978533+0000","last_peered":"2023-10-09T10:27:54.978533+0000","last_clean":"2023-10-09T10:24:39.553032+0000","last_became_active":"2023-10-09T10:27:07.549958+0000","last_became_peered":"2023-10-09T10:27:07.549958+0000","last_unstale":"2023-10-09T10:27:54.978533+0000","last_undegraded":"2023-10-09T10:27:54.978533+0000","last_fullsized":"2023-10-09T10:27:54.978533+0000","mapping_epoch":540,"log_start":"503'1976","ondisk_log_start":"503'1976","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22183936,"num_objects":339,"num_object_clones":0,"num_object_copies":1017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6713,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2333,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":["6","0","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":339}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":275,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.553282+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:27:37.553282+0000","last_peered":"2023-10-09T10:27:37.553282+0000","last_clean":"2023-10-09T10:27:37.553282+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:27:37.553282+0000","last_undegraded":"2023-10-09T10:27:37.553282+0000","last_fullsized":"2023-10-09T10:27:37.553282+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":138,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.544567+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:27:37.544567+0000","last_peered":"2023-10-09T10:27:37.544567+0000","last_clean":"2023-10-09T10:27:37.544567+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:27:37.544567+0000","last_undegraded":"2023-10-09T10:27:37.544567+0000","last_fullsized":"2023-10-09T10:27:37.544567+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"545'7634","reported_seq":14436,"reported_epoch":545,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:27:55.008016+0000","last_change":"2023-10-09T10:25:09.748624+0000","last_active":"2023-10-09T10:27:55.008016+0000","last_peered":"2023-10-09T10:27:55.008016+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:27:55.008016+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"504'2750","ondisk_log_start":"504'2750","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4884,"log_dups_size":0,"ondisk_log_size":4884,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26312704,"num_objects":402,"num_object_clones":0,"num_object_copies":1206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":834,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6690,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2240,"num_bytes_recovered":146800640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"5,6","objects":-432}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":258,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.554503+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:27:37.554503+0000","last_peered":"2023-10-09T10:27:37.554503+0000","last_clean":"2023-10-09T10:27:37.554503+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:27:37.554503+0000","last_undegraded":"2023-10-09T10:27:37.554503+0000","last_fullsized":"2023-10-09T10:27:37.554503+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":368,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.552835+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:27:37.552835+0000","last_peered":"2023-10-09T10:27:37.552835+0000","last_clean":"2023-10-09T10:27:37.552835+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:27:37.552835+0000","last_undegraded":"2023-10-09T10:27:37.552835+0000","last_fullsized":"2023-10-09T10:27:37.552835+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"545'7607","reported_seq":13970,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:56.907088+0000","last_change":"2023-10-09T10:27:21.732594+0000","last_active":"2023-10-09T10:27:56.907088+0000","last_peered":"2023-10-09T10:27:56.907088+0000","last_clean":"2023-10-09T10:27:56.907088+0000","last_became_active":"2023-10-09T10:27:21.699893+0000","last_became_peered":"2023-10-09T10:27:21.699893+0000","last_unstale":"2023-10-09T10:27:56.907088+0000","last_undegraded":"2023-10-09T10:27:56.907088+0000","last_fullsized":"2023-10-09T10:27:56.907088+0000","mapping_epoch":542,"log_start":"503'2557","ondisk_log_start":"503'2557","created":501,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":5050,"log_dups_size":0,"ondisk_log_size":5050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:06.239031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20250624,"num_objects":309,"num_object_clones":0,"num_object_copies":927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4670,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3137,"num_bytes_recovered":122093568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":44,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.533819+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:27:37.533819+0000","last_peered":"2023-10-09T10:27:37.533819+0000","last_clean":"2023-10-09T10:27:37.533819+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:27:37.533819+0000","last_undegraded":"2023-10-09T10:27:37.533819+0000","last_fullsized":"2023-10-09T10:27:37.533819+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":598,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.554763+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:27:37.554763+0000","last_peered":"2023-10-09T10:27:37.554763+0000","last_clean":"2023-10-09T10:27:37.554763+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:27:37.554763+0000","last_undegraded":"2023-10-09T10:27:37.554763+0000","last_fullsized":"2023-10-09T10:27:37.554763+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"545'7592","reported_seq":7664,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T10:27:55.052198+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:27:55.052198+0000","last_peered":"2023-10-09T10:27:55.052198+0000","last_clean":"2023-10-09T10:27:55.052198+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:27:55.052198+0000","last_undegraded":"2023-10-09T10:27:55.052198+0000","last_fullsized":"2023-10-09T10:27:55.052198+0000","mapping_epoch":501,"log_start":"503'2715","ondisk_log_start":"503'2715","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":4877,"log_dups_size":0,"ondisk_log_size":4877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4331,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":62,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.550591+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:27:37.550591+0000","last_peered":"2023-10-09T10:27:37.550591+0000","last_clean":"2023-10-09T10:27:37.550591+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:27:37.550591+0000","last_undegraded":"2023-10-09T10:27:37.550591+0000","last_fullsized":"2023-10-09T10:27:37.550591+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":165,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-09T10:27:37.543244+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:27:37.543244+0000","last_peered":"2023-10-09T10:27:37.543244+0000","last_clean":"2023-10-09T10:27:37.543244+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:27:37.543244+0000","last_undegraded":"2023-10-09T10:27:37.543244+0000","last_fullsized":"2023-10-09T10:27:37.543244+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":709984280,"num_objects":10836,"num_object_clones":0,"num_object_copies":32508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16048,"num_objects_misplaced":5383,"num_objects_unfound":0,"num_objects_dirty":10836,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":200110,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33547,"num_bytes_recovered":1936785408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5436219392,"data_stored":5436211248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133585,"ondisk_log_size":133585,"up":78,"acting":67,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430699,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626927,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1880412,"kb_used_data":363092,"kb_used_omap":0,"kb_used_meta":1517312,"kb_avail":91864740,"statfs":{"total":95995035648,"available":94069493760,"internally_reserved":0,"allocated":371806208,"data_stored":370831788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553727488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.625,"15min":1.5669999999999999},"min":{"1min":0.223,"5min":0.216,"15min":0.216},"max":{"1min":6.1849999999999996,"5min":9.4260000000000002,"15min":44.055},"last":6.7030000000000003},{"interface":"front","average":{"1min":2.9740000000000002,"5min":1.7130000000000001,"15min":1.9079999999999999},"min":{"1min":0.28799999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":9.3499999999999996,"5min":9.3499999999999996,"15min":43.869999999999997},"last":4.5780000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.3400000000000001,"15min":1.998},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":2.4049999999999998,"5min":6.3899999999999997,"15min":155.01400000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.482,"5min":1.3580000000000001,"15min":2.1739999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.6050000000000004,"5min":6.3369999999999997,"15min":154.71100000000001},"last":3.0030000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.887,"15min":1.833},"min":{"1min":0.315,"5min":0.309,"15min":0.24099999999999999},"max":{"1min":5.8300000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":3.149},{"interface":"front","average":{"1min":1.853,"5min":1.667,"15min":1.643},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4379999999999997,"5min":6.4379999999999997,"15min":11.720000000000001},"last":6.3550000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":2.9470000000000001,"5min":2.0459999999999998,"15min":1.6579999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":43.393000000000001,"5min":43.393000000000001,"15min":43.393000000000001},"last":2.0190000000000001},{"interface":"front","average":{"1min":3.4500000000000002,"5min":2.1829999999999998,"15min":1.806},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":43.466000000000001,"5min":43.466000000000001,"15min":43.466000000000001},"last":2.0299999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.0680000000000001,"15min":1.974},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":10.772,"5min":10.772,"15min":122.041},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.73,"15min":1.7769999999999999},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.1059999999999999,"5min":11.266,"15min":122.13200000000001},"last":0.35399999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.27,"15min":2.1200000000000001},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.5670000000000002,"5min":9.6890000000000001,"15min":121.98999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.405,"5min":1.4950000000000001,"15min":1.673},"min":{"1min":0.40500000000000003,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":4.835,"5min":7.0060000000000002,"15min":122.346},"last":0.51200000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.0219999999999998,"15min":1.966},"min":{"1min":0.38,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":12.612,"5min":49.991,"15min":130.66999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.5880000000000001,"5min":2.097,"15min":2.02},"min":{"1min":0.40799999999999997,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":12.659000000000001,"5min":50.023000000000003,"15min":130.702},"last":0.58899999999999997}]}]},{"osd":1,"up_from":0,"seq":360777253241,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416765,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2110740,"kb_used_data":1085644,"kb_used_omap":28,"kb_used_meta":1024867,"kb_avail":91634412,"statfs":{"total":95995035648,"available":93833637888,"internally_reserved":0,"allocated":1111699456,"data_stored":1109549723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1049464656},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,4,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:27:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.8049999999999999,"15min":1.6990000000000001},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.19800000000000001},"max":{"1min":13.077999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":6.9909999999999997},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.712,"15min":1.915},"min":{"1min":0.26500000000000001,"5min":0.214,"15min":0.192},"max":{"1min":12.349,"5min":12.349,"15min":56.950000000000003},"last":3.1880000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:27:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":1.845,"15min":2.0230000000000001},"min":{"1min":0.36099999999999999,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":8.859,"5min":17.093,"15min":53.988999999999997},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.286,"5min":2.081,"15min":2.1150000000000002},"min":{"1min":0.29999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.3739999999999997,"5min":17.053999999999998,"15min":54.119},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:27:05 2023","interfaces":[{"interface":"back","average":{"1min":3.7349999999999999,"5min":2.0649999999999999,"15min":1.655},"min":{"1min":0.248,"5min":0.217,"15min":0.217},"max":{"1min":29.835999999999999,"5min":29.835999999999999,"15min":56.984999999999999},"last":6.2169999999999996},{"interface":"front","average":{"1min":4.242,"5min":2.585,"15min":1.7749999999999999},"min":{"1min":0.32900000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":29.776,"5min":29.776,"15min":56.475999999999999},"last":1.6519999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":2.077,"15min":2.1579999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":9.2859999999999996,"5min":28.289000000000001,"15min":57.719000000000001},"last":3.2400000000000002},{"interface":"front","average":{"1min":2.5920000000000001,"5min":2.4710000000000001,"15min":2.1920000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.896000000000001,"5min":28.331,"15min":56.378},"last":1.698}]},{"osd":5,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":3.5830000000000002,"15min":3.4689999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":11.867000000000001,"5min":90.412000000000006,"15min":229.589},"last":1.677},{"interface":"front","average":{"1min":2.625,"5min":3.0270000000000001,"15min":3.194},"min":{"1min":0.35799999999999998,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":6.9539999999999997,"5min":90.393000000000001,"15min":228.88900000000001},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":2.0640000000000001,"15min":2.2690000000000001},"min":{"1min":0.41399999999999998,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":8.5719999999999992,"5min":9.7840000000000007,"15min":54.482999999999997},"last":0.628},{"interface":"front","average":{"1min":2.7989999999999999,"5min":2.2450000000000001,"15min":2.4049999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.34799999999999998},"max":{"1min":9.3439999999999994,"5min":9.3439999999999994,"15min":57.753},"last":3.2749999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.2749999999999999,"15min":2.2959999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.34999999999999998},"max":{"1min":5.6959999999999997,"5min":12.029999999999999,"15min":56.411999999999999},"last":1.7230000000000001},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.335,"15min":2.3410000000000002},"min":{"1min":0.47199999999999998,"5min":0.39900000000000002,"15min":0.38300000000000001},"max":{"1min":7.0490000000000004,"5min":17.129999999999999,"15min":54.451999999999998},"last":0.72599999999999998}]}]},{"osd":2,"up_from":217,"seq":932007903501,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1979204,"kb_used_data":1102652,"kb_used_omap":83,"kb_used_meta":876460,"kb_avail":91765948,"statfs":{"total":95995035648,"available":93968330752,"internally_reserved":0,"allocated":1129115648,"data_stored":1127165614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":897495535},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.5939999999999999,"15min":2.008},"min":{"1min":0.22900000000000001,"5min":0.192,"15min":0.185},"max":{"1min":14.666,"5min":48.990000000000002,"15min":48.990000000000002},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.8340000000000001,"5min":2.363,"15min":2.1379999999999999},"min":{"1min":0.23599999999999999,"5min":0.22600000000000001,"15min":0.20899999999999999},"max":{"1min":13.773,"5min":49.412999999999997,"15min":49.412999999999997},"last":0.64200000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":2.343,"15min":2.0230000000000001},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":14.632999999999999,"5min":14.632999999999999,"15min":34.893999999999998},"last":0.252},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.7689999999999999,"15min":1.823},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":13.356999999999999,"5min":13.356999999999999,"15min":34.962000000000003},"last":0.32000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.8879999999999999,"15min":1.4470000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.8410000000000002,"5min":15.115,"15min":35.043999999999997},"last":4.665},{"interface":"front","average":{"1min":2.395,"5min":1.776,"15min":1.4530000000000001},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.6820000000000004,"5min":11.206,"15min":50.406999999999996},"last":4.5990000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:27:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":1.899,"15min":1.6220000000000001},"min":{"1min":0.35899999999999999,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":8.1859999999999999,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.7639999999999998,"5min":2.2309999999999999,"15min":1.913},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":8.7219999999999995,"5min":47.704000000000001,"15min":47.704000000000001},"last":4.524}]},{"osd":5,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":1.8260000000000001,"15min":2.0169999999999999},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":8.6839999999999993,"5min":11.266999999999999,"15min":57.298000000000002},"last":4.5700000000000003},{"interface":"front","average":{"1min":3.8860000000000001,"5min":2.4199999999999999,"15min":2.1400000000000001},"min":{"1min":0.375,"5min":0.31,"15min":0.31},"max":{"1min":14.31,"5min":14.31,"15min":57.828000000000003},"last":0.58299999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":2.0310000000000001,"15min":1.9299999999999999},"min":{"1min":0.40300000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":7.9390000000000001,"5min":12.411,"15min":18.943999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.556,"5min":1.95,"15min":1.948},"min":{"1min":0.39300000000000002,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":8.7590000000000003,"5min":13.975,"15min":24.379000000000001},"last":4.6239999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.95,"5min":2.1139999999999999,"15min":2.1920000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":9.5199999999999996,"5min":11.704000000000001,"15min":35.008000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.1230000000000002,"15min":2.121},"min":{"1min":0.51400000000000001,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":8.8149999999999995,"5min":15.548,"15min":24.323},"last":1.4690000000000001}]}]},{"osd":3,"up_from":352,"seq":1511828488340,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1325264,"kb_used_data":747784,"kb_used_omap":15,"kb_used_meta":577392,"kb_avail":92419888,"statfs":{"total":95995035648,"available":94637965312,"internally_reserved":0,"allocated":765730816,"data_stored":763296354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":591249862},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.9740000000000002,"5min":3.4220000000000002,"15min":1.8859999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":13.661,"5min":70.004000000000005,"15min":70.004000000000005},"last":7.4740000000000002},{"interface":"front","average":{"1min":3.9849999999999999,"5min":3.1499999999999999,"15min":1.8779999999999999},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":10.313000000000001,"5min":66.802000000000007,"15min":66.802000000000007},"last":7.0529999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":1.8160000000000001,"15min":1.9490000000000001},"min":{"1min":0.42499999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.3100000000000005,"5min":25.370000000000001,"15min":105.401},"last":0.83999999999999997},{"interface":"front","average":{"1min":2.6800000000000002,"5min":2.0430000000000001,"15min":1.9299999999999999},"min":{"1min":0.28399999999999997,"5min":0.25900000000000001,"15min":0.23300000000000001},"max":{"1min":10.422000000000001,"5min":14.613,"15min":104.914},"last":1.1000000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2250000000000001,"5min":2.266,"15min":2.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":13.497999999999999,"5min":13.497999999999999,"15min":13.497999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.4689999999999999,"5min":2.2549999999999999,"15min":1.5780000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27600000000000002},"max":{"1min":10.339,"5min":25.338999999999999,"15min":25.338999999999999},"last":5.5369999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":2.016,"15min":1.5169999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.268000000000001,"5min":21.196000000000002,"15min":21.831},"last":0.45000000000000001},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.927,"15min":2.1179999999999999},"min":{"1min":0.34499999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":13.577999999999999,"5min":19.323,"15min":22.097999999999999},"last":0.67500000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5259999999999998,"5min":2.0609999999999999,"15min":1.4159999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":8.2710000000000008,"5min":25.251999999999999,"15min":25.251999999999999},"last":0.81699999999999995},{"interface":"front","average":{"1min":2.9319999999999999,"5min":2.3639999999999999,"15min":1.6910000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.388,"5min":10.388,"15min":13.394},"last":0.499}]},{"osd":6,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":2.262,"15min":1.845},"min":{"1min":0.375,"5min":0.36199999999999999,"15min":0.35599999999999998},"max":{"1min":13.545999999999999,"5min":19.114000000000001,"15min":19.114000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":2.758,"5min":2.0979999999999999,"15min":1.593},"min":{"1min":0.37,"5min":0.37,"15min":0.33900000000000002},"max":{"1min":10.117000000000001,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.79000000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.1589999999999998,"15min":1.643},"min":{"1min":0.41299999999999998,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":10.345000000000001,"5min":10.345000000000001,"15min":10.459},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.9060000000000001,"5min":2.3239999999999998,"15min":1.8660000000000001},"min":{"1min":0.41799999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.234,"5min":19.280000000000001,"15min":19.280000000000001},"last":0.75}]}]},{"osd":4,"up_from":266,"seq":1142461300970,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1862112,"kb_used_data":920792,"kb_used_omap":35,"kb_used_meta":941276,"kb_avail":91883040,"statfs":{"total":95995035648,"available":94088232960,"internally_reserved":0,"allocated":942891008,"data_stored":941910914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":963866862},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":8.9870000000000001,"5min":3.1680000000000001,"15min":2.6859999999999999},"min":{"1min":0.245,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":109.193,"5min":109.193,"15min":109.193},"last":7.6539999999999999},{"interface":"front","average":{"1min":8.6099999999999994,"5min":3.548,"15min":3.2469999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.224},"max":{"1min":110.438,"5min":110.438,"15min":110.438},"last":2.3999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":2.1040000000000001,"15min":2.2709999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.24299999999999999},"max":{"1min":10.696,"5min":12.209,"15min":71.457999999999998},"last":5.1029999999999998},{"interface":"front","average":{"1min":2.2799999999999998,"5min":2.7090000000000001,"15min":2.2330000000000001},"min":{"1min":0.32700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":9.2040000000000006,"5min":49.781999999999996,"15min":71.658000000000001},"last":3.968}]},{"osd":2,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.141,"15min":1.8160000000000001},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":5.4710000000000001,"5min":15.944000000000001,"15min":15.944000000000001},"last":6.016},{"interface":"front","average":{"1min":1.419,"5min":1.7430000000000001,"15min":1.766},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.0309999999999997,"5min":14.473000000000001,"15min":14.473000000000001},"last":1.466}]},{"osd":3,"last update":"Mon Oct 9 10:26:59 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":2.552,"15min":2.0659999999999998},"min":{"1min":0.40999999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.257,"5min":11.347,"15min":12.074},"last":5.141},{"interface":"front","average":{"1min":2.5600000000000001,"5min":2.786,"15min":1.839},"min":{"1min":0.50900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.1270000000000007,"5min":16.067,"15min":16.067},"last":3.9990000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.0369999999999999,"15min":2.0510000000000002},"min":{"1min":0.28000000000000003,"5min":0.26500000000000001,"15min":0.23799999999999999},"max":{"1min":3.8290000000000002,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.312},{"interface":"front","average":{"1min":1.0409999999999999,"5min":2.0619999999999998,"15min":1.871},"min":{"1min":0.317,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":5.9509999999999996,"5min":49.915999999999997,"15min":99.384},"last":0.32900000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.526,"15min":1.4079999999999999},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.0579999999999998,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.8600000000000001,"5min":2.2930000000000001,"15min":1.804},"min":{"1min":0.34000000000000002,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":10.824,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.60699999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":2.4569999999999999,"15min":1.913},"min":{"1min":0.39400000000000002,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":9.016,"5min":12.391999999999999,"15min":17.369},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.2350000000000001,"5min":2.431,"15min":1.778},"min":{"1min":0.37,"5min":0.37,"15min":0.314},"max":{"1min":3.9750000000000001,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.47399999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823154,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2676692,"kb_used_data":1175820,"kb_used_omap":0,"kb_used_meta":1500800,"kb_avail":91068460,"statfs":{"total":95995035648,"available":93254103040,"internally_reserved":0,"allocated":1204039680,"data_stored":1203067640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536819200},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,1,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":2.9260000000000002,"15min":2.0979999999999999},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":19.102,"5min":19.102,"15min":22.469999999999999},"last":5.0780000000000003},{"interface":"front","average":{"1min":2.7850000000000001,"5min":1.784,"15min":1.7270000000000001},"min":{"1min":0.253,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":17.887,"5min":17.887,"15min":22.437000000000001},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:27:43 2023","interfaces":[{"interface":"back","average":{"1min":3.3319999999999999,"5min":2.387,"15min":2.1680000000000001},"min":{"1min":0.36799999999999999,"5min":0.27900000000000003,"15min":0.23999999999999999},"max":{"1min":10.425000000000001,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.6299999999999999,"15min":1.788},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23499999999999999},"max":{"1min":5.2329999999999997,"5min":8.9529999999999994,"15min":29.065000000000001},"last":0.63800000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:27:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":2.5369999999999999,"15min":2.0710000000000002},"min":{"1min":0.311,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":7.8849999999999998,"5min":16.629999999999999,"15min":16.629999999999999},"last":5.0250000000000004},{"interface":"front","average":{"1min":2.7999999999999998,"5min":1.9399999999999999,"15min":1.774},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":7.1479999999999997,"5min":9.6929999999999996,"15min":15.929},"last":0.51100000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.8109999999999999,"5min":2.3940000000000001,"15min":1.9450000000000001},"min":{"1min":0.377,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":13.614000000000001,"5min":13.614000000000001,"15min":13.614000000000001},"last":3.4289999999999998},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.8660000000000001,"15min":1.7869999999999999},"min":{"1min":0.38400000000000001,"5min":0.35099999999999998,"15min":0.315},"max":{"1min":11.124000000000001,"5min":11.124000000000001,"15min":13.624000000000001},"last":3.3929999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.3679999999999999,"15min":1.6890000000000001},"min":{"1min":0.378,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":13.523999999999999,"5min":13.523999999999999,"15min":13.523999999999999},"last":1.9299999999999999},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.7130000000000001,"15min":1.4419999999999999},"min":{"1min":0.29399999999999998,"5min":0.28199999999999997,"15min":0.23599999999999999},"max":{"1min":4.1219999999999999,"5min":10.153,"15min":11.956},"last":0.35099999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":2.4430000000000001,"15min":1.8799999999999999},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":13.567,"5min":13.567,"15min":13.567},"last":1.9019999999999999},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.02,"15min":1.724},"min":{"1min":0.34499999999999997,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":7.2999999999999998,"5min":16.497,"15min":16.497},"last":0.36399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.9390000000000001,"15min":1.7450000000000001},"min":{"1min":0.38600000000000001,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":5.4160000000000004,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.089,"5min":1.0580000000000001,"15min":1.127},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":5.5519999999999996,"5min":9.0739999999999998,"15min":11.994},"last":0.47999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1095172096,"data_stored":1095172096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1111425024,"data_stored":1111425024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":750845952,"data_stored":750845952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":935137280,"data_stored":935133208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1195053056,"data_stored":1195048984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":348585984,"data_stored":348585984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:28:05.540 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:05.544 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:05 smithi012 ceph-mon[147926]: pgmap v1556: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 441 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 15432/21243 objects degraded (72.645%); 4915/21243 objects misplaced (23.137%); 0 B/s, 63 objects/s recovering 2023-10-09T10:28:05.544 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:05 smithi012 ceph-mon[152263]: pgmap v1556: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 441 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 15432/21243 objects degraded (72.645%); 4915/21243 objects misplaced (23.137%); 0 B/s, 63 objects/s recovering 2023-10-09T10:28:05.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:05.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:05.646 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:05.699 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:05.751 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:05.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:05.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:05.856 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:05.886 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:05 smithi110 ceph-mon[136032]: pgmap v1556: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 441 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 15432/21243 objects degraded (72.645%); 4915/21243 objects misplaced (23.137%); 0 B/s, 63 objects/s recovering 2023-10-09T10:28:05.893 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:05.950 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:05.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:05.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:06.058 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:06.086 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:06.097 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:06.142 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:06.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:06.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:06.247 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:06.295 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:06.346 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:06.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:06.452 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:06.471 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:28:06.471 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:28:06.566 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:06.857 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:06.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:06.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:06.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:28:06.966 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:07.105 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:07.332 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:07.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:07.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:07.443 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:07.489 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:07.536 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:07.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:07.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:07.643 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:07.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:07 smithi012 ceph-mon[152263]: pgmap v1557: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 326 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 15432/15714 objects degraded (98.205%); 4648/15714 objects misplaced (29.579%); 0 B/s, 63 objects/s recovering 2023-10-09T10:28:07.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:07 smithi012 ceph-mon[147926]: pgmap v1557: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 326 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 15432/15714 objects degraded (98.205%); 4648/15714 objects misplaced (29.579%); 0 B/s, 63 objects/s recovering 2023-10-09T10:28:07.700 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:07.741 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:07.745 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:07 smithi110 ceph-mon[136032]: pgmap v1557: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 326 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 784 op/s; 15432/15714 objects degraded (98.205%); 4648/15714 objects misplaced (29.579%); 0 B/s, 63 objects/s recovering 2023-10-09T10:28:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:07.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:07.768 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:28:07.847 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:07.956 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:28:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:28:07] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.33.4" 2023-10-09T10:28:07.966 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:07.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:28:08.027 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:08.079 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:08.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:08.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:08.192 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:08.306 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:08.491 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:08.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:08.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:08.613 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:08.626 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:28:08.656 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:08.667 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:28:08.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:08 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 15432/15714 objects degraded (98.205%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:08.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:08 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 15432/15714 objects degraded (98.205%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:08.705 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:28:08.712 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:08.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:08.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:08.743 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:28:08.785 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:28:08.817 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:08.824 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:28:08.855 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:28:08.890 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:08.895 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:28:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:28:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:28:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:28:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:28:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:28:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:28:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:28:08.918 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:28:08.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:08 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 15432/15714 objects degraded (98.205%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:09.270 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:09.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:09.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:09.376 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:09.411 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:09.462 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:09.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:09.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:09.569 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:09.614 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:09.663 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:09.669 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:09 smithi110 ceph-mon[136032]: pgmap v1558: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 219 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 15105/10581 objects degraded (142.756%); 4648/10581 objects misplaced (43.928%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:09.670 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/531043241' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:09.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:09.689 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Removed 112780 objects 2023-10-09T10:28:09.690 INFO:tasks.radosbench.radosbench.0.smithi012.stdout:Clean up completed and total clean up time :159.534 2023-10-09T10:28:09.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:09.798 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:09.830 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:09.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:09.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:09 smithi012 ceph-mon[147926]: pgmap v1558: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 219 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 15105/10581 objects degraded (142.756%); 4648/10581 objects misplaced (43.928%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:09.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/531043241' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:09.872 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:09 smithi012 ceph-mon[152263]: pgmap v1558: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 219 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 15105/10581 objects degraded (142.756%); 4648/10581 objects misplaced (43.928%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:09.872 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/531043241' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:09.934 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:09.973 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:10.191 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:10.232 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T10:28:10.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T10:28:10.245 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:10.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:10.309 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:10.352 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:10.379 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:10.412 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:10.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:10.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:10.516 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:10.541 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:10.574 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:10.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:10.678 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:10.710 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:10.952 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:10.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:10.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:11.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:11.085 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:11.399 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:28:11.399 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:28:11.399 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:28:11.399 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:28:11.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:11 smithi012 ceph-mon[147926]: pgmap v1559: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 171 MiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 810 op/s; 14849/8247 objects degraded (180.053%); 4392/8247 objects misplaced (53.256%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:11.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:11 smithi012 ceph-mon[152263]: pgmap v1559: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 171 MiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 810 op/s; 14849/8247 objects degraded (180.053%); 4392/8247 objects misplaced (53.256%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:11.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:11 smithi110 ceph-mon[136032]: pgmap v1559: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 171 MiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 810 op/s; 14849/8247 objects degraded (180.053%); 4392/8247 objects misplaced (53.256%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:11.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:28:12.323 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:12.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:12.429 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:12.461 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[152263]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[152263]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[152263]: osdmap e547: 8 total, 8 up, 6 in 2023-10-09T10:28:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[152263]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:12.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[147926]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[147926]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[147926]: osdmap e547: 8 total, 8 up, 6 in 2023-10-09T10:28:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[147926]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:12.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:12 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:12.697 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:12.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:12.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:12.740 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:28:12.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:12.857 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:12 smithi110 ceph-mon[136032]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:12 smithi110 ceph-mon[136032]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:12 smithi110 ceph-mon[136032]: osdmap e547: 8 total, 8 up, 6 in 2023-10-09T10:28:12.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:12 smithi110 ceph-mon[136032]: failed to encode map e547 with expected crc 2023-10-09T10:28:12.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:12.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:12 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:13.082 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:13.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:13.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:13.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:13.225 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:13.414 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:13.510 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:13.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:13.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:13.616 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:13.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:13.677 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:13.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:13.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:13.784 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[147926]: pgmap v1561: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 69 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 14849/3384 objects degraded (438.800%); 4178/3384 objects misplaced (123.463%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:13.784 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[147926]: failed to encode map e547 with expected crc 2023-10-09T10:28:13.784 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[147926]: osdmap e548: 8 total, 8 up, 6 in 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[147926]: failed to encode map e548 with expected crc 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[147926]: failed to encode map e548 with expected crc 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 14849/3384 objects degraded (438.800%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[147926]: osdmap e549: 8 total, 8 up, 6 in 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[152263]: pgmap v1561: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 69 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 14849/3384 objects degraded (438.800%); 4178/3384 objects misplaced (123.463%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[152263]: failed to encode map e547 with expected crc 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[152263]: osdmap e548: 8 total, 8 up, 6 in 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[152263]: failed to encode map e548 with expected crc 2023-10-09T10:28:13.785 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[152263]: failed to encode map e548 with expected crc 2023-10-09T10:28:13.786 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 14849/3384 objects degraded (438.800%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:13.786 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:13 smithi012 ceph-mon[152263]: osdmap e549: 8 total, 8 up, 6 in 2023-10-09T10:28:13.813 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:13 smithi110 ceph-mon[136032]: pgmap v1561: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 69 MiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 814 op/s; 14849/3384 objects degraded (438.800%); 4178/3384 objects misplaced (123.463%); 0 B/s, 70 objects/s recovering 2023-10-09T10:28:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:13 smithi110 ceph-mon[136032]: failed to encode map e547 with expected crc 2023-10-09T10:28:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:13 smithi110 ceph-mon[136032]: osdmap e548: 8 total, 8 up, 6 in 2023-10-09T10:28:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:13 smithi110 ceph-mon[136032]: failed to encode map e548 with expected crc 2023-10-09T10:28:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:13 smithi110 ceph-mon[136032]: failed to encode map e548 with expected crc 2023-10-09T10:28:13.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:13 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 14849/3384 objects degraded (438.800%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:13.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:13 smithi110 ceph-mon[136032]: osdmap e549: 8 total, 8 up, 6 in 2023-10-09T10:28:14.089 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:14.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:14.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:14.115 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:14.128 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:14.194 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:14.220 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:14.253 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:14.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:14.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:14.357 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:14.382 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:14.415 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:14.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:14.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:14.527 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:28:14.554 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4246097736' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[147926]: osdmap e550: 8 total, 8 up, 6 in 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4246097736' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:28:14.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:14 smithi012 ceph-mon[152263]: osdmap e550: 8 total, 8 up, 6 in 2023-10-09T10:28:14.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: failed to encode map e549 with expected crc 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4246097736' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:28:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:14 smithi110 ceph-mon[136032]: osdmap e550: 8 total, 8 up, 6 in 2023-10-09T10:28:15.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:15 smithi012 ceph-mon[147926]: pgmap v1564: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 14229/273 objects degraded (5212.088%); 4102/273 objects misplaced (1502.564%); 0 B/s, 90 objects/s recovering 2023-10-09T10:28:15.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:15 smithi012 ceph-mon[147926]: failed to encode map e550 with expected crc 2023-10-09T10:28:15.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:15 smithi012 ceph-mon[152263]: pgmap v1564: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 14229/273 objects degraded (5212.088%); 4102/273 objects misplaced (1502.564%); 0 B/s, 90 objects/s recovering 2023-10-09T10:28:15.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:15 smithi012 ceph-mon[152263]: failed to encode map e550 with expected crc 2023-10-09T10:28:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:15 smithi110 ceph-mon[136032]: pgmap v1564: 99 pgs: 1 active+undersized+remapped+backfill_wait, 1 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 14229/273 objects degraded (5212.088%); 4102/273 objects misplaced (1502.564%); 0 B/s, 90 objects/s recovering 2023-10-09T10:28:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:15 smithi110 ceph-mon[136032]: failed to encode map e550 with expected crc 2023-10-09T10:28:16.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:16 smithi012 ceph-mon[147926]: failed to encode map e551 with expected crc 2023-10-09T10:28:16.610 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:16 smithi012 ceph-mon[147926]: failed to encode map e551 with expected crc 2023-10-09T10:28:16.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:16 smithi012 ceph-mon[147926]: osdmap e551: 8 total, 8 up, 6 in 2023-10-09T10:28:16.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:16 smithi012 ceph-mon[152263]: failed to encode map e551 with expected crc 2023-10-09T10:28:16.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:16 smithi012 ceph-mon[152263]: failed to encode map e551 with expected crc 2023-10-09T10:28:16.926 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:16 smithi012 ceph-mon[152263]: osdmap e551: 8 total, 8 up, 6 in 2023-10-09T10:28:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:16 smithi110 ceph-mon[136032]: failed to encode map e551 with expected crc 2023-10-09T10:28:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:16 smithi110 ceph-mon[136032]: failed to encode map e551 with expected crc 2023-10-09T10:28:16.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:16 smithi110 ceph-mon[136032]: osdmap e551: 8 total, 8 up, 6 in 2023-10-09T10:28:17.120 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:17.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:17.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:17.226 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:17.259 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:17.296 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:17.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:17.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:17.439 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:17.723 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:28:17.725 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:28:17.876 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[147926]: pgmap v1567: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped+backfill_wait, 1 peering, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.6 MiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 12582/273 objects degraded (4608.791%); 4102/273 objects misplaced (1502.564%); 0 B/s, 547 objects/s recovering 2023-10-09T10:28:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[147926]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[147926]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[147926]: osdmap e552: 8 total, 8 up, 6 in 2023-10-09T10:28:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[147926]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[147926]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[152263]: pgmap v1567: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped+backfill_wait, 1 peering, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.6 MiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 12582/273 objects degraded (4608.791%); 4102/273 objects misplaced (1502.564%); 0 B/s, 547 objects/s recovering 2023-10-09T10:28:17.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[152263]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[152263]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[152263]: osdmap e552: 8 total, 8 up, 6 in 2023-10-09T10:28:17.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[152263]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.880 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-mon[152263]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:17.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:17 smithi110 ceph-mon[136032]: pgmap v1567: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped+backfill_wait, 1 peering, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.6 MiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 12582/273 objects degraded (4608.791%); 4102/273 objects misplaced (1502.564%); 0 B/s, 547 objects/s recovering 2023-10-09T10:28:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:17 smithi110 ceph-mon[136032]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:17 smithi110 ceph-mon[136032]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:17 smithi110 ceph-mon[136032]: osdmap e552: 8 total, 8 up, 6 in 2023-10-09T10:28:17.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:17 smithi110 ceph-mon[136032]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:17 smithi110 ceph-mon[136032]: failed to encode map e552 with expected crc 2023-10-09T10:28:17.982 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:18.014 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:18.046 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:28:18.050 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:18.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:18.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:18.070 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:28:18.091 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:28:18.114 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:28:18.141 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:28:18.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:18.158 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:28:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:28:17] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.33.4" 2023-10-09T10:28:18.163 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:28:18.185 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:28:18.192 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:18.208 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:28:18.210 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:28:18.210 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:28:18.210 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:28:18.210 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:28:18.210 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:28:18.210 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:28:18.210 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:28:18.210 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:28:18.375 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:18.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:18.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:18.482 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:18.514 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:18.551 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:18.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:18.655 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:18.682 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:18.715 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:18.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:18.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:18.819 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:18.843 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:18 smithi110 ceph-mon[136032]: failed to encode map e552 with expected crc 2023-10-09T10:28:18.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:18 smithi110 ceph-mon[136032]: from='client.47189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:18.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2610576826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:18.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:18 smithi110 ceph-mon[136032]: failed to encode map e552 with expected crc 2023-10-09T10:28:18.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:18 smithi110 ceph-mon[136032]: Health check update: Degraded data redundancy: 12582/273 objects degraded (4608.791%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:18.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2672589280' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:28:18.844 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:18 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:28:18.845 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:18.878 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:18.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:18.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:18.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[147926]: failed to encode map e552 with expected crc 2023-10-09T10:28:18.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[147926]: from='client.47189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:18.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2610576826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:18.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[147926]: failed to encode map e552 with expected crc 2023-10-09T10:28:18.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[147926]: Health check update: Degraded data redundancy: 12582/273 objects degraded (4608.791%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:18.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2672589280' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:28:18.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:28:18.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[152263]: failed to encode map e552 with expected crc 2023-10-09T10:28:18.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[152263]: from='client.47189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:18.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2610576826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:18.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[152263]: failed to encode map e552 with expected crc 2023-10-09T10:28:18.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[152263]: Health check update: Degraded data redundancy: 12582/273 objects degraded (4608.791%), 7 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:28:18.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2672589280' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:28:18.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:18 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:28:18.982 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:19.008 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:19.039 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:19.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:19.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:19.143 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:19.176 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:19.265 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:28:19.522 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'unique_pool_5' removed 2023-10-09T10:28:19.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:19 smithi012 ceph-mon[147926]: pgmap v1569: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 12220/273 objects degraded (4476.190%); 4102/273 objects misplaced (1502.564%); 0 B/s, 460 objects/s recovering 2023-10-09T10:28:19.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:19 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:28:19.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:19 smithi012 ceph-mon[147926]: osdmap e553: 8 total, 8 up, 6 in 2023-10-09T10:28:19.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:19 smithi012 ceph-mon[152263]: pgmap v1569: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 12220/273 objects degraded (4476.190%); 4102/273 objects misplaced (1502.564%); 0 B/s, 460 objects/s recovering 2023-10-09T10:28:19.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:19 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:28:19.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:19 smithi012 ceph-mon[152263]: osdmap e553: 8 total, 8 up, 6 in 2023-10-09T10:28:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:19 smithi110 ceph-mon[136032]: pgmap v1569: 99 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.6 MiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 12220/273 objects degraded (4476.190%); 4102/273 objects misplaced (1502.564%); 0 B/s, 460 objects/s recovering 2023-10-09T10:28:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:19 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:28:19.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:19 smithi110 ceph-mon[136032]: osdmap e553: 8 total, 8 up, 6 in 2023-10-09T10:28:20.531 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:20.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:20.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:20.648 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:20.674 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:20.706 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:20.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:20.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:20.810 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:20.836 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:20.846 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:28:20.847 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:28:20.867 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:20.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:20.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:20 smithi012 ceph-mon[147926]: failed to encode map e553 with expected crc 2023-10-09T10:28:20.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:20 smithi012 ceph-mon[147926]: failed to encode map e553 with expected crc 2023-10-09T10:28:20.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:20 smithi012 ceph-mon[152263]: failed to encode map e553 with expected crc 2023-10-09T10:28:20.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:20 smithi012 ceph-mon[152263]: failed to encode map e553 with expected crc 2023-10-09T10:28:20.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:20 smithi110 ceph-mon[136032]: failed to encode map e553 with expected crc 2023-10-09T10:28:20.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:20 smithi110 ceph-mon[136032]: failed to encode map e553 with expected crc 2023-10-09T10:28:20.971 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:21.000 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:21.340 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:21.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:21.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:21.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:21.474 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:21.834 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T10:28:21.835 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T10:28:21.835 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T10:28:21.835 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T10:28:21.835 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T10:28:21.835 DEBUG:teuthology.orchestra.run.smithi012:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:28:21.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:28:21.926 INFO:teuthology.orchestra.run.smithi012.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T10:28:21.926 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:28:21.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:21 smithi012 ceph-mon[152263]: pgmap v1571: 73 pgs: 73 active+clean; 582 KiB data, 9.2 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:21.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:21 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12220/273 objects degraded (4476.190%), 7 pgs degraded, 9 pgs undersized) 2023-10-09T10:28:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:21 smithi012 ceph-mon[147926]: pgmap v1571: 73 pgs: 73 active+clean; 582 KiB data, 9.2 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:21 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12220/273 objects degraded (4476.190%), 7 pgs degraded, 9 pgs undersized) 2023-10-09T10:28:21.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:21 smithi110 ceph-mon[136032]: pgmap v1571: 73 pgs: 73 active+clean; 582 KiB data, 9.2 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:21.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:21 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12220/273 objects degraded (4476.190%), 7 pgs degraded, 9 pgs undersized) 2023-10-09T10:28:21.964 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T10:28:21.964 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T10:28:22.021 INFO:tasks.workunit:timeout=3h 2023-10-09T10:28:22.021 INFO:tasks.workunit:cleanup=True 2023-10-09T10:28:22.022 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T10:28:22.022 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:22.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:22.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:22.107 INFO:tasks.workunit.client.0.smithi012.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T10:28:22.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:22.155 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:22.459 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:22.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:22.552 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:28:22.557 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1568,"stamp":"2023-10-09T10:28:16.610822+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4822944,"num_objects":91,"num_object_clones":0,"num_object_copies":273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12582,"num_objects_misplaced":4102,"num_objects_unfound":0,"num_objects_dirty":91,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":211226,"num_write_kb":6286126,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36605,"num_bytes_recovered":1939769084,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133935,"ondisk_log_size":133935,"up":297,"acting":288,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9650952,"kb_used_data":3197016,"kb_used_omap":163,"kb_used_meta":6453724,"kb_avail":552819960,"statfs":{"total":575970213888,"available":566087639040,"internally_reserved":0,"allocated":3273744384,"data_stored":3264180177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":6608614387},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-15007768,"num_objects":-230,"num_object_clones":0,"num_object_copies":-690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1723,"num_objects_misplaced":-76,"num_objects_unfound":0,"num_objects_dirty":-230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":230,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1647,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.048336"},"pg_stats":[{"pgid":"11.12","version":"546'8106","reported_seq":14428,"reported_epoch":547,"state":"peering","last_fresh":"2023-10-09T10:28:11.563641+0000","last_change":"2023-10-09T10:28:11.563761+0000","last_active":"2023-10-09T10:28:11.563641+0000","last_peered":"2023-10-09T10:28:10.665224+0000","last_clean":"2023-10-09T10:25:07.544332+0000","last_became_active":"2023-10-09T10:25:09.702724+0000","last_became_peered":"2023-10-09T10:25:09.702724+0000","last_unstale":"2023-10-09T10:28:11.563641+0000","last_undegraded":"2023-10-09T10:28:11.563641+0000","last_fullsized":"2023-10-09T10:28:11.563641+0000","mapping_epoch":547,"log_start":"504'3006","ondisk_log_start":"504'3006","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5100,"log_dups_size":0,"ondisk_log_size":5100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360448,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7158,"num_write_kb":229216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3119,"num_bytes_recovered":56885248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3,5],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":140,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.787999+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:28:00.787999+0000","last_peered":"2023-10-09T10:28:00.787999+0000","last_clean":"2023-10-09T10:28:00.787999+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:28:00.787999+0000","last_undegraded":"2023-10-09T10:28:00.787999+0000","last_fullsized":"2023-10-09T10:28:00.787999+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":501,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788415+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:28:00.788415+0000","last_peered":"2023-10-09T10:28:00.788415+0000","last_clean":"2023-10-09T10:28:00.788415+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:28:00.788415+0000","last_undegraded":"2023-10-09T10:28:00.788415+0000","last_fullsized":"2023-10-09T10:28:00.788415+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"546'8025","reported_seq":14907,"reported_epoch":546,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:28:09.687720+0000","last_change":"2023-10-09T10:25:09.734187+0000","last_active":"2023-10-09T10:28:09.687720+0000","last_peered":"2023-10-09T10:28:09.687720+0000","last_clean":"2023-10-09T10:24:39.562832+0000","last_became_active":"2023-10-09T10:25:09.727273+0000","last_became_peered":"2023-10-09T10:25:09.727273+0000","last_unstale":"2023-10-09T10:28:09.687720+0000","last_undegraded":"2023-10-09T10:25:09.536636+0000","last_fullsized":"2023-10-09T10:25:09.427838+0000","mapping_epoch":513,"log_start":"504'2999","ondisk_log_start":"504'2999","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5026,"log_dups_size":0,"ondisk_log_size":5026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":845,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7080,"num_write_kb":225984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,6","objects":-845}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":578,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788000+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:28:00.788000+0000","last_peered":"2023-10-09T10:28:00.788000+0000","last_clean":"2023-10-09T10:28:00.788000+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:28:00.788000+0000","last_undegraded":"2023-10-09T10:28:00.788000+0000","last_fullsized":"2023-10-09T10:28:00.788000+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":472,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.784897+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:28:00.784897+0000","last_peered":"2023-10-09T10:28:00.784897+0000","last_clean":"2023-10-09T10:28:00.784897+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:28:00.784897+0000","last_undegraded":"2023-10-09T10:28:00.784897+0000","last_fullsized":"2023-10-09T10:28:00.784897+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"546'7921","reported_seq":12479,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:09.687586+0000","last_change":"2023-10-09T10:25:49.265022+0000","last_active":"2023-10-09T10:28:09.687586+0000","last_peered":"2023-10-09T10:28:09.687586+0000","last_clean":"2023-10-09T10:28:09.687586+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:28:09.687586+0000","last_undegraded":"2023-10-09T10:28:09.687586+0000","last_fullsized":"2023-10-09T10:28:09.687586+0000","mapping_epoch":527,"log_start":"504'2905","ondisk_log_start":"504'2905","created":501,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5016,"log_dups_size":0,"ondisk_log_size":5016,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6984,"num_write_kb":223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2233,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":43,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.781893+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:28:00.781893+0000","last_peered":"2023-10-09T10:28:00.781893+0000","last_clean":"2023-10-09T10:28:00.781893+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:28:00.781893+0000","last_undegraded":"2023-10-09T10:28:00.781893+0000","last_fullsized":"2023-10-09T10:28:00.781893+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":124,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.784919+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:28:00.784919+0000","last_peered":"2023-10-09T10:28:00.784919+0000","last_clean":"2023-10-09T10:28:00.784919+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:28:00.784919+0000","last_undegraded":"2023-10-09T10:28:00.784919+0000","last_fullsized":"2023-10-09T10:28:00.784919+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":506,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.785328+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:28:00.785328+0000","last_peered":"2023-10-09T10:28:00.785328+0000","last_clean":"2023-10-09T10:28:00.785328+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:28:00.785328+0000","last_undegraded":"2023-10-09T10:28:00.785328+0000","last_fullsized":"2023-10-09T10:28:00.785328+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":459,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T10:28:14.522091+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:28:14.522091+0000","last_peered":"2023-10-09T10:28:14.522091+0000","last_clean":"2023-10-09T10:28:14.522091+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:28:14.522091+0000","last_undegraded":"2023-10-09T10:28:14.522091+0000","last_fullsized":"2023-10-09T10:28:14.522091+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":225,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.525554+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:28:13.525554+0000","last_peered":"2023-10-09T10:28:13.525554+0000","last_clean":"2023-10-09T10:28:13.525554+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:28:13.525554+0000","last_undegraded":"2023-10-09T10:28:13.525554+0000","last_fullsized":"2023-10-09T10:28:13.525554+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":459,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T10:28:14.522360+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:28:14.522360+0000","last_peered":"2023-10-09T10:28:14.522360+0000","last_clean":"2023-10-09T10:28:14.522360+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:28:14.522360+0000","last_undegraded":"2023-10-09T10:28:14.522360+0000","last_fullsized":"2023-10-09T10:28:14.522360+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":152,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788607+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:28:00.788607+0000","last_peered":"2023-10-09T10:28:00.788607+0000","last_clean":"2023-10-09T10:28:00.788607+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:28:00.788607+0000","last_undegraded":"2023-10-09T10:28:00.788607+0000","last_fullsized":"2023-10-09T10:28:00.788607+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":43,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.781492+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:28:00.781492+0000","last_peered":"2023-10-09T10:28:00.781492+0000","last_clean":"2023-10-09T10:28:00.781492+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:28:00.781492+0000","last_undegraded":"2023-10-09T10:28:00.781492+0000","last_fullsized":"2023-10-09T10:28:00.781492+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":198,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.789434+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:28:00.789434+0000","last_peered":"2023-10-09T10:28:00.789434+0000","last_clean":"2023-10-09T10:28:00.789434+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:28:00.789434+0000","last_undegraded":"2023-10-09T10:28:00.789434+0000","last_fullsized":"2023-10-09T10:28:00.789434+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":158,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788424+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:28:00.788424+0000","last_peered":"2023-10-09T10:28:00.788424+0000","last_clean":"2023-10-09T10:28:00.788424+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:28:00.788424+0000","last_undegraded":"2023-10-09T10:28:00.788424+0000","last_fullsized":"2023-10-09T10:28:00.788424+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":118,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788186+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:28:00.788186+0000","last_peered":"2023-10-09T10:28:00.788186+0000","last_clean":"2023-10-09T10:28:00.788186+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:28:00.788186+0000","last_undegraded":"2023-10-09T10:28:00.788186+0000","last_fullsized":"2023-10-09T10:28:00.788186+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":75,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.525279+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:28:13.525279+0000","last_peered":"2023-10-09T10:28:13.525279+0000","last_clean":"2023-10-09T10:28:13.525279+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:28:13.525279+0000","last_undegraded":"2023-10-09T10:28:13.525279+0000","last_fullsized":"2023-10-09T10:28:13.525279+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":471,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-09T10:28:11.564172+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:28:11.564172+0000","last_peered":"2023-10-09T10:28:11.564172+0000","last_clean":"2023-10-09T10:28:11.564172+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:28:11.564172+0000","last_undegraded":"2023-10-09T10:28:11.564172+0000","last_fullsized":"2023-10-09T10:28:11.564172+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":216,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.527563+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:28:13.527563+0000","last_peered":"2023-10-09T10:28:13.527563+0000","last_clean":"2023-10-09T10:28:13.527563+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:28:13.527563+0000","last_undegraded":"2023-10-09T10:28:13.527563+0000","last_fullsized":"2023-10-09T10:28:13.527563+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"546'8016","reported_seq":8833,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:28:13.523891+0000","last_change":"2023-10-09T10:25:12.768440+0000","last_active":"2023-10-09T10:28:13.523891+0000","last_peered":"2023-10-09T10:28:13.523891+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:12.730648+0000","last_became_peered":"2023-10-09T10:25:12.730648+0000","last_unstale":"2023-10-09T10:28:13.523891+0000","last_undegraded":"2023-10-09T10:25:12.723367+0000","last_fullsized":"2023-10-09T10:25:12.604254+0000","mapping_epoch":515,"log_start":"504'3096","ondisk_log_start":"504'3096","created":501,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4920,"log_dups_size":0,"ondisk_log_size":4920,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":819200,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1947,"num_objects_misplaced":3151,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7093,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":-3138}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":158,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788147+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:28:00.788147+0000","last_peered":"2023-10-09T10:28:00.788147+0000","last_clean":"2023-10-09T10:28:00.788147+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:28:00.788147+0000","last_undegraded":"2023-10-09T10:28:00.788147+0000","last_fullsized":"2023-10-09T10:28:00.788147+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":43,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.784081+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:28:00.784081+0000","last_peered":"2023-10-09T10:28:00.784081+0000","last_clean":"2023-10-09T10:28:00.784081+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:28:00.784081+0000","last_undegraded":"2023-10-09T10:28:00.784081+0000","last_fullsized":"2023-10-09T10:28:00.784081+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"546'8046","reported_seq":16569,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.523637+0000","last_change":"2023-10-09T10:26:39.522775+0000","last_active":"2023-10-09T10:28:13.523637+0000","last_peered":"2023-10-09T10:28:13.523637+0000","last_clean":"2023-10-09T10:28:13.523637+0000","last_became_active":"2023-10-09T10:26:39.509024+0000","last_became_peered":"2023-10-09T10:26:39.509024+0000","last_unstale":"2023-10-09T10:28:13.523637+0000","last_undegraded":"2023-10-09T10:28:13.523637+0000","last_fullsized":"2023-10-09T10:28:13.523637+0000","mapping_epoch":536,"log_start":"505'3197","ondisk_log_start":"505'3197","created":501,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4849,"log_dups_size":0,"ondisk_log_size":4849,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":950272,"num_objects":15,"num_object_clones":0,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7081,"num_write_kb":227040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3072,"num_bytes_recovered":201326592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":53,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.783061+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:28:00.783061+0000","last_peered":"2023-10-09T10:28:00.783061+0000","last_clean":"2023-10-09T10:28:00.783061+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:28:00.783061+0000","last_undegraded":"2023-10-09T10:28:00.783061+0000","last_fullsized":"2023-10-09T10:28:00.783061+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":50,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T10:28:14.524744+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:28:14.524744+0000","last_peered":"2023-10-09T10:28:14.524744+0000","last_clean":"2023-10-09T10:28:14.524744+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:28:14.524744+0000","last_undegraded":"2023-10-09T10:28:14.524744+0000","last_fullsized":"2023-10-09T10:28:14.524744+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"546'8156","reported_seq":8234,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.525202+0000","last_change":"2023-10-09T10:24:40.748797+0000","last_active":"2023-10-09T10:28:13.525202+0000","last_peered":"2023-10-09T10:28:13.525202+0000","last_clean":"2023-10-09T10:28:13.525202+0000","last_became_active":"2023-10-09T10:24:13.547552+0000","last_became_peered":"2023-10-09T10:24:13.547552+0000","last_unstale":"2023-10-09T10:28:13.525202+0000","last_undegraded":"2023-10-09T10:28:13.525202+0000","last_fullsized":"2023-10-09T10:28:13.525202+0000","mapping_epoch":494,"log_start":"505'3246","ondisk_log_start":"505'3246","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"506'3288","last_scrub_stamp":"2023-10-09T10:24:40.748676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:40.748676+0000","objects_scrubbed":0,"log_size":4910,"log_dups_size":0,"ondisk_log_size":4910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4868,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788681+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:28:00.788681+0000","last_peered":"2023-10-09T10:28:00.788681+0000","last_clean":"2023-10-09T10:28:00.788681+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:28:00.788681+0000","last_undegraded":"2023-10-09T10:28:00.788681+0000","last_fullsized":"2023-10-09T10:28:00.788681+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":225,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.523563+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:28:13.523563+0000","last_peered":"2023-10-09T10:28:13.523563+0000","last_clean":"2023-10-09T10:28:13.523563+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:28:13.523563+0000","last_undegraded":"2023-10-09T10:28:13.523563+0000","last_fullsized":"2023-10-09T10:28:13.523563+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"546'7661","reported_seq":13775,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:09.664261+0000","last_change":"2023-10-09T10:26:35.543001+0000","last_active":"2023-10-09T10:28:09.664261+0000","last_peered":"2023-10-09T10:28:09.664261+0000","last_clean":"2023-10-09T10:28:09.664261+0000","last_became_active":"2023-10-09T10:26:35.529481+0000","last_became_peered":"2023-10-09T10:26:35.529481+0000","last_unstale":"2023-10-09T10:28:09.664261+0000","last_undegraded":"2023-10-09T10:28:09.664261+0000","last_fullsized":"2023-10-09T10:28:09.664261+0000","mapping_epoch":534,"log_start":"503'2641","ondisk_log_start":"503'2641","created":494,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2776","last_scrub_stamp":"2023-10-09T10:24:31.689272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:31.689272+0000","objects_scrubbed":0,"log_size":5020,"log_dups_size":0,"ondisk_log_size":5020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:07.598234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4901,"num_write_kb":98944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3012,"num_bytes_recovered":174915584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":174,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788406+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:28:00.788406+0000","last_peered":"2023-10-09T10:28:00.788406+0000","last_clean":"2023-10-09T10:28:00.788406+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:28:00.788406+0000","last_undegraded":"2023-10-09T10:28:00.788406+0000","last_fullsized":"2023-10-09T10:28:00.788406+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":161,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788233+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:28:00.788233+0000","last_peered":"2023-10-09T10:28:00.788233+0000","last_clean":"2023-10-09T10:28:00.788233+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:28:00.788233+0000","last_undegraded":"2023-10-09T10:28:00.788233+0000","last_fullsized":"2023-10-09T10:28:00.788233+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"546'8023","reported_seq":16710,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T10:28:14.550066+0000","last_change":"2023-10-09T10:28:14.550066+0000","last_active":"2023-10-09T10:28:14.550066+0000","last_peered":"2023-10-09T10:28:14.550066+0000","last_clean":"2023-10-09T10:28:14.550066+0000","last_became_active":"2023-10-09T10:28:14.530356+0000","last_became_peered":"2023-10-09T10:28:14.530356+0000","last_unstale":"2023-10-09T10:28:14.550066+0000","last_undegraded":"2023-10-09T10:28:14.550066+0000","last_fullsized":"2023-10-09T10:28:14.550066+0000","mapping_epoch":549,"log_start":"504'3162","ondisk_log_start":"504'3162","created":494,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4861,"log_dups_size":0,"ondisk_log_size":4861,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7078,"num_write_kb":226080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3132,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":241,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.527273+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:28:13.527273+0000","last_peered":"2023-10-09T10:28:13.527273+0000","last_clean":"2023-10-09T10:28:13.527273+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:28:13.527273+0000","last_undegraded":"2023-10-09T10:28:13.527273+0000","last_fullsized":"2023-10-09T10:28:13.527273+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":142,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.524019+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:28:13.524019+0000","last_peered":"2023-10-09T10:28:13.524019+0000","last_clean":"2023-10-09T10:28:13.524019+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:28:13.524019+0000","last_undegraded":"2023-10-09T10:28:13.524019+0000","last_fullsized":"2023-10-09T10:28:13.524019+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"546'7977","reported_seq":12180,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.529415+0000","last_change":"2023-10-09T10:25:09.436926+0000","last_active":"2023-10-09T10:28:13.529415+0000","last_peered":"2023-10-09T10:28:13.529415+0000","last_clean":"2023-10-09T10:28:13.529415+0000","last_became_active":"2023-10-09T10:25:09.433635+0000","last_became_peered":"2023-10-09T10:25:09.433635+0000","last_unstale":"2023-10-09T10:28:13.529415+0000","last_undegraded":"2023-10-09T10:28:13.529415+0000","last_fullsized":"2023-10-09T10:28:13.529415+0000","mapping_epoch":513,"log_start":"503'2312","ondisk_log_start":"503'2312","created":494,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2923","last_scrub_stamp":"2023-10-09T10:24:33.395726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:33.395726+0000","objects_scrubbed":0,"log_size":5665,"log_dups_size":0,"ondisk_log_size":5665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5063,"num_write_kb":99136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2264,"num_bytes_recovered":148373504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":225,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.524898+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:28:13.524898+0000","last_peered":"2023-10-09T10:28:13.524898+0000","last_clean":"2023-10-09T10:28:13.524898+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:28:13.524898+0000","last_undegraded":"2023-10-09T10:28:13.524898+0000","last_fullsized":"2023-10-09T10:28:13.524898+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":457,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.782325+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:28:00.782325+0000","last_peered":"2023-10-09T10:28:00.782325+0000","last_clean":"2023-10-09T10:28:00.782325+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:28:00.782325+0000","last_undegraded":"2023-10-09T10:28:00.782325+0000","last_fullsized":"2023-10-09T10:28:00.782325+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"546'7940","reported_seq":9247,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:28:13.524081+0000","last_change":"2023-10-09T10:25:17.870308+0000","last_active":"2023-10-09T10:28:13.524081+0000","last_peered":"2023-10-09T10:28:13.524081+0000","last_clean":"2023-10-09T10:24:39.579605+0000","last_became_active":"2023-10-09T10:24:41.970641+0000","last_became_peered":"2023-10-09T10:24:41.970641+0000","last_unstale":"2023-10-09T10:28:13.524081+0000","last_undegraded":"2023-10-09T10:25:17.869248+0000","last_fullsized":"2023-10-09T10:24:41.650637+0000","mapping_epoch":507,"log_start":"504'3051","ondisk_log_start":"504'3051","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4889,"log_dups_size":0,"ondisk_log_size":4889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7019,"num_write_kb":224608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":611,"num_bytes_recovered":40042496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,5","objects":-1629}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":158,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.782236+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:28:00.782236+0000","last_peered":"2023-10-09T10:28:00.782236+0000","last_clean":"2023-10-09T10:28:00.782236+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:28:00.782236+0000","last_undegraded":"2023-10-09T10:28:00.782236+0000","last_fullsized":"2023-10-09T10:28:00.782236+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.789376+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:28:00.789376+0000","last_peered":"2023-10-09T10:28:00.789376+0000","last_clean":"2023-10-09T10:28:00.789376+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:28:00.789376+0000","last_undegraded":"2023-10-09T10:28:00.789376+0000","last_fullsized":"2023-10-09T10:28:00.789376+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"546'7955","reported_seq":8090,"reported_epoch":546,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:28:09.634711+0000","last_change":"2023-10-09T10:25:09.711381+0000","last_active":"2023-10-09T10:28:09.634711+0000","last_peered":"2023-10-09T10:28:09.634711+0000","last_clean":"2023-10-09T10:25:07.542933+0000","last_became_active":"2023-10-09T10:25:09.702666+0000","last_became_peered":"2023-10-09T10:25:09.702666+0000","last_unstale":"2023-10-09T10:28:09.634711+0000","last_undegraded":"2023-10-09T10:25:09.456447+0000","last_fullsized":"2023-10-09T10:25:09.427911+0000","mapping_epoch":513,"log_start":"504'2948","ondisk_log_start":"504'2948","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2897","last_scrub_stamp":"2023-10-09T10:24:32.360707+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:32.360707+0000","objects_scrubbed":1945,"log_size":5007,"log_dups_size":0,"ondisk_log_size":5007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:28:21.811310+0000","scrub_duration":0.46160963500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5067,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-3058}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":158,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.787876+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:28:00.787876+0000","last_peered":"2023-10-09T10:28:00.787876+0000","last_clean":"2023-10-09T10:28:00.787876+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:28:00.787876+0000","last_undegraded":"2023-10-09T10:28:00.787876+0000","last_fullsized":"2023-10-09T10:28:00.787876+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":124,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788495+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:28:00.788495+0000","last_peered":"2023-10-09T10:28:00.788495+0000","last_clean":"2023-10-09T10:28:00.788495+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:28:00.788495+0000","last_undegraded":"2023-10-09T10:28:00.788495+0000","last_fullsized":"2023-10-09T10:28:00.788495+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"546'7991","reported_seq":14844,"reported_epoch":546,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:28:09.690069+0000","last_change":"2023-10-09T10:25:09.734141+0000","last_active":"2023-10-09T10:28:09.690069+0000","last_peered":"2023-10-09T10:28:09.690069+0000","last_clean":"2023-10-09T10:24:39.589208+0000","last_became_active":"2023-10-09T10:25:09.725895+0000","last_became_peered":"2023-10-09T10:25:09.725895+0000","last_unstale":"2023-10-09T10:28:09.690069+0000","last_undegraded":"2023-10-09T10:25:09.554264+0000","last_fullsized":"2023-10-09T10:25:09.427860+0000","mapping_epoch":513,"log_start":"504'2965","ondisk_log_start":"504'2965","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5026,"log_dups_size":0,"ondisk_log_size":5026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1179648,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":806,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7046,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2254,"num_bytes_recovered":147718144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","2","4","6"],"object_location_counts":[{"shards":"0,6","objects":-788}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":140,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788397+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:28:00.788397+0000","last_peered":"2023-10-09T10:28:00.788397+0000","last_clean":"2023-10-09T10:28:00.788397+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:28:00.788397+0000","last_undegraded":"2023-10-09T10:28:00.788397+0000","last_fullsized":"2023-10-09T10:28:00.788397+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":167,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788234+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:28:00.788234+0000","last_peered":"2023-10-09T10:28:00.788234+0000","last_clean":"2023-10-09T10:28:00.788234+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:28:00.788234+0000","last_undegraded":"2023-10-09T10:28:00.788234+0000","last_fullsized":"2023-10-09T10:28:00.788234+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"546'7914","reported_seq":8363,"reported_epoch":546,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:28:09.657950+0000","last_change":"2023-10-09T10:25:12.617443+0000","last_active":"2023-10-09T10:28:09.657950+0000","last_peered":"2023-10-09T10:28:09.657950+0000","last_clean":"2023-10-09T10:24:39.566759+0000","last_became_active":"2023-10-09T10:24:41.968616+0000","last_became_peered":"2023-10-09T10:24:41.968616+0000","last_unstale":"2023-10-09T10:28:09.657950+0000","last_undegraded":"2023-10-09T10:25:12.615444+0000","last_fullsized":"2023-10-09T10:24:41.606427+0000","mapping_epoch":507,"log_start":"504'2873","ondisk_log_start":"504'2873","created":494,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"505'3109","last_scrub_stamp":"2023-10-09T10:24:38.227888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:38.227888+0000","objects_scrubbed":2167,"log_size":5041,"log_dups_size":0,"ondisk_log_size":5041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:53.911666+0000","scrub_duration":0.397029623,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4807,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,3","objects":-2081}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":502,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.523557+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:28:13.523557+0000","last_peered":"2023-10-09T10:28:13.523557+0000","last_clean":"2023-10-09T10:28:13.523557+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:28:13.523557+0000","last_undegraded":"2023-10-09T10:28:13.523557+0000","last_fullsized":"2023-10-09T10:28:13.523557+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":561,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788498+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:28:00.788498+0000","last_peered":"2023-10-09T10:28:00.788498+0000","last_clean":"2023-10-09T10:28:00.788498+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:28:00.788498+0000","last_undegraded":"2023-10-09T10:28:00.788498+0000","last_fullsized":"2023-10-09T10:28:00.788498+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"546'14096","reported_seq":14199,"reported_epoch":546,"state":"active+undersized+remapped+backfill_wait","last_fresh":"2023-10-09T10:28:09.693969+0000","last_change":"2023-10-09T10:28:12.568612+0000","last_active":"2023-10-09T10:28:09.693969+0000","last_peered":"2023-10-09T10:28:09.693969+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:17.922692+0000","last_became_peered":"2023-10-09T10:25:17.922692+0000","last_unstale":"2023-10-09T10:28:09.693969+0000","last_undegraded":"2023-10-09T10:25:17.689668+0000","last_fullsized":"2023-10-09T10:25:17.675189+0000","mapping_epoch":518,"log_start":"528'8000","ondisk_log_start":"528'8000","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":6096,"log_dups_size":0,"ondisk_log_size":6096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14096,"num_write_kb":451009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,5],"avail_no_missing":["4","2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":268,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.782193+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:28:00.782193+0000","last_peered":"2023-10-09T10:28:00.782193+0000","last_clean":"2023-10-09T10:28:00.782193+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:28:00.782193+0000","last_undegraded":"2023-10-09T10:28:00.782193+0000","last_fullsized":"2023-10-09T10:28:00.782193+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":234,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.527482+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:28:13.527482+0000","last_peered":"2023-10-09T10:28:13.527482+0000","last_clean":"2023-10-09T10:28:13.527482+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:28:13.527482+0000","last_undegraded":"2023-10-09T10:28:13.527482+0000","last_fullsized":"2023-10-09T10:28:13.527482+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":514,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.524044+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:28:13.524044+0000","last_peered":"2023-10-09T10:28:13.524044+0000","last_clean":"2023-10-09T10:28:13.524044+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:28:13.524044+0000","last_undegraded":"2023-10-09T10:28:13.524044+0000","last_fullsized":"2023-10-09T10:28:13.524044+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"546'14278","reported_seq":14341,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:09.687718+0000","last_change":"2023-10-09T10:23:53.843352+0000","last_active":"2023-10-09T10:28:09.687718+0000","last_peered":"2023-10-09T10:28:09.687718+0000","last_clean":"2023-10-09T10:28:09.687718+0000","last_became_active":"2023-10-09T10:23:53.842115+0000","last_became_peered":"2023-10-09T10:23:53.842115+0000","last_unstale":"2023-10-09T10:28:09.687718+0000","last_undegraded":"2023-10-09T10:28:09.687718+0000","last_fullsized":"2023-10-09T10:28:09.687718+0000","mapping_epoch":494,"log_start":"531'9206","ondisk_log_start":"531'9206","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5072,"log_dups_size":0,"ondisk_log_size":5072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:02.172950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14278,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":234,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.523967+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:28:13.523967+0000","last_peered":"2023-10-09T10:28:13.523967+0000","last_clean":"2023-10-09T10:28:13.523967+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:28:13.523967+0000","last_undegraded":"2023-10-09T10:28:13.523967+0000","last_fullsized":"2023-10-09T10:28:13.523967+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":149,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788339+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:28:00.788339+0000","last_peered":"2023-10-09T10:28:00.788339+0000","last_clean":"2023-10-09T10:28:00.788339+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:28:00.788339+0000","last_undegraded":"2023-10-09T10:28:00.788339+0000","last_fullsized":"2023-10-09T10:28:00.788339+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":462,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-09T10:28:12.568474+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:28:12.568474+0000","last_peered":"2023-10-09T10:28:12.568474+0000","last_clean":"2023-10-09T10:28:12.568474+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:28:12.568474+0000","last_undegraded":"2023-10-09T10:28:12.568474+0000","last_fullsized":"2023-10-09T10:28:12.568474+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"546'7989","reported_seq":16568,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.523497+0000","last_change":"2023-10-09T10:27:21.711478+0000","last_active":"2023-10-09T10:28:13.523497+0000","last_peered":"2023-10-09T10:28:13.523497+0000","last_clean":"2023-10-09T10:28:13.523497+0000","last_became_active":"2023-10-09T10:27:21.694370+0000","last_became_peered":"2023-10-09T10:27:21.694370+0000","last_unstale":"2023-10-09T10:28:13.523497+0000","last_undegraded":"2023-10-09T10:28:13.523497+0000","last_fullsized":"2023-10-09T10:28:13.523497+0000","mapping_epoch":542,"log_start":"504'3133","ondisk_log_start":"504'3133","created":494,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4856,"log_dups_size":0,"ondisk_log_size":4856,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7023,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3094,"num_bytes_recovered":202768384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":498,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.783188+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:28:00.783188+0000","last_peered":"2023-10-09T10:28:00.783188+0000","last_clean":"2023-10-09T10:28:00.783188+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:28:00.783188+0000","last_undegraded":"2023-10-09T10:28:00.783188+0000","last_fullsized":"2023-10-09T10:28:00.783188+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":115,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.783012+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:28:00.783012+0000","last_peered":"2023-10-09T10:28:00.783012+0000","last_clean":"2023-10-09T10:28:00.783012+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:28:00.783012+0000","last_undegraded":"2023-10-09T10:28:00.783012+0000","last_fullsized":"2023-10-09T10:28:00.783012+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":563,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.525449+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:28:13.525449+0000","last_peered":"2023-10-09T10:28:13.525449+0000","last_clean":"2023-10-09T10:28:13.525449+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:28:13.525449+0000","last_undegraded":"2023-10-09T10:28:13.525449+0000","last_fullsized":"2023-10-09T10:28:13.525449+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"546'14112","reported_seq":14201,"reported_epoch":549,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:28:13.523765+0000","last_change":"2023-10-09T10:26:37.693448+0000","last_active":"2023-10-09T10:28:13.523765+0000","last_peered":"2023-10-09T10:28:13.523765+0000","last_clean":"2023-10-09T10:24:39.579480+0000","last_became_active":"2023-10-09T10:24:41.772800+0000","last_became_peered":"2023-10-09T10:24:41.772800+0000","last_unstale":"2023-10-09T10:28:13.523765+0000","last_undegraded":"2023-10-09T10:28:13.523765+0000","last_fullsized":"2023-10-09T10:28:13.523765+0000","mapping_epoch":507,"log_start":"532'9200","ondisk_log_start":"532'9200","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4912,"log_dups_size":0,"ondisk_log_size":4912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14112,"num_write_kb":451584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,2],"avail_no_missing":["5","2","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":204,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.525051+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:28:13.525051+0000","last_peered":"2023-10-09T10:28:13.525051+0000","last_clean":"2023-10-09T10:28:13.525051+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:28:13.525051+0000","last_undegraded":"2023-10-09T10:28:13.525051+0000","last_fullsized":"2023-10-09T10:28:13.525051+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":741,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.781738+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:28:00.781738+0000","last_peered":"2023-10-09T10:28:00.781738+0000","last_clean":"2023-10-09T10:28:00.781738+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:28:00.781738+0000","last_undegraded":"2023-10-09T10:28:00.781738+0000","last_fullsized":"2023-10-09T10:28:00.781738+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":252,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.523763+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:28:13.523763+0000","last_peered":"2023-10-09T10:28:13.523763+0000","last_clean":"2023-10-09T10:28:13.523763+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:28:13.523763+0000","last_undegraded":"2023-10-09T10:28:13.523763+0000","last_fullsized":"2023-10-09T10:28:13.523763+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":158,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788583+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:28:00.788583+0000","last_peered":"2023-10-09T10:28:00.788583+0000","last_clean":"2023-10-09T10:28:00.788583+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:28:00.788583+0000","last_undegraded":"2023-10-09T10:28:00.788583+0000","last_fullsized":"2023-10-09T10:28:00.788583+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"546'8175","reported_seq":15132,"reported_epoch":549,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:28:13.523941+0000","last_change":"2023-10-09T10:27:22.518901+0000","last_active":"2023-10-09T10:28:13.523941+0000","last_peered":"2023-10-09T10:28:13.523941+0000","last_clean":"2023-10-09T10:23:52.601407+0000","last_became_active":"2023-10-09T10:25:13.751033+0000","last_became_peered":"2023-10-09T10:25:13.751033+0000","last_unstale":"2023-10-09T10:28:13.523941+0000","last_undegraded":"2023-10-09T10:28:13.523941+0000","last_fullsized":"2023-10-09T10:28:13.523941+0000","mapping_epoch":516,"log_start":"508'3288","ondisk_log_start":"508'3288","created":494,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4887,"log_dups_size":0,"ondisk_log_size":4887,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":951,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7251,"num_write_kb":231648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2306,"num_bytes_recovered":151126016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","6"],"object_location_counts":[{"shards":"3,5,6","objects":-951}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":232,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.783108+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:28:00.783108+0000","last_peered":"2023-10-09T10:28:00.783108+0000","last_clean":"2023-10-09T10:28:00.783108+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:28:00.783108+0000","last_undegraded":"2023-10-09T10:28:00.783108+0000","last_fullsized":"2023-10-09T10:28:00.783108+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":492,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.783355+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:28:00.783355+0000","last_peered":"2023-10-09T10:28:00.783355+0000","last_clean":"2023-10-09T10:28:00.783355+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:28:00.783355+0000","last_undegraded":"2023-10-09T10:28:00.783355+0000","last_fullsized":"2023-10-09T10:28:00.783355+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":43,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.783762+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:28:00.783762+0000","last_peered":"2023-10-09T10:28:00.783762+0000","last_clean":"2023-10-09T10:28:00.783762+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:28:00.783762+0000","last_undegraded":"2023-10-09T10:28:00.783762+0000","last_fullsized":"2023-10-09T10:28:00.783762+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"546'14212","reported_seq":14273,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:09.685252+0000","last_change":"2023-10-09T10:23:53.788657+0000","last_active":"2023-10-09T10:28:09.685252+0000","last_peered":"2023-10-09T10:28:09.685252+0000","last_clean":"2023-10-09T10:28:09.685252+0000","last_became_active":"2023-10-09T10:23:53.788330+0000","last_became_peered":"2023-10-09T10:23:53.788330+0000","last_unstale":"2023-10-09T10:28:09.685252+0000","last_undegraded":"2023-10-09T10:28:09.685252+0000","last_fullsized":"2023-10-09T10:28:09.685252+0000","mapping_epoch":494,"log_start":"529'8200","ondisk_log_start":"529'8200","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":6012,"log_dups_size":0,"ondisk_log_size":6012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14212,"num_write_kb":454784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":43,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.782289+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:28:00.782289+0000","last_peered":"2023-10-09T10:28:00.782289+0000","last_clean":"2023-10-09T10:28:00.782289+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:28:00.782289+0000","last_undegraded":"2023-10-09T10:28:00.782289+0000","last_fullsized":"2023-10-09T10:28:00.782289+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":531,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.525203+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:28:13.525203+0000","last_peered":"2023-10-09T10:28:13.525203+0000","last_clean":"2023-10-09T10:28:13.525203+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:28:13.525203+0000","last_undegraded":"2023-10-09T10:28:13.525203+0000","last_fullsized":"2023-10-09T10:28:13.525203+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":460,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.782600+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:28:00.782600+0000","last_peered":"2023-10-09T10:28:00.782600+0000","last_clean":"2023-10-09T10:28:00.782600+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:28:00.782600+0000","last_undegraded":"2023-10-09T10:28:00.782600+0000","last_fullsized":"2023-10-09T10:28:00.782600+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"546'13736","reported_seq":13799,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:09.689977+0000","last_change":"2023-10-09T10:23:53.789071+0000","last_active":"2023-10-09T10:28:09.689977+0000","last_peered":"2023-10-09T10:28:09.689977+0000","last_clean":"2023-10-09T10:28:09.689977+0000","last_became_active":"2023-10-09T10:23:53.788532+0000","last_became_peered":"2023-10-09T10:23:53.788532+0000","last_unstale":"2023-10-09T10:28:09.689977+0000","last_undegraded":"2023-10-09T10:28:09.689977+0000","last_fullsized":"2023-10-09T10:28:09.689977+0000","mapping_epoch":494,"log_start":"531'8676","ondisk_log_start":"531'8676","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5060,"log_dups_size":0,"ondisk_log_size":5060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.295207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13736,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":348,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.785194+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:28:00.785194+0000","last_peered":"2023-10-09T10:28:00.785194+0000","last_clean":"2023-10-09T10:28:00.785194+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:28:00.785194+0000","last_undegraded":"2023-10-09T10:28:00.785194+0000","last_fullsized":"2023-10-09T10:28:00.785194+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":528,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788148+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:28:00.788148+0000","last_peered":"2023-10-09T10:28:00.788148+0000","last_clean":"2023-10-09T10:28:00.788148+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:28:00.788148+0000","last_undegraded":"2023-10-09T10:28:00.788148+0000","last_fullsized":"2023-10-09T10:28:00.788148+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":43,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.781818+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:28:00.781818+0000","last_peered":"2023-10-09T10:28:00.781818+0000","last_clean":"2023-10-09T10:28:00.781818+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:28:00.781818+0000","last_undegraded":"2023-10-09T10:28:00.781818+0000","last_fullsized":"2023-10-09T10:28:00.781818+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.e","version":"546'14264","reported_seq":14325,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:09.685502+0000","last_change":"2023-10-09T10:23:53.842187+0000","last_active":"2023-10-09T10:28:09.685502+0000","last_peered":"2023-10-09T10:28:09.685502+0000","last_clean":"2023-10-09T10:28:09.685502+0000","last_became_active":"2023-10-09T10:23:53.841849+0000","last_became_peered":"2023-10-09T10:23:53.841849+0000","last_unstale":"2023-10-09T10:28:09.685502+0000","last_undegraded":"2023-10-09T10:28:09.685502+0000","last_fullsized":"2023-10-09T10:28:09.685502+0000","mapping_epoch":494,"log_start":"529'8200","ondisk_log_start":"529'8200","created":494,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":6064,"log_dups_size":0,"ondisk_log_size":6064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14264,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":53,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.528657+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:28:13.528657+0000","last_peered":"2023-10-09T10:28:13.528657+0000","last_clean":"2023-10-09T10:28:13.528657+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:28:13.528657+0000","last_undegraded":"2023-10-09T10:28:13.528657+0000","last_fullsized":"2023-10-09T10:28:13.528657+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":47,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.784194+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:28:00.784194+0000","last_peered":"2023-10-09T10:28:00.784194+0000","last_clean":"2023-10-09T10:28:00.784194+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:28:00.784194+0000","last_undegraded":"2023-10-09T10:28:00.784194+0000","last_fullsized":"2023-10-09T10:28:00.784194+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":230,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.523704+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:28:13.523704+0000","last_peered":"2023-10-09T10:28:13.523704+0000","last_clean":"2023-10-09T10:28:13.523704+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:28:13.523704+0000","last_undegraded":"2023-10-09T10:28:13.523704+0000","last_fullsized":"2023-10-09T10:28:13.523704+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"546'7851","reported_seq":7963,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:28:13.524226+0000","last_change":"2023-10-09T10:25:12.738037+0000","last_active":"2023-10-09T10:28:13.524226+0000","last_peered":"2023-10-09T10:28:13.524226+0000","last_clean":"2023-10-09T10:24:39.524718+0000","last_became_active":"2023-10-09T10:24:41.968333+0000","last_became_peered":"2023-10-09T10:24:41.968333+0000","last_unstale":"2023-10-09T10:28:13.524226+0000","last_undegraded":"2023-10-09T10:25:12.736858+0000","last_fullsized":"2023-10-09T10:24:41.611999+0000","mapping_epoch":507,"log_start":"504'2985","ondisk_log_start":"504'2985","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4866,"log_dups_size":0,"ondisk_log_size":4866,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32768,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6931,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,5","objects":-2215}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":43,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.782531+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:28:00.782531+0000","last_peered":"2023-10-09T10:28:00.782531+0000","last_clean":"2023-10-09T10:28:00.782531+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:28:00.782531+0000","last_undegraded":"2023-10-09T10:28:00.782531+0000","last_fullsized":"2023-10-09T10:28:00.782531+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":52,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.781874+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:28:00.781874+0000","last_peered":"2023-10-09T10:28:00.781874+0000","last_clean":"2023-10-09T10:28:00.781874+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:28:00.781874+0000","last_undegraded":"2023-10-09T10:28:00.781874+0000","last_fullsized":"2023-10-09T10:28:00.781874+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.14","version":"546'8000","reported_seq":12766,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.529278+0000","last_change":"2023-10-09T10:28:12.611992+0000","last_active":"2023-10-09T10:28:13.529278+0000","last_peered":"2023-10-09T10:28:13.529278+0000","last_clean":"2023-10-09T10:28:13.529278+0000","last_became_active":"2023-10-09T10:27:07.549958+0000","last_became_peered":"2023-10-09T10:27:07.549958+0000","last_unstale":"2023-10-09T10:28:13.529278+0000","last_undegraded":"2023-10-09T10:28:13.529278+0000","last_fullsized":"2023-10-09T10:28:13.529278+0000","mapping_epoch":540,"log_start":"503'2276","ondisk_log_start":"503'2276","created":501,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7076,"num_write_kb":225632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2333,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":280,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.524990+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:28:13.524990+0000","last_peered":"2023-10-09T10:28:13.524990+0000","last_clean":"2023-10-09T10:28:13.524990+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:28:13.524990+0000","last_undegraded":"2023-10-09T10:28:13.524990+0000","last_fullsized":"2023-10-09T10:28:13.524990+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":140,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.788499+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:28:00.788499+0000","last_peered":"2023-10-09T10:28:00.788499+0000","last_clean":"2023-10-09T10:28:00.788499+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:28:00.788499+0000","last_undegraded":"2023-10-09T10:28:00.788499+0000","last_fullsized":"2023-10-09T10:28:00.788499+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"546'8022","reported_seq":16499,"reported_epoch":550,"state":"active+undersized+remapped","last_fresh":"2023-10-09T10:28:15.021666+0000","last_change":"2023-10-09T10:28:15.021871+0000","last_active":"2023-10-09T10:28:15.021666+0000","last_peered":"2023-10-09T10:28:15.021666+0000","last_clean":"2023-10-09T10:24:39.496165+0000","last_became_active":"2023-10-09T10:25:09.741390+0000","last_became_peered":"2023-10-09T10:25:09.741390+0000","last_unstale":"2023-10-09T10:28:15.021666+0000","last_undegraded":"2023-10-09T10:25:09.538396+0000","last_fullsized":"2023-10-09T10:25:09.430861+0000","mapping_epoch":513,"log_start":"505'3150","ondisk_log_start":"505'3150","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:23:52.601407+0000","objects_scrubbed":0,"log_size":4872,"log_dups_size":0,"ondisk_log_size":4872,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":884736,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7078,"num_write_kb":226912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3074,"num_bytes_recovered":146800640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","3","4","6"],"object_location_counts":[{"shards":"5,6","objects":14}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":263,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.527366+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:28:13.527366+0000","last_peered":"2023-10-09T10:28:13.527366+0000","last_clean":"2023-10-09T10:28:13.527366+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:28:13.527366+0000","last_undegraded":"2023-10-09T10:28:13.527366+0000","last_fullsized":"2023-10-09T10:28:13.527366+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":370,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.782448+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:28:00.782448+0000","last_peered":"2023-10-09T10:28:00.782448+0000","last_clean":"2023-10-09T10:28:00.782448+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:28:00.782448+0000","last_undegraded":"2023-10-09T10:28:00.782448+0000","last_fullsized":"2023-10-09T10:28:00.782448+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.16","version":"546'7916","reported_seq":14281,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:09.685518+0000","last_change":"2023-10-09T10:27:21.732594+0000","last_active":"2023-10-09T10:28:09.685518+0000","last_peered":"2023-10-09T10:28:09.685518+0000","last_clean":"2023-10-09T10:28:09.685518+0000","last_became_active":"2023-10-09T10:27:21.699893+0000","last_became_peered":"2023-10-09T10:27:21.699893+0000","last_unstale":"2023-10-09T10:28:09.685518+0000","last_undegraded":"2023-10-09T10:28:09.685518+0000","last_fullsized":"2023-10-09T10:28:09.685518+0000","mapping_epoch":542,"log_start":"504'2857","ondisk_log_start":"504'2857","created":501,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"504'2947","last_scrub_stamp":"2023-10-09T10:24:34.732607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:34.732607+0000","objects_scrubbed":0,"log_size":5059,"log_dups_size":0,"ondisk_log_size":5059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:06.239031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4979,"num_write_kb":93504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3137,"num_bytes_recovered":122093568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":50,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T10:28:14.524568+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:28:14.524568+0000","last_peered":"2023-10-09T10:28:14.524568+0000","last_clean":"2023-10-09T10:28:14.524568+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:28:14.524568+0000","last_undegraded":"2023-10-09T10:28:14.524568+0000","last_fullsized":"2023-10-09T10:28:14.524568+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":600,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.783110+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:28:00.783110+0000","last_peered":"2023-10-09T10:28:00.783110+0000","last_clean":"2023-10-09T10:28:00.783110+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:28:00.783110+0000","last_undegraded":"2023-10-09T10:28:00.783110+0000","last_fullsized":"2023-10-09T10:28:00.783110+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.17","version":"546'7946","reported_seq":8023,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T10:28:13.525305+0000","last_change":"2023-10-09T10:24:41.837777+0000","last_active":"2023-10-09T10:28:13.525305+0000","last_peered":"2023-10-09T10:28:13.525305+0000","last_clean":"2023-10-09T10:28:13.525305+0000","last_became_active":"2023-10-09T10:24:13.556812+0000","last_became_peered":"2023-10-09T10:24:13.556812+0000","last_unstale":"2023-10-09T10:28:13.525305+0000","last_undegraded":"2023-10-09T10:28:13.525305+0000","last_fullsized":"2023-10-09T10:28:13.525305+0000","mapping_epoch":501,"log_start":"504'3015","ondisk_log_start":"504'3015","created":501,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":5,"last_scrub":"506'3261","last_scrub_stamp":"2023-10-09T10:24:41.837657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:23:52.601407+0000","last_clean_scrub_stamp":"2023-10-09T10:24:41.837657+0000","objects_scrubbed":0,"log_size":4931,"log_dups_size":0,"ondisk_log_size":4931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4685,"num_write_kb":76864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":64,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.784787+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:28:00.784787+0000","last_peered":"2023-10-09T10:28:00.784787+0000","last_clean":"2023-10-09T10:28:00.784787+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:28:00.784787+0000","last_undegraded":"2023-10-09T10:28:00.784787+0000","last_fullsized":"2023-10-09T10:28:00.784787+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":167,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T10:28:00.787782+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:28:00.787782+0000","last_peered":"2023-10-09T10:28:00.787782+0000","last_clean":"2023-10-09T10:28:00.787782+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:28:00.787782+0000","last_undegraded":"2023-10-09T10:28:00.787782+0000","last_fullsized":"2023-10-09T10:28:00.787782+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":4227072,"num_objects":67,"num_object_clones":0,"num_object_copies":201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12582,"num_objects_misplaced":4102,"num_objects_unfound":0,"num_objects_dirty":67,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":210966,"num_write_kb":6283681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36445,"num_bytes_recovered":1936785408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3234922496,"data_stored":3234922496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133741,"ondisk_log_size":133741,"up":78,"acting":69,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430703,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626931,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514828,"kb_used_data":5700,"kb_used_omap":0,"kb_used_meta":1509120,"kb_avail":92230324,"statfs":{"total":95995035648,"available":94443851776,"internally_reserved":0,"allocated":5836800,"data_stored":4846120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1545338880},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.625,"15min":1.5669999999999999},"min":{"1min":0.223,"5min":0.216,"15min":0.216},"max":{"1min":6.1849999999999996,"5min":9.4260000000000002,"15min":44.055},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.9740000000000002,"5min":1.7130000000000001,"15min":1.9079999999999999},"min":{"1min":0.28799999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":9.3499999999999996,"5min":9.3499999999999996,"15min":43.869999999999997},"last":0.34799999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.3400000000000001,"15min":1.998},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":2.4049999999999998,"5min":6.3899999999999997,"15min":155.01400000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.482,"5min":1.3580000000000001,"15min":2.1739999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.6050000000000004,"5min":6.3369999999999997,"15min":154.71100000000001},"last":0.64600000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.887,"15min":1.833},"min":{"1min":0.315,"5min":0.309,"15min":0.24099999999999999},"max":{"1min":5.8300000000000001,"5min":8.6280000000000001,"15min":29.367999999999999},"last":1.4750000000000001},{"interface":"front","average":{"1min":1.853,"5min":1.667,"15min":1.643},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4379999999999997,"5min":6.4379999999999997,"15min":11.720000000000001},"last":1.1339999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":2.1339999999999999,"15min":1.698},"min":{"1min":0.30599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.9450000000000003,"5min":43.393000000000001,"15min":43.393000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":2.4049999999999998,"5min":2.3069999999999999,"15min":1.867},"min":{"1min":0.36699999999999999,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":6.423,"5min":43.466000000000001,"15min":43.466000000000001},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":8.9589999999999996,"5min":3.2749999999999999,"15min":2.4500000000000002},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":177.929,"5min":177.929,"15min":177.929},"last":0.27900000000000003},{"interface":"front","average":{"1min":8.2530000000000001,"5min":2.8980000000000001,"15min":2.2490000000000001},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":178.16499999999999,"5min":178.16499999999999,"15min":178.16499999999999},"last":0.496}]},{"osd":5,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.1640000000000001,"15min":2.0910000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.788,"5min":11.788,"15min":121.98999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.2629999999999999,"15min":1.5760000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":2.2890000000000001,"5min":7.0060000000000002,"15min":122.346},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.0219999999999998,"15min":1.966},"min":{"1min":0.38,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":12.612,"5min":49.991,"15min":130.66999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.5880000000000001,"5min":2.097,"15min":2.02},"min":{"1min":0.40799999999999997,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":12.659000000000001,"5min":50.023000000000003,"15min":130.702},"last":0.59399999999999997}]}]},{"osd":1,"up_from":0,"seq":360777253244,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416768,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1731364,"kb_used_data":700700,"kb_used_omap":28,"kb_used_meta":1030627,"kb_avail":92013788,"statfs":{"total":95995035648,"available":94222118912,"internally_reserved":0,"allocated":717516800,"data_stored":715359237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1055362896},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:27:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.8049999999999999,"15min":1.6990000000000001},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.19800000000000001},"max":{"1min":13.077999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.712,"15min":1.915},"min":{"1min":0.26500000000000001,"5min":0.214,"15min":0.192},"max":{"1min":12.349,"5min":12.349,"15min":56.950000000000003},"last":0.438}]},{"osd":2,"last update":"Mon Oct 9 10:27:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":1.845,"15min":2.0230000000000001},"min":{"1min":0.36099999999999999,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":8.859,"5min":17.093,"15min":53.988999999999997},"last":0.315},{"interface":"front","average":{"1min":2.286,"5min":2.081,"15min":2.1150000000000002},"min":{"1min":0.29999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.3739999999999997,"5min":17.053999999999998,"15min":54.119},"last":0.34799999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:28:08 2023","interfaces":[{"interface":"back","average":{"1min":4.0599999999999996,"5min":2.605,"15min":1.8480000000000001},"min":{"1min":0.28299999999999997,"5min":0.217,"15min":0.217},"max":{"1min":15.882999999999999,"5min":29.835999999999999,"15min":56.984999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":3.6040000000000001,"5min":2.9729999999999999,"15min":1.944},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":9.8949999999999996,"5min":29.776,"15min":56.475999999999999},"last":0.33300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":2.077,"15min":2.1579999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":9.2859999999999996,"5min":28.289000000000001,"15min":57.719000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.5920000000000001,"5min":2.4710000000000001,"15min":2.1920000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.896000000000001,"5min":28.331,"15min":56.378},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":3.5830000000000002,"15min":3.4689999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":11.867000000000001,"5min":90.412000000000006,"15min":229.589},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.625,"5min":3.0270000000000001,"15min":3.194},"min":{"1min":0.35799999999999998,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":6.9539999999999997,"5min":90.393000000000001,"15min":228.88900000000001},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":2.0640000000000001,"15min":2.2690000000000001},"min":{"1min":0.41399999999999998,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":8.5719999999999992,"5min":9.7840000000000007,"15min":54.482999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.7989999999999999,"5min":2.2450000000000001,"15min":2.4049999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.34799999999999998},"max":{"1min":9.3439999999999994,"5min":9.3439999999999994,"15min":57.753},"last":0.53300000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.2749999999999999,"15min":2.2959999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.34999999999999998},"max":{"1min":5.6959999999999997,"5min":12.029999999999999,"15min":56.411999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.335,"15min":2.3410000000000002},"min":{"1min":0.47199999999999998,"5min":0.39900000000000002,"15min":0.38300000000000001},"max":{"1min":7.0490000000000004,"5min":17.129999999999999,"15min":54.451999999999998},"last":0.502}]}]},{"osd":2,"up_from":217,"seq":932007903504,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1645820,"kb_used_data":769268,"kb_used_omap":83,"kb_used_meta":876460,"kb_avail":92099332,"statfs":{"total":95995035648,"available":94309715968,"internally_reserved":0,"allocated":787730432,"data_stored":785756587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":897495535},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.5939999999999999,"15min":2.008},"min":{"1min":0.22900000000000001,"5min":0.192,"15min":0.185},"max":{"1min":14.666,"5min":48.990000000000002,"15min":48.990000000000002},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.8340000000000001,"5min":2.363,"15min":2.1379999999999999},"min":{"1min":0.23599999999999999,"5min":0.22600000000000001,"15min":0.20899999999999999},"max":{"1min":13.773,"5min":49.412999999999997,"15min":49.412999999999997},"last":0.48999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":2.343,"15min":2.0230000000000001},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":14.632999999999999,"5min":14.632999999999999,"15min":34.893999999999998},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.7689999999999999,"15min":1.823},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":13.356999999999999,"5min":13.356999999999999,"15min":34.962000000000003},"last":0.78400000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.8879999999999999,"15min":1.4470000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.8410000000000002,"5min":15.115,"15min":35.043999999999997},"last":0.35099999999999998},{"interface":"front","average":{"1min":2.395,"5min":1.776,"15min":1.4530000000000001},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.6820000000000004,"5min":11.206,"15min":50.406999999999996},"last":0.621}]},{"osd":4,"last update":"Mon Oct 9 10:27:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":1.899,"15min":1.6220000000000001},"min":{"1min":0.35899999999999999,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":8.1859999999999999,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.7639999999999998,"5min":2.2309999999999999,"15min":1.913},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":8.7219999999999995,"5min":47.704000000000001,"15min":47.704000000000001},"last":0.66100000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":1.8260000000000001,"15min":2.0169999999999999},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":8.6839999999999993,"5min":11.266999999999999,"15min":57.298000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":3.8860000000000001,"5min":2.4199999999999999,"15min":2.1400000000000001},"min":{"1min":0.375,"5min":0.31,"15min":0.31},"max":{"1min":14.31,"5min":14.31,"15min":57.828000000000003},"last":0.436}]},{"osd":6,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":2.0310000000000001,"15min":1.9299999999999999},"min":{"1min":0.40300000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":7.9390000000000001,"5min":12.411,"15min":18.943999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":2.556,"5min":1.95,"15min":1.948},"min":{"1min":0.39300000000000002,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":8.7590000000000003,"5min":13.975,"15min":24.379000000000001},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.95,"5min":2.1139999999999999,"15min":2.1920000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":9.5199999999999996,"5min":11.704000000000001,"15min":35.008000000000003},"last":0.749},{"interface":"front","average":{"1min":2.4820000000000002,"5min":2.1230000000000002,"15min":2.121},"min":{"1min":0.51400000000000001,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":8.8149999999999995,"5min":15.548,"15min":24.323},"last":0.76800000000000002}]}]},{"osd":3,"up_from":352,"seq":1511828488344,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":884136,"kb_used_data":292768,"kb_used_omap":15,"kb_used_meta":591344,"kb_avail":92861016,"statfs":{"total":95995035648,"available":95089680384,"internally_reserved":0,"allocated":299794432,"data_stored":297320693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":605536710},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":4.7489999999999997,"5min":3.5030000000000001,"15min":2.1280000000000001},"min":{"1min":0.318,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":40.965000000000003,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.23899999999999999},{"interface":"front","average":{"1min":5.7789999999999999,"5min":3.645,"15min":2.2010000000000001},"min":{"1min":0.27200000000000002,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":43.595999999999997,"5min":66.802000000000007,"15min":66.802000000000007},"last":0.27400000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.4059999999999999,"15min":1.9370000000000001},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.7759999999999998,"5min":11.785,"15min":105.401},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.008,"15min":2.0219999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23300000000000001},"max":{"1min":7.452,"5min":10.422000000000001,"15min":104.914},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.968,"5min":2.3759999999999999,"15min":2.0699999999999998},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":9.923,"5min":13.497999999999999,"15min":13.497999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.786,"5min":2.3500000000000001,"15min":1.6819999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":9.4039999999999999,"5min":16.475000000000001,"15min":25.338999999999999},"last":0.42799999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":12.550000000000001,"5min":3.8359999999999999,"15min":2.3100000000000001},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":178.72200000000001,"5min":178.72200000000001,"15min":178.72200000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":12.884,"5min":4.7800000000000002,"15min":2.891},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":179.08500000000001,"5min":179.08500000000001,"15min":179.08500000000001},"last":0.78200000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.617,"15min":1.4530000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.6870000000000003,"5min":11.363,"15min":25.251999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.0600000000000001,"5min":2.149,"15min":1.7809999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.3949999999999996,"5min":10.388,"15min":13.394},"last":0.76500000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.371,"5min":2.1549999999999998,"15min":1.915},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.8689999999999998,"5min":13.545999999999999,"15min":19.114000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.8069999999999999,"15min":1.6100000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":5.718,"5min":11.32,"15min":25.306999999999999},"last":0.73499999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":1.9730000000000001,"15min":1.7490000000000001},"min":{"1min":0.41299999999999998,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":7.4279999999999999,"5min":10.345000000000001,"15min":10.459},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.9260000000000002,"5min":2.2000000000000002,"15min":1.9710000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":12.4,"5min":12.4,"15min":19.280000000000001},"last":0.58099999999999996}]}]},{"osd":4,"up_from":266,"seq":1142461300973,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1575976,"kb_used_data":634656,"kb_used_omap":35,"kb_used_meta":941276,"kb_avail":92169176,"statfs":{"total":95995035648,"available":94381236224,"internally_reserved":0,"allocated":649887744,"data_stored":648907686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":963866862},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":3.1179999999999999,"15min":2.6320000000000001},"min":{"1min":0.22700000000000001,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":7.6539999999999999,"5min":109.193,"15min":109.193},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.629,"5min":3.625,"15min":3.2200000000000002},"min":{"1min":0.27100000000000002,"5min":0.26400000000000001,"15min":0.23300000000000001},"max":{"1min":10.289999999999999,"5min":110.438,"15min":110.438},"last":0.58299999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.9530000000000001,"15min":2.1469999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.24299999999999999},"max":{"1min":6.0259999999999998,"5min":12.209,"15min":71.457999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.913,"5min":2.6899999999999999,"15min":2.1899999999999999},"min":{"1min":0.34799999999999998,"5min":0.248,"15min":0.248},"max":{"1min":8.6899999999999995,"5min":49.781999999999996,"15min":71.658000000000001},"last":2.6819999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":2.153,"15min":1.758},"min":{"1min":0.41999999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":9.9039999999999999,"5min":15.944000000000001,"15min":15.944000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.903,"5min":1.7470000000000001,"15min":1.671},"min":{"1min":0.38600000000000001,"5min":0.255,"15min":0.255},"max":{"1min":6.2149999999999999,"5min":14.473000000000001,"15min":14.473000000000001},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:28:03 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.476,"15min":2.109},"min":{"1min":0.33000000000000002,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":5.141,"5min":11.347,"15min":12.074},"last":0.71299999999999997},{"interface":"front","average":{"1min":3.278,"5min":2.7120000000000002,"15min":1.986},"min":{"1min":0.38100000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.9779999999999998,"5min":16.067,"15min":16.067},"last":0.67300000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.8360000000000001,"15min":2.0209999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.252},"max":{"1min":10.007,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.439},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.7749999999999999,"15min":1.853},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":4.5620000000000003,"5min":49.915999999999997,"15min":99.384},"last":0.41699999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.446,"15min":1.393},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.8380000000000001,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.8109999999999999,"5min":2.1059999999999999,"15min":1.804},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":6.7329999999999997,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.68200000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":2.0880000000000001,"15min":1.835},"min":{"1min":0.36299999999999999,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":6.4279999999999999,"5min":12.391999999999999,"15min":17.369},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.9119999999999999,"5min":2.2200000000000002,"15min":1.7490000000000001},"min":{"1min":0.41399999999999998,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":9.5129999999999999,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.60599999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823158,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2298828,"kb_used_data":793924,"kb_used_omap":0,"kb_used_meta":1504896,"kb_avail":91446324,"statfs":{"total":95995035648,"available":93641035776,"internally_reserved":0,"allocated":812978176,"data_stored":811989854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541013504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5630000000000002,"5min":3.0760000000000001,"15min":2.1629999999999998},"min":{"1min":0.92600000000000005,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":10.323,"5min":19.102,"15min":22.469999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.9730000000000001,"5min":1.901,"15min":1.7430000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23200000000000001},"max":{"1min":5.0730000000000004,"5min":17.887,"15min":22.437000000000001},"last":0.52900000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:27:43 2023","interfaces":[{"interface":"back","average":{"1min":3.3319999999999999,"5min":2.387,"15min":2.1680000000000001},"min":{"1min":0.36799999999999999,"5min":0.27900000000000003,"15min":0.23999999999999999},"max":{"1min":10.425000000000001,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.6299999999999999,"15min":1.788},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23499999999999999},"max":{"1min":5.2329999999999997,"5min":8.9529999999999994,"15min":29.065000000000001},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:27:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":2.5369999999999999,"15min":2.0710000000000002},"min":{"1min":0.311,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":7.8849999999999998,"5min":16.629999999999999,"15min":16.629999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.7999999999999998,"5min":1.9399999999999999,"15min":1.774},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":7.1479999999999997,"5min":9.6929999999999996,"15min":15.929},"last":0.59399999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":2.3220000000000001,"15min":2.0139999999999998},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":8.1720000000000006,"5min":13.614000000000001,"15min":13.614000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.9740000000000002,"5min":2.1589999999999998,"15min":1.887},"min":{"1min":0.46300000000000002,"5min":0.376,"15min":0.315},"max":{"1min":9.8450000000000006,"5min":11.124000000000001,"15min":13.624000000000001},"last":0.63600000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":2.0230000000000001,"15min":1.6919999999999999},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":3.9500000000000002,"5min":13.523999999999999,"15min":13.523999999999999},"last":0.374},{"interface":"front","average":{"1min":1.391,"5min":1.5269999999999999,"15min":1.4690000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23599999999999999},"max":{"1min":4.3959999999999999,"5min":8.8620000000000001,"15min":11.956},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.929,"15min":1.8440000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":3.3679999999999999,"5min":13.567,"15min":13.567},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.74,"15min":1.696},"min":{"1min":0.34300000000000003,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":3.407,"5min":16.497,"15min":16.497},"last":0.57499999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.9390000000000001,"15min":1.7450000000000001},"min":{"1min":0.38600000000000001,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":5.4160000000000004,"5min":21.109000000000002,"15min":21.109000000000002},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.089,"5min":1.0580000000000001,"15min":1.127},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":5.5519999999999996,"5min":9.0739999999999998,"15min":11.994},"last":0.66300000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":710475776,"data_stored":710475776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":780861440,"data_stored":780861440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":292945920,"data_stored":292945920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":642777088,"data_stored":642777088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":807796736,"data_stored":807796736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":65536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:28:22.565 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:28:22.566 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:22.597 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:22.632 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:22.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:28:22.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:22.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:22.736 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:22.766 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:22.800 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:22.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:22.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:22.819 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:22.903 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:22.931 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:22.963 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:22.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:22.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:23.067 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:23.094 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:23.135 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:23.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:23.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:23.239 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:23.266 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:23.299 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:23.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:23.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:23.404 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:23.439 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:23.484 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:23.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:23.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:23.589 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:23.615 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:23.648 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:23.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:23.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:23.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:23 smithi012 ceph-mon[147926]: pgmap v1572: 73 pgs: 73 active+clean; 582 KiB data, 9.2 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:23.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:23 smithi012 ceph-mon[152263]: pgmap v1572: 73 pgs: 73 active+clean; 582 KiB data, 9.2 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:23.752 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:23.778 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:23.808 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:23.809 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:23.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:23.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:23.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:23.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:23 smithi110 ceph-mon[136032]: pgmap v1572: 73 pgs: 73 active+clean; 582 KiB data, 9.2 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:23.943 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:24.028 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:24.028 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:24.029 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:24.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:24.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:24.135 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:24.161 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:24.196 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:24.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:24.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:24.299 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:24.325 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:24.356 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:24.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:24.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:24.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:24.491 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:24.634 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:24.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:24.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:24.739 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:24.765 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:24.798 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:24.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:24.902 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:24.934 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:25.477 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:25.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:25.583 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:25.632 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[152263]: pgmap v1573: 73 pgs: 73 active+clean; 582 KiB data, 9.1 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[147926]: pgmap v1573: 73 pgs: 73 active+clean; 582 KiB data, 9.1 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:28:25.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:28:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:25 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:25 smithi110 ceph-mon[136032]: pgmap v1573: 73 pgs: 73 active+clean; 582 KiB data, 9.1 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:25 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:25 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:25 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:25 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:28:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:25 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:28:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:25 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:26.572 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:26.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:26.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:26.681 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:26.722 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:26.723 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:28:26.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/200582769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:26.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:26 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:26.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/200582769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:26.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:26 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:26.971 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T10:28:27.030 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:27.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:27.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:27.071 INFO:tasks.workunit.client.0.smithi012.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T10:28:27.073 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T10:28:27.073 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T10:28:27.135 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:27.165 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:27.190 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T10:28:27.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/200582769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:27.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:26 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:27.242 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T10:28:27.253 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:27.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:27.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:27.291 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T10:28:27.293 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T10:28:27.293 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T10:28:27.341 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T10:28:27.361 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:27.393 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:27.429 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:27.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:27.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:27.440 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T10:28:27.440 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T10:28:27.505 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:28:27.505 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:28:27.531 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T10:28:27.532 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T10:28:27.532 DEBUG:teuthology.orchestra.run.smithi012:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T10:28:27.534 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:27.564 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:27.600 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:27.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:27.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:27.736 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:27.736 INFO:tasks.workunit.client.0.smithi012.stdout:Running main() from gmock_main.cc 2023-10-09T10:28:27.737 INFO:tasks.workunit.client.0.smithi012.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T10:28:27.737 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] Global test environment set-up. 2023-10-09T10:28:27.737 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T10:28:27.773 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:27.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:27.847 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[152263]: pgmap v1574: 73 pgs: 73 active+clean; 582 KiB data, 9.0 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:27.847 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:27.847 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:27.847 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[152263]: osdmap e554: 8 total, 8 up, 6 in 2023-10-09T10:28:27.847 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[152263]: failed to encode map e554 with expected crc 2023-10-09T10:28:27.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[147926]: pgmap v1574: 73 pgs: 73 active+clean; 582 KiB data, 9.0 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:27.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:27.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:27.848 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[147926]: osdmap e554: 8 total, 8 up, 6 in 2023-10-09T10:28:27.848 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-mon[147926]: failed to encode map e554 with expected crc 2023-10-09T10:28:27.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:27.908 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:28:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:28:27] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.33.4" 2023-10-09T10:28:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:27 smithi110 ceph-mon[136032]: pgmap v1574: 73 pgs: 73 active+clean; 582 KiB data, 9.0 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:27 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:27 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:27 smithi110 ceph-mon[136032]: osdmap e554: 8 total, 8 up, 6 in 2023-10-09T10:28:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:27 smithi110 ceph-mon[136032]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.198 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:28.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:28.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:28.213 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:28:28.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:28:28.241 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:28:28.263 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:28:28.283 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:28:28.304 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:28.315 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:28:28.334 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:28:28.336 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:28.353 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:28:28.373 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:28.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:28.378 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:28:28.380 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:28:28.380 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:28:28.380 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:28:28.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:28:28.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:28:28.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:28:28.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:28:28.381 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:28:28.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:28.521 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:28.666 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:28:28.666 INFO:teuthology.orchestra.run.smithi012.stdout:{"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2025,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":554,"num_osds":8,"num_up_osds":8,"osd_up_since":1696846541,"num_in_osds":6,"osd_in_since":1696847079,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":24,"data_bytes":595872,"bytes_used":9696104448,"bytes_avail":566274109440,"bytes_total":575970213888},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi012.mbnrgj","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"}},"servicemap":{"epoch":33,"modified":"2023-10-09T10:27:11.837030+0000","services":{}},"progress_events":{"a6927860-49b2-4c3c-8108-2df0bcfea8a1":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 6m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T10:28:28.801 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:28.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[147926]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[147926]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[147926]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2259716966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:28.803 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/930236634' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:28:28.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[152263]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[152263]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[152263]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.804 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2259716966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:28.805 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:28 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/930236634' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:28:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:28.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:28.907 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:28.939 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:28.975 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:28.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:28 smithi110 ceph-mon[136032]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:28 smithi110 ceph-mon[136032]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:28 smithi110 ceph-mon[136032]: failed to encode map e554 with expected crc 2023-10-09T10:28:28.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2259716966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:28.977 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:28 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/930236634' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:28:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:28.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:29.079 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:29.110 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:29.144 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:29.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:29.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:29.169 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T10:28:29.169 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:28:29.226 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:28:29.250 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:29.280 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:29.321 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:29.403 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:29.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:29.509 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:29.542 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:29.585 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:29.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:29.689 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:29.728 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T10:28:29.740 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:29.749 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_all_features (24 ms) 2023-10-09T10:28:29.750 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T10:28:29.824 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:29.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:29.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:29.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:29.951 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.copyup (202 ms) 2023-10-09T10:28:29.951 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T10:28:29.966 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.sparse_copyup (16 ms) 2023-10-09T10:28:29.966 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T10:28:29.974 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-10-09T10:28:29.974 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T10:28:29.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[147926]: pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 9.0 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:29.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[147926]: failed to encode map e555 with expected crc 2023-10-09T10:28:29.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[147926]: failed to encode map e555 with expected crc 2023-10-09T10:28:29.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[147926]: osdmap e555: 8 total, 8 up, 6 in 2023-10-09T10:28:29.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/125198252' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:28:29.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:28:29.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:29.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:29.981 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[152263]: pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 9.0 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:29.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[152263]: failed to encode map e555 with expected crc 2023-10-09T10:28:29.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[152263]: failed to encode map e555 with expected crc 2023-10-09T10:28:29.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[152263]: osdmap e555: 8 total, 8 up, 6 in 2023-10-09T10:28:29.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/125198252' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:28:29.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:28:29.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:29.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:29.983 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:29.996 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.add_remove_child (22 ms) 2023-10-09T10:28:29.997 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T10:28:30.026 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:30.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:30.033 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.directory_methods (36 ms) 2023-10-09T10:28:30.034 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.create 2023-10-09T10:28:30.051 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.create (15 ms) 2023-10-09T10:28:30.051 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T10:28:30.052 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-10-09T10:28:30.052 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T10:28:30.052 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2023-10-09T10:28:30.052 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T10:28:30.052 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T10:28:30.052 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T10:28:30.053 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-10-09T10:28:30.054 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T10:28:30.055 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-10-09T10:28:30.055 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T10:28:30.065 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-10-09T10:28:30.065 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T10:28:30.068 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2023-10-09T10:28:30.068 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T10:28:30.069 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-10-09T10:28:30.069 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T10:28:30.087 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.protection_status (18 ms) 2023-10-09T10:28:30.087 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T10:28:30.096 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapshot_limits (9 ms) 2023-10-09T10:28:30.096 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T10:28:30.130 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:30.138 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.parents_v1 (42 ms) 2023-10-09T10:28:30.139 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T10:28:30.155 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:29 smithi110 ceph-mon[136032]: pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 9.0 GiB used, 527 GiB / 536 GiB avail 2023-10-09T10:28:30.156 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:29 smithi110 ceph-mon[136032]: failed to encode map e555 with expected crc 2023-10-09T10:28:30.156 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:29 smithi110 ceph-mon[136032]: failed to encode map e555 with expected crc 2023-10-09T10:28:30.156 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:29 smithi110 ceph-mon[136032]: osdmap e555: 8 total, 8 up, 6 in 2023-10-09T10:28:30.157 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/125198252' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:28:30.157 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:28:30.158 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:30.158 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:30.158 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:30.169 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.parents_v2 (32 ms) 2023-10-09T10:28:30.169 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T10:28:30.193 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:30.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:30.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:30.200 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapshots (31 ms) 2023-10-09T10:28:30.201 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T10:28:30.207 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapid_race (7 ms) 2023-10-09T10:28:30.207 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T10:28:30.215 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2023-10-09T10:28:30.216 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T10:28:30.216 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-10-09T10:28:30.217 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T10:28:30.222 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-10-09T10:28:30.222 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T10:28:30.227 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2023-10-09T10:28:30.228 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T10:28:30.228 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T10:28:30.228 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T10:28:30.232 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_snap_add (5 ms) 2023-10-09T10:28:30.233 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T10:28:30.236 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2023-10-09T10:28:30.236 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T10:28:30.242 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-10-09T10:28:30.242 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T10:28:30.265 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2023-10-09T10:28:30.265 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T10:28:30.272 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-10-09T10:28:30.272 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T10:28:30.297 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:30.306 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2023-10-09T10:28:30.306 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T10:28:30.324 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image (17 ms) 2023-10-09T10:28:30.324 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T10:28:30.328 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:30.371 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:30.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:30.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:30.458 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:30.476 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:30.519 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:30.592 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:30.593 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:30.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:30.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:30.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:30.737 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:30.887 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:28:30.922 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T10:28:30.922 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[147926]: osdmap e556: 8 total, 8 up, 6 in 2023-10-09T10:28:30.922 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[147926]: failed to encode map e556 with expected crc 2023-10-09T10:28:30.922 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[147926]: failed to encode map e556 with expected crc 2023-10-09T10:28:30.922 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[147926]: failed to encode map e556 with expected crc 2023-10-09T10:28:30.922 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[147926]: failed to encode map e556 with expected crc 2023-10-09T10:28:30.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T10:28:30.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[152263]: osdmap e556: 8 total, 8 up, 6 in 2023-10-09T10:28:30.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[152263]: failed to encode map e556 with expected crc 2023-10-09T10:28:30.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[152263]: failed to encode map e556 with expected crc 2023-10-09T10:28:30.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[152263]: failed to encode map e556 with expected crc 2023-10-09T10:28:30.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:30 smithi012 ceph-mon[152263]: failed to encode map e556 with expected crc 2023-10-09T10:28:30.947 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:30.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:30.966 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:28:30.967 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:28:30.967 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:28:30.967 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:28:31.054 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:31.107 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-441147-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T10:28:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:30 smithi110 ceph-mon[136032]: osdmap e556: 8 total, 8 up, 6 in 2023-10-09T10:28:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:30 smithi110 ceph-mon[136032]: failed to encode map e556 with expected crc 2023-10-09T10:28:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:30 smithi110 ceph-mon[136032]: failed to encode map e556 with expected crc 2023-10-09T10:28:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:30 smithi110 ceph-mon[136032]: failed to encode map e556 with expected crc 2023-10-09T10:28:31.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:30 smithi110 ceph-mon[136032]: failed to encode map e556 with expected crc 2023-10-09T10:28:31.311 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:31.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:31.417 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:31.449 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:31.482 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:31.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:31.586 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:31.623 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:31 smithi012 ceph-mon[147926]: pgmap v1579: 105 pgs: 1 creating+peering, 31 unknown, 73 active+clean; 582 KiB data, 8.9 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:31 smithi012 ceph-mon[147926]: failed to encode map e557 with expected crc 2023-10-09T10:28:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:31 smithi012 ceph-mon[147926]: osdmap e557: 8 total, 8 up, 6 in 2023-10-09T10:28:31.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:31 smithi012 ceph-mon[147926]: failed to encode map e557 with expected crc 2023-10-09T10:28:31.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:31 smithi012 ceph-mon[152263]: pgmap v1579: 105 pgs: 1 creating+peering, 31 unknown, 73 active+clean; 582 KiB data, 8.9 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:31.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:31 smithi012 ceph-mon[152263]: failed to encode map e557 with expected crc 2023-10-09T10:28:31.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:31 smithi012 ceph-mon[152263]: osdmap e557: 8 total, 8 up, 6 in 2023-10-09T10:28:31.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:31 smithi012 ceph-mon[152263]: failed to encode map e557 with expected crc 2023-10-09T10:28:32.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:31 smithi110 ceph-mon[136032]: pgmap v1579: 105 pgs: 1 creating+peering, 31 unknown, 73 active+clean; 582 KiB data, 8.9 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:31 smithi110 ceph-mon[136032]: failed to encode map e557 with expected crc 2023-10-09T10:28:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:31 smithi110 ceph-mon[136032]: osdmap e557: 8 total, 8 up, 6 in 2023-10-09T10:28:32.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:31 smithi110 ceph-mon[136032]: failed to encode map e557 with expected crc 2023-10-09T10:28:32.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:28:32.881 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:32.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:32.986 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:33.012 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:33.044 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:33.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:33.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:33.056 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image_status (2732 ms) 2023-10-09T10:28:33.056 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T10:28:33.148 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:33.153 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image_map (97 ms) 2023-10-09T10:28:33.153 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T10:28:33.163 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_instances (10 ms) 2023-10-09T10:28:33.163 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T10:28:33.176 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:33.177 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_snapshot (14 ms) 2023-10-09T10:28:33.177 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T10:28:33.179 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-10-09T10:28:33.179 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T10:28:33.181 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T10:28:33.182 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T10:28:33.185 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T10:28:33.185 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T10:28:33.192 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-10-09T10:28:33.192 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T10:28:33.196 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_remove (5 ms) 2023-10-09T10:28:33.196 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T10:28:33.203 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (6 ms) 2023-10-09T10:28:33.203 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T10:28:33.205 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-09T10:28:33.205 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T10:28:33.209 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-09T10:28:33.209 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T10:28:33.211 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:33.212 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-10-09T10:28:33.212 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T10:28:33.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:33.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:33.216 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-10-09T10:28:33.216 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T10:28:33.219 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-10-09T10:28:33.219 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T10:28:33.223 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-09T10:28:33.223 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T10:28:33.225 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2023-10-09T10:28:33.225 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T10:28:33.226 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (1 ms) 2023-10-09T10:28:33.226 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T10:28:33.228 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T10:28:33.228 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T10:28:33.230 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-09T10:28:33.230 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T10:28:33.233 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2023-10-09T10:28:33.234 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T10:28:33.235 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-10-09T10:28:33.236 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T10:28:33.239 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-10-09T10:28:33.239 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T10:28:33.255 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (17 ms) 2023-10-09T10:28:33.256 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T10:28:33.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:33.348 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:33.461 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (205 ms) 2023-10-09T10:28:33.461 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T10:28:33.464 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-10-09T10:28:33.464 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T10:28:33.466 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T10:28:33.466 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T10:28:33.473 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-10-09T10:28:33.474 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T10:28:33.480 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-10-09T10:28:33.480 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T10:28:33.503 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_parent (22 ms) 2023-10-09T10:28:33.503 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T10:28:33.511 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-10-09T10:28:33.511 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T10:28:33.523 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2023-10-09T10:28:33.523 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T10:28:33.530 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-10-09T10:28:33.530 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T10:28:33.540 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-10-09T10:28:33.540 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T10:28:33.548 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.migration_v1 (8 ms) 2023-10-09T10:28:33.548 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T10:28:34.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:33 smithi012 ceph-mon[147926]: pgmap v1581: 105 pgs: 1 creating+peering, 22 unknown, 82 active+clean; 582 KiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 1.6 MiB/s rd, 802 KiB/s wr, 55 op/s 2023-10-09T10:28:34.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:33 smithi012 ceph-mon[152263]: pgmap v1581: 105 pgs: 1 creating+peering, 22 unknown, 82 active+clean; 582 KiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 1.6 MiB/s rd, 802 KiB/s wr, 55 op/s 2023-10-09T10:28:34.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:33 smithi110 ceph-mon[136032]: pgmap v1581: 105 pgs: 1 creating+peering, 22 unknown, 82 active+clean; 582 KiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 1.6 MiB/s rd, 802 KiB/s wr, 55 op/s 2023-10-09T10:28:34.574 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:34.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:34.679 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:34.711 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:34.754 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:34.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:34.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:34.861 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:34.889 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:34.905 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:35.377 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:35.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:35.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:35.483 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:35.513 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:35.527 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1979 ms) 2023-10-09T10:28:35.527 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T10:28:35.562 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.sparsify (35 ms) 2023-10-09T10:28:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[147926]: pgmap v1582: 105 pgs: 1 creating+peering, 104 active+clean; 582 KiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 947 op/s 2023-10-09T10:28:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[147926]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[147926]: osdmap e558: 8 total, 8 up, 6 in 2023-10-09T10:28:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[147926]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[147926]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[152263]: pgmap v1582: 105 pgs: 1 creating+peering, 104 active+clean; 582 KiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 947 op/s 2023-10-09T10:28:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[152263]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[152263]: osdmap e558: 8 total, 8 up, 6 in 2023-10-09T10:28:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[152263]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:35 smithi012 ceph-mon[152263]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.716 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:35.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:35.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:35.758 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:35.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:35.850 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:35 smithi110 ceph-mon[136032]: pgmap v1582: 105 pgs: 1 creating+peering, 104 active+clean; 582 KiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 947 op/s 2023-10-09T10:28:35.850 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:35 smithi110 ceph-mon[136032]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.850 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:35 smithi110 ceph-mon[136032]: osdmap e558: 8 total, 8 up, 6 in 2023-10-09T10:28:35.850 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:35 smithi110 ceph-mon[136032]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.851 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:35 smithi110 ceph-mon[136032]: failed to encode map e558 with expected crc 2023-10-09T10:28:35.853 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:35.890 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:35.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:35.995 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:36.025 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:36.064 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:28:36.066 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:28:36.306 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:36.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:36.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:36.411 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:36.442 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:36.479 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:36.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:36.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:36.534 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] 70 tests from TestClsRbd (5838 ms total) 2023-10-09T10:28:36.534 INFO:tasks.workunit.client.0.smithi012.stdout: 2023-10-09T10:28:36.535 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] Global test environment tear-down 2023-10-09T10:28:36.535 INFO:tasks.workunit.client.0.smithi012.stdout:[==========] 70 tests from 1 test suite ran. (8797 ms total) 2023-10-09T10:28:36.535 INFO:tasks.workunit.client.0.smithi012.stdout:[ PASSED ] 70 tests. 2023-10-09T10:28:36.536 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T10:28:36.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T10:28:36.537 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:28:36.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[147926]: failed to encode map e559 with expected crc 2023-10-09T10:28:36.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[147926]: osdmap e559: 8 total, 8 up, 6 in 2023-10-09T10:28:36.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[147926]: failed to encode map e559 with expected crc 2023-10-09T10:28:36.564 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3617039592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:28:36.565 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:28:36.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[152263]: failed to encode map e559 with expected crc 2023-10-09T10:28:36.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[152263]: osdmap e559: 8 total, 8 up, 6 in 2023-10-09T10:28:36.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[152263]: failed to encode map e559 with expected crc 2023-10-09T10:28:36.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3617039592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:28:36.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:28:36.576 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T10:28:36.576 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T10:28:36.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:36.652 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:36 smithi110 ceph-mon[136032]: failed to encode map e559 with expected crc 2023-10-09T10:28:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:36 smithi110 ceph-mon[136032]: osdmap e559: 8 total, 8 up, 6 in 2023-10-09T10:28:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:36 smithi110 ceph-mon[136032]: failed to encode map e559 with expected crc 2023-10-09T10:28:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:36 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3617039592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:28:36.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:28:37.279 DEBUG:teuthology.parallel:result is None 2023-10-09T10:28:37.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:28:37.308 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:28:37.308 DEBUG:teuthology.orchestra.run.smithi012:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:28:37.365 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:28:37.365 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T10:28:37.365 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T10:28:37.365 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T10:28:37.366 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T10:28:37.366 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T10:28:37.366 DEBUG:teuthology.orchestra.run.smithi012:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:28:37.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:28:37.422 INFO:teuthology.orchestra.run.smithi012.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T10:28:37.422 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:28:37.487 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T10:28:37.488 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T10:28:37.530 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[147926]: pgmap v1585: 105 pgs: 105 active+clean; 582 KiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 1.07k op/s 2023-10-09T10:28:37.530 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[147926]: from='client.47228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:37.531 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[152263]: pgmap v1585: 105 pgs: 105 active+clean; 582 KiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 1.07k op/s 2023-10-09T10:28:37.531 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[152263]: from='client.47228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:37.531 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:37.531 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[152263]: failed to encode map e560 with expected crc 2023-10-09T10:28:37.549 INFO:tasks.workunit:timeout=3h 2023-10-09T10:28:37.549 INFO:tasks.workunit:cleanup=True 2023-10-09T10:28:37.550 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T10:28:37.607 INFO:tasks.workunit.client.0.smithi012.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T10:28:37.896 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:37.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:37.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[147926]: failed to encode map e560 with expected crc 2023-10-09T10:28:37.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:28:37.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[147926]: osdmap e560: 8 total, 8 up, 6 in 2023-10-09T10:28:37.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[147926]: failed to encode map e560 with expected crc 2023-10-09T10:28:37.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:28:37.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[152263]: osdmap e560: 8 total, 8 up, 6 in 2023-10-09T10:28:37.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-mon[152263]: failed to encode map e560 with expected crc 2023-10-09T10:28:37.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:37.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:37 smithi110 ceph-mon[136032]: pgmap v1585: 105 pgs: 105 active+clean; 582 KiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 1.07k op/s 2023-10-09T10:28:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:37 smithi110 ceph-mon[136032]: from='client.47228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:37 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:37 smithi110 ceph-mon[136032]: failed to encode map e560 with expected crc 2023-10-09T10:28:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:37 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:28:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:37 smithi110 ceph-mon[136032]: osdmap e560: 8 total, 8 up, 6 in 2023-10-09T10:28:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:37 smithi110 ceph-mon[136032]: failed to encode map e560 with expected crc 2023-10-09T10:28:38.001 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:38.026 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1585,"stamp":"2023-10-09T10:28:35.880985+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":596246,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":4352,"num_read_kb":13488,"num_write":2648,"num_write_kb":8870,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":160,"num_bytes_recovered":2983676,"num_keys_recovered":70,"num_objects_omap":46,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1982,"ondisk_log_size":1982,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9090572,"kb_used_data":2603852,"kb_used_omap":163,"kb_used_meta":6486492,"kb_avail":553380340,"statfs":{"total":575970213888,"available":566661468160,"internally_reserved":0,"allocated":2666344448,"data_stored":2656558812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":6642168819},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":374,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":4044,"num_read_kb":12976,"num_write":2388,"num_write_kb":6425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":32,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001273"},"pg_stats":[{"pgid":"12.15","version":"556'1","reported_seq":13,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.748330+0000","last_change":"2023-10-09T10:28:29.743218+0000","last_active":"2023-10-09T10:28:30.748330+0000","last_peered":"2023-10-09T10:28:30.748330+0000","last_clean":"2023-10-09T10:28:30.748330+0000","last_became_active":"2023-10-09T10:28:29.742773+0000","last_became_peered":"2023-10-09T10:28:29.742773+0000","last_unstale":"2023-10-09T10:28:30.748330+0000","last_undegraded":"2023-10-09T10:28:30.748330+0000","last_fullsized":"2023-10-09T10:28:30.748330+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":149,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.733505+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:28:29.733505+0000","last_peered":"2023-10-09T10:28:29.733505+0000","last_clean":"2023-10-09T10:28:29.733505+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:28:29.733505+0000","last_undegraded":"2023-10-09T10:28:29.733505+0000","last_fullsized":"2023-10-09T10:28:29.733505+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":510,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.735798+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:28:29.735798+0000","last_peered":"2023-10-09T10:28:29.735798+0000","last_clean":"2023-10-09T10:28:29.735798+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:28:29.735798+0000","last_undegraded":"2023-10-09T10:28:29.735798+0000","last_fullsized":"2023-10-09T10:28:29.735798+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.16","version":"556'22","reported_seq":42,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.272375+0000","last_change":"2023-10-09T10:28:29.745707+0000","last_active":"2023-10-09T10:28:30.272375+0000","last_peered":"2023-10-09T10:28:30.272375+0000","last_clean":"2023-10-09T10:28:30.272375+0000","last_became_active":"2023-10-09T10:28:29.742183+0000","last_became_peered":"2023-10-09T10:28:29.742183+0000","last_unstale":"2023-10-09T10:28:30.272375+0000","last_undegraded":"2023-10-09T10:28:30.272375+0000","last_fullsized":"2023-10-09T10:28:30.272375+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":23,"num_read_kb":44,"num_write":14,"num_write_kb":17,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":587,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.734051+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:28:29.734051+0000","last_peered":"2023-10-09T10:28:29.734051+0000","last_clean":"2023-10-09T10:28:29.734051+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:28:29.734051+0000","last_undegraded":"2023-10-09T10:28:29.734051+0000","last_fullsized":"2023-10-09T10:28:29.734051+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":482,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.722572+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:28:29.722572+0000","last_peered":"2023-10-09T10:28:29.722572+0000","last_clean":"2023-10-09T10:28:29.722572+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:28:29.722572+0000","last_undegraded":"2023-10-09T10:28:29.722572+0000","last_fullsized":"2023-10-09T10:28:29.722572+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"556'2","reported_seq":13,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.748298+0000","last_change":"2023-10-09T10:28:29.745190+0000","last_active":"2023-10-09T10:28:30.748298+0000","last_peered":"2023-10-09T10:28:30.748298+0000","last_clean":"2023-10-09T10:28:30.748298+0000","last_became_active":"2023-10-09T10:28:29.744805+0000","last_became_peered":"2023-10-09T10:28:29.744805+0000","last_unstale":"2023-10-09T10:28:30.748298+0000","last_undegraded":"2023-10-09T10:28:30.748298+0000","last_fullsized":"2023-10-09T10:28:30.748298+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":53,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.720600+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:28:29.720600+0000","last_peered":"2023-10-09T10:28:29.720600+0000","last_clean":"2023-10-09T10:28:29.720600+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:28:29.720600+0000","last_undegraded":"2023-10-09T10:28:29.720600+0000","last_fullsized":"2023-10-09T10:28:29.720600+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":135,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.744127+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:28:30.744127+0000","last_peered":"2023-10-09T10:28:30.744127+0000","last_clean":"2023-10-09T10:28:30.744127+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:28:30.744127+0000","last_undegraded":"2023-10-09T10:28:30.744127+0000","last_fullsized":"2023-10-09T10:28:30.744127+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.18","version":"556'15","reported_seq":54,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.200687+0000","last_change":"2023-10-09T10:28:29.743129+0000","last_active":"2023-10-09T10:28:30.200687+0000","last_peered":"2023-10-09T10:28:30.200687+0000","last_clean":"2023-10-09T10:28:30.200687+0000","last_became_active":"2023-10-09T10:28:29.742570+0000","last_became_peered":"2023-10-09T10:28:29.742570+0000","last_unstale":"2023-10-09T10:28:30.200687+0000","last_undegraded":"2023-10-09T10:28:30.200687+0000","last_fullsized":"2023-10-09T10:28:30.200687+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":86,"num_read_kb":82,"num_write":12,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":516,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.722199+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:28:29.722199+0000","last_peered":"2023-10-09T10:28:29.722199+0000","last_clean":"2023-10-09T10:28:29.722199+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:28:29.722199+0000","last_undegraded":"2023-10-09T10:28:29.722199+0000","last_fullsized":"2023-10-09T10:28:29.722199+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":465,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.716040+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:28:29.716040+0000","last_peered":"2023-10-09T10:28:29.716040+0000","last_clean":"2023-10-09T10:28:29.716040+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:28:29.716040+0000","last_undegraded":"2023-10-09T10:28:29.716040+0000","last_fullsized":"2023-10-09T10:28:29.716040+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"556'7","reported_seq":20,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.741088+0000","last_change":"2023-10-09T10:28:29.741387+0000","last_active":"2023-10-09T10:28:30.741088+0000","last_peered":"2023-10-09T10:28:30.741088+0000","last_clean":"2023-10-09T10:28:30.741088+0000","last_became_active":"2023-10-09T10:28:29.741084+0000","last_became_peered":"2023-10-09T10:28:29.741084+0000","last_unstale":"2023-10-09T10:28:30.741088+0000","last_undegraded":"2023-10-09T10:28:30.741088+0000","last_fullsized":"2023-10-09T10:28:30.741088+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:37.858166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":13,"num_read_kb":10,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":233,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.748358+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:28:30.748358+0000","last_peered":"2023-10-09T10:28:30.748358+0000","last_clean":"2023-10-09T10:28:30.748358+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:28:30.748358+0000","last_undegraded":"2023-10-09T10:28:30.748358+0000","last_fullsized":"2023-10-09T10:28:30.748358+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":465,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.717136+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:28:29.717136+0000","last_peered":"2023-10-09T10:28:29.717136+0000","last_clean":"2023-10-09T10:28:29.717136+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:28:29.717136+0000","last_undegraded":"2023-10-09T10:28:29.717136+0000","last_fullsized":"2023-10-09T10:28:29.717136+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1a","version":"0'0","reported_seq":10,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.746326+0000","last_change":"2023-10-09T10:28:29.744482+0000","last_active":"2023-10-09T10:28:29.746326+0000","last_peered":"2023-10-09T10:28:29.746326+0000","last_clean":"2023-10-09T10:28:29.746326+0000","last_became_active":"2023-10-09T10:28:29.744166+0000","last_became_peered":"2023-10-09T10:28:29.744166+0000","last_unstale":"2023-10-09T10:28:29.746326+0000","last_undegraded":"2023-10-09T10:28:29.746326+0000","last_fullsized":"2023-10-09T10:28:29.746326+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":161,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.736038+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:28:29.736038+0000","last_peered":"2023-10-09T10:28:29.736038+0000","last_clean":"2023-10-09T10:28:29.736038+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:28:29.736038+0000","last_undegraded":"2023-10-09T10:28:29.736038+0000","last_fullsized":"2023-10-09T10:28:29.736038+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721039+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:28:29.721039+0000","last_peered":"2023-10-09T10:28:29.721039+0000","last_clean":"2023-10-09T10:28:29.721039+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:28:29.721039+0000","last_undegraded":"2023-10-09T10:28:29.721039+0000","last_fullsized":"2023-10-09T10:28:29.721039+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.1b","version":"0'0","reported_seq":10,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.743743+0000","last_change":"2023-10-09T10:28:29.742195+0000","last_active":"2023-10-09T10:28:29.743743+0000","last_peered":"2023-10-09T10:28:29.743743+0000","last_clean":"2023-10-09T10:28:29.743743+0000","last_became_active":"2023-10-09T10:28:29.741858+0000","last_became_peered":"2023-10-09T10:28:29.741858+0000","last_unstale":"2023-10-09T10:28:29.743743+0000","last_undegraded":"2023-10-09T10:28:29.743743+0000","last_fullsized":"2023-10-09T10:28:29.743743+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":207,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.732401+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:28:29.732401+0000","last_peered":"2023-10-09T10:28:29.732401+0000","last_clean":"2023-10-09T10:28:29.732401+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:28:29.732401+0000","last_undegraded":"2023-10-09T10:28:29.732401+0000","last_fullsized":"2023-10-09T10:28:29.732401+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":167,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.732644+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:28:29.732644+0000","last_peered":"2023-10-09T10:28:29.732644+0000","last_clean":"2023-10-09T10:28:29.732644+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:28:29.732644+0000","last_undegraded":"2023-10-09T10:28:29.732644+0000","last_fullsized":"2023-10-09T10:28:29.732644+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1c","version":"0'0","reported_seq":11,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.742173+0000","last_change":"2023-10-09T10:28:29.735028+0000","last_active":"2023-10-09T10:28:30.742173+0000","last_peered":"2023-10-09T10:28:30.742173+0000","last_clean":"2023-10-09T10:28:30.742173+0000","last_became_active":"2023-10-09T10:28:29.734263+0000","last_became_peered":"2023-10-09T10:28:29.734263+0000","last_unstale":"2023-10-09T10:28:30.742173+0000","last_undegraded":"2023-10-09T10:28:30.742173+0000","last_fullsized":"2023-10-09T10:28:30.742173+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:33.203898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":127,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.732426+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:28:29.732426+0000","last_peered":"2023-10-09T10:28:29.732426+0000","last_clean":"2023-10-09T10:28:29.732426+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:28:29.732426+0000","last_undegraded":"2023-10-09T10:28:29.732426+0000","last_fullsized":"2023-10-09T10:28:29.732426+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":83,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747131+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:28:30.747131+0000","last_peered":"2023-10-09T10:28:30.747131+0000","last_clean":"2023-10-09T10:28:30.747131+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:28:30.747131+0000","last_undegraded":"2023-10-09T10:28:30.747131+0000","last_fullsized":"2023-10-09T10:28:30.747131+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1d","version":"556'2","reported_seq":13,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.742361+0000","last_change":"2023-10-09T10:28:29.738169+0000","last_active":"2023-10-09T10:28:30.742361+0000","last_peered":"2023-10-09T10:28:30.742361+0000","last_clean":"2023-10-09T10:28:30.742361+0000","last_became_active":"2023-10-09T10:28:29.737895+0000","last_became_peered":"2023-10-09T10:28:29.737895+0000","last_unstale":"2023-10-09T10:28:30.742361+0000","last_undegraded":"2023-10-09T10:28:30.742361+0000","last_fullsized":"2023-10-09T10:28:30.742361+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:25.498613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":481,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721436+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:28:29.721436+0000","last_peered":"2023-10-09T10:28:29.721436+0000","last_clean":"2023-10-09T10:28:29.721436+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:28:29.721436+0000","last_undegraded":"2023-10-09T10:28:29.721436+0000","last_fullsized":"2023-10-09T10:28:29.721436+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":224,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747365+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:28:30.747365+0000","last_peered":"2023-10-09T10:28:30.747365+0000","last_clean":"2023-10-09T10:28:30.747365+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:28:30.747365+0000","last_undegraded":"2023-10-09T10:28:30.747365+0000","last_fullsized":"2023-10-09T10:28:30.747365+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1e","version":"556'7","reported_seq":18,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.973929+0000","last_change":"2023-10-09T10:28:29.744373+0000","last_active":"2023-10-09T10:28:29.973929+0000","last_peered":"2023-10-09T10:28:29.973929+0000","last_clean":"2023-10-09T10:28:29.973929+0000","last_became_active":"2023-10-09T10:28:29.742762+0000","last_became_peered":"2023-10-09T10:28:29.742762+0000","last_unstale":"2023-10-09T10:28:29.973929+0000","last_undegraded":"2023-10-09T10:28:29.973929+0000","last_fullsized":"2023-10-09T10:28:29.973929+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":167,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.734188+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:28:29.734188+0000","last_peered":"2023-10-09T10:28:29.734188+0000","last_clean":"2023-10-09T10:28:29.734188+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:28:29.734188+0000","last_undegraded":"2023-10-09T10:28:29.734188+0000","last_fullsized":"2023-10-09T10:28:29.734188+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.722442+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:28:29.722442+0000","last_peered":"2023-10-09T10:28:29.722442+0000","last_clean":"2023-10-09T10:28:29.722442+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:28:29.722442+0000","last_undegraded":"2023-10-09T10:28:29.722442+0000","last_fullsized":"2023-10-09T10:28:29.722442+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.1f","version":"0'0","reported_seq":10,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.746011+0000","last_change":"2023-10-09T10:28:29.743904+0000","last_active":"2023-10-09T10:28:29.746011+0000","last_peered":"2023-10-09T10:28:29.746011+0000","last_clean":"2023-10-09T10:28:29.746011+0000","last_became_active":"2023-10-09T10:28:29.742532+0000","last_became_peered":"2023-10-09T10:28:29.742532+0000","last_unstale":"2023-10-09T10:28:29.746011+0000","last_undegraded":"2023-10-09T10:28:29.746011+0000","last_fullsized":"2023-10-09T10:28:29.746011+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":63,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.720696+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:28:29.720696+0000","last_peered":"2023-10-09T10:28:29.720696+0000","last_clean":"2023-10-09T10:28:29.720696+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:28:29.720696+0000","last_undegraded":"2023-10-09T10:28:29.720696+0000","last_fullsized":"2023-10-09T10:28:29.720696+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":56,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.716522+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:28:29.716522+0000","last_peered":"2023-10-09T10:28:29.716522+0000","last_clean":"2023-10-09T10:28:29.716522+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:28:29.716522+0000","last_undegraded":"2023-10-09T10:28:29.716522+0000","last_fullsized":"2023-10-09T10:28:29.716522+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"557'10","reported_seq":27,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:33.163308+0000","last_change":"2023-10-09T10:28:29.743224+0000","last_active":"2023-10-09T10:28:33.163308+0000","last_peered":"2023-10-09T10:28:33.163308+0000","last_clean":"2023-10-09T10:28:33.163308+0000","last_became_active":"2023-10-09T10:28:29.742905+0000","last_became_peered":"2023-10-09T10:28:29.742905+0000","last_unstale":"2023-10-09T10:28:33.163308+0000","last_undegraded":"2023-10-09T10:28:33.163308+0000","last_fullsized":"2023-10-09T10:28:33.163308+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":10,"num_read_kb":8,"num_write":6,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":57,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.735303+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:28:29.735303+0000","last_peered":"2023-10-09T10:28:29.735303+0000","last_clean":"2023-10-09T10:28:29.735303+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:28:29.735303+0000","last_undegraded":"2023-10-09T10:28:29.735303+0000","last_fullsized":"2023-10-09T10:28:29.735303+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":233,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.746702+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:28:30.746702+0000","last_peered":"2023-10-09T10:28:30.746702+0000","last_clean":"2023-10-09T10:28:30.746702+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:28:30.746702+0000","last_undegraded":"2023-10-09T10:28:30.746702+0000","last_fullsized":"2023-10-09T10:28:30.746702+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"557'23","reported_seq":36,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:33.512010+0000","last_change":"2023-10-09T10:28:29.741148+0000","last_active":"2023-10-09T10:28:33.512010+0000","last_peered":"2023-10-09T10:28:33.512010+0000","last_clean":"2023-10-09T10:28:33.512010+0000","last_became_active":"2023-10-09T10:28:29.740870+0000","last_became_peered":"2023-10-09T10:28:29.740870+0000","last_unstale":"2023-10-09T10:28:33.512010+0000","last_undegraded":"2023-10-09T10:28:33.512010+0000","last_fullsized":"2023-10-09T10:28:33.512010+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":34,"num_read_kb":34,"num_write":22,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":183,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.736236+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:28:29.736236+0000","last_peered":"2023-10-09T10:28:29.736236+0000","last_clean":"2023-10-09T10:28:29.736236+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:28:29.736236+0000","last_undegraded":"2023-10-09T10:28:29.736236+0000","last_fullsized":"2023-10-09T10:28:29.736236+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":170,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.734237+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:28:29.734237+0000","last_peered":"2023-10-09T10:28:29.734237+0000","last_clean":"2023-10-09T10:28:29.734237+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:28:29.734237+0000","last_undegraded":"2023-10-09T10:28:29.734237+0000","last_fullsized":"2023-10-09T10:28:29.734237+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"556'1","reported_seq":12,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.748804+0000","last_change":"2023-10-09T10:28:29.744989+0000","last_active":"2023-10-09T10:28:29.748804+0000","last_peered":"2023-10-09T10:28:29.748804+0000","last_clean":"2023-10-09T10:28:29.748804+0000","last_became_active":"2023-10-09T10:28:29.744671+0000","last_became_peered":"2023-10-09T10:28:29.744671+0000","last_unstale":"2023-10-09T10:28:29.748804+0000","last_undegraded":"2023-10-09T10:28:29.748804+0000","last_fullsized":"2023-10-09T10:28:29.748804+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":249,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747109+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:28:30.747109+0000","last_peered":"2023-10-09T10:28:30.747109+0000","last_clean":"2023-10-09T10:28:30.747109+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:28:30.747109+0000","last_undegraded":"2023-10-09T10:28:30.747109+0000","last_fullsized":"2023-10-09T10:28:30.747109+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":150,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747809+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:28:30.747809+0000","last_peered":"2023-10-09T10:28:30.747809+0000","last_clean":"2023-10-09T10:28:30.747809+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:28:30.747809+0000","last_undegraded":"2023-10-09T10:28:30.747809+0000","last_fullsized":"2023-10-09T10:28:30.747809+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":11,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.741868+0000","last_change":"2023-10-09T10:28:29.734833+0000","last_active":"2023-10-09T10:28:30.741868+0000","last_peered":"2023-10-09T10:28:30.741868+0000","last_clean":"2023-10-09T10:28:30.741868+0000","last_became_active":"2023-10-09T10:28:29.734500+0000","last_became_peered":"2023-10-09T10:28:29.734500+0000","last_unstale":"2023-10-09T10:28:30.741868+0000","last_undegraded":"2023-10-09T10:28:30.741868+0000","last_fullsized":"2023-10-09T10:28:30.741868+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:14.384780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":233,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.746923+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:28:30.746923+0000","last_peered":"2023-10-09T10:28:30.746923+0000","last_clean":"2023-10-09T10:28:30.746923+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:28:30.746923+0000","last_undegraded":"2023-10-09T10:28:30.746923+0000","last_fullsized":"2023-10-09T10:28:30.746923+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":467,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721332+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:28:29.721332+0000","last_peered":"2023-10-09T10:28:29.721332+0000","last_clean":"2023-10-09T10:28:29.721332+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:28:29.721332+0000","last_undegraded":"2023-10-09T10:28:29.721332+0000","last_fullsized":"2023-10-09T10:28:29.721332+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":10,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.745948+0000","last_change":"2023-10-09T10:28:29.743323+0000","last_active":"2023-10-09T10:28:29.745948+0000","last_peered":"2023-10-09T10:28:29.745948+0000","last_clean":"2023-10-09T10:28:29.745948+0000","last_became_active":"2023-10-09T10:28:29.742249+0000","last_became_peered":"2023-10-09T10:28:29.742249+0000","last_unstale":"2023-10-09T10:28:29.745948+0000","last_undegraded":"2023-10-09T10:28:29.745948+0000","last_fullsized":"2023-10-09T10:28:29.745948+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":168,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721265+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:28:29.721265+0000","last_peered":"2023-10-09T10:28:29.721265+0000","last_clean":"2023-10-09T10:28:29.721265+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:28:29.721265+0000","last_undegraded":"2023-10-09T10:28:29.721265+0000","last_fullsized":"2023-10-09T10:28:29.721265+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":167,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.732295+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:28:29.732295+0000","last_peered":"2023-10-09T10:28:29.732295+0000","last_clean":"2023-10-09T10:28:29.732295+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:28:29.732295+0000","last_undegraded":"2023-10-09T10:28:29.732295+0000","last_fullsized":"2023-10-09T10:28:29.732295+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":11,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.748185+0000","last_change":"2023-10-09T10:28:29.743058+0000","last_active":"2023-10-09T10:28:30.748185+0000","last_peered":"2023-10-09T10:28:30.748185+0000","last_clean":"2023-10-09T10:28:30.748185+0000","last_became_active":"2023-10-09T10:28:29.742375+0000","last_became_peered":"2023-10-09T10:28:29.742375+0000","last_unstale":"2023-10-09T10:28:30.748185+0000","last_undegraded":"2023-10-09T10:28:30.748185+0000","last_fullsized":"2023-10-09T10:28:30.748185+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":167,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.733347+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:28:29.733347+0000","last_peered":"2023-10-09T10:28:29.733347+0000","last_clean":"2023-10-09T10:28:29.733347+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:28:29.733347+0000","last_undegraded":"2023-10-09T10:28:29.733347+0000","last_fullsized":"2023-10-09T10:28:29.733347+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":133,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.737563+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:28:29.737563+0000","last_peered":"2023-10-09T10:28:29.737563+0000","last_clean":"2023-10-09T10:28:29.737563+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:28:29.737563+0000","last_undegraded":"2023-10-09T10:28:29.737563+0000","last_fullsized":"2023-10-09T10:28:29.737563+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"557'1355","reported_seq":1415,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:31.956619+0000","last_change":"2023-10-09T10:28:29.737530+0000","last_active":"2023-10-09T10:28:31.956619+0000","last_peered":"2023-10-09T10:28:31.956619+0000","last_clean":"2023-10-09T10:28:31.956619+0000","last_became_active":"2023-10-09T10:28:29.737220+0000","last_became_peered":"2023-10-09T10:28:29.737220+0000","last_unstale":"2023-10-09T10:28:31.956619+0000","last_undegraded":"2023-10-09T10:28:31.956619+0000","last_fullsized":"2023-10-09T10:28:31.956619+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":1355,"log_dups_size":0,"ondisk_log_size":1355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:27.872742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2834,"num_read_kb":2801,"num_write":1999,"num_write_kb":1962,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":149,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.734374+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:28:29.734374+0000","last_peered":"2023-10-09T10:28:29.734374+0000","last_clean":"2023-10-09T10:28:29.734374+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:28:29.734374+0000","last_undegraded":"2023-10-09T10:28:29.734374+0000","last_fullsized":"2023-10-09T10:28:29.734374+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":176,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.736173+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:28:29.736173+0000","last_peered":"2023-10-09T10:28:29.736173+0000","last_clean":"2023-10-09T10:28:29.736173+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:28:29.736173+0000","last_undegraded":"2023-10-09T10:28:29.736173+0000","last_fullsized":"2023-10-09T10:28:29.736173+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"557'34","reported_seq":70,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:33.176543+0000","last_change":"2023-10-09T10:28:29.742149+0000","last_active":"2023-10-09T10:28:33.176543+0000","last_peered":"2023-10-09T10:28:33.176543+0000","last_clean":"2023-10-09T10:28:33.176543+0000","last_became_active":"2023-10-09T10:28:29.741761+0000","last_became_peered":"2023-10-09T10:28:29.741761+0000","last_unstale":"2023-10-09T10:28:33.176543+0000","last_undegraded":"2023-10-09T10:28:33.176543+0000","last_fullsized":"2023-10-09T10:28:33.176543+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":72,"num_read_kb":71,"num_write":28,"num_write_kb":15,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":510,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.746639+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:28:30.746639+0000","last_peered":"2023-10-09T10:28:30.746639+0000","last_clean":"2023-10-09T10:28:30.746639+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:28:30.746639+0000","last_undegraded":"2023-10-09T10:28:30.746639+0000","last_fullsized":"2023-10-09T10:28:30.746639+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":570,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.735368+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:28:29.735368+0000","last_peered":"2023-10-09T10:28:29.735368+0000","last_clean":"2023-10-09T10:28:29.735368+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:28:29.735368+0000","last_undegraded":"2023-10-09T10:28:29.735368+0000","last_fullsized":"2023-10-09T10:28:29.735368+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.a","version":"556'14","reported_seq":31,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.741537+0000","last_change":"2023-10-09T10:28:29.727873+0000","last_active":"2023-10-09T10:28:30.741537+0000","last_peered":"2023-10-09T10:28:30.741537+0000","last_clean":"2023-10-09T10:28:30.741537+0000","last_became_active":"2023-10-09T10:28:29.727548+0000","last_became_peered":"2023-10-09T10:28:29.727548+0000","last_unstale":"2023-10-09T10:28:30.741537+0000","last_undegraded":"2023-10-09T10:28:30.741537+0000","last_fullsized":"2023-10-09T10:28:30.741537+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:15.187957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":44,"num_read_kb":35,"num_write":10,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":278,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.722588+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:28:29.722588+0000","last_peered":"2023-10-09T10:28:29.722588+0000","last_clean":"2023-10-09T10:28:29.722588+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:28:29.722588+0000","last_undegraded":"2023-10-09T10:28:29.722588+0000","last_fullsized":"2023-10-09T10:28:29.722588+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":242,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747287+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:28:30.747287+0000","last_peered":"2023-10-09T10:28:30.747287+0000","last_clean":"2023-10-09T10:28:30.747287+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:28:30.747287+0000","last_undegraded":"2023-10-09T10:28:30.747287+0000","last_fullsized":"2023-10-09T10:28:30.747287+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":522,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747881+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:28:30.747881+0000","last_peered":"2023-10-09T10:28:30.747881+0000","last_clean":"2023-10-09T10:28:30.747881+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:28:30.747881+0000","last_undegraded":"2023-10-09T10:28:30.747881+0000","last_fullsized":"2023-10-09T10:28:30.747881+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":11,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.742234+0000","last_change":"2023-10-09T10:28:29.741227+0000","last_active":"2023-10-09T10:28:30.742234+0000","last_peered":"2023-10-09T10:28:30.742234+0000","last_clean":"2023-10-09T10:28:30.742234+0000","last_became_active":"2023-10-09T10:28:29.741022+0000","last_became_peered":"2023-10-09T10:28:29.741022+0000","last_unstale":"2023-10-09T10:28:30.742234+0000","last_undegraded":"2023-10-09T10:28:30.742234+0000","last_fullsized":"2023-10-09T10:28:30.742234+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:02.294494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":242,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747763+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:28:30.747763+0000","last_peered":"2023-10-09T10:28:30.747763+0000","last_clean":"2023-10-09T10:28:30.747763+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:28:30.747763+0000","last_undegraded":"2023-10-09T10:28:30.747763+0000","last_fullsized":"2023-10-09T10:28:30.747763+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":158,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.732537+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:28:29.732537+0000","last_peered":"2023-10-09T10:28:29.732537+0000","last_clean":"2023-10-09T10:28:29.732537+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:28:29.732537+0000","last_undegraded":"2023-10-09T10:28:29.732537+0000","last_fullsized":"2023-10-09T10:28:29.732537+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":470,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.720845+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:28:29.720845+0000","last_peered":"2023-10-09T10:28:29.720845+0000","last_clean":"2023-10-09T10:28:29.720845+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:28:29.720845+0000","last_undegraded":"2023-10-09T10:28:29.720845+0000","last_fullsized":"2023-10-09T10:28:29.720845+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"557'30","reported_seq":55,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:33.203831+0000","last_change":"2023-10-09T10:28:29.740334+0000","last_active":"2023-10-09T10:28:33.203831+0000","last_peered":"2023-10-09T10:28:33.203831+0000","last_clean":"2023-10-09T10:28:33.203831+0000","last_became_active":"2023-10-09T10:28:29.739994+0000","last_became_peered":"2023-10-09T10:28:29.739994+0000","last_unstale":"2023-10-09T10:28:33.203831+0000","last_undegraded":"2023-10-09T10:28:33.203831+0000","last_fullsized":"2023-10-09T10:28:33.203831+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":51,"num_read_kb":8232,"num_write":29,"num_write_kb":4108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":508,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721095+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:28:29.721095+0000","last_peered":"2023-10-09T10:28:29.721095+0000","last_clean":"2023-10-09T10:28:29.721095+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:28:29.721095+0000","last_undegraded":"2023-10-09T10:28:29.721095+0000","last_fullsized":"2023-10-09T10:28:29.721095+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":126,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721534+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:28:29.721534+0000","last_peered":"2023-10-09T10:28:29.721534+0000","last_clean":"2023-10-09T10:28:29.721534+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:28:29.721534+0000","last_undegraded":"2023-10-09T10:28:29.721534+0000","last_fullsized":"2023-10-09T10:28:29.721534+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":571,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747876+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:28:30.747876+0000","last_peered":"2023-10-09T10:28:30.747876+0000","last_clean":"2023-10-09T10:28:30.747876+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:28:30.747876+0000","last_undegraded":"2023-10-09T10:28:30.747876+0000","last_fullsized":"2023-10-09T10:28:30.747876+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"556'30","reported_seq":65,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.232377+0000","last_change":"2023-10-09T10:28:29.743433+0000","last_active":"2023-10-09T10:28:30.232377+0000","last_peered":"2023-10-09T10:28:30.232377+0000","last_clean":"2023-10-09T10:28:30.232377+0000","last_became_active":"2023-10-09T10:28:29.742777+0000","last_became_peered":"2023-10-09T10:28:29.742777+0000","last_unstale":"2023-10-09T10:28:30.232377+0000","last_undegraded":"2023-10-09T10:28:30.232377+0000","last_fullsized":"2023-10-09T10:28:30.232377+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":139,"num_read_kb":104,"num_write":29,"num_write_kb":24,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":212,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.746998+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:28:30.746998+0000","last_peered":"2023-10-09T10:28:30.746998+0000","last_clean":"2023-10-09T10:28:30.746998+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:28:30.746998+0000","last_undegraded":"2023-10-09T10:28:30.746998+0000","last_fullsized":"2023-10-09T10:28:30.746998+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":751,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.720491+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:28:29.720491+0000","last_peered":"2023-10-09T10:28:29.720491+0000","last_clean":"2023-10-09T10:28:29.720491+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:28:29.720491+0000","last_undegraded":"2023-10-09T10:28:29.720491+0000","last_fullsized":"2023-10-09T10:28:29.720491+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":260,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747716+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:28:30.747716+0000","last_peered":"2023-10-09T10:28:30.747716+0000","last_clean":"2023-10-09T10:28:30.747716+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:28:30.747716+0000","last_undegraded":"2023-10-09T10:28:30.747716+0000","last_fullsized":"2023-10-09T10:28:30.747716+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":167,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.735911+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:28:29.735911+0000","last_peered":"2023-10-09T10:28:29.735911+0000","last_clean":"2023-10-09T10:28:29.735911+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:28:29.735911+0000","last_undegraded":"2023-10-09T10:28:29.735911+0000","last_fullsized":"2023-10-09T10:28:29.735911+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"556'9","reported_seq":48,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.741754+0000","last_change":"2023-10-09T10:28:29.738889+0000","last_active":"2023-10-09T10:28:30.741754+0000","last_peered":"2023-10-09T10:28:30.741754+0000","last_clean":"2023-10-09T10:28:30.741754+0000","last_became_active":"2023-10-09T10:28:29.738604+0000","last_became_peered":"2023-10-09T10:28:29.738604+0000","last_unstale":"2023-10-09T10:28:30.741754+0000","last_undegraded":"2023-10-09T10:28:30.741754+0000","last_fullsized":"2023-10-09T10:28:30.741754+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:30:59.129466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":32,"num_read_kb":52,"num_write":7,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":243,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721588+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:28:29.721588+0000","last_peered":"2023-10-09T10:28:29.721588+0000","last_clean":"2023-10-09T10:28:29.721588+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:28:29.721588+0000","last_undegraded":"2023-10-09T10:28:29.721588+0000","last_fullsized":"2023-10-09T10:28:29.721588+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":502,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721174+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:28:29.721174+0000","last_peered":"2023-10-09T10:28:29.721174+0000","last_clean":"2023-10-09T10:28:29.721174+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:28:29.721174+0000","last_undegraded":"2023-10-09T10:28:29.721174+0000","last_fullsized":"2023-10-09T10:28:29.721174+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":53,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.722253+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:28:29.722253+0000","last_peered":"2023-10-09T10:28:29.722253+0000","last_clean":"2023-10-09T10:28:29.722253+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:28:29.722253+0000","last_undegraded":"2023-10-09T10:28:29.722253+0000","last_fullsized":"2023-10-09T10:28:29.722253+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"556'37","reported_seq":73,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.739743+0000","last_change":"2023-10-09T10:28:29.741774+0000","last_active":"2023-10-09T10:28:30.739743+0000","last_peered":"2023-10-09T10:28:30.739743+0000","last_clean":"2023-10-09T10:28:30.739743+0000","last_became_active":"2023-10-09T10:28:29.741561+0000","last_became_peered":"2023-10-09T10:28:29.741561+0000","last_unstale":"2023-10-09T10:28:30.739743+0000","last_undegraded":"2023-10-09T10:28:30.739743+0000","last_fullsized":"2023-10-09T10:28:30.739743+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":37,"log_dups_size":0,"ondisk_log_size":37,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:46:09.768796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":215,"num_read_kb":174,"num_write":44,"num_write_kb":41,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":53,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721871+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:28:29.721871+0000","last_peered":"2023-10-09T10:28:29.721871+0000","last_clean":"2023-10-09T10:28:29.721871+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:28:29.721871+0000","last_undegraded":"2023-10-09T10:28:29.721871+0000","last_fullsized":"2023-10-09T10:28:29.721871+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":539,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747107+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:28:30.747107+0000","last_peered":"2023-10-09T10:28:30.747107+0000","last_clean":"2023-10-09T10:28:30.747107+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:28:30.747107+0000","last_undegraded":"2023-10-09T10:28:30.747107+0000","last_fullsized":"2023-10-09T10:28:30.747107+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":470,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721458+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:28:29.721458+0000","last_peered":"2023-10-09T10:28:29.721458+0000","last_clean":"2023-10-09T10:28:29.721458+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:28:29.721458+0000","last_undegraded":"2023-10-09T10:28:29.721458+0000","last_fullsized":"2023-10-09T10:28:29.721458+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"556'12","reported_seq":28,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.741366+0000","last_change":"2023-10-09T10:28:29.727989+0000","last_active":"2023-10-09T10:28:30.741366+0000","last_peered":"2023-10-09T10:28:30.741366+0000","last_clean":"2023-10-09T10:28:30.741366+0000","last_became_active":"2023-10-09T10:28:29.727510+0000","last_became_peered":"2023-10-09T10:28:29.727510+0000","last_unstale":"2023-10-09T10:28:30.741366+0000","last_undegraded":"2023-10-09T10:28:30.741366+0000","last_fullsized":"2023-10-09T10:28:30.741366+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:54.020718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":18,"num_read_kb":18,"num_write":9,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":358,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.722100+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:28:29.722100+0000","last_peered":"2023-10-09T10:28:29.722100+0000","last_clean":"2023-10-09T10:28:29.722100+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:28:29.722100+0000","last_undegraded":"2023-10-09T10:28:29.722100+0000","last_fullsized":"2023-10-09T10:28:29.722100+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":537,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.736098+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:28:29.736098+0000","last_peered":"2023-10-09T10:28:29.736098+0000","last_clean":"2023-10-09T10:28:29.736098+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:28:29.736098+0000","last_undegraded":"2023-10-09T10:28:29.736098+0000","last_fullsized":"2023-10-09T10:28:29.736098+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":53,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.720581+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:28:29.720581+0000","last_peered":"2023-10-09T10:28:29.720581+0000","last_clean":"2023-10-09T10:28:29.720581+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:28:29.720581+0000","last_undegraded":"2023-10-09T10:28:29.720581+0000","last_fullsized":"2023-10-09T10:28:29.720581+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.9","version":"557'155","reported_seq":170,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:33.460492+0000","last_change":"2023-10-09T10:28:29.739848+0000","last_active":"2023-10-09T10:28:33.460492+0000","last_peered":"2023-10-09T10:28:33.460492+0000","last_clean":"2023-10-09T10:28:33.460492+0000","last_became_active":"2023-10-09T10:28:29.739440+0000","last_became_peered":"2023-10-09T10:28:29.739440+0000","last_unstale":"2023-10-09T10:28:33.460492+0000","last_undegraded":"2023-10-09T10:28:33.460492+0000","last_fullsized":"2023-10-09T10:28:33.460492+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:20.699999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":421,"num_read_kb":1264,"num_write":155,"num_write_kb":155,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":61,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.748430+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:28:30.748430+0000","last_peered":"2023-10-09T10:28:30.748430+0000","last_clean":"2023-10-09T10:28:30.748430+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:28:30.748430+0000","last_undegraded":"2023-10-09T10:28:30.748430+0000","last_fullsized":"2023-10-09T10:28:30.748430+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":57,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.722552+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:28:29.722552+0000","last_peered":"2023-10-09T10:28:29.722552+0000","last_clean":"2023-10-09T10:28:29.722552+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:28:29.722552+0000","last_undegraded":"2023-10-09T10:28:29.722552+0000","last_fullsized":"2023-10-09T10:28:29.722552+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":238,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.746932+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:28:30.746932+0000","last_peered":"2023-10-09T10:28:30.746932+0000","last_clean":"2023-10-09T10:28:30.746932+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:28:30.746932+0000","last_undegraded":"2023-10-09T10:28:30.746932+0000","last_fullsized":"2023-10-09T10:28:30.746932+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.14","version":"557'12","reported_seq":28,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:33.466902+0000","last_change":"2023-10-09T10:28:29.740011+0000","last_active":"2023-10-09T10:28:33.466902+0000","last_peered":"2023-10-09T10:28:33.466902+0000","last_clean":"2023-10-09T10:28:33.466902+0000","last_became_active":"2023-10-09T10:28:29.739593+0000","last_became_peered":"2023-10-09T10:28:29.739593+0000","last_unstale":"2023-10-09T10:28:33.466902+0000","last_undegraded":"2023-10-09T10:28:33.466902+0000","last_fullsized":"2023-10-09T10:28:33.466902+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":34,"num_read_kb":33,"num_write":11,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":53,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721414+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:28:29.721414+0000","last_peered":"2023-10-09T10:28:29.721414+0000","last_clean":"2023-10-09T10:28:29.721414+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:28:29.721414+0000","last_undegraded":"2023-10-09T10:28:29.721414+0000","last_fullsized":"2023-10-09T10:28:29.721414+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":62,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.722193+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:28:29.722193+0000","last_peered":"2023-10-09T10:28:29.722193+0000","last_clean":"2023-10-09T10:28:29.722193+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:28:29.722193+0000","last_undegraded":"2023-10-09T10:28:29.722193+0000","last_fullsized":"2023-10-09T10:28:29.722193+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.13","version":"0'0","reported_seq":11,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.741657+0000","last_change":"2023-10-09T10:28:29.738519+0000","last_active":"2023-10-09T10:28:30.741657+0000","last_peered":"2023-10-09T10:28:30.741657+0000","last_clean":"2023-10-09T10:28:30.741657+0000","last_became_active":"2023-10-09T10:28:29.738251+0000","last_became_peered":"2023-10-09T10:28:29.738251+0000","last_unstale":"2023-10-09T10:28:30.741657+0000","last_undegraded":"2023-10-09T10:28:30.741657+0000","last_fullsized":"2023-10-09T10:28:30.741657+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:41:52.141289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":288,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747015+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:28:30.747015+0000","last_peered":"2023-10-09T10:28:30.747015+0000","last_clean":"2023-10-09T10:28:30.747015+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:28:30.747015+0000","last_undegraded":"2023-10-09T10:28:30.747015+0000","last_fullsized":"2023-10-09T10:28:30.747015+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":149,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.732746+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:28:29.732746+0000","last_peered":"2023-10-09T10:28:29.732746+0000","last_clean":"2023-10-09T10:28:29.732746+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:28:29.732746+0000","last_undegraded":"2023-10-09T10:28:29.732746+0000","last_fullsized":"2023-10-09T10:28:29.732746+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"556'1","reported_seq":13,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747543+0000","last_change":"2023-10-09T10:28:29.739951+0000","last_active":"2023-10-09T10:28:30.747543+0000","last_peered":"2023-10-09T10:28:30.747543+0000","last_clean":"2023-10-09T10:28:30.747543+0000","last_became_active":"2023-10-09T10:28:29.739629+0000","last_became_peered":"2023-10-09T10:28:29.739629+0000","last_unstale":"2023-10-09T10:28:30.747543+0000","last_undegraded":"2023-10-09T10:28:30.747543+0000","last_fullsized":"2023-10-09T10:28:30.747543+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":271,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.747203+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:28:30.747203+0000","last_peered":"2023-10-09T10:28:30.747203+0000","last_clean":"2023-10-09T10:28:30.747203+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:28:30.747203+0000","last_undegraded":"2023-10-09T10:28:30.747203+0000","last_fullsized":"2023-10-09T10:28:30.747203+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":380,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721271+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:28:29.721271+0000","last_peered":"2023-10-09T10:28:29.721271+0000","last_clean":"2023-10-09T10:28:29.721271+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:28:29.721271+0000","last_undegraded":"2023-10-09T10:28:29.721271+0000","last_fullsized":"2023-10-09T10:28:29.721271+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.11","version":"556'5","reported_seq":18,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.746684+0000","last_change":"2023-10-09T10:28:29.743035+0000","last_active":"2023-10-09T10:28:30.746684+0000","last_peered":"2023-10-09T10:28:30.746684+0000","last_clean":"2023-10-09T10:28:30.746684+0000","last_became_active":"2023-10-09T10:28:29.742736+0000","last_became_peered":"2023-10-09T10:28:29.742736+0000","last_unstale":"2023-10-09T10:28:30.746684+0000","last_undegraded":"2023-10-09T10:28:30.746684+0000","last_fullsized":"2023-10-09T10:28:30.746684+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":56,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.716959+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:28:29.716959+0000","last_peered":"2023-10-09T10:28:29.716959+0000","last_clean":"2023-10-09T10:28:29.716959+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:28:29.716959+0000","last_undegraded":"2023-10-09T10:28:29.716959+0000","last_fullsized":"2023-10-09T10:28:29.716959+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":610,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721019+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:28:29.721019+0000","last_peered":"2023-10-09T10:28:29.721019+0000","last_clean":"2023-10-09T10:28:29.721019+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:28:29.721019+0000","last_undegraded":"2023-10-09T10:28:29.721019+0000","last_fullsized":"2023-10-09T10:28:29.721019+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.10","version":"556'4","reported_seq":15,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T10:28:30.741094+0000","last_change":"2023-10-09T10:28:29.740477+0000","last_active":"2023-10-09T10:28:30.741094+0000","last_peered":"2023-10-09T10:28:30.741094+0000","last_clean":"2023-10-09T10:28:30.741094+0000","last_became_active":"2023-10-09T10:28:29.740139+0000","last_became_peered":"2023-10-09T10:28:29.740139+0000","last_unstale":"2023-10-09T10:28:30.741094+0000","last_undegraded":"2023-10-09T10:28:30.741094+0000","last_fullsized":"2023-10-09T10:28:30.741094+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":555,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:28:28.706130+0000","last_clean_scrub_stamp":"2023-10-09T10:28:28.706130+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:54.873914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":5,"num_read_kb":5,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":74,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.721897+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:28:29.721897+0000","last_peered":"2023-10-09T10:28:29.721897+0000","last_clean":"2023-10-09T10:28:29.721897+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:28:29.721897+0000","last_undegraded":"2023-10-09T10:28:29.721897+0000","last_fullsized":"2023-10-09T10:28:29.721897+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":176,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T10:28:29.732488+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:28:29.732488+0000","last_peered":"2023-10-09T10:28:29.732488+0000","last_clean":"2023-10-09T10:28:29.732488+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:28:29.732488+0000","last_undegraded":"2023-10-09T10:28:29.732488+0000","last_fullsized":"2023-10-09T10:28:29.732488+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":374,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":4044,"num_read_kb":12976,"num_write":2388,"num_write_kb":6425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":32,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":1190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1788,"ondisk_log_size":1788,"up":96,"acting":96,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430707,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626935,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1527140,"kb_used_data":5724,"kb_used_omap":0,"kb_used_meta":1521408,"kb_avail":92218012,"statfs":{"total":95995035648,"available":94431244288,"internally_reserved":0,"allocated":5861376,"data_stored":4844297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1557921792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2880000000000003,"5min":2.4009999999999998,"15min":1.7310000000000001},"min":{"1min":0.219,"5min":0.216,"15min":0.216},"max":{"1min":47.401000000000003,"5min":47.401000000000003,"15min":47.401000000000003},"last":0.27200000000000002},{"interface":"front","average":{"1min":4.7679999999999998,"5min":2.3380000000000001,"15min":2.0579999999999998},"min":{"1min":0.29399999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":47.311999999999998,"5min":47.311999999999998,"15min":47.311999999999998},"last":0.28899999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.3400000000000001,"15min":1.998},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":2.4049999999999998,"5min":6.3899999999999997,"15min":155.01400000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.482,"5min":1.3580000000000001,"15min":2.1739999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.6050000000000004,"5min":6.3369999999999997,"15min":154.71100000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:28:20 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.0609999999999999,"15min":1.8600000000000001},"min":{"1min":0.32100000000000001,"5min":0.309,"15min":0.24099999999999999},"max":{"1min":10.907,"5min":10.907,"15min":29.367999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.6319999999999999,"15min":1.524},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.4480000000000004,"5min":7.4480000000000004,"15min":8.7669999999999995},"last":0.45300000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":2.1339999999999999,"15min":1.698},"min":{"1min":0.30599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.9450000000000003,"5min":43.393000000000001,"15min":43.393000000000001},"last":0.434},{"interface":"front","average":{"1min":2.4049999999999998,"5min":2.3069999999999999,"15min":1.867},"min":{"1min":0.36699999999999999,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":6.423,"5min":43.466000000000001,"15min":43.466000000000001},"last":0.60399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":8.9589999999999996,"5min":3.2749999999999999,"15min":2.4500000000000002},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":177.929,"5min":177.929,"15min":177.929},"last":0.56200000000000006},{"interface":"front","average":{"1min":8.2530000000000001,"5min":2.8980000000000001,"15min":2.2490000000000001},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":178.16499999999999,"5min":178.16499999999999,"15min":178.16499999999999},"last":0.501}]},{"osd":5,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.1640000000000001,"15min":2.0910000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.788,"5min":11.788,"15min":121.98999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.2629999999999999,"15min":1.5760000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":2.2890000000000001,"5min":7.0060000000000002,"15min":122.346},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.0219999999999998,"15min":1.966},"min":{"1min":0.38,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":12.612,"5min":49.991,"15min":130.66999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.5880000000000001,"5min":2.097,"15min":2.02},"min":{"1min":0.40799999999999997,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":12.659000000000001,"5min":50.023000000000003,"15min":130.702},"last":0.59499999999999997}]}]},{"osd":1,"up_from":0,"seq":360777253248,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416772,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1643692,"kb_used_data":608932,"kb_used_omap":28,"kb_used_meta":1034723,"kb_avail":92101460,"statfs":{"total":95995035648,"available":94311895040,"internally_reserved":0,"allocated":623546368,"data_stored":621340483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1059557200},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:27:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.8049999999999999,"15min":1.6990000000000001},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.19800000000000001},"max":{"1min":13.077999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.712,"15min":1.915},"min":{"1min":0.26500000000000001,"5min":0.214,"15min":0.192},"max":{"1min":12.349,"5min":12.349,"15min":56.950000000000003},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":2.141,"15min":1.9379999999999999},"min":{"1min":0.26100000000000001,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":8.6440000000000001,"5min":17.093,"15min":53.988999999999997},"last":0.748},{"interface":"front","average":{"1min":2.3889999999999998,"5min":2.2970000000000002,"15min":1.9890000000000001},"min":{"1min":0.30399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.021000000000001,"5min":17.053999999999998,"15min":54.119},"last":0.82799999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:28:08 2023","interfaces":[{"interface":"back","average":{"1min":4.0599999999999996,"5min":2.605,"15min":1.8480000000000001},"min":{"1min":0.28299999999999997,"5min":0.217,"15min":0.217},"max":{"1min":15.882999999999999,"5min":29.835999999999999,"15min":56.984999999999999},"last":0.25800000000000001},{"interface":"front","average":{"1min":3.6040000000000001,"5min":2.9729999999999999,"15min":1.944},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":9.8949999999999996,"5min":29.776,"15min":56.475999999999999},"last":0.34399999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":1.823,"5min":1.8029999999999999,"15min":1.952},"min":{"1min":0.29199999999999998,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":11.653,"5min":11.653,"15min":57.719000000000001},"last":0.437},{"interface":"front","average":{"1min":2.548,"5min":2.2730000000000001,"15min":2.077},"min":{"1min":0.375,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.1359999999999992,"5min":11.896000000000001,"15min":56.378},"last":0.41999999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9159999999999999,"5min":3.6589999999999998,"15min":3.363},"min":{"1min":0.46300000000000002,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":8.25,"5min":90.412000000000006,"15min":229.589},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.504,"5min":2.9500000000000002,"15min":3.02},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.30199999999999999},"max":{"1min":6.1239999999999997,"5min":90.393000000000001,"15min":228.88900000000001},"last":0.98599999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.9990000000000001,"15min":2.089},"min":{"1min":0.38400000000000001,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":6.0789999999999997,"5min":9.7840000000000007,"15min":54.482999999999997},"last":0.96099999999999997},{"interface":"front","average":{"1min":2.1499999999999999,"5min":2.0880000000000001,"15min":2.1960000000000002},"min":{"1min":0.44500000000000001,"5min":0.373,"15min":0.34799999999999998},"max":{"1min":11.702999999999999,"5min":11.702999999999999,"15min":57.753},"last":0.37}]},{"osd":7,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.4550000000000001,"15min":2.2050000000000001},"min":{"1min":0.47799999999999998,"5min":0.38500000000000001,"15min":0.34999999999999998},"max":{"1min":8.1760000000000002,"5min":12.029999999999999,"15min":56.411999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.0910000000000002,"5min":2.3460000000000001,"15min":2.1779999999999999},"min":{"1min":0.502,"5min":0.39900000000000002,"15min":0.38300000000000001},"max":{"1min":8.4339999999999993,"5min":17.129999999999999,"15min":54.451999999999998},"last":0.93400000000000005}]}]},{"osd":2,"up_from":217,"seq":932007903508,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1460332,"kb_used_data":579684,"kb_used_omap":83,"kb_used_meta":880556,"kb_avail":92284820,"statfs":{"total":95995035648,"available":94499655680,"internally_reserved":0,"allocated":593596416,"data_stored":591573620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":901689839},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5800000000000001,"5min":2.3410000000000002,"15min":1.891},"min":{"1min":0.215,"5min":0.192,"15min":0.185},"max":{"1min":31.641999999999999,"5min":31.641999999999999,"15min":48.990000000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.9119999999999999,"15min":1.946},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.20899999999999999},"max":{"1min":9.968,"5min":13.773,"15min":49.412999999999997},"last":0.40600000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":2.4889999999999999,"15min":1.7729999999999999},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":27.097999999999999,"5min":27.097999999999999,"15min":34.893999999999998},"last":0.44},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.6519999999999999,"15min":1.6679999999999999},"min":{"1min":0.317,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.976,"5min":13.356999999999999,"15min":34.962000000000003},"last":0.46600000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.8879999999999999,"15min":1.4470000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.8410000000000002,"5min":15.115,"15min":35.043999999999997},"last":0.65900000000000003},{"interface":"front","average":{"1min":2.395,"5min":1.776,"15min":1.4530000000000001},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.6820000000000004,"5min":11.206,"15min":50.406999999999996},"last":0.61099999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.8200000000000001,"15min":1.5660000000000001},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":3.9089999999999998,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.73899999999999999},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.3610000000000002,"15min":1.8600000000000001},"min":{"1min":0.38700000000000001,"5min":0.34000000000000002,"15min":0.29299999999999998},"max":{"1min":7.532,"5min":47.704000000000001,"15min":47.704000000000001},"last":0.45200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.681,"15min":1.8540000000000001},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":7.4779999999999998,"5min":11.266999999999999,"15min":57.298000000000002},"last":0.67800000000000005},{"interface":"front","average":{"1min":2.6320000000000001,"5min":2.5230000000000001,"15min":2.0190000000000001},"min":{"1min":0.34899999999999998,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":26.997,"5min":26.997,"15min":57.828000000000003},"last":0.63}]},{"osd":6,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.6000000000000001,"15min":1.756},"min":{"1min":0.36599999999999999,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":3.8460000000000001,"5min":7.9390000000000001,"15min":12.411},"last":0.71399999999999997},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.915,"15min":1.7949999999999999},"min":{"1min":0.42499999999999999,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":7.5579999999999998,"5min":13.975,"15min":24.379000000000001},"last":0.72599999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":1.754,"15min":2.0110000000000001},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":4.0449999999999999,"5min":10.17,"15min":35.008000000000003},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.841,"5min":2.0880000000000001,"15min":1.869},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.35899999999999999},"max":{"1min":7.5830000000000002,"5min":15.548,"15min":24.323},"last":0.56899999999999995}]}]},{"osd":3,"up_from":352,"seq":1511828488347,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":854400,"kb_used_data":250744,"kb_used_omap":15,"kb_used_meta":603632,"kb_avail":92890752,"statfs":{"total":95995035648,"available":95120130048,"internally_reserved":0,"allocated":256761856,"data_stored":254247331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":618119622},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":4.7489999999999997,"5min":3.5030000000000001,"15min":2.1280000000000001},"min":{"1min":0.318,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":40.965000000000003,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.441},{"interface":"front","average":{"1min":5.7789999999999999,"5min":3.645,"15min":2.2010000000000001},"min":{"1min":0.27200000000000002,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":43.595999999999997,"5min":66.802000000000007,"15min":66.802000000000007},"last":0.29199999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.4059999999999999,"15min":1.9370000000000001},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.7759999999999998,"5min":11.785,"15min":105.401},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.008,"15min":2.0219999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23300000000000001},"max":{"1min":7.452,"5min":10.422000000000001,"15min":104.914},"last":0.41999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.968,"5min":2.3759999999999999,"15min":2.0699999999999998},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":9.923,"5min":13.497999999999999,"15min":13.497999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":2.786,"5min":2.3500000000000001,"15min":1.6819999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":9.4039999999999999,"5min":16.475000000000001,"15min":25.338999999999999},"last":0.48099999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":12.550000000000001,"5min":3.8359999999999999,"15min":2.3100000000000001},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":178.72200000000001,"5min":178.72200000000001,"15min":178.72200000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":12.884,"5min":4.7800000000000002,"15min":2.891},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":179.08500000000001,"5min":179.08500000000001,"15min":179.08500000000001},"last":0.498}]},{"osd":5,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.617,"15min":1.4530000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.6870000000000003,"5min":11.363,"15min":25.251999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.0600000000000001,"5min":2.149,"15min":1.7809999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.3949999999999996,"5min":10.388,"15min":13.394},"last":0.51900000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.371,"5min":2.1549999999999998,"15min":1.915},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.8689999999999998,"5min":13.545999999999999,"15min":19.114000000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.8069999999999999,"15min":1.6100000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":5.718,"5min":11.32,"15min":25.306999999999999},"last":0.60199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":1.9730000000000001,"15min":1.7490000000000001},"min":{"1min":0.41299999999999998,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":7.4279999999999999,"5min":10.345000000000001,"15min":10.459},"last":0.71099999999999997},{"interface":"front","average":{"1min":2.9260000000000002,"5min":2.2000000000000002,"15min":1.9710000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":12.4,"5min":12.4,"15min":19.280000000000001},"last":0.72599999999999998}]}]},{"osd":4,"up_from":266,"seq":1142461300977,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1428724,"kb_used_data":487388,"kb_used_omap":35,"kb_used_meta":941276,"kb_avail":92316428,"statfs":{"total":95995035648,"available":94532022272,"internally_reserved":0,"allocated":499085312,"data_stored":498066735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":963866862},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":3.1179999999999999,"15min":2.6320000000000001},"min":{"1min":0.22700000000000001,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":7.6539999999999999,"5min":109.193,"15min":109.193},"last":0.26900000000000002},{"interface":"front","average":{"1min":2.629,"5min":3.625,"15min":3.2200000000000002},"min":{"1min":0.27100000000000002,"5min":0.26400000000000001,"15min":0.23300000000000001},"max":{"1min":10.289999999999999,"5min":110.438,"15min":110.438},"last":0.248}]},{"osd":1,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.9530000000000001,"15min":2.1469999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.24299999999999999},"max":{"1min":6.0259999999999998,"5min":12.209,"15min":71.457999999999998},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.913,"5min":2.6899999999999999,"15min":2.1899999999999999},"min":{"1min":0.34799999999999998,"5min":0.248,"15min":0.248},"max":{"1min":8.6899999999999995,"5min":49.781999999999996,"15min":71.658000000000001},"last":0.371}]},{"osd":2,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":2.153,"15min":1.758},"min":{"1min":0.41999999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":9.9039999999999999,"5min":15.944000000000001,"15min":15.944000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.903,"5min":1.7470000000000001,"15min":1.671},"min":{"1min":0.38600000000000001,"5min":0.255,"15min":0.255},"max":{"1min":6.2149999999999999,"5min":14.473000000000001,"15min":14.473000000000001},"last":0.316}]},{"osd":3,"last update":"Mon Oct 9 10:28:03 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.476,"15min":2.109},"min":{"1min":0.33000000000000002,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":5.141,"5min":11.347,"15min":12.074},"last":0.752},{"interface":"front","average":{"1min":3.278,"5min":2.7120000000000002,"15min":1.986},"min":{"1min":0.38100000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.9779999999999998,"5min":16.067,"15min":16.067},"last":0.76500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.8360000000000001,"15min":2.0209999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.252},"max":{"1min":10.007,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.7749999999999999,"15min":1.853},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":4.5620000000000003,"5min":49.915999999999997,"15min":99.384},"last":0.35899999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.446,"15min":1.393},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.8380000000000001,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.8109999999999999,"5min":2.1059999999999999,"15min":1.804},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":6.7329999999999997,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.434}]},{"osd":7,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":2.0880000000000001,"15min":1.835},"min":{"1min":0.36299999999999999,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":6.4279999999999999,"5min":12.391999999999999,"15min":17.369},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.9119999999999999,"5min":2.2200000000000002,"15min":1.7490000000000001},"min":{"1min":0.41399999999999998,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":9.5129999999999999,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.73099999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823162,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2176284,"kb_used_data":671380,"kb_used_omap":0,"kb_used_meta":1504896,"kb_avail":91568868,"statfs":{"total":95995035648,"available":93766520832,"internally_reserved":0,"allocated":687493120,"data_stored":686486346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541013504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5630000000000002,"5min":3.0760000000000001,"15min":2.1629999999999998},"min":{"1min":0.92600000000000005,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":10.323,"5min":19.102,"15min":22.469999999999999},"last":0.38},{"interface":"front","average":{"1min":1.9730000000000001,"5min":1.901,"15min":1.7430000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23200000000000001},"max":{"1min":5.0730000000000004,"5min":17.887,"15min":22.437000000000001},"last":0.39700000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:27:43 2023","interfaces":[{"interface":"back","average":{"1min":3.3319999999999999,"5min":2.387,"15min":2.1680000000000001},"min":{"1min":0.36799999999999999,"5min":0.27900000000000003,"15min":0.23999999999999999},"max":{"1min":10.425000000000001,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.6299999999999999,"15min":1.788},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23499999999999999},"max":{"1min":5.2329999999999997,"5min":8.9529999999999994,"15min":29.065000000000001},"last":0.434}]},{"osd":2,"last update":"Mon Oct 9 10:28:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":2.149,"15min":2.0099999999999998},"min":{"1min":0.35499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.0759999999999996,"5min":16.629999999999999,"15min":16.629999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.8320000000000001,"15min":1.746},"min":{"1min":0.33100000000000002,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":6.819,"5min":9.6929999999999996,"15min":15.929},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":2.3220000000000001,"15min":2.0139999999999998},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":8.1720000000000006,"5min":13.614000000000001,"15min":13.614000000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":2.9740000000000002,"5min":2.1589999999999998,"15min":1.887},"min":{"1min":0.46300000000000002,"5min":0.376,"15min":0.315},"max":{"1min":9.8450000000000006,"5min":11.124000000000001,"15min":13.624000000000001},"last":0.60399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":2.0230000000000001,"15min":1.6919999999999999},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":3.9500000000000002,"5min":13.523999999999999,"15min":13.523999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.391,"5min":1.5269999999999999,"15min":1.4690000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23599999999999999},"max":{"1min":4.3959999999999999,"5min":8.8620000000000001,"15min":11.956},"last":0.64700000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.929,"15min":1.8440000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":3.3679999999999999,"5min":13.567,"15min":13.567},"last":0.754},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.74,"15min":1.696},"min":{"1min":0.34300000000000003,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":3.407,"5min":16.497,"15min":16.497},"last":0.624}]},{"osd":7,"last update":"Mon Oct 9 10:28:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.6379999999999999,"15min":1.7769999999999999},"min":{"1min":0.36899999999999999,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":7.3490000000000002,"5min":16.686,"15min":21.109000000000002},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.86499999999999999,"15min":1.1299999999999999},"min":{"1min":0.34799999999999998,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":2.375,"5min":8.9719999999999995,"15min":11.994},"last":0.56100000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:28:38.033 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:28:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:28:37] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.33.4" 2023-10-09T10:28:38.216 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:38.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:38.319 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:38.351 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:38.387 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:38.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:38.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:38.491 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:38.519 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:38.556 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:38.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:38.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:38.691 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:38.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:38 smithi110 ceph-mon[136032]: failed to encode map e560 with expected crc 2023-10-09T10:28:38.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:38 smithi110 ceph-mon[136032]: failed to encode map e560 with expected crc 2023-10-09T10:28:38.728 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:38.729 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:38 smithi012 ceph-mon[152263]: failed to encode map e560 with expected crc 2023-10-09T10:28:38.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:38 smithi012 ceph-mon[152263]: failed to encode map e560 with expected crc 2023-10-09T10:28:38.730 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:38 smithi012 ceph-mon[147926]: failed to encode map e560 with expected crc 2023-10-09T10:28:38.730 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:38 smithi012 ceph-mon[147926]: failed to encode map e560 with expected crc 2023-10-09T10:28:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:38.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:38.832 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:38.862 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:38.896 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:38.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:38.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:38.950 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:28:38.951 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:28:38.951 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:28:38.951 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:28:38.951 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:28:39.001 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:39.028 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:39.063 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:39.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:39.167 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:39.195 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:39.228 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:39.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:39.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:39.332 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:39.365 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:39.445 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:28:39.451 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:39.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:39.547 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:28:39.559 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:39.566 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:28:39.585 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:28:39.587 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:39.605 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:28:39.620 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:39.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:39.623 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:28:39.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:28:39.660 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:28:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:39 smithi012 ceph-mon[147926]: pgmap v1587: 73 pgs: 73 active+clean; 582 KiB data, 8.6 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2635195268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:39.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:39 smithi012 ceph-mon[152263]: pgmap v1587: 73 pgs: 73 active+clean; 582 KiB data, 8.6 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:39.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2635195268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:39.679 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:28:39.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:28:39.681 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:28:39.682 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:28:39.682 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:28:39.682 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:28:39.682 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:28:39.682 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:28:39.682 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:28:39.723 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:39.750 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:39.782 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:39.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:39.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:39.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:39.915 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:39.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:39 smithi110 ceph-mon[136032]: pgmap v1587: 73 pgs: 73 active+clean; 582 KiB data, 8.6 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2635195268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:40.008 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:40.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:40.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:40.113 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:40.140 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:40.173 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:40.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:40.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:40.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:28:40.202 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:40.276 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:40.305 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:40.341 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:40.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:40.445 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:40.472 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:40.504 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:40.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:40.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:40.638 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:40.673 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:40.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:40.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:40.806 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:40.830 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:40.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:40 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:40.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:40 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:40.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:40 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:40.979 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:40.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:41.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:28:41.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:41.117 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:41.210 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:41.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:41.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:41.314 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:41.347 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:41.839 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:41.843 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:41 smithi012 ceph-mon[147926]: pgmap v1588: 73 pgs: 73 active+clean; 582 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:41.843 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:41 smithi012 ceph-mon[152263]: pgmap v1588: 73 pgs: 73 active+clean; 582 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:41.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:41.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:41.883 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:28:41.883 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:28:41.945 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:41.974 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:41 smithi110 ceph-mon[136032]: pgmap v1588: 73 pgs: 73 active+clean; 582 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:41.977 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:42.013 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:42.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:42.023 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:28:42.117 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:42.145 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:42.190 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:42.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:42.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:42.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:42.327 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:42.852 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:42.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:42.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:42.893 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:42.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:42.970 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:28:42.991 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:43.079 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:43.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:43.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:43.147 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T10:28:43.164 INFO:tasks.workunit.client.0.smithi012.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T10:28:43.166 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T10:28:43.166 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T10:28:43.185 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:43.210 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:43.221 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T10:28:43.243 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:43.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:43.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:43.269 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T10:28:43.308 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T10:28:43.310 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T10:28:43.310 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T10:28:43.347 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:43.348 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T10:28:43.351 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2023-10-09T10:28:43.351 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T10:28:43.373 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:43.387 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:43.405 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T10:28:43.406 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T10:28:43.406 DEBUG:teuthology.orchestra.run.smithi012:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T10:28:43.407 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:43.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:43.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:43.467 INFO:tasks.workunit.client.0.smithi012.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T10:28:43.468 INFO:tasks.workunit.client.0.smithi012.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-09T10:28:43.468 INFO:tasks.workunit.client.0.smithi012.stderr:+ TMPDIR=/tmp/rbd_import_export_444239 2023-10-09T10:28:43.468 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm -rf /tmp/rbd_import_export_444239 2023-10-09T10:28:43.468 INFO:tasks.workunit.client.0.smithi012.stderr:+ mkdir /tmp/rbd_import_export_444239 2023-10-09T10:28:43.468 INFO:tasks.workunit.client.0.smithi012.stderr:+ trap 'rm -rf /tmp/rbd_import_export_444239' INT TERM EXIT 2023-10-09T10:28:43.469 INFO:tasks.workunit.client.0.smithi012.stderr:+ mkdir foo.444239 2023-10-09T10:28:43.469 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import foo.444239 foo.dir 2023-10-09T10:28:43.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:43.540 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:43.571 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:28:43.625 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:43.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:43.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:43.730 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:43.731 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:28:43.751 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: cannot import a directory 2023-10-09T10:28:43.751 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 0% complete...failed. 2023-10-09T10:28:43.751 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: import failed: (21) Is a directory 2023-10-09T10:28:43.754 INFO:tasks.workunit.client.0.smithi012.stderr:+ true 2023-10-09T10:28:43.754 INFO:tasks.workunit.client.0.smithi012.stderr:+ rmdir foo.444239 2023-10-09T10:28:43.757 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:43.762 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_444239/img bs=1k count=1 seek=10 2023-10-09T10:28:43.762 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records in 2023-10-09T10:28:43.763 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records out 2023-10-09T10:28:43.763 INFO:tasks.workunit.client.0.smithi012.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.8388e-05 s, 21.2 MB/s 2023-10-09T10:28:43.763 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_444239/img bs=1k count=10 seek=100 2023-10-09T10:28:43.763 INFO:tasks.workunit.client.0.smithi012.stderr:10+0 records in 2023-10-09T10:28:43.763 INFO:tasks.workunit.client.0.smithi012.stderr:10+0 records out 2023-10-09T10:28:43.764 INFO:tasks.workunit.client.0.smithi012.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.934e-05 s, 173 MB/s 2023-10-09T10:28:43.764 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_444239/img bs=1k count=100 seek=1000 2023-10-09T10:28:43.764 INFO:tasks.workunit.client.0.smithi012.stderr:70+1 records in 2023-10-09T10:28:43.764 INFO:tasks.workunit.client.0.smithi012.stderr:70+1 records out 2023-10-09T10:28:43.764 INFO:tasks.workunit.client.0.smithi012.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000198319 s, 363 MB/s 2023-10-09T10:28:43.764 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_444239/img bs=1k seek=10000 2023-10-09T10:28:43.765 INFO:tasks.workunit.client.0.smithi012.stderr:139+1 records in 2023-10-09T10:28:43.765 INFO:tasks.workunit.client.0.smithi012.stderr:139+1 records out 2023-10-09T10:28:43.765 INFO:tasks.workunit.client.0.smithi012.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000353998 s, 405 MB/s 2023-10-09T10:28:43.765 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_444239/img bs=1k seek=100000 2023-10-09T10:28:43.778 INFO:tasks.workunit.client.0.smithi012.stderr:70+1 records in 2023-10-09T10:28:43.778 INFO:tasks.workunit.client.0.smithi012.stderr:70+1 records out 2023-10-09T10:28:43.778 INFO:tasks.workunit.client.0.smithi012.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0117062 s, 6.2 MB/s 2023-10-09T10:28:43.779 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_444239/img bs=1k seek=1000000 2023-10-09T10:28:43.779 INFO:tasks.workunit.client.0.smithi012.stderr:189+1 records in 2023-10-09T10:28:43.780 INFO:tasks.workunit.client.0.smithi012.stderr:189+1 records out 2023-10-09T10:28:43.780 INFO:tasks.workunit.client.0.smithi012.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000497427 s, 390 MB/s 2023-10-09T10:28:43.780 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm testimg 2023-10-09T10:28:43.793 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:43.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:43.823 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 0% complete...failed. 2023-10-09T10:28:43.823 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T10:28:43.826 INFO:tasks.workunit.client.0.smithi012.stderr:+ true 2023-10-09T10:28:43.826 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format /tmp/rbd_import_export_444239/img testimg 2023-10-09T10:28:43.897 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:43.925 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:43.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:43 smithi012 ceph-mon[147926]: pgmap v1589: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:43.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:43 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4176460704' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:43.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:43 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:43.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:43 smithi012 ceph-mon[152263]: pgmap v1589: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:43.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:43 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4176460704' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:43.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:43 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:44.082 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:43 smithi110 ceph-mon[136032]: pgmap v1589: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:44.086 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:43 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4176460704' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:44.086 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:43 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:28:44.110 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:44.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:44.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:44.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:44.242 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:44.386 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:44.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:44.491 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:44.520 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:44.670 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:44.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:44.773 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:44.806 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:44.927 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:28:44.927 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:28:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:28:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:28:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:44.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: osdmap e561: 8 total, 8 up, 6 in 2023-10-09T10:28:44.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:44.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:44.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:28:44.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:28:44.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:44.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: osdmap e561: 8 total, 8 up, 6 in 2023-10-09T10:28:44.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:44.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:44.942 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:28:44.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: osdmap e561: 8 total, 8 up, 6 in 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: failed to encode map e561 with expected crc 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:44.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:44.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:44.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:45.046 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:45.078 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:45.206 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:28:45.208 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:28:45.223 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:45.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:45.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:45.281 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:28:45.305 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:28:45.323 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:28:45.326 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:45.342 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:28:45.356 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:45.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:28:45.378 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:28:45.388 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:28:45.394 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:45.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:45.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:45.398 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:28:45.415 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:28:45.416 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:28:45.416 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:28:45.416 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:28:45.417 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:28:45.417 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:28:45.417 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:28:45.417 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:28:45.417 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:28:45.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:45.524 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:45.558 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:45.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:45.641 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1592,"stamp":"2023-10-09T10:28:44.730501+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":160,"num_bytes_recovered":2983676,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":254,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8726424,"kb_used_data":2264296,"kb_used_omap":163,"kb_used_meta":6461916,"kb_avail":553744488,"statfs":{"total":575970213888,"available":567034355712,"internally_reserved":0,"allocated":2318639104,"data_stored":2308895537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":166925,"internal_metadata":6617002995},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.204290"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":152,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126142+0000","last_change":"2023-10-09T10:22:13.367431+0000","last_active":"2023-10-09T10:28:36.126142+0000","last_peered":"2023-10-09T10:28:36.126142+0000","last_clean":"2023-10-09T10:28:36.126142+0000","last_became_active":"2023-10-09T10:22:13.367070+0000","last_became_peered":"2023-10-09T10:22:13.367070+0000","last_unstale":"2023-10-09T10:28:36.126142+0000","last_undegraded":"2023-10-09T10:28:36.126142+0000","last_fullsized":"2023-10-09T10:28:36.126142+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":513,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126910+0000","last_change":"2023-10-09T10:24:40.661000+0000","last_active":"2023-10-09T10:28:36.126910+0000","last_peered":"2023-10-09T10:28:36.126910+0000","last_clean":"2023-10-09T10:28:36.126910+0000","last_became_active":"2023-10-09T10:24:40.660046+0000","last_became_peered":"2023-10-09T10:24:40.660046+0000","last_unstale":"2023-10-09T10:28:36.126910+0000","last_undegraded":"2023-10-09T10:28:36.126910+0000","last_fullsized":"2023-10-09T10:28:36.126910+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":590,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125706+0000","last_change":"2023-10-09T10:24:40.659440+0000","last_active":"2023-10-09T10:28:36.125706+0000","last_peered":"2023-10-09T10:28:36.125706+0000","last_clean":"2023-10-09T10:28:36.125706+0000","last_became_active":"2023-10-09T10:24:40.659203+0000","last_became_peered":"2023-10-09T10:24:40.659203+0000","last_unstale":"2023-10-09T10:28:36.125706+0000","last_undegraded":"2023-10-09T10:28:36.125706+0000","last_fullsized":"2023-10-09T10:28:36.125706+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":486,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.548818+0000","last_change":"2023-10-09T10:25:08.434772+0000","last_active":"2023-10-09T10:28:36.548818+0000","last_peered":"2023-10-09T10:28:36.548818+0000","last_clean":"2023-10-09T10:28:36.548818+0000","last_became_active":"2023-10-09T10:25:08.434330+0000","last_became_peered":"2023-10-09T10:25:08.434330+0000","last_unstale":"2023-10-09T10:28:36.548818+0000","last_undegraded":"2023-10-09T10:28:36.548818+0000","last_fullsized":"2023-10-09T10:28:36.548818+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:34.146197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":56,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125596+0000","last_change":"2023-10-09T10:25:12.615938+0000","last_active":"2023-10-09T10:28:36.125596+0000","last_peered":"2023-10-09T10:28:36.125596+0000","last_clean":"2023-10-09T10:28:36.125596+0000","last_became_active":"2023-10-09T10:25:12.615613+0000","last_became_peered":"2023-10-09T10:25:12.615613+0000","last_unstale":"2023-10-09T10:28:36.125596+0000","last_undegraded":"2023-10-09T10:28:36.125596+0000","last_fullsized":"2023-10-09T10:28:36.125596+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:52.269267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":138,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125873+0000","last_change":"2023-10-09T10:24:40.654015+0000","last_active":"2023-10-09T10:28:36.125873+0000","last_peered":"2023-10-09T10:28:36.125873+0000","last_clean":"2023-10-09T10:28:36.125873+0000","last_became_active":"2023-10-09T10:24:40.653749+0000","last_became_peered":"2023-10-09T10:24:40.653749+0000","last_unstale":"2023-10-09T10:28:36.125873+0000","last_undegraded":"2023-10-09T10:28:36.125873+0000","last_fullsized":"2023-10-09T10:28:36.125873+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:09.155262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":519,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126222+0000","last_change":"2023-10-09T10:25:08.434815+0000","last_active":"2023-10-09T10:28:36.126222+0000","last_peered":"2023-10-09T10:28:36.126222+0000","last_clean":"2023-10-09T10:28:36.126222+0000","last_became_active":"2023-10-09T10:25:08.434387+0000","last_became_peered":"2023-10-09T10:25:08.434387+0000","last_unstale":"2023-10-09T10:28:36.126222+0000","last_undegraded":"2023-10-09T10:28:36.126222+0000","last_fullsized":"2023-10-09T10:28:36.126222+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:34:49.963017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":468,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.523732+0000","last_change":"2023-10-09T10:24:40.660655+0000","last_active":"2023-10-09T10:28:35.523732+0000","last_peered":"2023-10-09T10:28:35.523732+0000","last_clean":"2023-10-09T10:28:35.523732+0000","last_became_active":"2023-10-09T10:24:40.660299+0000","last_became_peered":"2023-10-09T10:24:40.660299+0000","last_unstale":"2023-10-09T10:28:35.523732+0000","last_undegraded":"2023-10-09T10:28:35.523732+0000","last_fullsized":"2023-10-09T10:28:35.523732+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.834780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":235,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.530034+0000","last_change":"2023-10-09T10:25:08.435163+0000","last_active":"2023-10-09T10:28:35.530034+0000","last_peered":"2023-10-09T10:28:35.530034+0000","last_clean":"2023-10-09T10:28:35.530034+0000","last_became_active":"2023-10-09T10:25:08.434854+0000","last_became_peered":"2023-10-09T10:25:08.434854+0000","last_unstale":"2023-10-09T10:28:35.530034+0000","last_undegraded":"2023-10-09T10:28:35.530034+0000","last_fullsized":"2023-10-09T10:28:35.530034+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":468,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.523932+0000","last_change":"2023-10-09T10:24:40.644256+0000","last_active":"2023-10-09T10:28:35.523932+0000","last_peered":"2023-10-09T10:28:35.523932+0000","last_clean":"2023-10-09T10:28:35.523932+0000","last_became_active":"2023-10-09T10:24:40.643960+0000","last_became_peered":"2023-10-09T10:24:40.643960+0000","last_unstale":"2023-10-09T10:28:35.523932+0000","last_undegraded":"2023-10-09T10:28:35.523932+0000","last_fullsized":"2023-10-09T10:28:35.523932+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:11.848800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":164,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.127374+0000","last_change":"2023-10-09T10:23:26.110626+0000","last_active":"2023-10-09T10:28:36.127374+0000","last_peered":"2023-10-09T10:28:36.127374+0000","last_clean":"2023-10-09T10:28:36.127374+0000","last_became_active":"2023-10-09T10:23:25.612348+0000","last_became_peered":"2023-10-09T10:23:25.612348+0000","last_unstale":"2023-10-09T10:28:36.127374+0000","last_undegraded":"2023-10-09T10:28:36.127374+0000","last_fullsized":"2023-10-09T10:28:36.127374+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":56,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125609+0000","last_change":"2023-10-09T10:25:12.613598+0000","last_active":"2023-10-09T10:28:36.125609+0000","last_peered":"2023-10-09T10:28:36.125609+0000","last_clean":"2023-10-09T10:28:36.125609+0000","last_became_active":"2023-10-09T10:25:12.613265+0000","last_became_peered":"2023-10-09T10:25:12.613265+0000","last_unstale":"2023-10-09T10:28:36.125609+0000","last_undegraded":"2023-10-09T10:28:36.125609+0000","last_fullsized":"2023-10-09T10:28:36.125609+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:14.567802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":210,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126473+0000","last_change":"2023-10-09T10:24:40.672838+0000","last_active":"2023-10-09T10:28:36.126473+0000","last_peered":"2023-10-09T10:28:36.126473+0000","last_clean":"2023-10-09T10:28:36.126473+0000","last_became_active":"2023-10-09T10:24:40.656375+0000","last_became_peered":"2023-10-09T10:24:40.656375+0000","last_unstale":"2023-10-09T10:28:36.126473+0000","last_undegraded":"2023-10-09T10:28:36.126473+0000","last_fullsized":"2023-10-09T10:28:36.126473+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":170,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126163+0000","last_change":"2023-10-09T10:24:40.672038+0000","last_active":"2023-10-09T10:28:36.126163+0000","last_peered":"2023-10-09T10:28:36.126163+0000","last_clean":"2023-10-09T10:28:36.126163+0000","last_became_active":"2023-10-09T10:24:40.671715+0000","last_became_peered":"2023-10-09T10:24:40.671715+0000","last_unstale":"2023-10-09T10:28:36.126163+0000","last_undegraded":"2023-10-09T10:28:36.126163+0000","last_fullsized":"2023-10-09T10:28:36.126163+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":130,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125987+0000","last_change":"2023-10-09T10:24:40.683447+0000","last_active":"2023-10-09T10:28:36.125987+0000","last_peered":"2023-10-09T10:28:36.125987+0000","last_clean":"2023-10-09T10:28:36.125987+0000","last_became_active":"2023-10-09T10:24:40.657396+0000","last_became_peered":"2023-10-09T10:24:40.657396+0000","last_unstale":"2023-10-09T10:28:36.125987+0000","last_undegraded":"2023-10-09T10:28:36.125987+0000","last_fullsized":"2023-10-09T10:28:36.125987+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.529332+0000","last_change":"2023-10-09T10:23:25.452334+0000","last_active":"2023-10-09T10:28:35.529332+0000","last_peered":"2023-10-09T10:28:35.529332+0000","last_clean":"2023-10-09T10:28:35.529332+0000","last_became_active":"2023-10-09T10:23:25.451905+0000","last_became_peered":"2023-10-09T10:23:25.451905+0000","last_unstale":"2023-10-09T10:28:35.529332+0000","last_undegraded":"2023-10-09T10:28:35.529332+0000","last_fullsized":"2023-10-09T10:28:35.529332+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":484,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126034+0000","last_change":"2023-10-09T10:25:08.439360+0000","last_active":"2023-10-09T10:28:36.126034+0000","last_peered":"2023-10-09T10:28:36.126034+0000","last_clean":"2023-10-09T10:28:36.126034+0000","last_became_active":"2023-10-09T10:25:08.438950+0000","last_became_peered":"2023-10-09T10:25:08.438950+0000","last_unstale":"2023-10-09T10:28:36.126034+0000","last_undegraded":"2023-10-09T10:28:36.126034+0000","last_fullsized":"2023-10-09T10:28:36.126034+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:39.647084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":226,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.529611+0000","last_change":"2023-10-09T10:24:40.666508+0000","last_active":"2023-10-09T10:28:35.529611+0000","last_peered":"2023-10-09T10:28:35.529611+0000","last_clean":"2023-10-09T10:28:35.529611+0000","last_became_active":"2023-10-09T10:24:40.666057+0000","last_became_peered":"2023-10-09T10:24:40.666057+0000","last_unstale":"2023-10-09T10:28:35.529611+0000","last_undegraded":"2023-10-09T10:28:35.529611+0000","last_fullsized":"2023-10-09T10:28:35.529611+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":170,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125753+0000","last_change":"2023-10-09T10:24:40.658467+0000","last_active":"2023-10-09T10:28:36.125753+0000","last_peered":"2023-10-09T10:28:36.125753+0000","last_clean":"2023-10-09T10:28:36.125753+0000","last_became_active":"2023-10-09T10:24:40.656283+0000","last_became_peered":"2023-10-09T10:24:40.656283+0000","last_unstale":"2023-10-09T10:28:36.125753+0000","last_undegraded":"2023-10-09T10:28:36.125753+0000","last_fullsized":"2023-10-09T10:28:36.125753+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":56,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126114+0000","last_change":"2023-10-09T10:25:12.614480+0000","last_active":"2023-10-09T10:28:36.126114+0000","last_peered":"2023-10-09T10:28:36.126114+0000","last_clean":"2023-10-09T10:28:36.126114+0000","last_became_active":"2023-10-09T10:25:12.614127+0000","last_became_peered":"2023-10-09T10:25:12.614127+0000","last_unstale":"2023-10-09T10:28:36.126114+0000","last_undegraded":"2023-10-09T10:28:36.126114+0000","last_fullsized":"2023-10-09T10:28:36.126114+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:28.197727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":66,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125702+0000","last_change":"2023-10-09T10:25:12.755944+0000","last_active":"2023-10-09T10:28:36.125702+0000","last_peered":"2023-10-09T10:28:36.125702+0000","last_clean":"2023-10-09T10:28:36.125702+0000","last_became_active":"2023-10-09T10:25:12.613769+0000","last_became_peered":"2023-10-09T10:25:12.613769+0000","last_unstale":"2023-10-09T10:28:36.125702+0000","last_undegraded":"2023-10-09T10:28:36.125702+0000","last_fullsized":"2023-10-09T10:28:36.125702+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:20.764425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":59,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.525730+0000","last_change":"2023-10-09T10:25:08.429712+0000","last_active":"2023-10-09T10:28:35.525730+0000","last_peered":"2023-10-09T10:28:35.525730+0000","last_clean":"2023-10-09T10:28:35.525730+0000","last_became_active":"2023-10-09T10:25:08.429402+0000","last_became_peered":"2023-10-09T10:25:08.429402+0000","last_unstale":"2023-10-09T10:28:35.525730+0000","last_undegraded":"2023-10-09T10:28:35.525730+0000","last_fullsized":"2023-10-09T10:28:35.525730+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:13.587594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":60,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.128586+0000","last_change":"2023-10-09T10:25:12.823146+0000","last_active":"2023-10-09T10:28:36.128586+0000","last_peered":"2023-10-09T10:28:36.128586+0000","last_clean":"2023-10-09T10:28:36.128586+0000","last_became_active":"2023-10-09T10:25:12.616977+0000","last_became_peered":"2023-10-09T10:25:12.616977+0000","last_unstale":"2023-10-09T10:28:36.128586+0000","last_undegraded":"2023-10-09T10:28:36.128586+0000","last_fullsized":"2023-10-09T10:28:36.128586+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":235,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528421+0000","last_change":"2023-10-09T10:24:40.669539+0000","last_active":"2023-10-09T10:28:35.528421+0000","last_peered":"2023-10-09T10:28:35.528421+0000","last_clean":"2023-10-09T10:28:35.528421+0000","last_became_active":"2023-10-09T10:24:40.669157+0000","last_became_peered":"2023-10-09T10:24:40.669157+0000","last_unstale":"2023-10-09T10:28:35.528421+0000","last_undegraded":"2023-10-09T10:28:35.528421+0000","last_fullsized":"2023-10-09T10:28:35.528421+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":186,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.128339+0000","last_change":"2023-10-09T10:25:12.822026+0000","last_active":"2023-10-09T10:28:36.128339+0000","last_peered":"2023-10-09T10:28:36.128339+0000","last_clean":"2023-10-09T10:28:36.128339+0000","last_became_active":"2023-10-09T10:25:12.616972+0000","last_became_peered":"2023-10-09T10:25:12.616972+0000","last_unstale":"2023-10-09T10:28:36.128339+0000","last_undegraded":"2023-10-09T10:28:36.128339+0000","last_fullsized":"2023-10-09T10:28:36.128339+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":173,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125852+0000","last_change":"2023-10-09T10:25:08.436252+0000","last_active":"2023-10-09T10:28:36.125852+0000","last_peered":"2023-10-09T10:28:36.125852+0000","last_clean":"2023-10-09T10:28:36.125852+0000","last_became_active":"2023-10-09T10:25:08.435937+0000","last_became_peered":"2023-10-09T10:25:08.435937+0000","last_unstale":"2023-10-09T10:28:36.125852+0000","last_undegraded":"2023-10-09T10:28:36.125852+0000","last_fullsized":"2023-10-09T10:28:36.125852+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":251,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.529294+0000","last_change":"2023-10-09T10:24:40.741457+0000","last_active":"2023-10-09T10:28:35.529294+0000","last_peered":"2023-10-09T10:28:35.529294+0000","last_clean":"2023-10-09T10:28:35.529294+0000","last_became_active":"2023-10-09T10:24:40.676327+0000","last_became_peered":"2023-10-09T10:24:40.676327+0000","last_unstale":"2023-10-09T10:28:35.529294+0000","last_undegraded":"2023-10-09T10:28:35.529294+0000","last_fullsized":"2023-10-09T10:28:35.529294+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":152,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528901+0000","last_change":"2023-10-09T10:25:12.613758+0000","last_active":"2023-10-09T10:28:35.528901+0000","last_peered":"2023-10-09T10:28:35.528901+0000","last_clean":"2023-10-09T10:28:35.528901+0000","last_became_active":"2023-10-09T10:25:12.613470+0000","last_became_peered":"2023-10-09T10:25:12.613470+0000","last_unstale":"2023-10-09T10:28:35.528901+0000","last_undegraded":"2023-10-09T10:28:35.528901+0000","last_fullsized":"2023-10-09T10:28:35.528901+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":235,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528974+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:28:35.528974+0000","last_peered":"2023-10-09T10:28:35.528974+0000","last_clean":"2023-10-09T10:28:35.528974+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:28:35.528974+0000","last_undegraded":"2023-10-09T10:28:35.528974+0000","last_fullsized":"2023-10-09T10:28:35.528974+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":470,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.127177+0000","last_change":"2023-10-09T10:25:08.438851+0000","last_active":"2023-10-09T10:28:36.127177+0000","last_peered":"2023-10-09T10:28:36.127177+0000","last_clean":"2023-10-09T10:28:36.127177+0000","last_became_active":"2023-10-09T10:25:08.438504+0000","last_became_peered":"2023-10-09T10:25:08.438504+0000","last_unstale":"2023-10-09T10:28:36.127177+0000","last_undegraded":"2023-10-09T10:28:36.127177+0000","last_fullsized":"2023-10-09T10:28:36.127177+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:41.386349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":171,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.127051+0000","last_change":"2023-10-09T10:25:12.624885+0000","last_active":"2023-10-09T10:28:36.127051+0000","last_peered":"2023-10-09T10:28:36.127051+0000","last_clean":"2023-10-09T10:28:36.127051+0000","last_became_active":"2023-10-09T10:25:12.613708+0000","last_became_peered":"2023-10-09T10:25:12.613708+0000","last_unstale":"2023-10-09T10:28:36.127051+0000","last_undegraded":"2023-10-09T10:28:36.127051+0000","last_fullsized":"2023-10-09T10:28:36.127051+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:13.533717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126375+0000","last_change":"2023-10-09T10:24:40.661565+0000","last_active":"2023-10-09T10:28:36.126375+0000","last_peered":"2023-10-09T10:28:36.126375+0000","last_clean":"2023-10-09T10:28:36.126375+0000","last_became_active":"2023-10-09T10:24:40.661105+0000","last_became_peered":"2023-10-09T10:24:40.661105+0000","last_unstale":"2023-10-09T10:28:36.126375+0000","last_undegraded":"2023-10-09T10:28:36.126375+0000","last_fullsized":"2023-10-09T10:28:36.126375+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":170,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126031+0000","last_change":"2023-10-09T10:24:40.660603+0000","last_active":"2023-10-09T10:28:36.126031+0000","last_peered":"2023-10-09T10:28:36.126031+0000","last_clean":"2023-10-09T10:28:36.126031+0000","last_became_active":"2023-10-09T10:24:40.660286+0000","last_became_peered":"2023-10-09T10:24:40.660286+0000","last_unstale":"2023-10-09T10:28:36.126031+0000","last_undegraded":"2023-10-09T10:28:36.126031+0000","last_fullsized":"2023-10-09T10:28:36.126031+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":136,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.127003+0000","last_change":"2023-10-09T10:24:40.657694+0000","last_active":"2023-10-09T10:28:36.127003+0000","last_peered":"2023-10-09T10:28:36.127003+0000","last_clean":"2023-10-09T10:28:36.127003+0000","last_became_active":"2023-10-09T10:24:40.656731+0000","last_became_peered":"2023-10-09T10:24:40.656731+0000","last_unstale":"2023-10-09T10:28:36.127003+0000","last_undegraded":"2023-10-09T10:28:36.127003+0000","last_fullsized":"2023-10-09T10:28:36.127003+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":152,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125981+0000","last_change":"2023-10-09T10:23:25.427892+0000","last_active":"2023-10-09T10:28:36.125981+0000","last_peered":"2023-10-09T10:28:36.125981+0000","last_clean":"2023-10-09T10:28:36.125981+0000","last_became_active":"2023-10-09T10:23:25.424013+0000","last_became_peered":"2023-10-09T10:23:25.424013+0000","last_unstale":"2023-10-09T10:28:36.125981+0000","last_undegraded":"2023-10-09T10:28:36.125981+0000","last_fullsized":"2023-10-09T10:28:36.125981+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":179,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.128038+0000","last_change":"2023-10-09T10:24:40.660189+0000","last_active":"2023-10-09T10:28:36.128038+0000","last_peered":"2023-10-09T10:28:36.128038+0000","last_clean":"2023-10-09T10:28:36.128038+0000","last_became_active":"2023-10-09T10:24:40.659687+0000","last_became_peered":"2023-10-09T10:24:40.659687+0000","last_unstale":"2023-10-09T10:28:36.128038+0000","last_undegraded":"2023-10-09T10:28:36.128038+0000","last_fullsized":"2023-10-09T10:28:36.128038+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":512,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528337+0000","last_change":"2023-10-09T10:25:08.435170+0000","last_active":"2023-10-09T10:28:35.528337+0000","last_peered":"2023-10-09T10:28:35.528337+0000","last_clean":"2023-10-09T10:28:35.528337+0000","last_became_active":"2023-10-09T10:25:08.434380+0000","last_became_peered":"2023-10-09T10:25:08.434380+0000","last_unstale":"2023-10-09T10:28:35.528337+0000","last_undegraded":"2023-10-09T10:28:35.528337+0000","last_fullsized":"2023-10-09T10:28:35.528337+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":573,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.128660+0000","last_change":"2023-10-09T10:23:25.468888+0000","last_active":"2023-10-09T10:28:36.128660+0000","last_peered":"2023-10-09T10:28:36.128660+0000","last_clean":"2023-10-09T10:28:36.128660+0000","last_became_active":"2023-10-09T10:23:25.468583+0000","last_became_peered":"2023-10-09T10:23:25.468583+0000","last_unstale":"2023-10-09T10:28:36.128660+0000","last_undegraded":"2023-10-09T10:28:36.128660+0000","last_fullsized":"2023-10-09T10:28:36.128660+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":282,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.548688+0000","last_change":"2023-10-09T10:25:08.434463+0000","last_active":"2023-10-09T10:28:36.548688+0000","last_peered":"2023-10-09T10:28:36.548688+0000","last_clean":"2023-10-09T10:28:36.548688+0000","last_became_active":"2023-10-09T10:25:08.434105+0000","last_became_peered":"2023-10-09T10:25:08.434105+0000","last_unstale":"2023-10-09T10:28:36.548688+0000","last_undegraded":"2023-10-09T10:28:36.548688+0000","last_fullsized":"2023-10-09T10:28:36.548688+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:43.861717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":244,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.529498+0000","last_change":"2023-10-09T10:25:08.434608+0000","last_active":"2023-10-09T10:28:35.529498+0000","last_peered":"2023-10-09T10:28:35.529498+0000","last_clean":"2023-10-09T10:28:35.529498+0000","last_became_active":"2023-10-09T10:25:08.434034+0000","last_became_peered":"2023-10-09T10:25:08.434034+0000","last_unstale":"2023-10-09T10:28:35.529498+0000","last_undegraded":"2023-10-09T10:28:35.529498+0000","last_fullsized":"2023-10-09T10:28:35.529498+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":524,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528948+0000","last_change":"2023-10-09T10:22:13.354432+0000","last_active":"2023-10-09T10:28:35.528948+0000","last_peered":"2023-10-09T10:28:35.528948+0000","last_clean":"2023-10-09T10:28:35.528948+0000","last_became_active":"2023-10-09T10:22:13.353991+0000","last_became_peered":"2023-10-09T10:22:13.353991+0000","last_unstale":"2023-10-09T10:28:35.528948+0000","last_undegraded":"2023-10-09T10:28:35.528948+0000","last_fullsized":"2023-10-09T10:28:35.528948+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":244,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528865+0000","last_change":"2023-10-09T10:24:40.663231+0000","last_active":"2023-10-09T10:28:35.528865+0000","last_peered":"2023-10-09T10:28:35.528865+0000","last_clean":"2023-10-09T10:28:35.528865+0000","last_became_active":"2023-10-09T10:24:40.649858+0000","last_became_peered":"2023-10-09T10:24:40.649858+0000","last_unstale":"2023-10-09T10:28:35.528865+0000","last_undegraded":"2023-10-09T10:28:35.528865+0000","last_fullsized":"2023-10-09T10:28:35.528865+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":161,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126029+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:28:36.126029+0000","last_peered":"2023-10-09T10:28:36.126029+0000","last_clean":"2023-10-09T10:28:36.126029+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:28:36.126029+0000","last_undegraded":"2023-10-09T10:28:36.126029+0000","last_fullsized":"2023-10-09T10:28:36.126029+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":474,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.775252+0000","last_change":"2023-10-09T10:25:08.431989+0000","last_active":"2023-10-09T10:28:36.775252+0000","last_peered":"2023-10-09T10:28:36.775252+0000","last_clean":"2023-10-09T10:28:36.775252+0000","last_became_active":"2023-10-09T10:25:08.431645+0000","last_became_peered":"2023-10-09T10:25:08.431645+0000","last_unstale":"2023-10-09T10:28:36.775252+0000","last_undegraded":"2023-10-09T10:28:36.775252+0000","last_fullsized":"2023-10-09T10:28:36.775252+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:48.055140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":511,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125553+0000","last_change":"2023-10-09T10:25:08.436449+0000","last_active":"2023-10-09T10:28:36.125553+0000","last_peered":"2023-10-09T10:28:36.125553+0000","last_clean":"2023-10-09T10:28:36.125553+0000","last_became_active":"2023-10-09T10:25:08.434838+0000","last_became_peered":"2023-10-09T10:25:08.434838+0000","last_unstale":"2023-10-09T10:28:36.125553+0000","last_undegraded":"2023-10-09T10:28:36.125553+0000","last_fullsized":"2023-10-09T10:28:36.125553+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:02.724566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":129,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125664+0000","last_change":"2023-10-09T10:24:40.646065+0000","last_active":"2023-10-09T10:28:36.125664+0000","last_peered":"2023-10-09T10:28:36.125664+0000","last_clean":"2023-10-09T10:28:36.125664+0000","last_became_active":"2023-10-09T10:24:40.645723+0000","last_became_peered":"2023-10-09T10:24:40.645723+0000","last_unstale":"2023-10-09T10:28:36.125664+0000","last_undegraded":"2023-10-09T10:28:36.125664+0000","last_fullsized":"2023-10-09T10:28:36.125664+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:32.434766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":573,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.530197+0000","last_change":"2023-10-09T10:25:12.611801+0000","last_active":"2023-10-09T10:28:35.530197+0000","last_peered":"2023-10-09T10:28:35.530197+0000","last_clean":"2023-10-09T10:28:35.530197+0000","last_became_active":"2023-10-09T10:25:12.611450+0000","last_became_peered":"2023-10-09T10:25:12.611450+0000","last_unstale":"2023-10-09T10:28:35.530197+0000","last_undegraded":"2023-10-09T10:28:35.530197+0000","last_fullsized":"2023-10-09T10:28:35.530197+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":214,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528976+0000","last_change":"2023-10-09T10:23:25.437421+0000","last_active":"2023-10-09T10:28:35.528976+0000","last_peered":"2023-10-09T10:28:35.528976+0000","last_clean":"2023-10-09T10:28:35.528976+0000","last_became_active":"2023-10-09T10:23:25.437067+0000","last_became_peered":"2023-10-09T10:23:25.437067+0000","last_unstale":"2023-10-09T10:28:35.528976+0000","last_undegraded":"2023-10-09T10:28:35.528976+0000","last_fullsized":"2023-10-09T10:28:35.528976+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":754,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125299+0000","last_change":"2023-10-09T10:25:49.264790+0000","last_active":"2023-10-09T10:28:36.125299+0000","last_peered":"2023-10-09T10:28:36.125299+0000","last_clean":"2023-10-09T10:28:36.125299+0000","last_became_active":"2023-10-09T10:25:49.258010+0000","last_became_peered":"2023-10-09T10:25:49.258010+0000","last_unstale":"2023-10-09T10:28:36.125299+0000","last_undegraded":"2023-10-09T10:28:36.125299+0000","last_fullsized":"2023-10-09T10:28:36.125299+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:07.231152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":262,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528782+0000","last_change":"2023-10-09T10:25:08.436413+0000","last_active":"2023-10-09T10:28:35.528782+0000","last_peered":"2023-10-09T10:28:35.528782+0000","last_clean":"2023-10-09T10:28:35.528782+0000","last_became_active":"2023-10-09T10:25:08.436085+0000","last_became_peered":"2023-10-09T10:25:08.436085+0000","last_unstale":"2023-10-09T10:28:35.528782+0000","last_undegraded":"2023-10-09T10:28:35.528782+0000","last_fullsized":"2023-10-09T10:28:35.528782+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":170,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.127274+0000","last_change":"2023-10-09T10:24:40.658112+0000","last_active":"2023-10-09T10:28:36.127274+0000","last_peered":"2023-10-09T10:28:36.127274+0000","last_clean":"2023-10-09T10:28:36.127274+0000","last_became_active":"2023-10-09T10:24:40.657193+0000","last_became_peered":"2023-10-09T10:24:40.657193+0000","last_unstale":"2023-10-09T10:28:36.127274+0000","last_undegraded":"2023-10-09T10:28:36.127274+0000","last_fullsized":"2023-10-09T10:28:36.127274+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":246,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125753+0000","last_change":"2023-10-09T10:23:25.413739+0000","last_active":"2023-10-09T10:28:36.125753+0000","last_peered":"2023-10-09T10:28:36.125753+0000","last_clean":"2023-10-09T10:28:36.125753+0000","last_became_active":"2023-10-09T10:23:25.413215+0000","last_became_peered":"2023-10-09T10:23:25.413215+0000","last_unstale":"2023-10-09T10:28:36.125753+0000","last_undegraded":"2023-10-09T10:28:36.125753+0000","last_fullsized":"2023-10-09T10:28:36.125753+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:24.149009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":505,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125791+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:28:36.125791+0000","last_peered":"2023-10-09T10:28:36.125791+0000","last_clean":"2023-10-09T10:28:36.125791+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:28:36.125791+0000","last_undegraded":"2023-10-09T10:28:36.125791+0000","last_fullsized":"2023-10-09T10:28:36.125791+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":56,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125931+0000","last_change":"2023-10-09T10:25:12.614996+0000","last_active":"2023-10-09T10:28:36.125931+0000","last_peered":"2023-10-09T10:28:36.125931+0000","last_clean":"2023-10-09T10:28:36.125931+0000","last_became_active":"2023-10-09T10:25:12.614658+0000","last_became_peered":"2023-10-09T10:25:12.614658+0000","last_unstale":"2023-10-09T10:28:36.125931+0000","last_undegraded":"2023-10-09T10:28:36.125931+0000","last_fullsized":"2023-10-09T10:28:36.125931+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.133098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":56,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126049+0000","last_change":"2023-10-09T10:25:12.613842+0000","last_active":"2023-10-09T10:28:36.126049+0000","last_peered":"2023-10-09T10:28:36.126049+0000","last_clean":"2023-10-09T10:28:36.126049+0000","last_became_active":"2023-10-09T10:25:12.613301+0000","last_became_peered":"2023-10-09T10:25:12.613301+0000","last_unstale":"2023-10-09T10:28:36.126049+0000","last_undegraded":"2023-10-09T10:28:36.126049+0000","last_fullsized":"2023-10-09T10:28:36.126049+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:06.130372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":541,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.529269+0000","last_change":"2023-10-09T10:25:12.613863+0000","last_active":"2023-10-09T10:28:35.529269+0000","last_peered":"2023-10-09T10:28:35.529269+0000","last_clean":"2023-10-09T10:28:35.529269+0000","last_became_active":"2023-10-09T10:25:12.613519+0000","last_became_peered":"2023-10-09T10:25:12.613519+0000","last_unstale":"2023-10-09T10:28:35.529269+0000","last_undegraded":"2023-10-09T10:28:35.529269+0000","last_fullsized":"2023-10-09T10:28:35.529269+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":473,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.127369+0000","last_change":"2023-10-09T10:25:17.866073+0000","last_active":"2023-10-09T10:28:36.127369+0000","last_peered":"2023-10-09T10:28:36.127369+0000","last_clean":"2023-10-09T10:28:36.127369+0000","last_became_active":"2023-10-09T10:25:17.865734+0000","last_became_peered":"2023-10-09T10:25:17.865734+0000","last_unstale":"2023-10-09T10:28:36.127369+0000","last_undegraded":"2023-10-09T10:28:36.127369+0000","last_fullsized":"2023-10-09T10:28:36.127369+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.843668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":361,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126052+0000","last_change":"2023-10-09T10:23:25.498121+0000","last_active":"2023-10-09T10:28:36.126052+0000","last_peered":"2023-10-09T10:28:36.126052+0000","last_clean":"2023-10-09T10:28:36.126052+0000","last_became_active":"2023-10-09T10:23:25.432983+0000","last_became_peered":"2023-10-09T10:23:25.432983+0000","last_unstale":"2023-10-09T10:28:36.126052+0000","last_undegraded":"2023-10-09T10:28:36.126052+0000","last_fullsized":"2023-10-09T10:28:36.126052+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:00.918290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":540,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.127264+0000","last_change":"2023-10-09T10:24:40.660010+0000","last_active":"2023-10-09T10:28:36.127264+0000","last_peered":"2023-10-09T10:28:36.127264+0000","last_clean":"2023-10-09T10:28:36.127264+0000","last_became_active":"2023-10-09T10:24:40.659248+0000","last_became_peered":"2023-10-09T10:24:40.659248+0000","last_unstale":"2023-10-09T10:28:36.127264+0000","last_undegraded":"2023-10-09T10:28:36.127264+0000","last_fullsized":"2023-10-09T10:28:36.127264+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125491+0000","last_change":"2023-10-09T10:25:12.615295+0000","last_active":"2023-10-09T10:28:36.125491+0000","last_peered":"2023-10-09T10:28:36.125491+0000","last_clean":"2023-10-09T10:28:36.125491+0000","last_became_active":"2023-10-09T10:25:12.614856+0000","last_became_peered":"2023-10-09T10:25:12.614856+0000","last_unstale":"2023-10-09T10:28:36.125491+0000","last_undegraded":"2023-10-09T10:28:36.125491+0000","last_fullsized":"2023-10-09T10:28:36.125491+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:10.631513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.532256+0000","last_change":"2023-10-09T10:24:40.649928+0000","last_active":"2023-10-09T10:28:35.532256+0000","last_peered":"2023-10-09T10:28:35.532256+0000","last_clean":"2023-10-09T10:28:35.532256+0000","last_became_active":"2023-10-09T10:24:40.649583+0000","last_became_peered":"2023-10-09T10:24:40.649583+0000","last_unstale":"2023-10-09T10:28:35.532256+0000","last_undegraded":"2023-10-09T10:28:35.532256+0000","last_fullsized":"2023-10-09T10:28:35.532256+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":60,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126225+0000","last_change":"2023-10-09T10:25:12.762734+0000","last_active":"2023-10-09T10:28:36.126225+0000","last_peered":"2023-10-09T10:28:36.126225+0000","last_clean":"2023-10-09T10:28:36.126225+0000","last_became_active":"2023-10-09T10:25:12.617046+0000","last_became_peered":"2023-10-09T10:25:12.617046+0000","last_unstale":"2023-10-09T10:28:36.126225+0000","last_undegraded":"2023-10-09T10:28:36.126225+0000","last_fullsized":"2023-10-09T10:28:36.126225+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.618486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":240,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.528773+0000","last_change":"2023-10-09T10:24:40.668504+0000","last_active":"2023-10-09T10:28:35.528773+0000","last_peered":"2023-10-09T10:28:35.528773+0000","last_clean":"2023-10-09T10:28:35.528773+0000","last_became_active":"2023-10-09T10:24:40.668232+0000","last_became_peered":"2023-10-09T10:24:40.668232+0000","last_unstale":"2023-10-09T10:28:35.528773+0000","last_undegraded":"2023-10-09T10:28:35.528773+0000","last_fullsized":"2023-10-09T10:28:35.528773+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":56,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.127328+0000","last_change":"2023-10-09T10:25:12.614496+0000","last_active":"2023-10-09T10:28:36.127328+0000","last_peered":"2023-10-09T10:28:36.127328+0000","last_clean":"2023-10-09T10:28:36.127328+0000","last_became_active":"2023-10-09T10:25:12.613998+0000","last_became_peered":"2023-10-09T10:25:12.613998+0000","last_unstale":"2023-10-09T10:28:36.127328+0000","last_undegraded":"2023-10-09T10:28:36.127328+0000","last_fullsized":"2023-10-09T10:28:36.127328+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:31.485349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":65,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126585+0000","last_change":"2023-10-09T10:25:17.866918+0000","last_active":"2023-10-09T10:28:36.126585+0000","last_peered":"2023-10-09T10:28:36.126585+0000","last_clean":"2023-10-09T10:28:36.126585+0000","last_became_active":"2023-10-09T10:25:17.866471+0000","last_became_peered":"2023-10-09T10:25:17.866471+0000","last_unstale":"2023-10-09T10:28:36.126585+0000","last_undegraded":"2023-10-09T10:28:36.126585+0000","last_fullsized":"2023-10-09T10:28:36.126585+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:12.663064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":290,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.529076+0000","last_change":"2023-10-09T10:24:40.738927+0000","last_active":"2023-10-09T10:28:35.529076+0000","last_peered":"2023-10-09T10:28:35.529076+0000","last_clean":"2023-10-09T10:28:35.529076+0000","last_became_active":"2023-10-09T10:24:40.670788+0000","last_became_peered":"2023-10-09T10:24:40.670788+0000","last_unstale":"2023-10-09T10:28:35.529076+0000","last_undegraded":"2023-10-09T10:28:35.529076+0000","last_fullsized":"2023-10-09T10:28:35.529076+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":152,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.126268+0000","last_change":"2023-10-09T10:22:13.366506+0000","last_active":"2023-10-09T10:28:36.126268+0000","last_peered":"2023-10-09T10:28:36.126268+0000","last_clean":"2023-10-09T10:28:36.126268+0000","last_became_active":"2023-10-09T10:22:13.366101+0000","last_became_peered":"2023-10-09T10:22:13.366101+0000","last_unstale":"2023-10-09T10:28:36.126268+0000","last_undegraded":"2023-10-09T10:28:36.126268+0000","last_fullsized":"2023-10-09T10:28:36.126268+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":273,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.529398+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:28:35.529398+0000","last_peered":"2023-10-09T10:28:35.529398+0000","last_clean":"2023-10-09T10:28:35.529398+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:28:35.529398+0000","last_undegraded":"2023-10-09T10:28:35.529398+0000","last_fullsized":"2023-10-09T10:28:35.529398+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":383,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125879+0000","last_change":"2023-10-09T10:19:43.291008+0000","last_active":"2023-10-09T10:28:36.125879+0000","last_peered":"2023-10-09T10:28:36.125879+0000","last_clean":"2023-10-09T10:28:36.125879+0000","last_became_active":"2023-10-09T10:19:43.290659+0000","last_became_peered":"2023-10-09T10:19:43.290659+0000","last_unstale":"2023-10-09T10:28:36.125879+0000","last_undegraded":"2023-10-09T10:28:36.125879+0000","last_fullsized":"2023-10-09T10:28:36.125879+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:14.261562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":59,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:35.525731+0000","last_change":"2023-10-09T10:25:08.430429+0000","last_active":"2023-10-09T10:28:35.525731+0000","last_peered":"2023-10-09T10:28:35.525731+0000","last_clean":"2023-10-09T10:28:35.525731+0000","last_became_active":"2023-10-09T10:25:08.430102+0000","last_became_peered":"2023-10-09T10:25:08.430102+0000","last_unstale":"2023-10-09T10:28:35.525731+0000","last_undegraded":"2023-10-09T10:28:35.525731+0000","last_fullsized":"2023-10-09T10:28:35.525731+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:14.267777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":613,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125440+0000","last_change":"2023-10-09T10:25:08.435954+0000","last_active":"2023-10-09T10:28:36.125440+0000","last_peered":"2023-10-09T10:28:36.125440+0000","last_clean":"2023-10-09T10:28:36.125440+0000","last_became_active":"2023-10-09T10:25:08.434562+0000","last_became_peered":"2023-10-09T10:25:08.434562+0000","last_unstale":"2023-10-09T10:28:36.125440+0000","last_undegraded":"2023-10-09T10:28:36.125440+0000","last_fullsized":"2023-10-09T10:28:36.125440+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:21.438242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":77,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125772+0000","last_change":"2023-10-09T10:25:17.871223+0000","last_active":"2023-10-09T10:28:36.125772+0000","last_peered":"2023-10-09T10:28:36.125772+0000","last_clean":"2023-10-09T10:28:36.125772+0000","last_became_active":"2023-10-09T10:25:17.865006+0000","last_became_peered":"2023-10-09T10:25:17.865006+0000","last_unstale":"2023-10-09T10:28:36.125772+0000","last_undegraded":"2023-10-09T10:28:36.125772+0000","last_fullsized":"2023-10-09T10:28:36.125772+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:36.933256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":7202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T10:28:36.125652+0000","last_change":"2023-10-09T10:25:08.435584+0000","last_active":"2023-10-09T10:28:36.125652+0000","last_peered":"2023-10-09T10:28:36.125652+0000","last_clean":"2023-10-09T10:28:36.125652+0000","last_became_active":"2023-10-09T10:25:08.435226+0000","last_became_peered":"2023-10-09T10:25:08.435226+0000","last_unstale":"2023-10-09T10:28:36.125652+0000","last_undegraded":"2023-10-09T10:28:36.125652+0000","last_fullsized":"2023-10-09T10:28:36.125652+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":31836,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430708,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626936,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1510768,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":1505024,"kb_avail":92234384,"statfs":{"total":95995035648,"available":94448009216,"internally_reserved":0,"allocated":5873664,"data_stored":4867890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541144576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2880000000000003,"5min":2.4009999999999998,"15min":1.7310000000000001},"min":{"1min":0.219,"5min":0.216,"15min":0.216},"max":{"1min":47.401000000000003,"5min":47.401000000000003,"15min":47.401000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":4.7679999999999998,"5min":2.3380000000000001,"15min":2.0579999999999998},"min":{"1min":0.29399999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":47.311999999999998,"5min":47.311999999999998,"15min":47.311999999999998},"last":0.41499999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.3400000000000001,"15min":1.998},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":2.4049999999999998,"5min":6.3899999999999997,"15min":155.01400000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.482,"5min":1.3580000000000001,"15min":2.1739999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":5.6050000000000004,"5min":6.3369999999999997,"15min":154.71100000000001},"last":0.46500000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:28:20 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.0609999999999999,"15min":1.8600000000000001},"min":{"1min":0.32100000000000001,"5min":0.309,"15min":0.24099999999999999},"max":{"1min":10.907,"5min":10.907,"15min":29.367999999999999},"last":0.505},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.6319999999999999,"15min":1.524},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.4480000000000004,"5min":7.4480000000000004,"15min":8.7669999999999995},"last":0.56399999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":2.1339999999999999,"15min":1.698},"min":{"1min":0.30599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.9450000000000003,"5min":43.393000000000001,"15min":43.393000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.4049999999999998,"5min":2.3069999999999999,"15min":1.867},"min":{"1min":0.36699999999999999,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":6.423,"5min":43.466000000000001,"15min":43.466000000000001},"last":0.49099999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":8.9589999999999996,"5min":3.2749999999999999,"15min":2.4500000000000002},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":177.929,"5min":177.929,"15min":177.929},"last":0.42999999999999999},{"interface":"front","average":{"1min":8.2530000000000001,"5min":2.8980000000000001,"15min":2.2490000000000001},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":178.16499999999999,"5min":178.16499999999999,"15min":178.16499999999999},"last":0.68999999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.1640000000000001,"15min":2.0910000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.788,"5min":11.788,"15min":121.98999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.2629999999999999,"15min":1.5760000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":2.2890000000000001,"5min":7.0060000000000002,"15min":122.346},"last":0.59299999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.0219999999999998,"15min":1.966},"min":{"1min":0.38,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":12.612,"5min":49.991,"15min":130.66999999999999},"last":0.751},{"interface":"front","average":{"1min":1.5880000000000001,"5min":2.097,"15min":2.02},"min":{"1min":0.40799999999999997,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":12.659000000000001,"5min":50.023000000000003,"15min":130.702},"last":0.81899999999999995}]}]},{"osd":1,"up_from":0,"seq":360777253250,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":79,"seq":339302416774,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1571020,"kb_used_data":532164,"kb_used_omap":28,"kb_used_meta":1038819,"kb_avail":92174132,"statfs":{"total":95995035648,"available":94386311168,"internally_reserved":0,"allocated":544935936,"data_stored":542721824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1063751504},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:27:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.8049999999999999,"15min":1.6990000000000001},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.19800000000000001},"max":{"1min":13.077999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.712,"15min":1.915},"min":{"1min":0.26500000000000001,"5min":0.214,"15min":0.192},"max":{"1min":12.349,"5min":12.349,"15min":56.950000000000003},"last":0.34899999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":2.141,"15min":1.9379999999999999},"min":{"1min":0.26100000000000001,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":8.6440000000000001,"5min":17.093,"15min":53.988999999999997},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.3889999999999998,"5min":2.2970000000000002,"15min":1.9890000000000001},"min":{"1min":0.30399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.021000000000001,"5min":17.053999999999998,"15min":54.119},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:28:08 2023","interfaces":[{"interface":"back","average":{"1min":4.0599999999999996,"5min":2.605,"15min":1.8480000000000001},"min":{"1min":0.28299999999999997,"5min":0.217,"15min":0.217},"max":{"1min":15.882999999999999,"5min":29.835999999999999,"15min":56.984999999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":3.6040000000000001,"5min":2.9729999999999999,"15min":1.944},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":9.8949999999999996,"5min":29.776,"15min":56.475999999999999},"last":0.32800000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":1.823,"5min":1.8029999999999999,"15min":1.952},"min":{"1min":0.29199999999999998,"5min":0.28399999999999997,"15min":0.251},"max":{"1min":11.653,"5min":11.653,"15min":57.719000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.548,"5min":2.2730000000000001,"15min":2.077},"min":{"1min":0.375,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.1359999999999992,"5min":11.896000000000001,"15min":56.378},"last":0.70399999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9159999999999999,"5min":3.6589999999999998,"15min":3.363},"min":{"1min":0.46300000000000002,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":8.25,"5min":90.412000000000006,"15min":229.589},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.504,"5min":2.9500000000000002,"15min":3.02},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.30199999999999999},"max":{"1min":6.1239999999999997,"5min":90.393000000000001,"15min":228.88900000000001},"last":0.64200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.9990000000000001,"15min":2.089},"min":{"1min":0.38400000000000001,"5min":0.379,"15min":0.34000000000000002},"max":{"1min":6.0789999999999997,"5min":9.7840000000000007,"15min":54.482999999999997},"last":0.67300000000000004},{"interface":"front","average":{"1min":2.1499999999999999,"5min":2.0880000000000001,"15min":2.1960000000000002},"min":{"1min":0.44500000000000001,"5min":0.373,"15min":0.34799999999999998},"max":{"1min":11.702999999999999,"5min":11.702999999999999,"15min":57.753},"last":0.61699999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:28:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.4550000000000001,"15min":2.2050000000000001},"min":{"1min":0.47799999999999998,"5min":0.38500000000000001,"15min":0.34999999999999998},"max":{"1min":8.1760000000000002,"5min":12.029999999999999,"15min":56.411999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.0910000000000002,"5min":2.3460000000000001,"15min":2.1779999999999999},"min":{"1min":0.502,"5min":0.39900000000000002,"15min":0.38300000000000001},"max":{"1min":8.4339999999999993,"5min":17.129999999999999,"15min":54.451999999999998},"last":0.71499999999999997}]}]},{"osd":2,"up_from":217,"seq":932007903510,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1363380,"kb_used_data":482732,"kb_used_omap":83,"kb_used_meta":880556,"kb_avail":92381772,"statfs":{"total":95995035648,"available":94598934528,"internally_reserved":0,"allocated":494317568,"data_stored":492310403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":901689839},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5800000000000001,"5min":2.3410000000000002,"15min":1.891},"min":{"1min":0.215,"5min":0.192,"15min":0.185},"max":{"1min":31.641999999999999,"5min":31.641999999999999,"15min":48.990000000000002},"last":0.315},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.9119999999999999,"15min":1.946},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.20899999999999999},"max":{"1min":9.968,"5min":13.773,"15min":49.412999999999997},"last":0.33100000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":2.4889999999999999,"15min":1.7729999999999999},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.20300000000000001},"max":{"1min":27.097999999999999,"5min":27.097999999999999,"15min":34.893999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.6519999999999999,"15min":1.6679999999999999},"min":{"1min":0.317,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.976,"5min":13.356999999999999,"15min":34.962000000000003},"last":0.375}]},{"osd":3,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.8879999999999999,"15min":1.4470000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.8410000000000002,"5min":15.115,"15min":35.043999999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.395,"5min":1.776,"15min":1.4530000000000001},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.6820000000000004,"5min":11.206,"15min":50.406999999999996},"last":0.34200000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.8200000000000001,"15min":1.5660000000000001},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":3.9089999999999998,"5min":47.503999999999998,"15min":47.503999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.3610000000000002,"15min":1.8600000000000001},"min":{"1min":0.38700000000000001,"5min":0.34000000000000002,"15min":0.29299999999999998},"max":{"1min":7.532,"5min":47.704000000000001,"15min":47.704000000000001},"last":0.40899999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.681,"15min":1.8540000000000001},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.30199999999999999},"max":{"1min":7.4779999999999998,"5min":11.266999999999999,"15min":57.298000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.6320000000000001,"5min":2.5230000000000001,"15min":2.0190000000000001},"min":{"1min":0.34899999999999998,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":26.997,"5min":26.997,"15min":57.828000000000003},"last":0.54700000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.6000000000000001,"15min":1.756},"min":{"1min":0.36599999999999999,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":3.8460000000000001,"5min":7.9390000000000001,"15min":12.411},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.915,"15min":1.7949999999999999},"min":{"1min":0.42499999999999999,"5min":0.38300000000000001,"15min":0.36699999999999999},"max":{"1min":7.5579999999999998,"5min":13.975,"15min":24.379000000000001},"last":0.47999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:28:21 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":1.754,"15min":2.0110000000000001},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":4.0449999999999999,"5min":10.17,"15min":35.008000000000003},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.841,"5min":2.0880000000000001,"15min":1.869},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.35899999999999999},"max":{"1min":7.5830000000000002,"5min":15.548,"15min":24.323},"last":0.442}]}]},{"osd":3,"up_from":352,"seq":1511828488349,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803732,"kb_used_data":212364,"kb_used_omap":15,"kb_used_meta":591344,"kb_avail":92941420,"statfs":{"total":95995035648,"available":95172014080,"internally_reserved":0,"allocated":217460736,"data_stored":214950918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":605536710},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":4.7489999999999997,"5min":3.5030000000000001,"15min":2.1280000000000001},"min":{"1min":0.318,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":40.965000000000003,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.41399999999999998},{"interface":"front","average":{"1min":5.7789999999999999,"5min":3.645,"15min":2.2010000000000001},"min":{"1min":0.27200000000000002,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":43.595999999999997,"5min":66.802000000000007,"15min":66.802000000000007},"last":0.54000000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.4059999999999999,"15min":1.9370000000000001},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.7759999999999998,"5min":11.785,"15min":105.401},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.008,"15min":2.0219999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23300000000000001},"max":{"1min":7.452,"5min":10.422000000000001,"15min":104.914},"last":0.44400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.968,"5min":2.3759999999999999,"15min":2.0699999999999998},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":9.923,"5min":13.497999999999999,"15min":13.497999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.786,"5min":2.3500000000000001,"15min":1.6819999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":9.4039999999999999,"5min":16.475000000000001,"15min":25.338999999999999},"last":0.51300000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":12.550000000000001,"5min":3.8359999999999999,"15min":2.3100000000000001},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":178.72200000000001,"5min":178.72200000000001,"15min":178.72200000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":12.884,"5min":4.7800000000000002,"15min":2.891},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":179.08500000000001,"5min":179.08500000000001,"15min":179.08500000000001},"last":0.622}]},{"osd":5,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.617,"15min":1.4530000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.6870000000000003,"5min":11.363,"15min":25.251999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.0600000000000001,"5min":2.149,"15min":1.7809999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.3949999999999996,"5min":10.388,"15min":13.394},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.371,"5min":2.1549999999999998,"15min":1.915},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.8689999999999998,"5min":13.545999999999999,"15min":19.114000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.8069999999999999,"15min":1.6100000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":5.718,"5min":11.32,"15min":25.306999999999999},"last":0.59599999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:28:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":1.9730000000000001,"15min":1.7490000000000001},"min":{"1min":0.41299999999999998,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":7.4279999999999999,"5min":10.345000000000001,"15min":10.459},"last":0.64000000000000001},{"interface":"front","average":{"1min":2.9260000000000002,"5min":2.2000000000000002,"15min":1.9710000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":12.4,"5min":12.4,"15min":19.280000000000001},"last":0.58299999999999996}]}]},{"osd":4,"up_from":266,"seq":1142461300979,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1339632,"kb_used_data":398312,"kb_used_omap":35,"kb_used_meta":941276,"kb_avail":92405520,"statfs":{"total":95995035648,"available":94623252480,"internally_reserved":0,"allocated":407871488,"data_stored":406864112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":963866862},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":3.1179999999999999,"15min":2.6320000000000001},"min":{"1min":0.22700000000000001,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":7.6539999999999999,"5min":109.193,"15min":109.193},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.629,"5min":3.625,"15min":3.2200000000000002},"min":{"1min":0.27100000000000002,"5min":0.26400000000000001,"15min":0.23300000000000001},"max":{"1min":10.289999999999999,"5min":110.438,"15min":110.438},"last":0.372}]},{"osd":1,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.9530000000000001,"15min":2.1469999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.24299999999999999},"max":{"1min":6.0259999999999998,"5min":12.209,"15min":71.457999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.913,"5min":2.6899999999999999,"15min":2.1899999999999999},"min":{"1min":0.34799999999999998,"5min":0.248,"15min":0.248},"max":{"1min":8.6899999999999995,"5min":49.781999999999996,"15min":71.658000000000001},"last":0.38700000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":2.153,"15min":1.758},"min":{"1min":0.41999999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":9.9039999999999999,"5min":15.944000000000001,"15min":15.944000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.903,"5min":1.7470000000000001,"15min":1.671},"min":{"1min":0.38600000000000001,"5min":0.255,"15min":0.255},"max":{"1min":6.2149999999999999,"5min":14.473000000000001,"15min":14.473000000000001},"last":0.443}]},{"osd":3,"last update":"Mon Oct 9 10:28:03 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.476,"15min":2.109},"min":{"1min":0.33000000000000002,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":5.141,"5min":11.347,"15min":12.074},"last":0.503},{"interface":"front","average":{"1min":3.278,"5min":2.7120000000000002,"15min":1.986},"min":{"1min":0.38100000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.9779999999999998,"5min":16.067,"15min":16.067},"last":0.45900000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.8360000000000001,"15min":2.0209999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.252},"max":{"1min":10.007,"5min":16.094000000000001,"15min":99.585999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.7749999999999999,"15min":1.853},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":4.5620000000000003,"5min":49.915999999999997,"15min":99.384},"last":0.623}]},{"osd":6,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.446,"15min":1.393},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.8380000000000001,"5min":14.499000000000001,"15min":14.499000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.8109999999999999,"5min":2.1059999999999999,"15min":1.804},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":6.7329999999999997,"5min":18.952999999999999,"15min":18.952999999999999},"last":0.64900000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:28:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":2.0880000000000001,"15min":1.835},"min":{"1min":0.36299999999999999,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":6.4279999999999999,"5min":12.391999999999999,"15min":17.369},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.9119999999999999,"5min":2.2200000000000002,"15min":1.7490000000000001},"min":{"1min":0.41399999999999998,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":9.5129999999999999,"5min":51.026000000000003,"15min":51.026000000000003},"last":0.56799999999999995}]}]},{"osd":5,"up_from":36,"seq":154618823163,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2137892,"kb_used_data":632988,"kb_used_omap":0,"kb_used_meta":1504896,"kb_avail":91607260,"statfs":{"total":95995035648,"available":93805834240,"internally_reserved":0,"allocated":648179712,"data_stored":647180390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541013504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5630000000000002,"5min":3.0760000000000001,"15min":2.1629999999999998},"min":{"1min":0.92600000000000005,"5min":0.23200000000000001,"15min":0.219},"max":{"1min":10.323,"5min":19.102,"15min":22.469999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.9730000000000001,"5min":1.901,"15min":1.7430000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23200000000000001},"max":{"1min":5.0730000000000004,"5min":17.887,"15min":22.437000000000001},"last":0.42299999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:27:43 2023","interfaces":[{"interface":"back","average":{"1min":3.3319999999999999,"5min":2.387,"15min":2.1680000000000001},"min":{"1min":0.36799999999999999,"5min":0.27900000000000003,"15min":0.23999999999999999},"max":{"1min":10.425000000000001,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.6299999999999999,"15min":1.788},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23499999999999999},"max":{"1min":5.2329999999999997,"5min":8.9529999999999994,"15min":29.065000000000001},"last":0.48699999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:28:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":2.149,"15min":2.0099999999999998},"min":{"1min":0.35499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.0759999999999996,"5min":16.629999999999999,"15min":16.629999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.8320000000000001,"15min":1.746},"min":{"1min":0.33100000000000002,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":6.819,"5min":9.6929999999999996,"15min":15.929},"last":0.37}]},{"osd":3,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":2.3220000000000001,"15min":2.0139999999999998},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.28199999999999997},"max":{"1min":8.1720000000000006,"5min":13.614000000000001,"15min":13.614000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.9740000000000002,"5min":2.1589999999999998,"15min":1.887},"min":{"1min":0.46300000000000002,"5min":0.376,"15min":0.315},"max":{"1min":9.8450000000000006,"5min":11.124000000000001,"15min":13.624000000000001},"last":0.45800000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":2.0230000000000001,"15min":1.6919999999999999},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":3.9500000000000002,"5min":13.523999999999999,"15min":13.523999999999999},"last":0.443},{"interface":"front","average":{"1min":1.391,"5min":1.5269999999999999,"15min":1.4690000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23599999999999999},"max":{"1min":4.3959999999999999,"5min":8.8620000000000001,"15min":11.956},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.929,"15min":1.8440000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":3.3679999999999999,"5min":13.567,"15min":13.567},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.74,"15min":1.696},"min":{"1min":0.34300000000000003,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":3.407,"5min":16.497,"15min":16.497},"last":0.47499999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:28:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.6379999999999999,"15min":1.7769999999999999},"min":{"1min":0.36899999999999999,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":7.3490000000000002,"5min":16.686,"15min":21.109000000000002},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.86499999999999999,"15min":1.1299999999999999},"min":{"1min":0.34799999999999998,"5min":0.318,"15min":0.27500000000000002},"max":{"1min":2.375,"5min":8.9719999999999995,"15min":11.994},"last":0.54600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:28:45.647 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T10:28:45.647 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:28:45.647 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.3173423938122134 2023-10-09T10:28:45.648 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 0.3173423938122134 2023-10-09T10:28:45.662 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:45.690 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:45.724 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:45.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:45.829 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:45.855 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:45.891 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:45.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:45.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[147926]: pgmap v1591: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:45.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[147926]: failed to encode map e562 with expected crc 2023-10-09T10:28:45.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[147926]: osdmap e562: 8 total, 8 up, 6 in 2023-10-09T10:28:45.918 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[147926]: failed to encode map e562 with expected crc 2023-10-09T10:28:45.918 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/763447496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:45.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[152263]: pgmap v1591: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:45.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[152263]: failed to encode map e562 with expected crc 2023-10-09T10:28:45.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[152263]: osdmap e562: 8 total, 8 up, 6 in 2023-10-09T10:28:45.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[152263]: failed to encode map e562 with expected crc 2023-10-09T10:28:45.918 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/763447496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:45 smithi110 ceph-mon[136032]: pgmap v1591: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2023-10-09T10:28:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:45 smithi110 ceph-mon[136032]: failed to encode map e562 with expected crc 2023-10-09T10:28:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:45 smithi110 ceph-mon[136032]: osdmap e562: 8 total, 8 up, 6 in 2023-10-09T10:28:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:45 smithi110 ceph-mon[136032]: failed to encode map e562 with expected crc 2023-10-09T10:28:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/763447496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:45.996 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:46.003 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:28:46.031 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:46.126 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:46.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:46.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:46.231 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:46.258 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:46.294 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:46.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:46.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:46.318 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T10:28:46.318 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:28:46.321 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export testimg /tmp/rbd_import_export_444239/img2 2023-10-09T10:28:46.327 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:46.397 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:46.425 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:46.436 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:28:46.606 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T10:28:46.610 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export testimg - 2023-10-09T10:28:46.669 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:46.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:46.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:46.802 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:46.937 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:46.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:46.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:47.042 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:47.069 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:46 smithi012 ceph-mon[147926]: from='client.47270 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:47.069 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:46 smithi012 ceph-mon[147926]: failed to encode map e563 with expected crc 2023-10-09T10:28:47.070 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:46 smithi012 ceph-mon[147926]: failed to encode map e563 with expected crc 2023-10-09T10:28:47.070 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:46 smithi012 ceph-mon[147926]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T10:28:47.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:46 smithi012 ceph-mon[152263]: from='client.47270 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:47.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:46 smithi012 ceph-mon[152263]: failed to encode map e563 with expected crc 2023-10-09T10:28:47.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:46 smithi012 ceph-mon[152263]: failed to encode map e563 with expected crc 2023-10-09T10:28:47.070 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:46 smithi012 ceph-mon[152263]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T10:28:47.073 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:47.106 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:28:47.106 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:28:47.106 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:28:47.107 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:28:47.107 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:28:47.128 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:46 smithi110 ceph-mon[136032]: from='client.47270 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:28:47.129 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:46 smithi110 ceph-mon[136032]: failed to encode map e563 with expected crc 2023-10-09T10:28:47.129 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:46 smithi110 ceph-mon[136032]: failed to encode map e563 with expected crc 2023-10-09T10:28:47.129 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:46 smithi110 ceph-mon[136032]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T10:28:47.361 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T10:28:47.362 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm testimg 2023-10-09T10:28:47.512 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T10:28:47.515 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp /tmp/rbd_import_export_444239/img /tmp/rbd_import_export_444239/img2 2023-10-09T10:28:47.518 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:28:47.548 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:47.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:47.652 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:47.686 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:47.858 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:47 smithi012 ceph-mon[147926]: pgmap v1594: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 170 B/s wr, 0 op/s 2023-10-09T10:28:47.859 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:47 smithi012 ceph-mon[152263]: pgmap v1594: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 170 B/s wr, 0 op/s 2023-10-09T10:28:47.877 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:47.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:47.923 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:28:47.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:47 smithi110 ceph-mon[136032]: pgmap v1594: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 170 B/s wr, 0 op/s 2023-10-09T10:28:47.983 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:48.012 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:48.046 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:48.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:48.114 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:28:48.150 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:28:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:28:47] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.33.4" 2023-10-09T10:28:48.183 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:48.313 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp /tmp/rbd_import_export_444239/img /tmp/rbd_import_export_444239/img3 2023-10-09T10:28:48.431 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:48.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:48.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:48.536 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:48.541 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:48.565 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:48.600 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:48.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:48.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:48.704 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:48.730 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:48.815 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:48.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:48.854 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img2 /tmp/rbd_import_export_444239/img3 2023-10-09T10:28:48.920 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:48.948 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:49.240 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:28:49.240 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:28:49.240 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:28:49.240 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:28:49.240 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.240 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:176158 11.19 11:9ceb3d43:::benchmark_data_smithi012_404402_object63375:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:28:49.240 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:09.553641+0000", 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.683461222000005, 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.137821208, 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176158 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.241 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.553641+0000", 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.553641+0000", 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.553640+0000", 2023-10-09T10:28:49.242 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2150000000000001e-06 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.553644+0000", 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7599999999999999e-06 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.553649+0000", 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3599999999999998e-06 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.243 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.553654+0000", 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6910999999999999e-05 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.553701+0000", 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017072999999999999 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.553871+0000", 2023-10-09T10:28:49.244 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00054031299999999995 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.554412+0000", 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027907700000000001 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.557202+0000", 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0076793210000000002 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.245 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.564882+0000", 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046324799999999998 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.565345+0000", 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.124726133 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.690071+0000", 2023-10-09T10:28:49.246 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7094e-05 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.690118+0000", 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001343849 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.691462+0000", 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.137821208 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.247 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:176166 11.17 11:ed3bfd93:::benchmark_data_smithi012_404402_object63383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:09.562436+0000", 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.674665771999997, 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13066319400000001, 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:49.248 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176166 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.562436+0000", 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.562436+0000", 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:28:49.249 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.562435+0000", 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0270000000000001e-06 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.562441+0000", 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5829999999999996e-06 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.250 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.562450+0000", 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3789999999999997e-06 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.562456+0000", 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8174000000000003e-05 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.562504+0000", 2023-10-09T10:28:49.251 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012261300000000001 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.562626+0000", 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046949399999999999 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.563096+0000", 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0062361079999999998 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.252 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569332+0000", 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1264000000000003e-05 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569373+0000", 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.123645267 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.693019+0000", 2023-10-09T10:28:49.253 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7608999999999999e-05 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.693056+0000", 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3349000000000001e-05 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.693100+0000", 2023-10-09T10:28:49.254 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.13066319400000001 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:176171 11.7 11:e521039b:::benchmark_data_smithi012_404402_object63388:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:09.569704+0000", 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.667398014, 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12888223800000001, 2023-10-09T10:28:49.255 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176171 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569704+0000", 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.256 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569704+0000", 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569703+0000", 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8140000000000003e-06 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.257 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569710+0000", 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0813e-05 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569720+0000", 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1770000000000003e-06 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569727+0000", 2023-10-09T10:28:49.258 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1810000000000006e-05 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569788+0000", 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000107513 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569896+0000", 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00047472999999999998 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.259 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.570371+0000", 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026520390000000001 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.573023+0000", 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026157540000000001 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.575638+0000", 2023-10-09T10:28:49.260 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.122848285 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.698487+0000", 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7210999999999998e-05 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.698544+0000", 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2528999999999999e-05 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.261 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.698586+0000", 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12888223800000001 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:176173 11.19 11:9ebdb961:::benchmark_data_smithi012_404402_object63390:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:09.569865+0000", 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.667236747999993, 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.130471581, 2023-10-09T10:28:49.262 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176173 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569865+0000", 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.263 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569865+0000", 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569864+0000", 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2750000000000001e-06 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.264 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569871+0000", 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0414000000000001e-05 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569881+0000", 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1650000000000003e-06 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569887+0000", 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9327000000000003e-05 2023-10-09T10:28:49.265 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.569947+0000", 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017648100000000001 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.570123+0000", 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058329599999999997 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.266 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.570706+0000", 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027443260000000001 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.573451+0000", 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1695000000000001e-05 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.573462+0000", 2023-10-09T10:28:49.267 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0019337569999999999 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.575396+0000", 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.12316149 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.698558+0000", 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5935e-05 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.268 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.698604+0000", 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0017333520000000001 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.700337+0000", 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.130471581 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.269 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:176323 11.7 11:e3fbf91f:::benchmark_data_smithi012_404402_object63540:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:09.873406+0000", 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.363695808000003, 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.090953996999999995, 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176323 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.270 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873406+0000", 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873406+0000", 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.271 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873405+0000", 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1243000000000001e-05 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873416+0000", 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5645e-05 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873432+0000", 2023-10-09T10:28:49.272 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.409e-06 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873438+0000", 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8208999999999999e-05 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873486+0000", 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000125108 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.273 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873611+0000", 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00048127100000000002 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.874093+0000", 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003488194 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.274 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.877581+0000", 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00070370800000000005 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.878285+0000", 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086010280999999994 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.964295+0000", 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6677000000000001e-05 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.275 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.964331+0000", 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8841999999999999e-05 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.964360+0000", 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.090953996999999995 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.276 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:176329 11.17 11:ecae452e:::benchmark_data_smithi012_404402_object63546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:09.873517+0000", 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 99.363585552000004, 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.090892097000000005, 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176329 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.277 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873517+0000", 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873517+0000", 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.278 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873516+0000", 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5929999999999996e-06 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873521+0000", 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0825999999999999e-05 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873532+0000", 2023-10-09T10:28:49.279 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0339999999999999e-06 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873539+0000", 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000459073 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.873998+0000", 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012306799999999999 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.280 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.874121+0000", 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044064199999999999 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.874562+0000", 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030255780000000001 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.877587+0000", 2023-10-09T10:28:49.281 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000661336 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.878249+0000", 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.086072380000000004 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.964321+0000", 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7482999999999999e-05 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.282 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.964369+0000", 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0074999999999998e-05 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.964409+0000", 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.090892097000000005 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.283 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:196793 11.19 11:9af370a5:::benchmark_data_smithi012_404402_object84010:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:35.244927+0000", 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 73.992174900999998, 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.089409354999999996, 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 196793 2023-10-09T10:28:49.284 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.244927+0000", 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.244927+0000", 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.285 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.244927+0000", 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6600000000000001e-06 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.244930+0000", 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9059999999999996e-06 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.244937+0000", 2023-10-09T10:28:49.286 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4950000000000002e-06 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.244942+0000", 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0063038759999999999 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.251246+0000", 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017615600000000001 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.287 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.251422+0000", 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020182800000000001 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.251624+0000", 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025656720000000002 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.288 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.254189+0000", 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0040324970000000003 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.258222+0000", 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015368280000000001 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.289 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.259759+0000", 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.073153897999999995 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.332912+0000", 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015227200000000001 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.333065+0000", 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012718429999999999 2023-10-09T10:28:49.290 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:35.334337+0000", 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.089409354999999996 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:214927 11.17 11:e966a91e:::benchmark_data_smithi012_404402_object102144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e545)", 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:56.636305+0000", 2023-10-09T10:28:49.291 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 52.600797174, 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.093565390999999998, 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 214927 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636305+0000", 2023-10-09T10:28:49.292 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636305+0000", 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636305+0000", 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.26e-06 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.293 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.294 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.294 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636307+0000", 2023-10-09T10:28:49.294 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2109999999999997e-06 2023-10-09T10:28:49.294 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.294 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636312+0000", 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6400000000000001e-06 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636315+0000", 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0018999999999997e-05 2023-10-09T10:28:49.295 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636365+0000", 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013842800000000001 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636503+0000", 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017232599999999999 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,4", 2023-10-09T10:28:49.296 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.636676+0000", 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0030038080000000002 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.639679+0000", 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058622850000000001 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.645542+0000", 2023-10-09T10:28:49.297 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.084239497999999996 2023-10-09T10:28:49.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.298 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.298 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.298 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.729781+0000", 2023-10-09T10:28:49.298 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9332000000000001e-05 2023-10-09T10:28:49.298 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.729830+0000", 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9907000000000003e-05 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:56.729870+0000", 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.093565390999999998 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.299 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47210.0:2940 12.14 12:286b461f:::group_id_snap_remove:head [create] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:33.461697+0000", 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.775404682, 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001022202, 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47210", 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/125198252", 2023-10-09T10:28:49.300 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 2940 2023-10-09T10:28:49.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.301 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.301 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.301 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.301 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461697+0000", 2023-10-09T10:28:49.301 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.301 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461697+0000", 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461696+0000", 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7159999999999992e-06 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.302 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461705+0000", 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4897e-05 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461720+0000", 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8889999999999999e-06 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461725+0000", 2023-10-09T10:28:49.303 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3449999999999999e-05 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461769+0000", 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011233 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461881+0000", 2023-10-09T10:28:49.304 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000108121 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.461989+0000", 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031785899999999998 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.462307+0000", 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00031388100000000001 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.305 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.462621+0000", 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0263000000000001e-05 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.462651+0000", 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5945999999999998e-05 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.306 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.462687+0000", 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2320000000000002e-05 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.462720+0000", 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001022202 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.307 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47210.0:3002 12.1 12:86efd170:::image43:head [call rbd.create in=9b] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:33.503817+0000", 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.733285062, 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099771600000000005, 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47210", 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/125198252", 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 3002 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.308 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.503817+0000", 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.503817+0000", 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.503816+0000", 2023-10-09T10:28:49.309 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7869999999999998e-06 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.503822+0000", 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1861e-05 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.503834+0000", 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8980000000000004e-06 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.310 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.503838+0000", 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2171e-05 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.503881+0000", 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5201e-05 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.311 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.503966+0000", 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017327 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.504139+0000", 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030943300000000002 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.504449+0000", 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020799800000000001 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.312 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.504657+0000", 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0001054 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.504762+0000", 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9692999999999998e-05 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.504792+0000", 2023-10-09T10:28:49.313 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3204999999999999e-05 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.504815+0000", 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00099771600000000005 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47264.0:8 2.2 2:5cea7035:::rbd_info:head [write 0~19] snapc 0=[] ondisk+write+known_if_redirected e563)", 2023-10-09T10:28:49.314 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:45.736080+0000", 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 3.5010221029999999, 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037330850000000001, 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47264", 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3962656744", 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 8 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.315 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.736080+0000", 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.736080+0000", 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.736078+0000", 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0045e-05 2023-10-09T10:28:49.316 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.736088+0000", 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3737000000000001e-05 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.736102+0000", 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0654e-05 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.317 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.736113+0000", 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023664279999999998 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.738479+0000", 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8740999999999995e-05 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.738578+0000", 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019759500000000001 2023-10-09T10:28:49.318 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.738776+0000", 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00057542100000000001 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.739351+0000", 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024859900000000001 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.319 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.739600+0000", 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011294999999999999 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.739713+0000", 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5285999999999999e-05 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.739768+0000", 2023-10-09T10:28:49.320 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5225000000000002e-05 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.739813+0000", 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0037330850000000001 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47264.0:33 2.3 2:d1e37dd3:::rbd_data.b8a0aba70585.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 8192~4096 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e563)", 2023-10-09T10:28:49.321 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:45.767760+0000", 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 3.4693421500000001, 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001656067, 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47264", 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3962656744", 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 33 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.322 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767760+0000", 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767760+0000", 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999948 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767755+0000", 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4620999999999999e-05 2023-10-09T10:28:49.323 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767769+0000", 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8090000000000001e-05 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767788+0000", 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6252e-05 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.324 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767804+0000", 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0974999999999998e-05 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767865+0000", 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000102129 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767967+0000", 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012896000000000001 2023-10-09T10:28:49.325 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.768096+0000", 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012109849999999999 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769307+0000", 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4999000000000002e-05 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.326 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769342+0000", 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0149000000000001e-05 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769352+0000", 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3566999999999999e-05 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769386+0000", 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0501000000000002e-05 2023-10-09T10:28:49.327 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769416+0000", 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001656067 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47264.0:34 2.3 2:d1e37dd3:::rbd_data.b8a0aba70585.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 102400~12288 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e563)", 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:45.767832+0000", 2023-10-09T10:28:49.328 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 3.4692697419999998, 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001707964, 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47264", 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3962656744", 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 34 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767832+0000", 2023-10-09T10:28:49.329 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767832+0000", 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767832+0000", 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4958000000000001e-05 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.330 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767857+0000", 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4370000000000007e-06 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767865+0000", 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9028000000000001e-05 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.331 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767884+0000", 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00051931500000000001 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.768403+0000", 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.3171999999999998e-05 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.768456+0000", 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000118479 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.332 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.768575+0000", 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00085006199999999995 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769425+0000", 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7253000000000001e-05 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769452+0000", 2023-10-09T10:28:49.333 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1328e-05 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769474+0000", 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9417999999999999e-05 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769503+0000", 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7339000000000003e-05 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.334 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769540+0000", 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001707964 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47264.0:35 2.3 2:d1e37dd3:::rbd_data.b8a0aba70585.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 1024000~73728 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e563)", 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:45.767895+0000", 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 3.4692071979999999, 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020498000000000001, 2023-10-09T10:28:49.335 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47264", 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3962656744", 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 35 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767895+0000", 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.336 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767895+0000", 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767894+0000", 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7347e-05 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.337 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767932+0000", 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3130000000000001e-06 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767939+0000", 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6099999999999997e-06 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.767945+0000", 2023-10-09T10:28:49.338 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00087504 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.768820+0000", 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5253000000000001e-05 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.768875+0000", 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010946999999999999 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.339 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.768984+0000", 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00056673599999999998 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769551+0000", 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030939500000000002 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769860+0000", 2023-10-09T10:28:49.340 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3847000000000001e-05 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769884+0000", 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6021e-05 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769920+0000", 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4482e-05 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.341 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.769945+0000", 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020498000000000001 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47264.0:39 2.3 2:cd6d5f7d:::rbd_data.b8a0aba70585.0000000000000018:head [set-alloc-hint object_size 4194304 write_size 4194304,write 1736704~73728 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e563)", 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:45.823327+0000", 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 3.4137749579999999, 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013572499999999999, 2023-10-09T10:28:49.342 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47264", 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3962656744", 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 39 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823327+0000", 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.343 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823327+0000", 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823326+0000", 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1745000000000001e-05 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.344 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823368+0000", 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2503e-05 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823380+0000", 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9269999999999999e-06 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823387+0000", 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9867000000000001e-05 2023-10-09T10:28:49.345 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823417+0000", 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1383000000000003e-05 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823499+0000", 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012099000000000001 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.346 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823620+0000", 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037905600000000001 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.823999+0000", 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00053273199999999995 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.824531+0000", 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.2948999999999994e-05 2023-10-09T10:28:49.347 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.824624+0000", 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5574000000000002e-05 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.824660+0000", 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4522999999999999e-05 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.348 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:45.824684+0000", 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013572499999999999 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37344.0:51 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_add in=12b] snapc 0=[] ondisk+write+known_if_redirected+full_try e563)", 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:47.481339+0000", 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.7557634850000001, 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013165760000000001, 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.349 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37344", 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2719945035", 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 51 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481339+0000", 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.350 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481339+0000", 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481336+0000", 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3536000000000001e-05 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481350+0000", 2023-10-09T10:28:49.351 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0429999999999998e-05 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481380+0000", 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2743e-05 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481393+0000", 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000149741 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.352 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481543+0000", 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000196548 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481739+0000", 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014711800000000001 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.481886+0000", 2023-10-09T10:28:49.353 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000296886 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.482183+0000", 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037398899999999997 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.482557+0000", 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4283999999999999e-05 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.354 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.482592+0000", 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7635999999999997e-05 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.482629+0000", 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6050999999999998e-05 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.355 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.482655+0000", 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013165760000000001 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37344.0:54 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_state_set in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try e563)", 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:47.484695+0000", 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.7524073069999999, 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001238329, 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.356 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37344", 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2719945035", 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 54 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.484695+0000", 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.357 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.484695+0000", 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.484692+0000", 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7592999999999999e-05 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.484710+0000", 2023-10-09T10:28:49.358 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0788999999999999e-05 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.484741+0000", 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2967e-05 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.484754+0000", 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6738999999999999e-05 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.359 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.484821+0000", 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000141791 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.484962+0000", 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000222995 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.485185+0000", 2023-10-09T10:28:49.360 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026290299999999999 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.485448+0000", 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038532600000000002 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.485834+0000", 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1942e-05 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.361 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.485866+0000", 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4418999999999997e-05 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.485900+0000", 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3235000000000003e-05 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.485933+0000", 2023-10-09T10:28:49.362 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001238329 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37344.0:57 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_state_set in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try e563)", 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:47.486529+0000", 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.750572748, 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010559810000000001, 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.363 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37344", 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2719945035", 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 57 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486529+0000", 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.364 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486529+0000", 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486529+0000", 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1980000000000001e-06 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486532+0000", 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0490000000000004e-06 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.365 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486540+0000", 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0920000000000002e-06 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486543+0000", 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5452e-05 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.366 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486569+0000", 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8299e-05 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486617+0000", 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012104000000000001 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486738+0000", 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00021044600000000001 2023-10-09T10:28:49.367 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.486948+0000", 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00055330699999999995 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.487502+0000", 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1402000000000002e-05 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.368 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.487533+0000", 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0423999999999999e-05 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.487563+0000", 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1889e-05 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.487585+0000", 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0010559810000000001 2023-10-09T10:28:49.369 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37344.0:80 2.3 2:d1e37dd3:::rbd_data.b8a0aba70585.0000000000000000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+full_try e563)", 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:47.499256+0000", 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.7378464789999999, 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014259279999999999, 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.370 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37344", 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2719945035", 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 80 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499256+0000", 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499256+0000", 2023-10-09T10:28:49.371 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499255+0000", 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.947e-06 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499261+0000", 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1326e-05 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.372 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499272+0000", 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0250000000000002e-06 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499277+0000", 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6848999999999999e-05 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499314+0000", 2023-10-09T10:28:49.373 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9679999999999996e-05 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499393+0000", 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0932000000000003e-05 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499484+0000", 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00053682100000000004 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.374 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500021+0000", 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000417255 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500438+0000", 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000183859 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.375 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500622+0000", 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2147999999999998e-05 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500654+0000", 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7166999999999999e-05 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500682+0000", 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014259279999999999 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.376 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37344.0:82 2.3 2:cd6d5f7d:::rbd_data.b8a0aba70585.0000000000000018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+full_try e563)", 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:47.499337+0000", 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 1.7377654520000001, 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014745489999999999, 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37344", 2023-10-09T10:28:49.377 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/2719945035", 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 82 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499337+0000", 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499337+0000", 2023-10-09T10:28:49.378 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499336+0000", 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.33e-06 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499340+0000", 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9460000000000001e-06 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.379 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499347+0000", 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5580000000000001e-06 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499356+0000", 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00034645500000000002 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499702+0000", 2023-10-09T10:28:49.380 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6244999999999999e-05 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499779+0000", 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3759999999999998e-05 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.499862+0000", 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024478 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.381 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500107+0000", 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00060584799999999998 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500713+0000", 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9894e-05 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500763+0000", 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6244000000000001e-05 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500789+0000", 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.2110999999999999e-05 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:47.500811+0000", 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014745489999999999 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.383 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.384 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.384 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:49.384 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:49.384 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:28:49.626 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:28:49.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:49 smithi110 ceph-mon[136032]: pgmap v1595: 73 pgs: 73 active+clean; 3.2 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 94 KiB/s rd, 57 KiB/s wr, 23 op/s 2023-10-09T10:28:49.967 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:49.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:49.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:50.071 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:50.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:49 smithi012 ceph-mon[147926]: pgmap v1595: 73 pgs: 73 active+clean; 3.2 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 94 KiB/s rd, 57 KiB/s wr, 23 op/s 2023-10-09T10:28:50.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:49 smithi012 ceph-mon[152263]: pgmap v1595: 73 pgs: 73 active+clean; 3.2 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 94 KiB/s rd, 57 KiB/s wr, 23 op/s 2023-10-09T10:28:50.102 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:50.219 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:28:51.500 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:28:51.500 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:28:51.500 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:28:51.500 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:28:51.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:28:51.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:51 smithi110 ceph-mon[136032]: pgmap v1596: 73 pgs: 73 active+clean; 3.2 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 92 KiB/s rd, 56 KiB/s wr, 22 op/s 2023-10-09T10:28:52.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:51 smithi012 ceph-mon[147926]: pgmap v1596: 73 pgs: 73 active+clean; 3.2 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 92 KiB/s rd, 56 KiB/s wr, 22 op/s 2023-10-09T10:28:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:51 smithi012 ceph-mon[152263]: pgmap v1596: 73 pgs: 73 active+clean; 3.2 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 92 KiB/s rd, 56 KiB/s wr, 22 op/s 2023-10-09T10:28:52.431 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:28:52.879 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:52.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:52.983 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:53.017 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:53.624 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:28:53.624 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:28:53.624 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:28:53.624 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:28:53.624 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:28:54.043 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:53 smithi110 ceph-mon[136032]: pgmap v1597: 73 pgs: 73 active+clean; 3.2 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 112 KiB/s rd, 63 KiB/s wr, 20 op/s 2023-10-09T10:28:54.043 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:53 smithi110 ceph-mon[136032]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T10:28:54.044 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:28:54.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:53 smithi012 ceph-mon[152263]: pgmap v1597: 73 pgs: 73 active+clean; 3.2 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 112 KiB/s rd, 63 KiB/s wr, 20 op/s 2023-10-09T10:28:54.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:53 smithi012 ceph-mon[152263]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T10:28:54.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:53 smithi012 ceph-mon[147926]: pgmap v1597: 73 pgs: 73 active+clean; 3.2 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 112 KiB/s rd, 63 KiB/s wr, 20 op/s 2023-10-09T10:28:54.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:53 smithi012 ceph-mon[147926]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T10:28:54.345 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:54.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:54.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:54.450 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:54.477 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:54.510 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:54.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:54.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:54.614 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:54.640 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:54.659 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:28:54.669 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:28:54.673 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:54.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:54.778 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:54.811 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:54.998 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:55.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:55.060 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:54 smithi110 ceph-mon[136032]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.060 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:54 smithi110 ceph-mon[136032]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.060 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:54 smithi110 ceph-mon[136032]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.102 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:55.132 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:54 smithi012 ceph-mon[147926]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:54 smithi012 ceph-mon[147926]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:54 smithi012 ceph-mon[147926]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:54 smithi012 ceph-mon[152263]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:54 smithi012 ceph-mon[152263]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:54 smithi012 ceph-mon[152263]: failed to encode map e564 with expected crc 2023-10-09T10:28:55.369 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:55.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:55.473 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:55.502 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:55.603 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:55.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:55.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:55.707 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:55.735 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:55.770 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:55.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:55.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:55.874 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:55.886 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:28:55.886 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:28:55.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72716 11.4 11.2af64ba4 (undecoded) ondisk+retry+write+known_if_redirected e506)", 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.611957+0000", 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 256.27129529400003, 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0282683029999999, 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.887 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72716 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.611957+0000", 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.611957+0000", 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:28:55.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.611955+0000", 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022969399999999999 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612185+0000", 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5172e-05 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612200+0000", 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.3059999999999998e-06 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612208+0000", 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9839000000000002e-05 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612258+0000", 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3360000000000001e-06 2023-10-09T10:28:55.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.612268+0000", 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0279472949999999 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:40.640215+0000", 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.044e-05 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:40.640225+0000", 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0282683029999999 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:72741 11.4 11.c1ae8004 (undecoded) ondisk+write+known_if_redirected e506)", 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:24:39.628049+0000", 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 256.25520303299999, 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0122071070000001, 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.892 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 72741 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628049+0000", 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628049+0000", 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628047+0000", 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8661000000000001e-05 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628086+0000", 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6623e-05 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628102+0000", 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5379999999999999e-06 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628109+0000", 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1010999999999997e-05 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628160+0000", 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0148e-05 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:39.628170+0000", 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0120785800000001 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:40.640249+0000", 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6459999999999993e-06 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:24:40.640256+0000", 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0122071070000001 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.897 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:157305 11.4 11:208b8c8b:::benchmark_data_smithi012_404402_object44522:head [delete] snapc 0=[] ondisk+write+known_if_redirected e537)", 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:40.571713+0000", 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 135.31153904199999, 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.47087400299999999, 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 157305 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571713+0000", 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571713+0000", 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571712+0000", 2023-10-09T10:28:55.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5790000000000003e-06 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571717+0000", 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5520000000000005e-06 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571726+0000", 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.493e-06 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.571730+0000", 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044968710000000004 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576227+0000", 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023127900000000001 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576458+0000", 2023-10-09T10:28:55.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043961500000000003 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576898+0000", 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0048120180000000004 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.581710+0000", 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.96e-06 2023-10-09T10:28:55.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.581718+0000", 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.460789267 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042507+0000", 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4877999999999999e-05 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042562+0000", 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5174e-05 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042587+0000", 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.47087400299999999 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.904 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:157308 11.4 11:2062cb3d:::benchmark_data_smithi012_404402_object44525:head [delete] snapc 0=[] ondisk+write+known_if_redirected e537)", 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:40.576237+0000", 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 135.30701573600001, 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46641645599999998, 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.905 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 157308 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576237+0000", 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576237+0000", 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:55.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576235+0000", 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6810000000000003e-06 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576241+0000", 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9400999999999993e-05 2023-10-09T10:28:55.907 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.908 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576330+0000", 2023-10-09T10:28:55.908 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0400000000000004e-06 2023-10-09T10:28:55.908 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.908 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.908 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.576337+0000", 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002802999 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.579140+0000", 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112234 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.579252+0000", 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000386172 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.579638+0000", 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021832420000000002 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.581822+0000", 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00050634300000000003 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.582328+0000", 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46028158600000002 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042610+0000", 2023-10-09T10:28:55.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3995999999999999e-05 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042634+0000", 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.9375e-05 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042653+0000", 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46641645599999998 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.912 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174154 11.14 11:29a9bfc4:::benchmark_data_smithi012_404402_object61371:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e540)", 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.542266+0000", 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 109.340986696, 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.038844511, 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "reached pg", 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.913 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174154 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.542266+0000", 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.542266+0000", 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:55.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.542264+0000", 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4279999999999999e-06 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.542272+0000", 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0959e-05 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.542283+0000", 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4040000000000003e-06 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.542288+0000", 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2254000000000004e-05 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.542350+0000", 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3130000000000001e-06 2023-10-09T10:28:55.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.542358+0000", 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.02534985 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.567707+0000", 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011858699999999999 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for missing object", 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.567826+0000", 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013204465 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.581030+0000", 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9611999999999996e-05 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.581110+0000", 2023-10-09T10:28:55.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.038844511 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174163 11.14 11:2a23e156:::benchmark_data_smithi012_404402_object61380:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.547718+0000", 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 109.33553459399999, 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0329484470000001, 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.919 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174163 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547718+0000", 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547718+0000", 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547716+0000", 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0305999999999999e-05 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547726+0000", 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2295e-05 2023-10-09T10:28:55.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547739+0000", 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6699999999999997e-06 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547745+0000", 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0032529519999999999 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.550998+0000", 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0084e-05 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.551008+0000", 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0171245209999999 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.568133+0000", 2023-10-09T10:28:55.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7958000000000001e-05 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.568211+0000", 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5470000000000006e-05 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.568306+0000", 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0065014590000000002 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.574808+0000", 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001461787 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.576270+0000", 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0042915269999999998 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.580561+0000", 2023-10-09T10:28:55.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8386000000000001e-05 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.580600+0000", 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.6594000000000006e-05 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.580666+0000", 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0329484470000001 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.926 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174165 11.14 11:2ed50a59:::benchmark_data_smithi012_404402_object61382:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.547769+0000", 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 109.33548318, 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0411274829999999, 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.927 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174165 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547769+0000", 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547769+0000", 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.928 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547769+0000", 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3819999999999998e-06 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547771+0000", 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1839999999999998e-06 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547776+0000", 2023-10-09T10:28:55.929 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.6059999999999999e-06 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.547780+0000", 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.003411833 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.551192+0000", 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.4250000000000004e-06 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.930 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.551201+0000", 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.019086677 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.570288+0000", 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3848000000000002e-05 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.931 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.570362+0000", 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9524000000000006e-05 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.570441+0000", 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0056699480000000002 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.576111+0000", 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043952640000000003 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.932 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.580506+0000", 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082092109999999992 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.588716+0000", 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5479000000000002e-05 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.933 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.588771+0000", 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000125523 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.588897+0000", 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0411274829999999 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.934 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174176 11.14 11:289f6d3b:::benchmark_data_smithi012_404402_object61393:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.558981+0000", 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 109.32427170299999, 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0355769889999999, 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174176 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.935 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.558981+0000", 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.558981+0000", 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.558979+0000", 2023-10-09T10:28:55.936 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4730000000000003e-06 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.558986+0000", 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4319e-05 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.559001+0000", 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.985e-06 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.937 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.559008+0000", 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00019307800000000001 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.559201+0000", 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9199999999999999e-06 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.559211+0000", 2023-10-09T10:28:55.938 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.01268618 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.571897+0000", 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000117234 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.572014+0000", 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000142578 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.939 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.572157+0000", 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0039605660000000004 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.576117+0000", 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012557656 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.588675+0000", 2023-10-09T10:28:55.940 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057291909999999998 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594404+0000", 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4423999999999999e-05 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594459+0000", 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.9024999999999995e-05 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.941 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594558+0000", 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0355769889999999 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174202 11.14 11:2f6b14fd:::benchmark_data_smithi012_404402_object61419:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.599053+0000", 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 109.284199059, 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99570522500000003, 2023-10-09T10:28:55.942 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174202 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.599053+0000", 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.943 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.599053+0000", 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.599051+0000", 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7329999999999992e-06 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.944 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.599060+0000", 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2632e-05 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.599073+0000", 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9709999999999998e-06 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.599080+0000", 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9276e-05 2023-10-09T10:28:55.945 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.599139+0000", 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0856000000000001e-05 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.599150+0000", 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.97434701499999998 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.946 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.573497+0000", 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1623000000000001e-05 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.573579+0000", 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103185 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.947 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.573682+0000", 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0025991260000000002 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.576281+0000", 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012672719000000001 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.949 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.588954+0000", 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0057135229999999999 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594667+0000", 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9453000000000001e-05 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594697+0000", 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1888000000000001e-05 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.950 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594758+0000", 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.99570522500000003 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174287 11.14 11:28c1adf7:::benchmark_data_smithi012_404402_object61504:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.711534+0000", 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 109.171718109, 2023-10-09T10:28:55.951 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88339943200000004, 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174287 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.711534+0000", 2023-10-09T10:28:55.952 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.711534+0000", 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.711532+0000", 2023-10-09T10:28:55.953 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.8480000000000007e-06 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.711541+0000", 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7864e-05 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.711559+0000", 2023-10-09T10:28:55.954 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9959999999999996e-06 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.711566+0000", 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.0469000000000001e-05 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.711616+0000", 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0910999999999999e-05 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.955 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.711627+0000", 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.86319767000000003 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.574825+0000", 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7911000000000007e-05 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.574903+0000", 2023-10-09T10:28:55.956 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5248999999999996e-05 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.574998+0000", 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00544634 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.580444+0000", 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085754019999999993 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.957 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.589020+0000", 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0058311539999999999 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594851+0000", 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.5967999999999999e-05 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594877+0000", 2023-10-09T10:28:55.958 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6842999999999998e-05 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594934+0000", 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.88339943200000004 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174409 11.14 11:2f189c1e:::benchmark_data_smithi012_404402_object61626:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.959 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.877150+0000", 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 109.006101903, 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.71795140300000004, 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174409 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.960 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.877150+0000", 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.877150+0000", 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.877149+0000", 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1300000000000003e-06 2023-10-09T10:28:55.961 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.877156+0000", 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6918e-05 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.877173+0000", 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5350000000000001e-06 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.962 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.877179+0000", 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4138999999999998e-05 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.877234+0000", 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1160000000000005e-06 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.877243+0000", 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.69906364399999998 2023-10-09T10:28:55.963 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.576306+0000", 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000122271 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.576429+0000", 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014346000000000001 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.964 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.576572+0000", 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0041026400000000003 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.580675+0000", 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013941370999999999 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594616+0000", 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036793900000000002 2023-10-09T10:28:55.965 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594984+0000", 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6524999999999999e-05 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.595030+0000", 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1261999999999996e-05 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.966 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.595102+0000", 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.71795140300000004 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174418 11.14 11:28d07a8b:::benchmark_data_smithi012_404402_object61635:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.893779+0000", 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.989473395, 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70153699899999999, 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.967 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174418 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.893779+0000", 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.968 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.893779+0000", 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.893777+0000", 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0831e-05 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.969 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.893788+0000", 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.6019e-05 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.893804+0000", 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5429999999999999e-06 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.893810+0000", 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001906952 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.970 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.895717+0000", 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5279999999999999e-05 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.895733+0000", 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.68245215800000003 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.971 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.578185+0000", 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000129047 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.578314+0000", 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015801899999999999 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.578472+0000", 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0026352419999999999 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.972 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.581107+0000", 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013700485 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594808+0000", 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00035610699999999998 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.973 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.595164+0000", 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.7837000000000003e-05 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.595212+0000", 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000104349 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.595316+0000", 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.70153699899999999 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.974 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174430 11.14 11:2ea095f9:::benchmark_data_smithi012_404402_object61647:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.910543+0000", 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.972708756, 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.68869227600000005, 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.975 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174430 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.910543+0000", 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.910543+0000", 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:55.976 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.910542+0000", 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4799999999999998e-06 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.910549+0000", 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3155e-05 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.977 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.910562+0000", 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3069999999999997e-06 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.910569+0000", 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8397999999999999e-05 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.910628+0000", 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0033e-05 2023-10-09T10:28:55.978 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.910638+0000", 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.67051048499999999 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.581148+0000", 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013152300000000001 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.979 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.581280+0000", 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000138658 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.581418+0000", 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0072659689999999997 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.588684+0000", 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010417164 2023-10-09T10:28:55.980 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.599101+0000", 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.3979000000000001e-05 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.599135+0000", 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5641e-05 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.981 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.599171+0000", 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.4720000000000004e-05 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.599236+0000", 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.68869227600000005 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.982 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174467 11.14 11:28d2a4ce:::benchmark_data_smithi012_404402_object61684:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:06.961721+0000", 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.92153082199999, 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.64188026799999998, 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174467 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.983 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.961721+0000", 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.961721+0000", 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.984 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.961720+0000", 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5769999999999994e-06 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.961728+0000", 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5540999999999999e-05 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.961744+0000", 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7109999999999998e-06 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.985 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.961751+0000", 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6937000000000001e-05 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.961798+0000", 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3880000000000006e-06 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.986 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:06.961807+0000", 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.62179014700000002 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.583597+0000", 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112988 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.583710+0000", 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012776500000000001 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.987 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.583838+0000", 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051241849999999999 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.588962+0000", 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.014185018000000001 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.988 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603147+0000", 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032258599999999998 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603470+0000", 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.2106999999999993e-05 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603532+0000", 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.9998000000000003e-05 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.989 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603602+0000", 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.64188026799999998 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174545 11.14 11:2cbc8c91:::benchmark_data_smithi012_404402_object61762:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:07.131340+0000", 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.75191178199999, 2023-10-09T10:28:55.990 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.47240815600000002, 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174545 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.131340+0000", 2023-10-09T10:28:55.991 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.131340+0000", 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.131339+0000", 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.272e-06 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.992 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.131346+0000", 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0778000000000001e-05 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.131357+0000", 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3499999999999999e-06 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.131364+0000", 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7197000000000001e-05 2023-10-09T10:28:55.993 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.131421+0000", 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5230000000000001e-06 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.131431+0000", 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.45473704700000001 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.994 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.586168+0000", 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012433 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.586292+0000", 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013177100000000001 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.586424+0000", 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051724010000000001 2023-10-09T10:28:55.995 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.591596+0000", 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011601027 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603197+0000", 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00044390100000000002 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.996 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603641+0000", 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2175999999999998e-05 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603673+0000", 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5135999999999993e-05 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603749+0000", 2023-10-09T10:28:55.997 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.47240815600000002 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174566 11.14 11:2b72153a:::benchmark_data_smithi012_404402_object61783:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:07.164520+0000", 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.71873177800001, 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.43934627199999998, 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:55.998 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174566 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.164520+0000", 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:55.999 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.164520+0000", 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.164519+0000", 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9300000000000003e-06 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.164527+0000", 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2323e-05 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.000 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.164539+0000", 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0230000000000004e-06 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.164546+0000", 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8366000000000003e-05 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.001 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.164605+0000", 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8320000000000001e-06 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.164614+0000", 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42443619599999999 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.589051+0000", 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4441e-05 2023-10-09T10:28:56.002 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.589135+0000", 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010022900000000001 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.589235+0000", 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0053909179999999998 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:56.003 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594626+0000", 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0086231469999999994 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603249+0000", 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00053304800000000005 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603782+0000", 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6055e-05 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.004 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603808+0000", 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8347999999999998e-05 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603867+0000", 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.43934627199999998 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.005 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174571 11.14 11:2f03299a:::benchmark_data_smithi012_404402_object61788:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:07.174761+0000", 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.708491159, 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42924498100000003, 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174571 2023-10-09T10:28:56.006 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.174761+0000", 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.174761+0000", 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.007 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.174760+0000", 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3099999999999997e-06 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.174766+0000", 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2719e-05 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.174779+0000", 2023-10-09T10:28:56.008 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1220000000000003e-06 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.174784+0000", 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.9524999999999997e-05 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.174843+0000", 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2955999999999999e-05 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.009 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.174856+0000", 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.41677286699999999 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.591629+0000", 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012692899999999999 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.591756+0000", 2023-10-09T10:28:56.010 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014778099999999999 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.591904+0000", 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029140630000000002 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.594818+0000", 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085208190000000007 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.011 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603339+0000", 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00055926700000000001 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603898+0000", 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7552999999999999e-05 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:56.012 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603926+0000", 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.0420000000000006e-05 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.604006+0000", 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.42924498100000003 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.013 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174582 11.14 11:2a0bfd03:::benchmark_data_smithi012_404402_object61799:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:07.199746+0000", 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.68350655499999, 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.40441645599999998, 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174582 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:56.014 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.199746+0000", 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.199746+0000", 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:56.015 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.199744+0000", 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0920000000000003e-06 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.199751+0000", 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2204e-05 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.199763+0000", 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1850000000000001e-06 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.016 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.199769+0000", 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5676000000000001e-05 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.199815+0000", 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.9899999999999997e-06 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:56.017 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.199823+0000", 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.39554033799999999 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.595363+0000", 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.4975999999999994e-05 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.595448+0000", 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9896999999999997e-05 2023-10-09T10:28:56.018 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.595538+0000", 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0035343869999999999 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.599072+0000", 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0043442480000000002 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.019 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.603416+0000", 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00063506700000000001 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.604051+0000", 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5052999999999998e-05 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.604087+0000", 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5610000000000003e-05 2023-10-09T10:28:56.020 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.604162+0000", 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.40441645599999998 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:174614 11.14 11:2b8b1d81:::benchmark_data_smithi012_404402_object61831:head [delete] snapc 0=[] ondisk+write+known_if_redirected e540)", 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:07.287906+0000", 2023-10-09T10:28:56.021 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 108.595345995, 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.32166062499999998, 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 174614 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:56.022 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.287906+0000", 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.287906+0000", 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.287905+0000", 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2440000000000001e-06 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.023 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.287910+0000", 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0424e-05 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.287921+0000", 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4410000000000003e-06 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:56.024 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.287925+0000", 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.2434e-05 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.287978+0000", 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.287989+0000", 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30946169600000001 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.025 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.597450+0000", 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010026699999999999 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.597551+0000", 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000101829 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:56.026 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.597653+0000", 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015910950000000001 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.599244+0000", 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0084433979999999995 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.607687+0000", 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0016726930000000001 2023-10-09T10:28:56.027 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.609360+0000", 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.1975000000000002e-05 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.609422+0000", 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014516600000000001 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.028 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:07.609567+0000", 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.32166062499999998 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:176180 11.4 11:22df2737:::benchmark_data_smithi012_404402_object63397:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:27:09.581788+0000", 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 106.301464168, 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.106535409, 2023-10-09T10:28:56.029 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 176180 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.581788+0000", 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:28:56.030 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.581788+0000", 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.581787+0000", 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.8469999999999998e-06 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.031 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.581791+0000", 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.2579999999999995e-06 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.581799+0000", 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7809999999999999e-06 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.581803+0000", 2023-10-09T10:28:56.032 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.8010000000000001e-05 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.581831+0000", 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1738999999999998e-05 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.581913+0000", 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024876899999999998 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.033 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.582161+0000", 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015361249999999999 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.583697+0000", 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046979699999999999 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.584167+0000", 2023-10-09T10:28:56.034 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.104080588 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.688248+0000", 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3856999999999997e-05 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.688292+0000", 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1794000000000001e-05 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:28:56.035 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:27:09.688323+0000", 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.106535409 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:28:56.036 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:28:56.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:56.038 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:56.038 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:56.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:56.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:55 smithi110 ceph-mon[136032]: pgmap v1599: 73 pgs: 73 active+clean; 582 KiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 146 KiB/s rd, 63 KiB/s wr, 37 op/s 2023-10-09T10:28:56.038 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:55 smithi110 ceph-mon[136032]: failed to encode map e564 with expected crc 2023-10-09T10:28:56.139 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:56.167 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:55 smithi012 ceph-mon[147926]: pgmap v1599: 73 pgs: 73 active+clean; 582 KiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 146 KiB/s rd, 63 KiB/s wr, 37 op/s 2023-10-09T10:28:56.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:55 smithi012 ceph-mon[147926]: failed to encode map e564 with expected crc 2023-10-09T10:28:56.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:55 smithi012 ceph-mon[152263]: pgmap v1599: 73 pgs: 73 active+clean; 582 KiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 146 KiB/s rd, 63 KiB/s wr, 37 op/s 2023-10-09T10:28:56.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:55 smithi012 ceph-mon[152263]: failed to encode map e564 with expected crc 2023-10-09T10:28:56.274 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:56.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:56.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:56.283 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:28:56.378 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:56.467 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:56.501 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:28:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:56.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:28:56.606 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:56.638 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:56.675 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:56.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:56.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:56.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:28:56.817 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:56.903 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format - testimg 2023-10-09T10:28:56.924 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:28:57.054 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:28:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:57.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:28:57.161 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:57.196 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:57.346 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:28:57.347 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:28:57.350 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export testimg /tmp/rbd_import_export_444239/img2 2023-10-09T10:28:57.391 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:57.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:57.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:57.500 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:28:57.529 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:57.619 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:28:57.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:57.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:28:57.639 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T10:28:57.643 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export testimg - 2023-10-09T10:28:57.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:28:57.781 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:57.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:57 smithi012 ceph-mon[147926]: pgmap v1600: 73 pgs: 73 active+clean; 582 KiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-10-09T10:28:57.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:57 smithi012 ceph-mon[152263]: pgmap v1600: 73 pgs: 73 active+clean; 582 KiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-10-09T10:28:57.981 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:28:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:57.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:28:58.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:28:58.121 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:58.141 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:28:58.142 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:28:58.142 INFO:teuthology.orchestra.run.smithi110.stdout: "num_ops": 0 2023-10-09T10:28:58.142 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:28:58.167 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:57 smithi110 ceph-mon[136032]: pgmap v1600: 73 pgs: 73 active+clean; 582 KiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-10-09T10:28:58.172 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:28:58.175 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:28:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:28:57] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.33.4" 2023-10-09T10:28:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:58.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:28:58.280 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:58.312 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:58.352 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:58.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:58.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:58.450 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T10:28:58.454 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm testimg 2023-10-09T10:28:58.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:58.488 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:58.553 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T10:28:58.556 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:28:58.556 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp /tmp/rbd_import_export_444239/img /tmp/rbd_import_export_444239/img2 2023-10-09T10:28:58.629 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:58.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:58.698 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:28:58.717 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:28:58.734 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:58.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:28:58.759 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:28:58.763 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:58.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/958099152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.3173423938122134}]: dispatch 2023-10-09T10:28:58.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/45458415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:58.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:58 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:58.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/418236704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:58.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/958099152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.3173423938122134}]: dispatch 2023-10-09T10:28:58.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/45458415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:58.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:58 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:58.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/418236704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:58.779 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.5 to 0.317342 (513d) 2023-10-09T10:28:58.786 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is set 2023-10-09T10:28:58.795 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:28:58.808 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:58.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:58.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:58.817 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:28:58.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:28:58.855 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:28:58.857 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:28:58.857 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:28:58.857 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:28:58.857 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:28:58.857 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:28:58.857 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:28:58.857 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:28:58.858 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:28:58.911 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:28:58.938 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:59.024 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:28:59.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/958099152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.3173423938122134}]: dispatch 2023-10-09T10:28:59.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/45458415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:59.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:58 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:28:59.026 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/418236704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:28:59.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:59.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:28:59.072 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:28:59.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:28:59.162 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:59.300 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp /tmp/rbd_import_export_444239/img /tmp/rbd_import_export_444239/img3 2023-10-09T10:28:59.664 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:28:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:59.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:28:59.768 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:28:59.808 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:28:59.847 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img /tmp/rbd_import_export_444239/img2 /tmp/rbd_import_export_444239/img3 2023-10-09T10:28:59.851 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:28:59.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:28:59.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:28:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: pgmap v1601: 73 pgs: 73 active+clean; 1.6 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 65 KiB/s rd, 26 KiB/s wr, 20 op/s 2023-10-09T10:28:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/958099152' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.3173423938122134}]': finished 2023-10-09T10:28:59.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[147926]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: pgmap v1601: 73 pgs: 73 active+clean; 1.6 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 65 KiB/s rd, 26 KiB/s wr, 20 op/s 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/958099152' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.3173423938122134}]': finished 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T10:28:59.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:59.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:28:59 smithi012 ceph-mon[152263]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T10:28:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: pgmap v1601: 73 pgs: 73 active+clean; 1.6 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 65 KiB/s rd, 26 KiB/s wr, 20 op/s 2023-10-09T10:28:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:28:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/958099152' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.3173423938122134}]': finished 2023-10-09T10:28:59.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: failed to encode map e565 with expected crc 2023-10-09T10:28:59.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:28:59.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:28:59.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:28:59 smithi110 ceph-mon[136032]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T10:28:59.956 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:28:59.984 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:00.069 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:00.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:00.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:00.146 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd help export 2023-10-09T10:29:00.146 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep -q export-format 2023-10-09T10:29:00.166 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_444239/img bs=1k count=10 seek=100 2023-10-09T10:29:00.166 INFO:tasks.workunit.client.0.smithi012.stderr:10+0 records in 2023-10-09T10:29:00.166 INFO:tasks.workunit.client.0.smithi012.stderr:10+0 records out 2023-10-09T10:29:00.166 INFO:tasks.workunit.client.0.smithi012.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.1016e-05 s, 144 MB/s 2023-10-09T10:29:00.167 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format /tmp/rbd_import_export_444239/img testimg 2023-10-09T10:29:00.174 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:00.204 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:00.218 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:29:00.219 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:29:00.221 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap create testimg@snap 2023-10-09T10:29:00.246 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:00.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:29:00.351 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:00.379 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:00.616 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:00.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:00.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:29:00.723 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:29:00.755 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:00.795 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:29:00.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:00.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:29:00.800 INFO:tasks.workunit.client.0.smithi012.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T10:29:00.808 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T10:29:00.857 INFO:tasks.workunit.client.0.smithi012.stderr:++ rbd image-meta list testimg 2023-10-09T10:29:00.896 INFO:tasks.workunit.client.0.smithi012.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-09T10:29:00.896 INFO:tasks.workunit.client.0.smithi012.stderr: 2023-10-09T10:29:00.897 INFO:tasks.workunit.client.0.smithi012.stderr:Key Value 2023-10-09T10:29:00.897 INFO:tasks.workunit.client.0.smithi012.stderr:key1 value1' 2023-10-09T10:29:00.897 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_444239/img_v2 2023-10-09T10:29:00.899 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:00.925 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:00 smithi110 ceph-mon[136032]: failed to encode map e566 with expected crc 2023-10-09T10:29:00.925 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:00 smithi110 ceph-mon[136032]: failed to encode map e566 with expected crc 2023-10-09T10:29:00.925 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:00 smithi110 ceph-mon[136032]: failed to encode map e566 with expected crc 2023-10-09T10:29:00.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:00 smithi110 ceph-mon[136032]: failed to encode map e566 with expected crc 2023-10-09T10:29:00.926 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:00 smithi110 ceph-mon[136032]: pgmap v1604: 73 pgs: 1 peering, 72 active+clean; 1.6 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 4.7 KiB/s rd, 15 KiB/s wr, 7 op/s 2023-10-09T10:29:00.932 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:00.936 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:00.939 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_444239/img_v2 testimg_import 2023-10-09T10:29:00.962 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:00.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:00.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:01.066 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:29:01.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[147926]: failed to encode map e566 with expected crc 2023-10-09T10:29:01.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[147926]: failed to encode map e566 with expected crc 2023-10-09T10:29:01.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[147926]: failed to encode map e566 with expected crc 2023-10-09T10:29:01.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[147926]: failed to encode map e566 with expected crc 2023-10-09T10:29:01.092 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[147926]: pgmap v1604: 73 pgs: 1 peering, 72 active+clean; 1.6 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 4.7 KiB/s rd, 15 KiB/s wr, 7 op/s 2023-10-09T10:29:01.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[152263]: failed to encode map e566 with expected crc 2023-10-09T10:29:01.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[152263]: failed to encode map e566 with expected crc 2023-10-09T10:29:01.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[152263]: failed to encode map e566 with expected crc 2023-10-09T10:29:01.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[152263]: failed to encode map e566 with expected crc 2023-10-09T10:29:01.093 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:00 smithi012 ceph-mon[152263]: pgmap v1604: 73 pgs: 1 peering, 72 active+clean; 1.6 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 4.7 KiB/s rd, 15 KiB/s wr, 7 op/s 2023-10-09T10:29:01.099 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:01.128 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:29:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:01.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:29:01.232 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:01.269 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:01.301 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:01.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:01.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:01.327 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:29:01.405 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:01.408 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:29:01.437 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:01.524 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:01.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:01.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:01.628 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:01.657 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:01.744 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:01.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:01.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:01.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:01 smithi012 ceph-mon[147926]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T10:29:01.784 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:01 smithi012 ceph-mon[147926]: failed to encode map e567 with expected crc 2023-10-09T10:29:01.810 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T10:29:01.812 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd info testimg_import 2023-10-09T10:29:01.830 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:29:01.846 INFO:tasks.workunit.client.0.smithi012.stdout:rbd image 'testimg_import': 2023-10-09T10:29:01.846 INFO:tasks.workunit.client.0.smithi012.stdout: size 110 KiB in 1 objects 2023-10-09T10:29:01.846 INFO:tasks.workunit.client.0.smithi012.stdout: order 22 (4 MiB objects) 2023-10-09T10:29:01.847 INFO:tasks.workunit.client.0.smithi012.stdout: snapshot_count: 1 2023-10-09T10:29:01.847 INFO:tasks.workunit.client.0.smithi012.stdout: id: 922b4d0e3c1f 2023-10-09T10:29:01.847 INFO:tasks.workunit.client.0.smithi012.stdout: block_name_prefix: rbd_data.922b4d0e3c1f 2023-10-09T10:29:01.847 INFO:tasks.workunit.client.0.smithi012.stdout: format: 2 2023-10-09T10:29:01.847 INFO:tasks.workunit.client.0.smithi012.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T10:29:01.847 INFO:tasks.workunit.client.0.smithi012.stdout: op_features: 2023-10-09T10:29:01.847 INFO:tasks.workunit.client.0.smithi012.stdout: flags: 2023-10-09T10:29:01.848 INFO:tasks.workunit.client.0.smithi012.stdout: create_timestamp: Mon Oct 9 10:29:00 2023 2023-10-09T10:29:01.848 INFO:tasks.workunit.client.0.smithi012.stdout: access_timestamp: Mon Oct 9 10:29:00 2023 2023-10-09T10:29:01.848 INFO:tasks.workunit.client.0.smithi012.stdout: modify_timestamp: Mon Oct 9 10:29:00 2023 2023-10-09T10:29:01.849 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:01.850 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd info testimg_import@snap 2023-10-09T10:29:01.879 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:01.885 INFO:tasks.workunit.client.0.smithi012.stdout:rbd image 'testimg_import': 2023-10-09T10:29:01.885 INFO:tasks.workunit.client.0.smithi012.stdout: size 110 KiB in 1 objects 2023-10-09T10:29:01.885 INFO:tasks.workunit.client.0.smithi012.stdout: order 22 (4 MiB objects) 2023-10-09T10:29:01.885 INFO:tasks.workunit.client.0.smithi012.stdout: snapshot_count: 1 2023-10-09T10:29:01.885 INFO:tasks.workunit.client.0.smithi012.stdout: id: 922b4d0e3c1f 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: block_name_prefix: rbd_data.922b4d0e3c1f 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: format: 2 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: op_features: 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: flags: 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: create_timestamp: Mon Oct 9 10:29:00 2023 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: access_timestamp: Mon Oct 9 10:29:00 2023 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: modify_timestamp: Mon Oct 9 10:29:00 2023 2023-10-09T10:29:01.886 INFO:tasks.workunit.client.0.smithi012.stdout: protected: False 2023-10-09T10:29:01.889 INFO:tasks.workunit.client.0.smithi012.stderr:++ rbd image-meta list testimg_import 2023-10-09T10:29:01.945 INFO:tasks.workunit.client.0.smithi012.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-09T10:29:01.945 INFO:tasks.workunit.client.0.smithi012.stderr: 2023-10-09T10:29:01.945 INFO:tasks.workunit.client.0.smithi012.stderr:Key Value 2023-10-09T10:29:01.945 INFO:tasks.workunit.client.0.smithi012.stderr:key1 value1' 2023-10-09T10:29:01.945 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-09T10:29:01.945 INFO:tasks.workunit.client.0.smithi012.stderr: 2023-10-09T10:29:01.945 INFO:tasks.workunit.client.0.smithi012.stderr:Key Value 2023-10-09T10:29:01.946 INFO:tasks.workunit.client.0.smithi012.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-09T10:29:01.946 INFO:tasks.workunit.client.0.smithi012.stderr: 2023-10-09T10:29:01.946 INFO:tasks.workunit.client.0.smithi012.stderr:Key Value 2023-10-09T10:29:01.946 INFO:tasks.workunit.client.0.smithi012.stderr:key1 value1' ']' 2023-10-09T10:29:01.946 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export testimg_import /tmp/rbd_import_export_444239/img_import 2023-10-09T10:29:01.980 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 100% complete...done. 2023-10-09T10:29:01.983 INFO:tasks.workunit.client.0.smithi012.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_444239/img /tmp/rbd_import_export_444239/img_import 2023-10-09T10:29:01.984 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp -l /tmp/rbd_import_export_444239/img /tmp/rbd_import_export_444239/img_import 2023-10-09T10:29:01.984 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/img --format %b 2023-10-09T10:29:01.985 INFO:tasks.workunit.client.0.smithi012.stderr:+ origsize=24 2023-10-09T10:29:01.986 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/img_import --format %b 2023-10-09T10:29:01.986 INFO:tasks.workunit.client.0.smithi012.stderr:+ exportsize=224 2023-10-09T10:29:01.987 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=200 2023-10-09T10:29:01.987 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=200 2023-10-09T10:29:01.987 INFO:tasks.workunit.client.0.smithi012.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T10:29:01.987 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_444239/img_snap 2023-10-09T10:29:02.021 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 100% complete...done. 2023-10-09T10:29:02.024 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_444239/img_snap_import 2023-10-09T10:29:02.036 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:02.053 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:02.055 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:01 smithi110 ceph-mon[136032]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T10:29:02.056 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:01 smithi110 ceph-mon[136032]: failed to encode map e567 with expected crc 2023-10-09T10:29:02.056 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:01 smithi110 ceph-mon[136032]: failed to encode map e567 with expected crc 2023-10-09T10:29:02.056 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:01 smithi110 ceph-mon[136032]: failed to encode map e567 with expected crc 2023-10-09T10:29:02.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:02.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:02.067 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 100% complete...done. 2023-10-09T10:29:02.076 INFO:tasks.workunit.client.0.smithi012.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_444239/img_snap /tmp/rbd_import_export_444239/img_snap_import 2023-10-09T10:29:02.076 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp -l /tmp/rbd_import_export_444239/img_snap /tmp/rbd_import_export_444239/img_snap_import 2023-10-09T10:29:02.077 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/img_snap --format %b 2023-10-09T10:29:02.078 INFO:tasks.workunit.client.0.smithi012.stderr:+ origsize=224 2023-10-09T10:29:02.078 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/img_snap_import --format %b 2023-10-09T10:29:02.079 INFO:tasks.workunit.client.0.smithi012.stderr:+ exportsize=224 2023-10-09T10:29:02.079 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:02.079 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:02.079 INFO:tasks.workunit.client.0.smithi012.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T10:29:02.079 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img_v2 2023-10-09T10:29:02.080 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img_import 2023-10-09T10:29:02.081 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img_snap 2023-10-09T10:29:02.082 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img_snap_import 2023-10-09T10:29:02.082 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T10:29:02.159 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:01 smithi012 ceph-mon[147926]: failed to encode map e567 with expected crc 2023-10-09T10:29:02.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:01 smithi012 ceph-mon[147926]: failed to encode map e567 with expected crc 2023-10-09T10:29:02.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:01 smithi012 ceph-mon[152263]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T10:29:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:01 smithi012 ceph-mon[152263]: failed to encode map e567 with expected crc 2023-10-09T10:29:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:01 smithi012 ceph-mon[152263]: failed to encode map e567 with expected crc 2023-10-09T10:29:02.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:01 smithi012 ceph-mon[152263]: failed to encode map e567 with expected crc 2023-10-09T10:29:02.194 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:02.230 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:02.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:02.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:02.333 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:29:02.363 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:02.482 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:29:02.482 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:29:02.505 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:29:02.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:02.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:29:02.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:02.644 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:02.647 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:02.766 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:02.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:02.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:02.807 INFO:tasks.workunit.client.0.smithi012.stderr: Removing snap: 100% complete...done. 2023-10-09T10:29:02.820 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg_import 2023-10-09T10:29:02.842 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:02 smithi110 ceph-mon[136032]: failed to encode map e568 with expected crc 2023-10-09T10:29:02.842 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:02 smithi110 ceph-mon[136032]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T10:29:02.842 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:02 smithi110 ceph-mon[136032]: failed to encode map e568 with expected crc 2023-10-09T10:29:02.842 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:02 smithi110 ceph-mon[136032]: failed to encode map e568 with expected crc 2023-10-09T10:29:02.842 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:02 smithi110 ceph-mon[136032]: failed to encode map e568 with expected crc 2023-10-09T10:29:02.843 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:02 smithi110 ceph-mon[136032]: pgmap v1607: 73 pgs: 1 peering, 72 active+clean; 1.6 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 20 KiB/s rd, 6.2 KiB/s wr, 34 op/s 2023-10-09T10:29:02.846 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:29:02.872 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:29:02.922 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:02.926 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap rm testimg@snap 2023-10-09T10:29:02.931 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:03.044 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[147926]: failed to encode map e568 with expected crc 2023-10-09T10:29:03.044 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[147926]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T10:29:03.044 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[147926]: failed to encode map e568 with expected crc 2023-10-09T10:29:03.044 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[147926]: failed to encode map e568 with expected crc 2023-10-09T10:29:03.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[147926]: failed to encode map e568 with expected crc 2023-10-09T10:29:03.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[147926]: pgmap v1607: 73 pgs: 1 peering, 72 active+clean; 1.6 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 20 KiB/s rd, 6.2 KiB/s wr, 34 op/s 2023-10-09T10:29:03.045 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[152263]: failed to encode map e568 with expected crc 2023-10-09T10:29:03.045 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[152263]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T10:29:03.045 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[152263]: failed to encode map e568 with expected crc 2023-10-09T10:29:03.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[152263]: failed to encode map e568 with expected crc 2023-10-09T10:29:03.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[152263]: failed to encode map e568 with expected crc 2023-10-09T10:29:03.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:02 smithi012 ceph-mon[152263]: pgmap v1607: 73 pgs: 1 peering, 72 active+clean; 1.6 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 20 KiB/s rd, 6.2 KiB/s wr, 34 op/s 2023-10-09T10:29:03.172 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:29:03.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:03.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:29:03.277 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:03.305 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:03.338 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:03.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:03.442 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:03.468 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:03.500 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:03.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:03.604 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:03.632 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:03.663 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:03.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:03.767 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:29:03.801 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:03.810 INFO:tasks.workunit.client.0.smithi012.stderr: Removing snap: 100% complete...done. 2023-10-09T10:29:03.818 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm testimg 2023-10-09T10:29:03.892 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:29:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:03.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:29:03.920 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:03.924 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --order 20 /tmp/rbd_import_export_444239/img testimg 2023-10-09T10:29:03.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:03 smithi110 ceph-mon[136032]: failed to encode map e569 with expected crc 2023-10-09T10:29:03.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:03 smithi110 ceph-mon[136032]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T10:29:03.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:03 smithi110 ceph-mon[136032]: failed to encode map e569 with expected crc 2023-10-09T10:29:03.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:03 smithi110 ceph-mon[136032]: failed to encode map e569 with expected crc 2023-10-09T10:29:03.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:03 smithi110 ceph-mon[136032]: failed to encode map e569 with expected crc 2023-10-09T10:29:03.979 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:29:03.980 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T10:29:03.982 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_444239/img_v2 2023-10-09T10:29:03.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:29:04.023 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:04.025 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:29:04.026 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_444239/img_v2 testimg_import 2023-10-09T10:29:04.032 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:04.048 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:29:04.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:29:04.085 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:29:04.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[147926]: failed to encode map e569 with expected crc 2023-10-09T10:29:04.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[147926]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T10:29:04.089 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[147926]: failed to encode map e569 with expected crc 2023-10-09T10:29:04.089 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[147926]: failed to encode map e569 with expected crc 2023-10-09T10:29:04.089 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[147926]: failed to encode map e569 with expected crc 2023-10-09T10:29:04.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[152263]: failed to encode map e569 with expected crc 2023-10-09T10:29:04.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[152263]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T10:29:04.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[152263]: failed to encode map e569 with expected crc 2023-10-09T10:29:04.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[152263]: failed to encode map e569 with expected crc 2023-10-09T10:29:04.090 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:03 smithi012 ceph-mon[152263]: failed to encode map e569 with expected crc 2023-10-09T10:29:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:04.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:29:04.100 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:29:04.105 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T10:29:04.108 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd info testimg_import 2023-10-09T10:29:04.109 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep order 2023-10-09T10:29:04.109 INFO:tasks.workunit.client.0.smithi012.stderr:+ awk '{print $2}' 2023-10-09T10:29:04.109 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep 20 2023-10-09T10:29:04.120 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:29:04.140 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:29:04.147 INFO:tasks.workunit.client.0.smithi012.stdout:20 2023-10-09T10:29:04.147 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img_v2 2023-10-09T10:29:04.148 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg_import 2023-10-09T10:29:04.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:29:04.165 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:04.181 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:29:04.183 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:29:04.183 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:29:04.183 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:29:04.183 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:29:04.183 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:29:04.183 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:29:04.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:29:04.184 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:29:04.191 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:04.218 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:04.254 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:04.256 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:04.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:04.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:04.259 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg 2023-10-09T10:29:04.351 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:04.355 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_444239/img testimg 2023-10-09T10:29:04.358 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:04.384 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:04.413 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:29:04.416 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:04.417 INFO:tasks.workunit.client.0.smithi012.stderr:++ rbd info testimg 2023-10-09T10:29:04.417 INFO:tasks.workunit.client.0.smithi012.stderr:++ grep features 2023-10-09T10:29:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:04.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:04.454 INFO:tasks.workunit.client.0.smithi012.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-09T10:29:04.454 INFO:tasks.workunit.client.0.smithi012.stderr: op_features: ' 2023-10-09T10:29:04.454 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_444239/img_v2 2023-10-09T10:29:04.490 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:04.493 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_444239/img_v2 testimg_import 2023-10-09T10:29:04.520 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:04.546 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T10:29:04.549 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:04.550 INFO:tasks.workunit.client.0.smithi012.stderr:++ rbd info testimg_import 2023-10-09T10:29:04.550 INFO:tasks.workunit.client.0.smithi012.stderr:++ grep features 2023-10-09T10:29:04.580 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:04.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:04.586 INFO:tasks.workunit.client.0.smithi012.stderr:+ FEATURES_AFTER=' features: layering 2023-10-09T10:29:04.586 INFO:tasks.workunit.client.0.smithi012.stderr: op_features: ' 2023-10-09T10:29:04.586 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' ' features: layering 2023-10-09T10:29:04.586 INFO:tasks.workunit.client.0.smithi012.stderr: op_features: ' '!=' ' features: layering 2023-10-09T10:29:04.586 INFO:tasks.workunit.client.0.smithi012.stderr: op_features: ' ']' 2023-10-09T10:29:04.586 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img_v2 2023-10-09T10:29:04.587 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg_import 2023-10-09T10:29:04.657 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:04.662 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg 2023-10-09T10:29:04.684 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:29:04.713 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4, 6] out_osds: [1, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:29:04.714 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:29:04.714 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:29:04.714 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-10-09T10:29:04.714 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0 2023-10-09T10:29:04.718 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:04.742 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:04.747 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_444239/img testimg 2023-10-09T10:29:04.817 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is set 2023-10-09T10:29:04.825 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:29:04.829 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_444239/img_v2 2023-10-09T10:29:04.874 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:04.879 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_444239/img_v2 testimg_import 2023-10-09T10:29:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[147926]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T10:29:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[147926]: failed to encode map e570 with expected crc 2023-10-09T10:29:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[147926]: failed to encode map e570 with expected crc 2023-10-09T10:29:04.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[147926]: failed to encode map e570 with expected crc 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[147926]: pgmap v1610: 73 pgs: 1 peering, 72 active+clean; 692 KiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 420 KiB/s rd, 118 KiB/s wr, 242 op/s; 108 B/s, 2 objects/s recovering 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4022728832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[147926]: failed to encode map e570 with expected crc 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3562748521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[152263]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[152263]: failed to encode map e570 with expected crc 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[152263]: failed to encode map e570 with expected crc 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[152263]: failed to encode map e570 with expected crc 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[152263]: pgmap v1610: 73 pgs: 1 peering, 72 active+clean; 692 KiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 420 KiB/s rd, 118 KiB/s wr, 242 op/s; 108 B/s, 2 objects/s recovering 2023-10-09T10:29:04.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4022728832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:04.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[152263]: failed to encode map e570 with expected crc 2023-10-09T10:29:04.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:04 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3562748521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:04.946 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T10:29:04.950 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd info testimg_import 2023-10-09T10:29:04.950 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep 'stripe unit' 2023-10-09T10:29:04.950 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-09T10:29:04.965 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:29:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:04.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:29:04.992 INFO:tasks.workunit.client.0.smithi012.stdout: stripe unit: 4 KiB 2023-10-09T10:29:04.993 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd info testimg_import 2023-10-09T10:29:04.993 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep 'stripe count' 2023-10-09T10:29:04.993 INFO:tasks.workunit.client.0.smithi012.stderr:+ awk '{print $3}' 2023-10-09T10:29:04.993 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep 1000 2023-10-09T10:29:05.028 INFO:tasks.workunit.client.0.smithi012.stdout:1000 2023-10-09T10:29:05.029 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/img_v2 2023-10-09T10:29:05.029 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg_import 2023-10-09T10:29:05.070 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:29:05.101 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:05.122 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T10:29:05.125 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg 2023-10-09T10:29:05.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:04 smithi110 ceph-mon[136032]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T10:29:05.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:04 smithi110 ceph-mon[136032]: failed to encode map e570 with expected crc 2023-10-09T10:29:05.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:04 smithi110 ceph-mon[136032]: failed to encode map e570 with expected crc 2023-10-09T10:29:05.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:04 smithi110 ceph-mon[136032]: failed to encode map e570 with expected crc 2023-10-09T10:29:05.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:04 smithi110 ceph-mon[136032]: pgmap v1610: 73 pgs: 1 peering, 72 active+clean; 692 KiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 420 KiB/s rd, 118 KiB/s wr, 242 op/s; 108 B/s, 2 objects/s recovering 2023-10-09T10:29:05.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4022728832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:05.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:04 smithi110 ceph-mon[136032]: failed to encode map e570 with expected crc 2023-10-09T10:29:05.194 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:04 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3562748521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:05.210 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:29:05.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:05.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:29:05.223 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T10:29:05.226 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_444239/img testimg 2023-10-09T10:29:05.280 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:29:05.283 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap create testimg@snap1 2023-10-09T10:29:05.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:29:05.347 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:05.437 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:29:05.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:05.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:29:05.542 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:29:05.571 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:05.663 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:29:05.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:05.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:29:05.768 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:05.795 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:05.824 INFO:tasks.workunit.client.0.smithi012.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T10:29:05.831 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap create testimg@snap2 2023-10-09T10:29:05.831 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:05.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:05.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:05.887 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:05.888 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:29:05.889 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:29:05.889 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:29:05.889 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:29:05.935 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:05.962 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:05.993 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:05.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:06.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[147926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:29:06.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[147926]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[147926]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3562748521' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:06.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[147926]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T10:29:06.068 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[147926]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.069 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[152263]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:29:06.069 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[152263]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.069 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[152263]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.069 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3562748521' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:06.069 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[152263]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T10:29:06.069 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:05 smithi012 ceph-mon[152263]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.097 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:06.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:05 smithi110 ceph-mon[136032]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:29:06.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:05 smithi110 ceph-mon[136032]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:05 smithi110 ceph-mon[136032]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:05 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3562748521' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:06.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:05 smithi110 ceph-mon[136032]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T10:29:06.120 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:05 smithi110 ceph-mon[136032]: failed to encode map e571 with expected crc 2023-10-09T10:29:06.122 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:06.155 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:06.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:06.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:06.259 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:06.285 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:06.317 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:06.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:06.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:06.400 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:29:06.421 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:06.447 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:06.478 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:06.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:06.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:06.582 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:06.640 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:06.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:06.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:06.744 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:06.770 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:06.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:29:06.802 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:06.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:06.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:06.838 INFO:tasks.workunit.client.0.smithi012.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T10:29:06.845 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap protect testimg@snap2 2023-10-09T10:29:06.887 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_444239/snap_protect 2023-10-09T10:29:06.905 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:29:06.929 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:06.933 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_444239/snap_protect testimg_import 2023-10-09T10:29:06.936 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:06.980 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:29:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:06.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:29:06.985 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[147926]: failed to encode map e572 with expected crc 2023-10-09T10:29:06.985 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[147926]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T10:29:06.985 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[147926]: failed to encode map e572 with expected crc 2023-10-09T10:29:06.985 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[147926]: failed to encode map e572 with expected crc 2023-10-09T10:29:06.985 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[147926]: pgmap v1613: 73 pgs: 73 active+clean; 694 KiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 537 KiB/s rd, 128 KiB/s wr, 365 op/s; 108 B/s, 2 objects/s recovering 2023-10-09T10:29:06.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[152263]: failed to encode map e572 with expected crc 2023-10-09T10:29:06.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[152263]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T10:29:06.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[152263]: failed to encode map e572 with expected crc 2023-10-09T10:29:06.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[152263]: failed to encode map e572 with expected crc 2023-10-09T10:29:06.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:06 smithi012 ceph-mon[152263]: pgmap v1613: 73 pgs: 73 active+clean; 694 KiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 537 KiB/s rd, 128 KiB/s wr, 365 op/s; 108 B/s, 2 objects/s recovering 2023-10-09T10:29:07.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:29:07.115 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:06 smithi110 ceph-mon[136032]: failed to encode map e572 with expected crc 2023-10-09T10:29:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:06 smithi110 ceph-mon[136032]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T10:29:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:06 smithi110 ceph-mon[136032]: failed to encode map e572 with expected crc 2023-10-09T10:29:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:06 smithi110 ceph-mon[136032]: failed to encode map e572 with expected crc 2023-10-09T10:29:07.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:06 smithi110 ceph-mon[136032]: pgmap v1613: 73 pgs: 73 active+clean; 694 KiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 537 KiB/s rd, 128 KiB/s wr, 365 op/s; 108 B/s, 2 objects/s recovering 2023-10-09T10:29:07.209 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:07.221 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:29:07.250 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:29:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:07.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:29:07.358 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:07.386 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:07.419 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:07.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:07.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:07.523 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:07.549 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:07.582 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:07.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:07.686 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:29:07.716 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:07.728 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:07.753 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:29:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:07.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:29:07.839 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.5 primary-affinity to 0 (802) 2023-10-09T10:29:07.860 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:29:07.889 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:08.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:29:07] "GET /metrics HTTP/1.1" 200 34239 "" "Prometheus/2.33.4" 2023-10-09T10:29:08.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[147926]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[147926]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T10:29:08.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[147926]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[147926]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/343403493' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T10:29:08.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[152263]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[152263]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T10:29:08.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[152263]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[152263]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:07 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/343403493' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T10:29:08.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:07 smithi110 ceph-mon[136032]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:07 smithi110 ceph-mon[136032]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T10:29:08.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:07 smithi110 ceph-mon[136032]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:07 smithi110 ceph-mon[136032]: failed to encode map e573 with expected crc 2023-10-09T10:29:08.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:07 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/343403493' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T10:29:08.337 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:29:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:08.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:29:08.442 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:29:08.516 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:08.670 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:08.766 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:29:08.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:08.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:29:08.871 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:08.898 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:08.931 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:08.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:08.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:08.967 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:29:08.968 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:29:08.968 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:29:08.968 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:29:08.968 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:29:08.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[147926]: failed to encode map e574 with expected crc 2023-10-09T10:29:08.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[147926]: failed to encode map e574 with expected crc 2023-10-09T10:29:08.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/343403493' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T10:29:08.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[147926]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T10:29:08.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[147926]: pgmap v1616: 73 pgs: 73 active+clean; 584 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 337 KiB/s rd, 34 KiB/s wr, 428 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:29:08.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[152263]: failed to encode map e574 with expected crc 2023-10-09T10:29:08.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[152263]: failed to encode map e574 with expected crc 2023-10-09T10:29:08.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/343403493' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T10:29:08.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[152263]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T10:29:08.982 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:08 smithi012 ceph-mon[152263]: pgmap v1616: 73 pgs: 73 active+clean; 584 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 337 KiB/s rd, 34 KiB/s wr, 428 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:29:09.035 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:29:09.082 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:09.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:08 smithi110 ceph-mon[136032]: failed to encode map e574 with expected crc 2023-10-09T10:29:09.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:08 smithi110 ceph-mon[136032]: failed to encode map e574 with expected crc 2023-10-09T10:29:09.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:08 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/343403493' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T10:29:09.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:08 smithi110 ceph-mon[136032]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T10:29:09.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:08 smithi110 ceph-mon[136032]: pgmap v1616: 73 pgs: 73 active+clean; 584 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 337 KiB/s rd, 34 KiB/s wr, 428 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:29:09.857 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T10:29:09.861 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd info testimg_import@snap1 2023-10-09T10:29:09.861 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep 'protected: False' 2023-10-09T10:29:09.897 INFO:tasks.workunit.client.0.smithi012.stdout: protected: False 2023-10-09T10:29:09.898 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd info testimg_import@snap2 2023-10-09T10:29:09.898 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep 'protected: True' 2023-10-09T10:29:09.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:09 smithi012 ceph-mon[147926]: failed to encode map e575 with expected crc 2023-10-09T10:29:09.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:09 smithi012 ceph-mon[147926]: failed to encode map e575 with expected crc 2023-10-09T10:29:09.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:09 smithi012 ceph-mon[147926]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T10:29:09.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:09 smithi012 ceph-mon[147926]: failed to encode map e575 with expected crc 2023-10-09T10:29:09.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:09 smithi012 ceph-mon[152263]: failed to encode map e575 with expected crc 2023-10-09T10:29:09.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:09 smithi012 ceph-mon[152263]: failed to encode map e575 with expected crc 2023-10-09T10:29:09.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:09 smithi012 ceph-mon[152263]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T10:29:09.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:09 smithi012 ceph-mon[152263]: failed to encode map e575 with expected crc 2023-10-09T10:29:09.934 INFO:tasks.workunit.client.0.smithi012.stdout: protected: True 2023-10-09T10:29:09.935 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/snap_protect 2023-10-09T10:29:09.935 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T10:29:09.951 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:29:09.970 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:29:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:09.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:29:09.986 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T10:29:10.037 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap purge testimg 2023-10-09T10:29:10.077 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:10.105 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:09 smithi110 ceph-mon[136032]: failed to encode map e575 with expected crc 2023-10-09T10:29:10.105 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:09 smithi110 ceph-mon[136032]: failed to encode map e575 with expected crc 2023-10-09T10:29:10.105 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:09 smithi110 ceph-mon[136032]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T10:29:10.105 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:09 smithi110 ceph-mon[136032]: failed to encode map e575 with expected crc 2023-10-09T10:29:10.108 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:10.143 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:10.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:10.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:10.247 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:10.275 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:10.309 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:10.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:10.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:10.413 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:29:10.444 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:10.485 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:29:10.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:10.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:29:10.488 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:29:10.509 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:29:10.529 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:29:10.548 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:29:10.566 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:29:10.584 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:29:10.589 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:10.602 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:29:10.615 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:10.620 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:29:10.622 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:29:10.622 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:29:10.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:29:10.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:29:10.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:29:10.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:29:10.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:29:10.623 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:29:10.648 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:10.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:10.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:10.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:29:10.785 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:10 smithi110 ceph-mon[136032]: failed to encode map e575 with expected crc 2023-10-09T10:29:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:10 smithi110 ceph-mon[136032]: failed to encode map e576 with expected crc 2023-10-09T10:29:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:10 smithi110 ceph-mon[136032]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T10:29:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:10 smithi110 ceph-mon[136032]: failed to encode map e576 with expected crc 2023-10-09T10:29:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:10 smithi110 ceph-mon[136032]: failed to encode map e576 with expected crc 2023-10-09T10:29:10.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:10 smithi110 ceph-mon[136032]: pgmap v1619: 73 pgs: 1 peering, 72 active+clean; 694 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 270 KiB/s rd, 32 KiB/s wr, 374 op/s 2023-10-09T10:29:10.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:10 smithi110 ceph-mon[136032]: failed to encode map e576 with expected crc 2023-10-09T10:29:10.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3573033204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:10.969 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:11.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[147926]: failed to encode map e575 with expected crc 2023-10-09T10:29:11.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[147926]: failed to encode map e576 with expected crc 2023-10-09T10:29:11.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[147926]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T10:29:11.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[147926]: failed to encode map e576 with expected crc 2023-10-09T10:29:11.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[147926]: failed to encode map e576 with expected crc 2023-10-09T10:29:11.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[147926]: pgmap v1619: 73 pgs: 1 peering, 72 active+clean; 694 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 270 KiB/s rd, 32 KiB/s wr, 374 op/s 2023-10-09T10:29:11.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[147926]: failed to encode map e576 with expected crc 2023-10-09T10:29:11.027 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3573033204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:11.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[152263]: failed to encode map e575 with expected crc 2023-10-09T10:29:11.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[152263]: failed to encode map e576 with expected crc 2023-10-09T10:29:11.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[152263]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T10:29:11.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[152263]: failed to encode map e576 with expected crc 2023-10-09T10:29:11.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[152263]: failed to encode map e576 with expected crc 2023-10-09T10:29:11.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[152263]: pgmap v1619: 73 pgs: 1 peering, 72 active+clean; 694 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 270 KiB/s rd, 32 KiB/s wr, 374 op/s 2023-10-09T10:29:11.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[152263]: failed to encode map e576 with expected crc 2023-10-09T10:29:11.028 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3573033204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:11.032 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:29:11.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:11.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:29:11.137 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:11.163 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:11.195 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:11.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:11.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:11.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:29:11.328 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:11.413 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:29:11.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:11.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:29:11.517 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:11.544 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:11.578 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:11.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:11.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:11.681 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:11.708 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:11.740 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:11.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:11.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:11.844 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:11.859 INFO:tasks.workunit.client.0.smithi012.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T10:29:11.859 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:29:11.869 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd snap purge testimg_import 2023-10-09T10:29:11.871 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:11.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:11.906 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:11.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:12.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:29:12.042 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:12.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:12.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[147926]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T10:29:12.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[147926]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[147926]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[147926]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1345770686' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:12.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[147926]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:12.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[152263]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T10:29:12.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[152263]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[152263]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[152263]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1345770686' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:12.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:11 smithi012 ceph-mon[152263]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:11 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:12.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:11 smithi110 ceph-mon[136032]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T10:29:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:11 smithi110 ceph-mon[136032]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:11 smithi110 ceph-mon[136032]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:11 smithi110 ceph-mon[136032]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:11 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1345770686' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:12.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:11 smithi110 ceph-mon[136032]: failed to encode map e577 with expected crc 2023-10-09T10:29:12.213 INFO:teuthology.orchestra.run.smithi012.stdout:ERROR: (22) Invalid argument 2023-10-09T10:29:12.213 INFO:teuthology.orchestra.run.smithi012.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:29:12.649 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:29:12.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:12.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:29:12.689 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:29:12.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:12.785 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:12.820 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:12.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:12.925 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[147926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:29:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1345770686' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:29:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[147926]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T10:29:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[147926]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[147926]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[147926]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[147926]: pgmap v1622: 73 pgs: 1 peering, 72 active+clean; 694 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 96 KiB/s rd, 14 KiB/s wr, 133 op/s 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[152263]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1345770686' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[152263]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[152263]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[152263]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[152263]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:12 smithi012 ceph-mon[152263]: pgmap v1622: 73 pgs: 1 peering, 72 active+clean; 694 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 96 KiB/s rd, 14 KiB/s wr, 133 op/s 2023-10-09T10:29:12.951 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:12.983 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:12.985 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:12 smithi110 ceph-mon[136032]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:29:12.985 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1345770686' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:29:12.985 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:12 smithi110 ceph-mon[136032]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T10:29:12.985 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:12 smithi110 ceph-mon[136032]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.985 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:12 smithi110 ceph-mon[136032]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.986 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:12 smithi110 ceph-mon[136032]: failed to encode map e578 with expected crc 2023-10-09T10:29:12.986 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:12 smithi110 ceph-mon[136032]: pgmap v1622: 73 pgs: 1 peering, 72 active+clean; 694 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 96 KiB/s rd, 14 KiB/s wr, 133 op/s 2023-10-09T10:29:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:12.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:13.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:29:13.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:29:13.116 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:13.152 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:29:13.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:13.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:29:13.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:29:13.282 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:13.307 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:29:13.316 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:29:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:13.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:29:13.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:13.444 INFO:tasks.thrashosds.thrasher:in_osds: [3, 0, 2, 5, 4, 6] out_osds: [1, 7] dead_osds: [] live_osds: [0, 1, 5, 6, 7, 2, 4, 3] 2023-10-09T10:29:13.444 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:29:13.445 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T10:29:13.445 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:29:13.445 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:29:13.445 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T10:29:13.446 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:13.481 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:13.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:13.536 INFO:tasks.workunit.client.0.smithi012.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T10:29:13.547 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg 2023-10-09T10:29:13.584 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:13.620 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:13.642 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:13.645 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd remove testimg_import 2023-10-09T10:29:13.646 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:13.658 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:13.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:13.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:13.734 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:13.738 INFO:tasks.workunit.client.0.smithi012.stderr:+ tiered=0 2023-10-09T10:29:13.738 INFO:tasks.workunit.client.0.smithi012.stderr:+ ceph osd dump 2023-10-09T10:29:13.738 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep '^pool' 2023-10-09T10:29:13.738 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep ''\''rbd'\''' 2023-10-09T10:29:13.738 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep tier 2023-10-09T10:29:13.762 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:29:13.789 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:13.823 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:29:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:13.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:29:13.927 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:13.954 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:13.983 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:29:13.992 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:13.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:13.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:14.097 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:14.121 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:13 smithi110 ceph-mon[136032]: failed to encode map e578 with expected crc 2023-10-09T10:29:14.121 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:13 smithi110 ceph-mon[136032]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.121 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:13 smithi110 ceph-mon[136032]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.121 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:13 smithi110 ceph-mon[136032]: osdmap e579: 8 total, 8 up, 6 in 2023-10-09T10:29:14.121 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:13 smithi110 ceph-mon[136032]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.121 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:13 smithi110 ceph-mon[136032]: osdmap e580: 8 total, 8 up, 6 in 2023-10-09T10:29:14.123 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:14.157 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[147926]: failed to encode map e578 with expected crc 2023-10-09T10:29:14.157 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[147926]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.157 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[147926]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[147926]: osdmap e579: 8 total, 8 up, 6 in 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[147926]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[147926]: osdmap e580: 8 total, 8 up, 6 in 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[152263]: failed to encode map e578 with expected crc 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[152263]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[152263]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[152263]: osdmap e579: 8 total, 8 up, 6 in 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[152263]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.158 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:13 smithi012 ceph-mon[152263]: osdmap e580: 8 total, 8 up, 6 in 2023-10-09T10:29:14.264 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:14.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:14.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:14.368 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:14.398 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:14.400 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_444239/sparse1 2023-10-09T10:29:14.406 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records in 2023-10-09T10:29:14.406 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records out 2023-10-09T10:29:14.406 INFO:tasks.workunit.client.0.smithi012.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00507054 s, 207 MB/s 2023-10-09T10:29:14.406 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_444239/sparse2 2023-10-09T10:29:14.412 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records in 2023-10-09T10:29:14.412 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records out 2023-10-09T10:29:14.412 INFO:tasks.workunit.client.0.smithi012.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00500203 s, 210 MB/s 2023-10-09T10:29:14.412 INFO:tasks.workunit.client.0.smithi012.stderr:+ truncate /tmp/rbd_import_export_444239/sparse2 -s 2M 2023-10-09T10:29:14.466 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse1 2023-10-09T10:29:14.502 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 0% complete...failed. 2023-10-09T10:29:14.502 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T10:29:14.504 INFO:tasks.workunit.client.0.smithi012.stderr:+ true 2023-10-09T10:29:14.505 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_444239/sparse1 2023-10-09T10:29:14.555 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:14.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:14.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:14.561 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T10:29:14.562 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:29:14.562 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T10:29:14.565 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd ls -l 2023-10-09T10:29:14.565 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep sparse1 2023-10-09T10:29:14.565 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T10:29:14.602 INFO:tasks.workunit.client.0.smithi012.stdout:sparse1 2 MiB 2 2023-10-09T10:29:14.603 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects sparse1 2023-10-09T10:29:14.603 INFO:tasks.workunit.client.0.smithi012.stderr:++ image=sparse1 2023-10-09T10:29:14.603 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rbd info sparse1 2023-10-09T10:29:14.603 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep block_name_prefix 2023-10-09T10:29:14.604 INFO:tasks.workunit.client.0.smithi012.stderr:+++ awk '{print $NF;}' 2023-10-09T10:29:14.647 INFO:tasks.workunit.client.0.smithi012.stderr:++ prefix=rbd_data.935a250018f9 2023-10-09T10:29:14.649 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep rbd_data.935a250018f9 2023-10-09T10:29:14.649 INFO:tasks.workunit.client.0.smithi012.stderr:++++ get_image_data_pool sparse1 2023-10-09T10:29:14.649 INFO:tasks.workunit.client.0.smithi012.stderr:++++ image=sparse1 2023-10-09T10:29:14.649 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sed -e 's/rbd_data.935a250018f9\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T10:29:14.649 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sort -u 2023-10-09T10:29:14.649 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ rbd info sparse1 2023-10-09T10:29:14.649 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ grep 'data_pool: ' 2023-10-09T10:29:14.649 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T10:29:14.659 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:14.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:29:14.690 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool= 2023-10-09T10:29:14.691 INFO:tasks.workunit.client.0.smithi012.stderr:++++ '[' -z ']' 2023-10-09T10:29:14.691 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool=rbd 2023-10-09T10:29:14.691 INFO:tasks.workunit.client.0.smithi012.stderr:++++ echo rbd 2023-10-09T10:29:14.691 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rados ls -p rbd 2023-10-09T10:29:14.693 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:14.726 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects=1 2023-10-09T10:29:14.726 INFO:tasks.workunit.client.0.smithi012.stderr:++ echo 1 2023-10-09T10:29:14.726 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T10:29:14.726 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export sparse1 /tmp/rbd_import_export_444239/sparse1.out 2023-10-09T10:29:14.762 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:14.767 INFO:tasks.workunit.client.0.smithi012.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_444239/sparse1 /tmp/rbd_import_export_444239/sparse1.out 2023-10-09T10:29:14.767 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp -l /tmp/rbd_import_export_444239/sparse1 /tmp/rbd_import_export_444239/sparse1.out 2023-10-09T10:29:14.769 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse1 --format %b 2023-10-09T10:29:14.770 INFO:tasks.workunit.client.0.smithi012.stderr:+ origsize=2048 2023-10-09T10:29:14.770 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse1.out --format %b 2023-10-09T10:29:14.772 INFO:tasks.workunit.client.0.smithi012.stderr:+ exportsize=2048 2023-10-09T10:29:14.772 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:14.772 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:14.772 INFO:tasks.workunit.client.0.smithi012.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T10:29:14.772 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/sparse1.out 2023-10-09T10:29:14.772 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse1 2023-10-09T10:29:14.782 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:14.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:14.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:14.880 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T10:29:14.882 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse2 2023-10-09T10:29:14.887 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:14.916 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:14.925 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 0% complete...failed. 2023-10-09T10:29:14.925 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T10:29:14.930 INFO:tasks.workunit.client.0.smithi012.stderr:+ true 2023-10-09T10:29:14.930 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_444239/sparse2 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: failed to encode map e579 with expected crc 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: failed to encode map e580 with expected crc 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: failed to encode map e580 with expected crc 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: failed to encode map e580 with expected crc 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: pgmap v1625: 73 pgs: 73 active+clean; 802 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 186 op/s 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2339824367' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:29:14.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:14 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2339824367' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:29:14.950 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:14.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:14.990 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T10:29:14.990 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:29:14.990 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T10:29:14.994 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd ls -l 2023-10-09T10:29:14.994 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep sparse2 2023-10-09T10:29:14.994 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T10:29:15.031 INFO:tasks.workunit.client.0.smithi012.stdout:sparse2 2 MiB 2 2023-10-09T10:29:15.032 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects sparse2 2023-10-09T10:29:15.032 INFO:tasks.workunit.client.0.smithi012.stderr:++ image=sparse2 2023-10-09T10:29:15.033 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rbd info sparse2 2023-10-09T10:29:15.033 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep block_name_prefix 2023-10-09T10:29:15.033 INFO:tasks.workunit.client.0.smithi012.stderr:+++ awk '{print $NF;}' 2023-10-09T10:29:15.047 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0: {} 2023-10-09T10:29:15.054 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:15.062 INFO:teuthology.orchestra.run.smithi110.stdout:ERROR: (22) Invalid argument 2023-10-09T10:29:15.062 INFO:teuthology.orchestra.run.smithi110.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:29:15.071 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1: {} 2023-10-09T10:29:15.076 INFO:tasks.workunit.client.0.smithi012.stderr:++ prefix=rbd_data.ba74dfb9ef50 2023-10-09T10:29:15.077 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep rbd_data.ba74dfb9ef50 2023-10-09T10:29:15.077 INFO:tasks.workunit.client.0.smithi012.stderr:++++ get_image_data_pool sparse2 2023-10-09T10:29:15.077 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sed -e 's/rbd_data.ba74dfb9ef50\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T10:29:15.077 INFO:tasks.workunit.client.0.smithi012.stderr:++++ image=sparse2 2023-10-09T10:29:15.077 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sort -u 2023-10-09T10:29:15.078 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ rbd info sparse2 2023-10-09T10:29:15.078 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ grep 'data_pool: ' 2023-10-09T10:29:15.079 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T10:29:15.087 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:15.102 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2: {} 2023-10-09T10:29:15.119 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool= 2023-10-09T10:29:15.119 INFO:tasks.workunit.client.0.smithi012.stderr:++++ '[' -z ']' 2023-10-09T10:29:15.119 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool=rbd 2023-10-09T10:29:15.120 INFO:tasks.workunit.client.0.smithi012.stderr:++++ echo rbd 2023-10-09T10:29:15.120 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rados ls -p rbd 2023-10-09T10:29:15.125 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3: {} 2023-10-09T10:29:15.144 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4: {} 2023-10-09T10:29:15.151 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects=0 2023-10-09T10:29:15.151 INFO:tasks.workunit.client.0.smithi012.stderr:++ echo 0 2023-10-09T10:29:15.151 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T10:29:15.151 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export sparse2 /tmp/rbd_import_export_444239/sparse2.out 2023-10-09T10:29:15.164 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5: {} 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: failed to encode map e579 with expected crc 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: failed to encode map e580 with expected crc 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: failed to encode map e580 with expected crc 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: failed to encode map e580 with expected crc 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: pgmap v1625: 73 pgs: 73 active+clean; 802 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 186 op/s 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2339824367' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:29:15.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2339824367' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: failed to encode map e579 with expected crc 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: failed to encode map e580 with expected crc 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: failed to encode map e580 with expected crc 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: failed to encode map e580 with expected crc 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: pgmap v1625: 73 pgs: 73 active+clean; 802 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 186 op/s 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2339824367' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:29:15.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:14 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2339824367' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:29:15.185 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6: {} 2023-10-09T10:29:15.188 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:15.194 INFO:tasks.workunit.client.0.smithi012.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_444239/sparse2 /tmp/rbd_import_export_444239/sparse2.out 2023-10-09T10:29:15.194 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp -l /tmp/rbd_import_export_444239/sparse2 /tmp/rbd_import_export_444239/sparse2.out 2023-10-09T10:29:15.196 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse2 --format %b 2023-10-09T10:29:15.197 INFO:tasks.workunit.client.0.smithi012.stderr:+ origsize=2048 2023-10-09T10:29:15.198 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse2.out --format %b 2023-10-09T10:29:15.199 INFO:tasks.workunit.client.0.smithi012.stderr:+ exportsize=2048 2023-10-09T10:29:15.199 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:15.199 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:15.199 INFO:tasks.workunit.client.0.smithi012.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T10:29:15.199 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/sparse2.out 2023-10-09T10:29:15.200 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse2 2023-10-09T10:29:15.205 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {} 2023-10-09T10:29:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:29:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:29:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:29:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:29:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:29:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:29:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:29:15.207 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:29:15.235 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:15.305 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T10:29:15.308 INFO:tasks.workunit.client.0.smithi012.stderr:+ truncate /tmp/rbd_import_export_444239/sparse1 -s 10M 2023-10-09T10:29:15.309 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T10:29:15.350 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:15.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:15.394 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:29:15.394 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:29:15.394 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T10:29:15.395 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd ls -l 2023-10-09T10:29:15.395 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep sparse1 2023-10-09T10:29:15.395 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-09T10:29:15.430 INFO:tasks.workunit.client.0.smithi012.stdout:sparse1 10 MiB 2 2023-10-09T10:29:15.431 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects sparse1 2023-10-09T10:29:15.431 INFO:tasks.workunit.client.0.smithi012.stderr:++ image=sparse1 2023-10-09T10:29:15.431 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rbd info sparse1 2023-10-09T10:29:15.431 INFO:tasks.workunit.client.0.smithi012.stderr:+++ awk '{print $NF;}' 2023-10-09T10:29:15.432 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep block_name_prefix 2023-10-09T10:29:15.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:29:15.466 INFO:tasks.workunit.client.0.smithi012.stderr:++ prefix=rbd_data.baa7b5ee6517 2023-10-09T10:29:15.467 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep rbd_data.baa7b5ee6517 2023-10-09T10:29:15.467 INFO:tasks.workunit.client.0.smithi012.stderr:++++ get_image_data_pool sparse1 2023-10-09T10:29:15.467 INFO:tasks.workunit.client.0.smithi012.stderr:++++ image=sparse1 2023-10-09T10:29:15.467 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sed -e 's/rbd_data.baa7b5ee6517\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T10:29:15.467 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sort -u 2023-10-09T10:29:15.469 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ rbd info sparse1 2023-10-09T10:29:15.470 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ grep 'data_pool: ' 2023-10-09T10:29:15.470 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T10:29:15.483 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:15.503 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool= 2023-10-09T10:29:15.503 INFO:tasks.workunit.client.0.smithi012.stderr:++++ '[' -z ']' 2023-10-09T10:29:15.503 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool=rbd 2023-10-09T10:29:15.504 INFO:tasks.workunit.client.0.smithi012.stderr:++++ echo rbd 2023-10-09T10:29:15.504 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rados ls -p rbd 2023-10-09T10:29:15.537 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects=1 2023-10-09T10:29:15.538 INFO:tasks.workunit.client.0.smithi012.stderr:++ echo 1 2023-10-09T10:29:15.538 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T10:29:15.538 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export sparse1 /tmp/rbd_import_export_444239/sparse1.out 2023-10-09T10:29:15.573 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:15.577 INFO:tasks.workunit.client.0.smithi012.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_444239/sparse1 /tmp/rbd_import_export_444239/sparse1.out 2023-10-09T10:29:15.577 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp -l /tmp/rbd_import_export_444239/sparse1 /tmp/rbd_import_export_444239/sparse1.out 2023-10-09T10:29:15.586 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse1 --format %b 2023-10-09T10:29:15.587 INFO:tasks.workunit.client.0.smithi012.stderr:+ origsize=2048 2023-10-09T10:29:15.587 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse1.out --format %b 2023-10-09T10:29:15.588 INFO:tasks.workunit.client.0.smithi012.stderr:+ exportsize=2048 2023-10-09T10:29:15.588 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:15.588 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:15.588 INFO:tasks.workunit.client.0.smithi012.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T10:29:15.588 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/sparse1.out 2023-10-09T10:29:15.589 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse1 2023-10-09T10:29:15.619 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:29:15.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:15.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:29:15.652 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:29:15.681 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:29:15.681 INFO:teuthology.orchestra.run.smithi110.stdout: "success": "" 2023-10-09T10:29:15.681 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:29:15.685 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T10:29:15.688 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_444239/sparse2 oflag=append conv=notrunc 2023-10-09T10:29:15.700 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records in 2023-10-09T10:29:15.700 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records out 2023-10-09T10:29:15.701 INFO:tasks.workunit.client.0.smithi012.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.011264 s, 186 MB/s 2023-10-09T10:29:15.701 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T10:29:15.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:29:15.760 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:15.789 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:29:15.789 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:29:15.790 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T10:29:15.793 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd ls -l 2023-10-09T10:29:15.793 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep sparse2 2023-10-09T10:29:15.793 INFO:tasks.workunit.client.0.smithi012.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-09T10:29:15.829 INFO:tasks.workunit.client.0.smithi012.stdout:sparse2 4 MiB 2 2023-10-09T10:29:15.829 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects sparse2 2023-10-09T10:29:15.829 INFO:tasks.workunit.client.0.smithi012.stderr:++ image=sparse2 2023-10-09T10:29:15.830 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rbd info sparse2 2023-10-09T10:29:15.830 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep block_name_prefix 2023-10-09T10:29:15.830 INFO:tasks.workunit.client.0.smithi012.stderr:+++ awk '{print $NF;}' 2023-10-09T10:29:15.853 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:29:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:15.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:29:15.868 INFO:tasks.workunit.client.0.smithi012.stderr:++ prefix=rbd_data.bad4f44caf82 2023-10-09T10:29:15.869 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep rbd_data.bad4f44caf82 2023-10-09T10:29:15.869 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sed -e 's/rbd_data.bad4f44caf82\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T10:29:15.869 INFO:tasks.workunit.client.0.smithi012.stderr:++++ get_image_data_pool sparse2 2023-10-09T10:29:15.869 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sort -u 2023-10-09T10:29:15.869 INFO:tasks.workunit.client.0.smithi012.stderr:++++ image=sparse2 2023-10-09T10:29:15.871 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ rbd info sparse2 2023-10-09T10:29:15.872 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ grep 'data_pool: ' 2023-10-09T10:29:15.872 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T10:29:15.907 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool= 2023-10-09T10:29:15.907 INFO:tasks.workunit.client.0.smithi012.stderr:++++ '[' -z ']' 2023-10-09T10:29:15.908 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool=rbd 2023-10-09T10:29:15.908 INFO:tasks.workunit.client.0.smithi012.stderr:++++ echo rbd 2023-10-09T10:29:15.908 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rados ls -p rbd 2023-10-09T10:29:15.941 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects='0 2023-10-09T10:29:15.941 INFO:tasks.workunit.client.0.smithi012.stderr:2 2023-10-09T10:29:15.941 INFO:tasks.workunit.client.0.smithi012.stderr:3' 2023-10-09T10:29:15.942 INFO:tasks.workunit.client.0.smithi012.stderr:++ echo 0 2 3 2023-10-09T10:29:15.942 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-09T10:29:15.942 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export sparse2 /tmp/rbd_import_export_444239/sparse2.out 2023-10-09T10:29:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:15 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:15.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3463535627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:15.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:29:15.979 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T10:29:15.982 INFO:tasks.workunit.client.0.smithi012.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_444239/sparse2 /tmp/rbd_import_export_444239/sparse2.out 2023-10-09T10:29:15.983 INFO:tasks.workunit.client.0.smithi012.stderr:+ cmp -l /tmp/rbd_import_export_444239/sparse2 /tmp/rbd_import_export_444239/sparse2.out 2023-10-09T10:29:15.986 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse2 --format %b 2023-10-09T10:29:15.987 INFO:tasks.workunit.client.0.smithi012.stderr:+ origsize=6144 2023-10-09T10:29:15.987 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse2.out --format %b 2023-10-09T10:29:15.988 INFO:tasks.workunit.client.0.smithi012.stderr:+ exportsize=6144 2023-10-09T10:29:15.988 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:15.988 INFO:tasks.workunit.client.0.smithi012.stderr:+ difference=0 2023-10-09T10:29:15.989 INFO:tasks.workunit.client.0.smithi012.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T10:29:15.989 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/sparse2.out 2023-10-09T10:29:15.989 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:15.989 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse2 2023-10-09T10:29:16.086 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T10:29:16.089 INFO:tasks.workunit.client.0.smithi012.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-09T10:29:16.090 INFO:tasks.workunit.client.0.smithi012.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T10:29:16.090 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_444239/sparse1 sparse 2023-10-09T10:29:16.148 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T10:29:16.149 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:29:16.149 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T10:29:16.152 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects sparse 2023-10-09T10:29:16.152 INFO:tasks.workunit.client.0.smithi012.stderr:++ image=sparse 2023-10-09T10:29:16.152 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rbd info sparse 2023-10-09T10:29:16.153 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep block_name_prefix 2023-10-09T10:29:16.153 INFO:tasks.workunit.client.0.smithi012.stderr:+++ awk '{print $NF;}' 2023-10-09T10:29:16.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:15 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:16.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3463535627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:16.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:15 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:16.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3463535627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:29:16.189 INFO:tasks.workunit.client.0.smithi012.stderr:++ prefix=rbd_data.94176a5b1592 2023-10-09T10:29:16.190 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep rbd_data.94176a5b1592 2023-10-09T10:29:16.190 INFO:tasks.workunit.client.0.smithi012.stderr:++++ get_image_data_pool sparse 2023-10-09T10:29:16.190 INFO:tasks.workunit.client.0.smithi012.stderr:++++ image=sparse 2023-10-09T10:29:16.190 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sed -e 's/rbd_data.94176a5b1592\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T10:29:16.190 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sort -u 2023-10-09T10:29:16.191 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ rbd info sparse 2023-10-09T10:29:16.191 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ grep 'data_pool: ' 2023-10-09T10:29:16.192 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T10:29:16.225 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool= 2023-10-09T10:29:16.226 INFO:tasks.workunit.client.0.smithi012.stderr:++++ '[' -z ']' 2023-10-09T10:29:16.226 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool=rbd 2023-10-09T10:29:16.226 INFO:tasks.workunit.client.0.smithi012.stderr:++++ echo rbd 2023-10-09T10:29:16.226 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rados ls -p rbd 2023-10-09T10:29:16.254 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:29:16.257 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects=1 2023-10-09T10:29:16.257 INFO:tasks.workunit.client.0.smithi012.stderr:++ echo 1 2023-10-09T10:29:16.257 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T10:29:16.257 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse 2023-10-09T10:29:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:16.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:29:16.342 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T10:29:16.344 INFO:tasks.workunit.client.0.smithi012.stdout:zeros import through stdin to sparse image 2023-10-09T10:29:16.344 INFO:tasks.workunit.client.0.smithi012.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-09T10:29:16.345 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T10:29:16.345 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd import --new-format - sparse 2023-10-09T10:29:16.349 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:29:16.359 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:16.388 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:16.393 INFO:tasks.workunit.client.0.smithi012.stderr:4+0 records in 2023-10-09T10:29:16.393 INFO:tasks.workunit.client.0.smithi012.stderr:4+0 records out 2023-10-09T10:29:16.393 INFO:tasks.workunit.client.0.smithi012.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0480339 s, 87.3 MB/s 2023-10-09T10:29:16.406 INFO:tasks.workunit.client.0.smithi012.stderr: Importing image: 100% complete...done. 2023-10-09T10:29:16.406 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:29:16.409 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects sparse 2023-10-09T10:29:16.409 INFO:tasks.workunit.client.0.smithi012.stderr:++ image=sparse 2023-10-09T10:29:16.410 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rbd info sparse 2023-10-09T10:29:16.410 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep block_name_prefix 2023-10-09T10:29:16.410 INFO:tasks.workunit.client.0.smithi012.stderr:+++ awk '{print $NF;}' 2023-10-09T10:29:16.446 INFO:tasks.workunit.client.0.smithi012.stderr:++ prefix=rbd_data.94386dbd50ae 2023-10-09T10:29:16.447 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep rbd_data.94386dbd50ae 2023-10-09T10:29:16.447 INFO:tasks.workunit.client.0.smithi012.stderr:++++ get_image_data_pool sparse 2023-10-09T10:29:16.447 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sed -e 's/rbd_data.94386dbd50ae\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T10:29:16.447 INFO:tasks.workunit.client.0.smithi012.stderr:++++ image=sparse 2023-10-09T10:29:16.447 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sort -u 2023-10-09T10:29:16.447 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ rbd info sparse 2023-10-09T10:29:16.447 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ grep 'data_pool: ' 2023-10-09T10:29:16.447 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T10:29:16.478 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:16.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:16.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:16.483 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool= 2023-10-09T10:29:16.483 INFO:tasks.workunit.client.0.smithi012.stderr:++++ '[' -z ']' 2023-10-09T10:29:16.483 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool=rbd 2023-10-09T10:29:16.483 INFO:tasks.workunit.client.0.smithi012.stderr:++++ echo rbd 2023-10-09T10:29:16.483 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rados ls -p rbd 2023-10-09T10:29:16.514 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects= 2023-10-09T10:29:16.514 INFO:tasks.workunit.client.0.smithi012.stderr:++ echo 2023-10-09T10:29:16.515 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-09T10:29:16.515 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse 2023-10-09T10:29:16.582 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:29:16.606 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:16.609 INFO:tasks.workunit.client.0.smithi012.stdout:zeros export to sparse file 2023-10-09T10:29:16.609 INFO:tasks.workunit.client.0.smithi012.stderr:+ echo 'zeros export to sparse file' 2023-10-09T10:29:16.609 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T10:29:16.610 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:16.645 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:29:16.646 INFO:tasks.workunit.client.0.smithi012.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T10:29:16.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:16.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:29:16.649 INFO:tasks.workunit.client.0.smithi012.stderr:++ rbd info sparse 2023-10-09T10:29:16.649 INFO:tasks.workunit.client.0.smithi012.stderr:++ grep block_name_prefix 2023-10-09T10:29:16.650 INFO:tasks.workunit.client.0.smithi012.stderr:++ awk '{print $NF;}' 2023-10-09T10:29:16.686 INFO:tasks.workunit.client.0.smithi012.stderr:+ prefix=rbd_data.bb40e21ff18b 2023-10-09T10:29:16.686 INFO:tasks.workunit.client.0.smithi012.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T10:29:16.686 INFO:tasks.workunit.client.0.smithi012.stderr:++ get_image_data_pool sparse 2023-10-09T10:29:16.686 INFO:tasks.workunit.client.0.smithi012.stderr:++ image=sparse 2023-10-09T10:29:16.687 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rbd info sparse 2023-10-09T10:29:16.687 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep 'data_pool: ' 2023-10-09T10:29:16.687 INFO:tasks.workunit.client.0.smithi012.stderr:+++ awk -F: '{ print $NF }' 2023-10-09T10:29:16.725 INFO:tasks.workunit.client.0.smithi012.stderr:++ data_pool= 2023-10-09T10:29:16.725 INFO:tasks.workunit.client.0.smithi012.stderr:++ '[' -z ']' 2023-10-09T10:29:16.726 INFO:tasks.workunit.client.0.smithi012.stderr:++ data_pool=rbd 2023-10-09T10:29:16.726 INFO:tasks.workunit.client.0.smithi012.stderr:++ echo rbd 2023-10-09T10:29:16.726 INFO:tasks.workunit.client.0.smithi012.stderr:+ rados -p rbd put rbd_data.bb40e21ff18b.000000000000 - 2023-10-09T10:29:16.749 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:29:16.755 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records in 2023-10-09T10:29:16.755 INFO:tasks.workunit.client.0.smithi012.stderr:1+0 records out 2023-10-09T10:29:16.755 INFO:tasks.workunit.client.0.smithi012.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0688087 s, 61.0 MB/s 2023-10-09T10:29:16.773 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects sparse 2023-10-09T10:29:16.773 INFO:tasks.workunit.client.0.smithi012.stderr:++ image=sparse 2023-10-09T10:29:16.773 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rbd info sparse 2023-10-09T10:29:16.773 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep block_name_prefix 2023-10-09T10:29:16.773 INFO:tasks.workunit.client.0.smithi012.stderr:+++ awk '{print $NF;}' 2023-10-09T10:29:16.786 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:16.816 INFO:tasks.workunit.client.0.smithi012.stderr:++ prefix=rbd_data.bb40e21ff18b 2023-10-09T10:29:16.816 INFO:tasks.workunit.client.0.smithi012.stderr:+++ grep rbd_data.bb40e21ff18b 2023-10-09T10:29:16.816 INFO:tasks.workunit.client.0.smithi012.stderr:++++ get_image_data_pool sparse 2023-10-09T10:29:16.816 INFO:tasks.workunit.client.0.smithi012.stderr:++++ image=sparse 2023-10-09T10:29:16.816 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sed -e 's/rbd_data.bb40e21ff18b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T10:29:16.816 INFO:tasks.workunit.client.0.smithi012.stderr:+++ sort -u 2023-10-09T10:29:16.817 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ grep 'data_pool: ' 2023-10-09T10:29:16.818 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T10:29:16.818 INFO:tasks.workunit.client.0.smithi012.stderr:+++++ rbd info sparse 2023-10-09T10:29:16.829 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:29:16.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:16.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:29:16.854 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool= 2023-10-09T10:29:16.854 INFO:tasks.workunit.client.0.smithi012.stderr:++++ '[' -z ']' 2023-10-09T10:29:16.854 INFO:tasks.workunit.client.0.smithi012.stderr:++++ data_pool=rbd 2023-10-09T10:29:16.854 INFO:tasks.workunit.client.0.smithi012.stderr:++++ echo rbd 2023-10-09T10:29:16.855 INFO:tasks.workunit.client.0.smithi012.stderr:+++ rados ls -p rbd 2023-10-09T10:29:16.889 INFO:tasks.workunit.client.0.smithi012.stderr:++ objects=0 2023-10-09T10:29:16.889 INFO:tasks.workunit.client.0.smithi012.stderr:++ echo 0 2023-10-09T10:29:16.889 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T10:29:16.890 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/sparse 2023-10-09T10:29:16.890 INFO:tasks.workunit.client.0.smithi012.stderr:rm: cannot remove '/tmp/rbd_import_export_444239/sparse': No such file or directory 2023-10-09T10:29:16.891 INFO:tasks.workunit.client.0.smithi012.stderr:+ true 2023-10-09T10:29:16.891 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd export sparse /tmp/rbd_import_export_444239/sparse 2023-10-09T10:29:16.924 INFO:tasks.workunit.client.0.smithi012.stderr: Exporting image: 100% complete...done. 2023-10-09T10:29:16.927 INFO:tasks.workunit.client.0.smithi012.stderr:++ stat /tmp/rbd_import_export_444239/sparse --format=%b 2023-10-09T10:29:16.928 INFO:tasks.workunit.client.0.smithi012.stderr:+ '[' 0 = 0 ']' 2023-10-09T10:29:16.929 INFO:tasks.workunit.client.0.smithi012.stderr:+ rbd rm sparse 2023-10-09T10:29:16.933 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:29:16.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:16 smithi110 ceph-mon[136032]: pgmap v1626: 73 pgs: 73 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 92 KiB/s rd, 11 KiB/s wr, 129 op/s 2023-10-09T10:29:16.961 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:17.017 INFO:tasks.workunit.client.0.smithi012.stderr: Removing image: 100% complete...done. 2023-10-09T10:29:17.019 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm /tmp/rbd_import_export_444239/sparse /tmp/rbd_import_export_444239/sparse1 /tmp/rbd_import_export_444239/sparse2 /tmp/rbd_import_export_444239/sparse3 2023-10-09T10:29:17.022 INFO:tasks.workunit.client.0.smithi012.stderr:rm: cannot remove '/tmp/rbd_import_export_444239/sparse3': No such file or directory 2023-10-09T10:29:17.022 INFO:tasks.workunit.client.0.smithi012.stderr:+ true 2023-10-09T10:29:17.023 INFO:tasks.workunit.client.0.smithi012.stderr:+ echo OK 2023-10-09T10:29:17.023 INFO:tasks.workunit.client.0.smithi012.stdout:OK 2023-10-09T10:29:17.024 INFO:tasks.workunit.client.0.smithi012.stderr:+ rm -rf /tmp/rbd_import_export_444239 2023-10-09T10:29:17.025 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T10:29:17.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T10:29:17.047 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:29:17.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:17.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:29:17.090 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:16 smithi012 ceph-mon[147926]: pgmap v1626: 73 pgs: 73 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 92 KiB/s rd, 11 KiB/s wr, 129 op/s 2023-10-09T10:29:17.091 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:16 smithi012 ceph-mon[152263]: pgmap v1626: 73 pgs: 73 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 92 KiB/s rd, 11 KiB/s wr, 129 op/s 2023-10-09T10:29:17.093 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T10:29:17.094 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T10:29:17.151 DEBUG:teuthology.orchestra.run.smithi110:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:29:17.178 INFO:teuthology.orchestra.run.smithi110.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:17.224 INFO:teuthology.orchestra.run.smithi110.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:29:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:17.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi110 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:29:17.328 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:29:17.356 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:17.364 DEBUG:teuthology.parallel:result is None 2023-10-09T10:29:17.365 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:29:17.390 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:29:17.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:17.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:29:17.417 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:29:17.417 DEBUG:teuthology.orchestra.run.smithi012:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:29:17.473 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:29:17.473 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T10:29:17.473 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T10:29:17.474 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T10:29:17.474 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T10:29:17.474 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T10:29:17.474 DEBUG:teuthology.orchestra.run.smithi012:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:29:17.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:29:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:29:17.531 INFO:teuthology.orchestra.run.smithi012.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T10:29:17.531 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:29:17.541 INFO:teuthology.orchestra.run.smithi012.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:29:17.549 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:29:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [], 2023-10-09T10:29:17.549 INFO:teuthology.orchestra.run.smithi110.stdout: "complaint_time": 30, 2023-10-09T10:29:17.550 INFO:teuthology.orchestra.run.smithi110.stdout: "num_blocked_ops": 0 2023-10-09T10:29:17.550 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:29:17.557 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T10:29:17.558 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T10:29:17.575 INFO:teuthology.orchestra.run.smithi012.stderr:Error: no container with name or ID "ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:29:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:29:17.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi012 with status 125: 'sudo docker kill -s 1 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:29:17.592 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T10:29:17.890 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:29:17.975 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:29:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:29:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:29:17] "GET /metrics HTTP/1.1" 200 34241 "" "Prometheus/2.33.4" 2023-10-09T10:29:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:17 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1171089164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:17 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:17 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1171089164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:18.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:17 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:18.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:17 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1171089164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:18.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:17 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:18.574 INFO:teuthology.orchestra.run.smithi110.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:29:19.062 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:29:19.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:29:19.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:19.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: osdmap e581: 8 total, 8 up, 6 in 2023-10-09T10:29:19.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: pgmap v1628: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1.1 MiB/s rd, 873 KiB/s wr, 420 op/s 2023-10-09T10:29:19.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.063 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:18 smithi110 ceph-mon[136032]: osdmap e582: 8 total, 8 up, 6 in 2023-10-09T10:29:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:29:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:29:19.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: osdmap e581: 8 total, 8 up, 6 in 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: pgmap v1628: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1.1 MiB/s rd, 873 KiB/s wr, 420 op/s 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[147926]: osdmap e582: 8 total, 8 up, 6 in 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:29:19.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: osdmap e581: 8 total, 8 up, 6 in 2023-10-09T10:29:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: pgmap v1628: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1.1 MiB/s rd, 873 KiB/s wr, 420 op/s 2023-10-09T10:29:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: failed to encode map e581 with expected crc 2023-10-09T10:29:19.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:18 smithi012 ceph-mon[152263]: osdmap e582: 8 total, 8 up, 6 in 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: "size": 20, 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 600, 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: "ops": [ 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158234 10.2 10:41c2e2c6:::benchmark_data_smithi012_366312_object46905:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.041780+0000", 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 426.71356309499998, 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4942225009999999, 2023-10-09T10:29:19.759 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158234 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041780+0000", 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.760 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041780+0000", 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041779+0000", 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.9330000000000007e-06 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.761 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041788+0000", 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6970000000000005e-06 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041798+0000", 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0059999999999998e-06 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.041804+0000", 2023-10-09T10:29:19.762 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0027629389999999998 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.044567+0000", 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2949999999999996e-06 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op must wait for map", 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.044574+0000", 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0085963280000000003 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.053170+0000", 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6649999999999992e-06 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.053179+0000", 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29553761699999997 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.348717+0000", 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5679999999999999e-05 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.348732+0000", 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.167581639 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.516314+0000", 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011239 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.516426+0000", 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00039882999999999997 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.516825+0000", 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021040099999999999 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.766 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.518929+0000", 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0021660400000000002 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.521095+0000", 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001823414 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.767 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.522919+0000", 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.012894950000000001 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.535814+0000", 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7305000000000001e-05 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.535871+0000", 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013163699999999999 2023-10-09T10:29:19.768 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.536003+0000", 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.4942225009999999 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158238 10.4 10:21665022:::benchmark_data_smithi012_366312_object46909:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:29:19.769 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.046937+0000", 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 426.708406674, 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.571134107, 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158238 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.770 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046937+0000", 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046937+0000", 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999981 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.771 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046935+0000", 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.1009999999999998e-06 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046944+0000", 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8352e-05 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046962+0000", 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1799999999999996e-06 2023-10-09T10:29:19.772 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.046970+0000", 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.013846494000000001 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.060817+0000", 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.7120000000000006e-06 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.773 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.060826+0000", 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.291419873 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352245+0000", 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.5809999999999999e-06 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.774 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352253+0000", 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.238754991 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.591008+0000", 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000105715 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.591114+0000", 2023-10-09T10:29:19.775 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042542200000000001 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3", 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.591539+0000", 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.006005074 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.597544+0000", 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0044712470000000002 2023-10-09T10:29:19.776 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.602015+0000", 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051608729999999998 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.607176+0000", 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.010717366000000001 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.777 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.617894+0000", 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6595000000000003e-05 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.617950+0000", 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000120309 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.778 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618071+0000", 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.571134107 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158239 10.4 10:23993120:::benchmark_data_smithi012_366312_object46910:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.051553+0000", 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 426.70379024599998, 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.566721287, 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.779 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.780 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.780 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:29:19.780 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:29:19.780 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158239 2023-10-09T10:29:19.780 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.780 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.780 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051553+0000", 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051553+0000", 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.781 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051552+0000", 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0249999999999997e-06 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051559+0000", 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0814e-05 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.782 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051569+0000", 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.8150000000000006e-06 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.051577+0000", 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.009272789 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.783 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.060850+0000", 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.311e-06 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.060857+0000", 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29142648199999999 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.784 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352284+0000", 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0299999999999999e-06 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352293+0000", 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.241035501 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.593328+0000", 2023-10-09T10:29:19.785 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010223 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.593431+0000", 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00033623600000000001 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3", 2023-10-09T10:29:19.786 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.593767+0000", 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059739930000000004 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.599741+0000", 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0023597510000000002 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.787 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.602101+0000", 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051075469999999996 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.607208+0000", 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0109065 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618115+0000", 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1894000000000002e-05 2023-10-09T10:29:19.788 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618156+0000", 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011776400000000001 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618274+0000", 2023-10-09T10:29:19.789 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.566721287 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158249 10.14 10:2e30cd7c:::benchmark_data_smithi012_366312_object46920:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.076034+0000", 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 426.67930901300002, 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.542011485, 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.790 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158249 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.791 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076034+0000", 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076034+0000", 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.792 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076033+0000", 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0129999999999996e-06 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076040+0000", 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0220999999999999e-05 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.793 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076050+0000", 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.913e-06 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076056+0000", 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5566999999999998e-05 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.794 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076111+0000", 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0560000000000001e-05 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.076122+0000", 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.276081191 2023-10-09T10:29:19.795 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352203+0000", 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.7429999999999997e-06 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352213+0000", 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.234992906 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.796 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.587206+0000", 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00012481999999999999 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.587331+0000", 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00036589800000000003 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3", 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.587697+0000", 2023-10-09T10:29:19.797 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0078394450000000004 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595536+0000", 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9175000000000001e-05 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.798 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595565+0000", 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0020002290000000001 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.597565+0000", 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.020290898000000002 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.799 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.617856+0000", 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7027000000000001e-05 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.617913+0000", 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132366 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618046+0000", 2023-10-09T10:29:19.800 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.542011485 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:158251 10.4 10:231b664c:::benchmark_data_smithi012_366312_object46922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:22:13.082556+0000", 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 426.67278736499998, 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5359710120000001, 2023-10-09T10:29:19.801 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 158251 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.802 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082556+0000", 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082556+0000", 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.803 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082554+0000", 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2155e-05 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082566+0000", 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3308e-05 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082580+0000", 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3039999999999999e-06 2023-10-09T10:29:19.804 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082587+0000", 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3467000000000007e-05 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082661+0000", 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.5810000000000003e-06 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.805 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.082670+0000", 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.26964073300000002 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352311+0000", 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.7440000000000007e-06 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:13.352319+0000", 2023-10-09T10:29:19.806 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.2432332580000001 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595552+0000", 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3499999999999996e-05 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595645+0000", 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028873199999999999 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.807 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 1,2,3", 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.595934+0000", 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0060391780000000001 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.601973+0000", 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00016949700000000001 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.808 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.602143+0000", 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0051037610000000001 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.607246+0000", 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.011098742999999999 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618345+0000", 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9413999999999998e-05 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.809 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618395+0000", 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00013218 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:22:14.618527+0000", 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5359710120000001 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.810 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36474.0:210964 10.d 10:b494e22a:::benchmark_data_smithi012_366312_object99635:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:23:24.454180+0000", 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 355.30116325199998, 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8139022659999999, 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36474", 2023-10-09T10:29:19.811 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3417927471", 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 210964 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454180+0000", 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454180+0000", 2023-10-09T10:29:19.812 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454180+0000", 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.779e-06 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454182+0000", 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6509999999999998e-06 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.813 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454188+0000", 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.118e-06 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.454193+0000", 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.002309167 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.814 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.456502+0000", 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5301000000000001e-05 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for peered", 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:24.456518+0000", 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.251861756 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.708379+0000", 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000112889 2023-10-09T10:29:19.815 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.708492+0000", 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00037031599999999998 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,3,7", 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.708863+0000", 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001691901 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.816 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.710554+0000", 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6291999999999994e-05 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.710631+0000", 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015809680000000001 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.817 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:25.712212+0000", 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.55551082600000001 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.267723+0000", 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8658999999999994e-05 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.267791+0000", 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00029103199999999999 2023-10-09T10:29:19.818 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:23:26.268082+0000", 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8139022659999999 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:139105 11.e 11:78ebf9f2:::benchmark_data_smithi012_404402_object26322:head [delete] snapc 0=[] ondisk+write+known_if_redirected e530)", 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:11.617492+0000", 2023-10-09T10:29:19.819 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 188.13785164500001, 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30184161900000001, 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 139105 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.820 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617492+0000", 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617492+0000", 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617490+0000", 2023-10-09T10:29:19.821 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.567e-06 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617496+0000", 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.8885e-05 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617515+0000", 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.6500000000000002e-06 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.822 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617524+0000", 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5328000000000003e-05 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617579+0000", 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000132127 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.823 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617711+0000", 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015152699999999999 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.617863+0000", 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0029933770000000002 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.824 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.620856+0000", 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00062096099999999995 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.621477+0000", 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.29774380700000003 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.919221+0000", 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.5549000000000005e-05 2023-10-09T10:29:19.825 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.919287+0000", 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.6467000000000001e-05 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:11.919333+0000", 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.30184161900000001 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.826 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.36870.0:157303 11.e 11:762be816:::benchmark_data_smithi012_404402_object44520:head [delete] snapc 0=[] ondisk+write+known_if_redirected e537)", 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:26:40.565070+0000", 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 159.19027277399999, 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.47729162000000003, 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.36870", 2023-10-09T10:29:19.827 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/435602176", 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 157303 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565070+0000", 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565070+0000", 2023-10-09T10:29:19.828 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565069+0000", 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.312e-06 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565078+0000", 2023-10-09T10:29:19.829 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.5141e-05 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565093+0000", 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.712e-06 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.830 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565101+0000", 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1703999999999998e-05 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565153+0000", 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000103034 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565256+0000", 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000116803 2023-10-09T10:29:19.831 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.565373+0000", 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0082170920000000005 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.573590+0000", 2023-10-09T10:29:19.832 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0059502160000000004 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:40.579540+0000", 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.46273191699999999 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042272+0000", 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8179000000000001e-05 2023-10-09T10:29:19.833 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042320+0000", 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.2101000000000002e-05 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:26:41.042362+0000", 2023-10-09T10:29:19.834 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.47729162000000003 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47210.0:2984 12.7 12:e00bc771:::image42:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:33.490372+0000", 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 46.264970759999997, 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012784420000000001, 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.835 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47210", 2023-10-09T10:29:19.836 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/125198252", 2023-10-09T10:29:19.836 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 2984 2023-10-09T10:29:19.836 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.836 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.836 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.836 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.490372+0000", 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.490372+0000", 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.837 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.490372+0000", 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2219999999999998e-06 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.490375+0000", 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7680000000000003e-06 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.838 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.490382+0000", 2023-10-09T10:29:19.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.433e-06 2023-10-09T10:29:19.839 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.839 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.839 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.839 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.490385+0000", 2023-10-09T10:29:19.839 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027972000000000001 2023-10-09T10:29:19.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.840 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.490665+0000", 2023-10-09T10:29:19.840 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2089999999999996e-05 2023-10-09T10:29:19.840 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.840 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.840 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.490737+0000", 2023-10-09T10:29:19.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00032251900000000001 2023-10-09T10:29:19.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.841 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.841 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:29:19.841 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.491060+0000", 2023-10-09T10:29:19.841 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00027141099999999999 2023-10-09T10:29:19.841 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.491331+0000", 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000216066 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.491547+0000", 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9125999999999997e-05 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.842 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.491596+0000", 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0229000000000001e-05 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.491626+0000", 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4477e-05 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.491651+0000", 2023-10-09T10:29:19.843 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0012784420000000001 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.47210.0:2998 12.7 12:e00bc771:::image42:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:33.500631+0000", 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 46.254711979, 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00141712, 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.844 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.47210", 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/125198252", 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 2998 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.500631+0000", 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.845 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.500631+0000", 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.500631+0000", 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.1300000000000001e-06 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.500634+0000", 2023-10-09T10:29:19.846 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.3690000000000001e-06 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.500641+0000", 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0920000000000002e-06 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.500644+0000", 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00022801399999999999 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.847 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.500872+0000", 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.1134000000000002e-05 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.500923+0000", 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00018562799999999999 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:29:19.848 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.501109+0000", 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000265445 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.501374+0000", 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00023491699999999999 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.501609+0000", 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038477599999999998 2023-10-09T10:29:19.849 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.501994+0000", 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0623999999999997e-05 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.502025+0000", 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3642000000000002e-05 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.850 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:33.502048+0000", 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00141712 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_repop(client.47210.0:3077 12.e e558/555 12:725d3449:::image47:head v 558'32, mlcod=558'29)", 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:28:34.526097+0000", 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 45.229246193000002, 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014964360000000001, 2023-10-09T10:29:19.851 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.526097+0000", 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.526097+0000", 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:29:19.852 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.526096+0000", 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.5399999999999996e-06 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.526105+0000", 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.8339999999999994e-06 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.853 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.526115+0000", 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.8139999999999997e-06 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.526120+0000", 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9492e-05 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.526170+0000", 2023-10-09T10:29:19.854 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3770000000000001e-05 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.526194+0000", 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013608139999999999 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.527554+0000", 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9032000000000002e-05 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.855 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:28:34.527593+0000", 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014964360000000001 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37548.0:6 2.6 2:7d15d1b1:::rbd_id.testimg_import:head [create,call rbd.set_id in=9b] snapc 0=[] ondisk+write+known_if_redirected e570)", 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:04.527582+0000", 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.227761264, 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013538109999999999, 2023-10-09T10:29:19.856 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37548", 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1777016500", 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 6 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.527582+0000", 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.857 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.527582+0000", 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.527581+0000", 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.263e-06 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.858 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.527587+0000", 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3565e-05 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.527601+0000", 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.101e-06 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.527606+0000", 2023-10-09T10:29:19.859 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.5130000000000002e-05 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.527661+0000", 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00010614499999999999 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.527767+0000", 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00026969199999999998 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.860 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.528037+0000", 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00043741 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.528474+0000", 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000319337 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.861 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.528794+0000", 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5980999999999999e-05 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.528840+0000", 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4786e-05 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.528894+0000", 2023-10-09T10:29:19.862 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1393999999999997e-05 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.528936+0000", 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0013538109999999999 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37548.0:30 2.6 2:645a329c:::rbd_data.92acede67799.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,write 102400~10240 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected e570)", 2023-10-09T10:29:19.863 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:04.544494+0000", 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 15.210849367, 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001496233, 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37548", 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1777016500", 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 30 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.864 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.544494+0000", 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.544494+0000", 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.544493+0000", 2023-10-09T10:29:19.865 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.0176e-05 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.544504+0000", 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.5430000000000004e-06 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.544508+0000", 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.0429999999999996e-06 2023-10-09T10:29:19.866 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.544517+0000", 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00045664300000000001 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.544974+0000", 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6417999999999999e-05 2023-10-09T10:29:19.867 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.545050+0000", 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017110899999999999 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.545221+0000", 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00058816200000000004 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.868 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.545810+0000", 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.7948999999999996e-05 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.545877+0000", 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3037999999999999e-05 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.869 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.545901+0000", 2023-10-09T10:29:19.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8372999999999999e-05 2023-10-09T10:29:19.870 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.870 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.870 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.870 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.545949+0000", 2023-10-09T10:29:19.870 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.1158000000000001e-05 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.545990+0000", 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001496233 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.871 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37572.0:25 2.6 2:6e96215d:::rbd_header.92c4e31ac46:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected e570)", 2023-10-09T10:29:19.872 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:04.810929+0000", 2023-10-09T10:29:19.872 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 14.944413829, 2023-10-09T10:29:19.872 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001355591, 2023-10-09T10:29:19.872 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.872 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.872 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.872 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37572", 2023-10-09T10:29:19.872 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/4255752951", 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 25 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.810929+0000", 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.810929+0000", 2023-10-09T10:29:19.873 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.999999 2023-10-09T10:29:19.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.874 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.810929+0000", 2023-10-09T10:29:19.874 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.0099999999999997e-06 2023-10-09T10:29:19.874 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.874 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.874 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.810933+0000", 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 8.1720000000000007e-06 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.810941+0000", 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.7189999999999999e-06 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.875 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.810944+0000", 2023-10-09T10:29:19.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00017051699999999999 2023-10-09T10:29:19.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.876 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.876 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.876 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.811115+0000", 2023-10-09T10:29:19.876 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3818e-05 2023-10-09T10:29:19.876 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.811159+0000", 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000121199 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.811280+0000", 2023-10-09T10:29:19.877 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00028211700000000002 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.811562+0000", 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000253033 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.811815+0000", 2023-10-09T10:29:19.878 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00038069800000000001 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.812196+0000", 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.4318999999999999e-05 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.879 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.812250+0000", 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.4786999999999997e-05 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:04.812285+0000", 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001355591 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.880 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.57577.0:6 2.6 2:7d15d1b1:::rbd_id.testimg_import:head [create,call rbd.set_id in=9b] snapc 0=[] ondisk+write+known_if_redirected e573)", 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:06.984776+0000", 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 12.770566786, 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014984079999999999, 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.57577", 2023-10-09T10:29:19.881 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/1798920313", 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 6 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.984776+0000", 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.984776+0000", 2023-10-09T10:29:19.882 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999976 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.984774+0000", 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3699999999999999e-05 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.984788+0000", 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2069000000000001e-05 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.883 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.984820+0000", 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7243000000000001e-05 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.984837+0000", 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014125800000000001 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.884 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.984978+0000", 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015083100000000001 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.985129+0000", 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000207369 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.985337+0000", 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046885200000000002 2023-10-09T10:29:19.885 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.985805+0000", 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025667499999999999 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.986062+0000", 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00014697199999999999 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.886 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.986209+0000", 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.5201999999999999e-05 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.986244+0000", 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.0527000000000001e-05 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:06.986275+0000", 2023-10-09T10:29:19.887 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0014984079999999999 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37815.0:26 2.6 2:6787094c:::rbd_header.ba74dfb9ef50:head [watch unwatch cookie 140665279235552] snapc 0=[] ondisk+write+known_if_redirected+full_try e580)", 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:15.248263+0000", 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.5070800540000002, 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015218230000000001, 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.888 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37815", 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3588130515", 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 26 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248263+0000", 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.889 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248263+0000", 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248263+0000", 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.1739999999999999e-06 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248265+0000", 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.9140000000000001e-06 2023-10-09T10:29:19.890 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248269+0000", 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.3760000000000002e-06 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248271+0000", 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046556700000000001 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.891 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248737+0000", 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.8776000000000002e-05 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248786+0000", 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.3752000000000006e-05 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248879+0000", 2023-10-09T10:29:19.892 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00042631599999999999 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249306+0000", 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.4926999999999996e-05 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249381+0000", 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000230492 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.893 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249611+0000", 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.0511999999999999e-05 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249672+0000", 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011340099999999999 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.894 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249785+0000", 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0015218230000000001 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37815.0:27 2.6 2:6787094c:::rbd_header.ba74dfb9ef50:head [notify-ack] snapc 0=[] ondisk+read+known_if_redirected+full_try e580)", 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:15.248575+0000", 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.5067678149999999, 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001328079, 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.895 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "started", 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37815", 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3588130515", 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 27 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248575+0000", 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.896 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248575+0000", 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248575+0000", 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.2880000000000001e-06 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.897 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248578+0000", 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.0129999999999996e-06 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248585+0000", 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 3.196e-06 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.248588+0000", 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00063164299999999998 2023-10-09T10:29:19.898 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249220+0000", 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.6156999999999996e-05 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for rw locks", 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249296+0000", 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00051479300000000004 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.899 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249811+0000", 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3028000000000002e-05 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249854+0000", 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.9521000000000002e-05 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.249903+0000", 2023-10-09T10:29:19.900 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001328079 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37815.0:41 2.6 2:6787094c:::rbd_header.ba74dfb9ef50:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+full_try e580)", 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:15.258240+0000", 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.4971027890000004, 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001704971, 2023-10-09T10:29:19.901 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37815", 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3588130515", 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 41 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258240+0000", 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.902 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258240+0000", 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258240+0000", 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.931e-06 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.903 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258243+0000", 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.8569999999999997e-06 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258250+0000", 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 6.3169999999999998e-06 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258256+0000", 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00046812399999999999 2023-10-09T10:29:19.904 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258724+0000", 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.4452000000000001e-05 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258768+0000", 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00011144199999999999 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:29:19.905 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.258880+0000", 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00024729299999999999 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.259127+0000", 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00020762600000000001 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.906 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.259335+0000", 2023-10-09T10:29:19.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00056209499999999998 2023-10-09T10:29:19.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.909 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.909 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.259897+0000", 2023-10-09T10:29:19.909 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.7787999999999999e-05 2023-10-09T10:29:19.909 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.909 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.259925+0000", 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.0642e-05 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.259945+0000", 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001704971 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.910 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37815.0:49 2.6 2:6787094c:::rbd_header.ba74dfb9ef50:head [watch unwatch cookie 140665212162592] snapc 0=[] ondisk+write+known_if_redirected+full_try e580)", 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:15.264745+0000", 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.4905979800000004, 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022687950000000001, 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.911 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37815", 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3588130515", 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 49 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.264745+0000", 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.912 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.264745+0000", 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.264745+0000", 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.6740000000000001e-06 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.913 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.264747+0000", 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.3574000000000001e-05 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.264761+0000", 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4.3089999999999997e-06 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.914 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.264765+0000", 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000153299 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.264919+0000", 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.147e-05 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.264970+0000", 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 9.6926e-05 2023-10-09T10:29:19.915 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.265067+0000", 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00025509799999999999 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.265322+0000", 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000200414 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.916 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.265522+0000", 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001405173 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.266928+0000", 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9241e-05 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.917 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.266957+0000", 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.7164999999999999e-05 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.267014+0000", 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.0022687950000000001 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.918 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: "description": "osd_op(client.37815.0:86 2.6 2:6787094c:::rbd_header.ba74dfb9ef50:head [watch unwatch cookie 140665212120832] snapc 0=[] ondisk+write+known_if_redirected+full_try e580)", 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: "initiated_at": "2023-10-09T10:29:15.298080+0000", 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: "age": 4.4572634239999998, 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001323437, 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: "type_data": { 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:19.919 INFO:teuthology.orchestra.run.smithi110.stdout: "client_info": { 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: "client": "client.37815", 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: "client_addr": "172.21.15.12:0/3588130515", 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: "tid": 86 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: "events": [ 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "initiated", 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298080+0000", 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.920 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "throttled", 2023-10-09T10:29:19.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298080+0000", 2023-10-09T10:29:19.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:19.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.921 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.921 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "header_read", 2023-10-09T10:29:19.921 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298079+0000", 2023-10-09T10:29:19.921 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 1.7969999999999999e-06 2023-10-09T10:29:19.921 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "all_read", 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298081+0000", 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.1840000000000002e-06 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "dispatched", 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298088+0000", 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.4509999999999999e-06 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.922 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "queued_for_pg", 2023-10-09T10:29:19.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298091+0000", 2023-10-09T10:29:19.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00030348499999999999 2023-10-09T10:29:19.923 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.923 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.923 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "reached_pg", 2023-10-09T10:29:19.923 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298394+0000", 2023-10-09T10:29:19.923 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 7.2088999999999994e-05 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "started", 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298466+0000", 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000165687 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298632+0000", 2023-10-09T10:29:19.924 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000273687 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "op_commit", 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.298906+0000", 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.000258583 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.925 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.299164+0000", 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.00015303800000000001 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.299317+0000", 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 2.9300999999999999e-05 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.926 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "commit_sent", 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.299347+0000", 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 5.6499000000000002e-05 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: }, 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: { 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: "event": "done", 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: "time": "2023-10-09T10:29:15.299403+0000", 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: "duration": 0.001323437 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.927 INFO:teuthology.orchestra.run.smithi110.stdout: } 2023-10-09T10:29:19.928 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2023-10-09T10:29:19.928 INFO:teuthology.orchestra.run.smithi110.stdout:} 2023-10-09T10:29:20.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:19 smithi110 ceph-mon[136032]: failed to encode map e581 with expected crc 2023-10-09T10:29:20.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:19 smithi110 ceph-mon[136032]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:19 smithi110 ceph-mon[136032]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:19 smithi110 ceph-mon[136032]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:19 smithi110 ceph-mon[136032]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.098 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:19 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:20.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:29:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[147926]: failed to encode map e581 with expected crc 2023-10-09T10:29:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[147926]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[147926]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[147926]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[147926]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:20.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[152263]: failed to encode map e581 with expected crc 2023-10-09T10:29:20.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[152263]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[152263]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[152263]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[152263]: failed to encode map e582 with expected crc 2023-10-09T10:29:20.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:19 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:20.326 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:29:20.721 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:21.109 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T10:29:21.110 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T10:29:21.110 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T10:29:21.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:20 smithi012 ceph-mon[147926]: failed to encode map e582 with expected crc 2023-10-09T10:29:21.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:20 smithi012 ceph-mon[147926]: failed to encode map e582 with expected crc 2023-10-09T10:29:21.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:20 smithi012 ceph-mon[147926]: pgmap v1630: 73 pgs: 73 active+clean; 4.6 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 394 op/s 2023-10-09T10:29:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:20 smithi012 ceph-mon[152263]: failed to encode map e582 with expected crc 2023-10-09T10:29:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:20 smithi012 ceph-mon[152263]: failed to encode map e582 with expected crc 2023-10-09T10:29:21.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:20 smithi012 ceph-mon[152263]: pgmap v1630: 73 pgs: 73 active+clean; 4.6 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 394 op/s 2023-10-09T10:29:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:20 smithi110 ceph-mon[136032]: failed to encode map e582 with expected crc 2023-10-09T10:29:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:20 smithi110 ceph-mon[136032]: failed to encode map e582 with expected crc 2023-10-09T10:29:21.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:20 smithi110 ceph-mon[136032]: pgmap v1630: 73 pgs: 73 active+clean; 4.6 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 394 op/s 2023-10-09T10:29:21.383 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:22.138 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:22.739 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:29:22.740 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:29:22.740 INFO:teuthology.orchestra.run.smithi012.stdout: "num_ops": 0 2023-10-09T10:29:22.740 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:29:23.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:22 smithi012 ceph-mon[147926]: pgmap v1631: 73 pgs: 73 active+clean; 4.6 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1006 KiB/s rd, 1.4 MiB/s wr, 317 op/s 2023-10-09T10:29:23.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:22 smithi012 ceph-mon[152263]: pgmap v1631: 73 pgs: 73 active+clean; 4.6 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1006 KiB/s rd, 1.4 MiB/s wr, 317 op/s 2023-10-09T10:29:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:22 smithi110 ceph-mon[136032]: pgmap v1631: 73 pgs: 73 active+clean; 4.6 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1006 KiB/s rd, 1.4 MiB/s wr, 317 op/s 2023-10-09T10:29:23.819 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:29:24.438 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:24.562 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.7. 2023-10-09T10:29:24.565 INFO:teuthology.orchestra.run.smithi012.stderr:noscrub is unset 2023-10-09T10:29:24.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:29:24.391+0000 7f85a99d3700 -1 mon.a@0(leader).osd e583 definitely_dead 0 2023-10-09T10:29:24.676 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[147926]: failed to encode map e583 with expected crc 2023-10-09T10:29:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[147926]: osdmap e583: 8 total, 8 up, 6 in 2023-10-09T10:29:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4182406865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1151855783' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:29:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:29:24.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[152263]: failed to encode map e583 with expected crc 2023-10-09T10:29:24.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[152263]: osdmap e583: 8 total, 8 up, 6 in 2023-10-09T10:29:24.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4182406865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:24.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:24.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1151855783' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:29:24.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:24 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:29:24.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:24 smithi110 ceph-mon[136032]: failed to encode map e583 with expected crc 2023-10-09T10:29:24.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:24 smithi110 ceph-mon[136032]: osdmap e583: 8 total, 8 up, 6 in 2023-10-09T10:29:24.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4182406865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:24.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:29:24.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1151855783' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:29:24.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:24 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:29:25.229 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T10:29:25.229 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T10:29:25.229 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T10:29:25.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T10:29:25.633 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:29:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 259 KiB/s rd, 1.0 MiB/s wr, 121 op/s 2023-10-09T10:29:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[152263]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T10:29:25.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 259 KiB/s rd, 1.0 MiB/s wr, 121 op/s 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:29:25.929 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:29:25.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:25 smithi012 ceph-mon[147926]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T10:29:25.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 259 KiB/s rd, 1.0 MiB/s wr, 121 op/s 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e583 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: failed to encode map e584 with expected crc 2023-10-09T10:29:25.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:29:25.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:29:25.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:25 smithi110 ceph-mon[136032]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T10:29:26.201 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:26.203 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:29:26.203 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [], 2023-10-09T10:29:26.203 INFO:teuthology.orchestra.run.smithi012.stdout: "complaint_time": 30, 2023-10-09T10:29:26.203 INFO:teuthology.orchestra.run.smithi012.stdout: "num_blocked_ops": 0 2023-10-09T10:29:26.203 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:29:26.564 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[152263]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[152263]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[152263]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[152263]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[152263]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T10:29:26.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[152263]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.565 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:29:26.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:29:26.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[147926]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[147926]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[147926]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[147926]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[147926]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T10:29:26.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[147926]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.877 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:29:26.878 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[147926]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T10:29:26.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:26 smithi012 ceph-mon[152263]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T10:29:26.878 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:26 smithi110 ceph-mon[136032]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:26 smithi110 ceph-mon[136032]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:26 smithi110 ceph-mon[136032]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:26 smithi110 ceph-mon[136032]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:26 smithi110 ceph-mon[136032]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T10:29:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:26 smithi110 ceph-mon[136032]: failed to encode map e585 with expected crc 2023-10-09T10:29:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:26 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:29:26.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:26 smithi110 ceph-mon[136032]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T10:29:27.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[147926]: pgmap v1636: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 14 KiB/s rd, 0 B/s wr, 20 op/s 2023-10-09T10:29:27.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[147926]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[147926]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[147926]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[147926]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[152263]: pgmap v1636: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 14 KiB/s rd, 0 B/s wr, 20 op/s 2023-10-09T10:29:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[152263]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[152263]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[152263]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[152263]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.679 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:27.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:27 smithi110 ceph-mon[136032]: pgmap v1636: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 14 KiB/s rd, 0 B/s wr, 20 op/s 2023-10-09T10:29:27.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:27 smithi110 ceph-mon[136032]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:27 smithi110 ceph-mon[136032]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:27 smithi110 ceph-mon[136032]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:27 smithi110 ceph-mon[136032]: failed to encode map e586 with expected crc 2023-10-09T10:29:27.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:27 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:29:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:29:27] "GET /metrics HTTP/1.1" 200 34241 "" "Prometheus/2.33.4" 2023-10-09T10:29:28.684 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:28.771 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[152263]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.771 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[152263]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[152263]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[152263]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[152263]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[152263]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[152263]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[147926]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[147926]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[147926]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[147926]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[147926]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[147926]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-mon[147926]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:28 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:29:28.667+0000 7f85a99d3700 -1 mon.a@0(leader).osd e588 definitely_dead 0 2023-10-09T10:29:28.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:28 smithi110 ceph-mon[136032]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:28 smithi110 ceph-mon[136032]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T10:29:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:28 smithi110 ceph-mon[136032]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:28 smithi110 ceph-mon[136032]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:28 smithi110 ceph-mon[136032]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:28 smithi110 ceph-mon[136032]: failed to encode map e587 with expected crc 2023-10-09T10:29:28.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:28 smithi110 ceph-mon[136032]: failed to encode map e587 with expected crc 2023-10-09T10:29:29.606 INFO:teuthology.orchestra.run.smithi012.stderr:nodeep-scrub is unset 2023-10-09T10:29:29.606 INFO:teuthology.orchestra.run.smithi012.stderr:marked in osd.1. 2023-10-09T10:29:29.639 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: pgmap v1639: 73 pgs: 3 active+undersized+remapped, 7 peering, 63 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 144 KiB/s, 4 objects/s recovering 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3875153065' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:29.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:29.641 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/4052831729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:29.641 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:29.641 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: pgmap v1639: 73 pgs: 3 active+undersized+remapped, 7 peering, 63 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 144 KiB/s, 4 objects/s recovering 2023-10-09T10:29:29.641 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.641 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T10:29:29.641 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.641 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.641 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.642 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.642 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3875153065' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:29:29.642 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:29.642 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:29.642 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/4052831729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:29.642 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:29 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: pgmap v1639: 73 pgs: 3 active+undersized+remapped, 7 peering, 63 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 144 KiB/s, 4 objects/s recovering 2023-10-09T10:29:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T10:29:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: failed to encode map e588 with expected crc 2023-10-09T10:29:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3875153065' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:29:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:29.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:29.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/4052831729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:29.945 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:29 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:29:30.385 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2023-10-09T10:29:30.385 INFO:teuthology.orchestra.run.smithi012.stdout: "size": 20, 2023-10-09T10:29:30.385 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 600, 2023-10-09T10:29:30.385 INFO:teuthology.orchestra.run.smithi012.stdout: "ops": [ 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27047 10.4 e428/424 10:2e4baf45:::benchmark_data_smithi012_366312_object27046:head v 428'1731, mlcod=428'1729)", 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.70377015700001, 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16898123100000001, 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.386 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680251+0000", 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680249+0000", 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:30.387 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680286+0000", 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6359000000000002e-05 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680298+0000", 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2072999999999999e-05 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.388 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680305+0000", 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.2400000000000001e-06 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680357+0000", 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1610000000000002e-05 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.389 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680379+0000", 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.1732000000000001e-05 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849191+0000", 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.168812924 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.390 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849232+0000", 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0663e-05 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27057 10.5 e428/424 10:ac997561:::benchmark_data_smithi012_366312_object27056:head v 428'1679, mlcod=428'1677)", 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:29:30.391 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.70318519499995, 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.170571852, 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680836+0000", 2023-10-09T10:29:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680835+0000", 2023-10-09T10:29:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:29:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680874+0000", 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9499000000000002e-05 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680887+0000", 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.278e-05 2023-10-09T10:29:30.394 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680895+0000", 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.644e-06 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.680981+0000", 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.6104000000000001e-05 2023-10-09T10:29:30.395 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.681001+0000", 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0417000000000001e-05 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851371+0000", 2023-10-09T10:29:30.396 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.170369565 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851408+0000", 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.6757000000000001e-05 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.397 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36474.0:27067 10.5 e428/424 10:a3c37c8c:::benchmark_data_smithi012_366312_object27066:head v 428'1680, mlcod=428'1677)", 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 571.69972371999995, 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16713378700000001, 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:29:30.398 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684297+0000", 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684296+0000", 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.399 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.684335+0000", 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9345999999999999e-05 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848078+0000", 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.16374229300000001 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.400 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.848092+0000", 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4222e-05 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849076+0000", 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00098447500000000006 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.849097+0000", 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0258000000000002e-05 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.401 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851412+0000", 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.002315517 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:19:58.851431+0000", 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8884000000000001e-05 2023-10-09T10:29:30.402 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:139095 11.b e530/494 11:d93b600e:::benchmark_data_smithi012_404402_object26312:head v 530'8780, mlcod=530'8778)", 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:11.607303+0000", 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 198.77671808100001, 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.30654307200000003, 2023-10-09T10:29:30.403 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.405 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.405 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.405 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.405 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.405 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607303+0000", 2023-10-09T10:29:30.405 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.405 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.406 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607303+0000", 2023-10-09T10:29:30.406 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.406 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.406 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.406 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607302+0000", 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607322+0000", 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9579e-05 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.407 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607334+0000", 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2566e-05 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607343+0000", 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.5129999999999996e-06 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607847+0000", 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00050401700000000005 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.408 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607868+0000", 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0968e-05 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.610799+0000", 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0029315399999999998 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.409 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913846+0000", 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.30304684700000001 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:139094 11.14 e530/507 11:2f14ddda:::benchmark_data_smithi012_404402_object26311:head v 530'5291, mlcod=498'1)", 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:11.607358+0000", 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 198.77666260399999, 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.30646980000000001, 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.410 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607358+0000", 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607358+0000", 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.411 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607358+0000", 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607362+0000", 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1960000000000001e-06 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.412 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607368+0000", 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.1109999999999996e-06 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607373+0000", 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1529999999999999e-06 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607450+0000", 2023-10-09T10:29:30.413 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.7618000000000001e-05 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607482+0000", 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1461000000000003e-05 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.414 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.610696+0000", 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0032143580000000001 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913828+0000", 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.30313225999999999 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.415 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:139092 11.10 e530/527 11:0be16d14:::benchmark_data_smithi012_404402_object26309:head v 530'5238, mlcod=530'5236)", 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:11.607384+0000", 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 198.77663748099999, 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.30949074399999998, 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607384+0000", 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.416 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607384+0000", 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607383+0000", 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967296 2023-10-09T10:29:30.417 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607390+0000", 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.9009999999999997e-06 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607395+0000", 2023-10-09T10:29:30.418 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.0560000000000004e-06 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.607400+0000", 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5650000000000003e-06 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609524+0000", 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0021238110000000002 2023-10-09T10:29:30.419 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609539+0000", 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5554e-05 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.916849+0000", 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.30730986799999999 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.420 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.916874+0000", 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5222999999999999e-05 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:139098 11.18 e530/513 11:191f3209:::benchmark_data_smithi012_404402_object26315:head v 530'5335, mlcod=503'2373)", 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:11.609766+0000", 2023-10-09T10:29:30.421 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 198.77425507699999, 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.31040261899999999, 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609766+0000", 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.422 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609766+0000", 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609765+0000", 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.423 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609773+0000", 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.8169999999999999e-06 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609784+0000", 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0647e-05 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609790+0000", 2023-10-09T10:29:30.424 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6499999999999999e-06 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609832+0000", 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2147000000000002e-05 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.609864+0000", 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.1316999999999998e-05 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.425 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.920129+0000", 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.31026568500000001 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.920169+0000", 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9153000000000003e-05 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.426 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:139100 11.b e530/494 11:d331c725:::benchmark_data_smithi012_404402_object26317:head v 530'8781, mlcod=530'8778)", 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:11.611371+0000", 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 198.77264977900001, 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.30775719400000001, 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.611371+0000", 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.427 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.611371+0000", 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.611370+0000", 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.428 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.611380+0000", 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3300000000000005e-06 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913805+0000", 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.30242548000000002 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.429 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913817+0000", 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2295e-05 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913881+0000", 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.3207999999999995e-05 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913906+0000", 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.5181000000000002e-05 2023-10-09T10:29:30.430 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.919099+0000", 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0051930190000000001 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.919128+0000", 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9635999999999999e-05 2023-10-09T10:29:30.431 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.432 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.432 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.432 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.432 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.432 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:139105 11.e e530/494 11:78ebf9f2:::benchmark_data_smithi012_404402_object26322:head v 530'8791, mlcod=530'8789)", 2023-10-09T10:29:30.432 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:11.617971+0000", 2023-10-09T10:29:30.432 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 198.76605043999999, 2023-10-09T10:29:30.432 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.301064744, 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.617971+0000", 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.617971+0000", 2023-10-09T10:29:30.433 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.434 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.434 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.617969+0000", 2023-10-09T10:29:30.434 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:30.434 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.434 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.617985+0000", 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5928999999999999e-05 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913814+0000", 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.29582834299999999 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.435 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913821+0000", 2023-10-09T10:29:30.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.1400000000000002e-06 2023-10-09T10:29:30.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.436 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.436 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913888+0000", 2023-10-09T10:29:30.436 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.7292999999999994e-05 2023-10-09T10:29:30.436 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.436 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.913908+0000", 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.0225e-05 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.918993+0000", 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.005084664 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.437 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:11.919035+0000", 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2373999999999998e-05 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:157281 11.15 e537/513 11:abd3c1b6:::benchmark_data_smithi012_404402_object44498:head v 537'5873, mlcod=503'2340)", 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:40.565022+0000", 2023-10-09T10:29:30.438 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 169.81899951, 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.47237212200000001, 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565022+0000", 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.439 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565022+0000", 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565021+0000", 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:30.440 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565027+0000", 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.648e-06 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565033+0000", 2023-10-09T10:29:30.441 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6490000000000002e-06 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565038+0000", 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8049999999999997e-06 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.442 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.567023+0000", 2023-10-09T10:29:30.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019848859999999999 2023-10-09T10:29:30.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.443 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.443 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.443 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.567053+0000", 2023-10-09T10:29:30.443 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.0402000000000001e-05 2023-10-09T10:29:30.443 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.444 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.444 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.037366+0000", 2023-10-09T10:29:30.444 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.470312495 2023-10-09T10:29:30.444 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.444 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.037394+0000", 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.7637e-05 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:157273 11.5 e537/513 11:a73349e3:::benchmark_data_smithi012_404402_object44490:head v 537'5910, mlcod=503'2363)", 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:40.565075+0000", 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 169.818946461, 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.47234822399999998, 2023-10-09T10:29:30.445 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565075+0000", 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.446 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565075+0000", 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565074+0000", 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565081+0000", 2023-10-09T10:29:30.447 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6209999999999998e-06 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565091+0000", 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.9739999999999997e-06 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.565097+0000", 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.6669999999999999e-06 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.448 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.567096+0000", 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0019988580000000001 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.567113+0000", 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.6923999999999999e-05 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.037397+0000", 2023-10-09T10:29:30.449 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.470283799 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.037423+0000", 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6015000000000001e-05 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.450 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:157302 11.a e537/507 11:5b82f599:::benchmark_data_smithi012_404402_object44519:head v 537'9838, mlcod=537'9836)", 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:40.568258+0000", 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 169.815763441, 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.47225721599999998, 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568258+0000", 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.451 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568258+0000", 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568257+0000", 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.452 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568275+0000", 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.8185999999999998e-05 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568288+0000", 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.343e-05 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568297+0000", 2023-10-09T10:29:30.453 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.3760000000000008e-06 2023-10-09T10:29:30.454 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568456+0000", 2023-10-09T10:29:30.456 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.00015987799999999999 2023-10-09T10:29:30.456 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.456 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.456 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.456 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568483+0000", 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.6706000000000001e-05 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.040480+0000", 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.471997265 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.040515+0000", 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4434000000000003e-05 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.457 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:157291 11.9 e537/516 11:91b56ad7:::benchmark_data_smithi012_404402_object44508:head v 537'5929, mlcod=503'2296)", 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:40.568339+0000", 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 169.81568195200001, 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.475203453, 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.458 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568339+0000", 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568339+0000", 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568339+0000", 2023-10-09T10:29:30.459 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568349+0000", 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0522e-05 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568360+0000", 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0329999999999999e-05 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.460 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568365+0000", 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.3619999999999997e-06 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034767+0000", 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.46640227000000001 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034796+0000", 2023-10-09T10:29:30.461 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.9218999999999999e-05 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.043503+0000", 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0087069060000000004 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.043543+0000", 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.9205000000000002e-05 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.462 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:157287 11.8 e537/513 11:1044c9e8:::benchmark_data_smithi012_404402_object44504:head v 537'5860, mlcod=503'2340)", 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:40.568378+0000", 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 169.81564325900001, 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.47378436400000001, 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.463 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568378+0000", 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568378+0000", 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568377+0000", 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:30.464 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568382+0000", 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1529999999999999e-06 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568387+0000", 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8340000000000004e-06 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.465 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568392+0000", 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.8269999999999997e-06 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034711+0000", 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.46631824799999999 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034743+0000", 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.2895000000000003e-05 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.466 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.042117+0000", 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0073735210000000001 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.042162+0000", 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.5142000000000002e-05 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.467 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:157303 11.e e537/494 11:762be816:::benchmark_data_smithi012_404402_object44520:head v 537'9989, mlcod=537'9987)", 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:40.568867+0000", 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 169.81515380600001, 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.47325159900000002, 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568867+0000", 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.468 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568867+0000", 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568866+0000", 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.469 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568875+0000", 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.2330000000000006e-06 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568886+0000", 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.1134999999999999e-05 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568893+0000", 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 7.238e-06 2023-10-09T10:29:30.470 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.568958+0000", 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.5241999999999996e-05 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034637+0000", 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.465678437 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.471 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.042068+0000", 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0074309859999999997 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.042119+0000", 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1156000000000003e-05 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.472 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:157304 11.a e537/507 11:5a209d15:::benchmark_data_smithi012_404402_object44521:head v 537'9839, mlcod=537'9836)", 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:40.571553+0000", 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 169.812468002, 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.47400799100000002, 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.571553+0000", 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.473 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.571553+0000", 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.571552+0000", 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.999999 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.571567+0000", 2023-10-09T10:29:30.474 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4887e-05 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034644+0000", 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.46307670200000001 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034659+0000", 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5576000000000001e-05 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.475 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.037303+0000", 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0026440159999999999 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.037346+0000", 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.2629000000000001e-05 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.045527+0000", 2023-10-09T10:29:30.476 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0081810110000000002 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.045561+0000", 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 3.4116999999999999e-05 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:157305 11.4 e537/513 11:208b8c8b:::benchmark_data_smithi012_404402_object44522:head v 537'5809, mlcod=537'5807)", 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:26:40.576997+0000", 2023-10-09T10:29:30.477 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 169.807023593, 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.46528743299999997, 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.576997+0000", 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.478 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.576997+0000", 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.576996+0000", 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:40.577010+0000", 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.4100000000000001e-05 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.479 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034957+0000", 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.45794704200000003 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.034967+0000", 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.3179999999999996e-06 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.480 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.035061+0000", 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.4881000000000002e-05 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.035081+0000", 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.9409999999999999e-05 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.042256+0000", 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.007175201 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.481 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:26:41.042285+0000", 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.8824999999999999e-05 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGPush(11.8 541/513 [PushOp(11:11c28b4c:::benchmark_data_smithi012_404402_object70339:head, version: 505'3162, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:11c28b4c:::benchmark_data_smithi012_404402_object70339:head@505'3162, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:09.600015+0000", 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 140.78400594999999, 2023-10-09T10:29:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.089840398000000002, 2023-10-09T10:29:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:29:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.600015+0000", 2023-10-09T10:29:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.600015+0000", 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.600014+0000", 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.600039+0000", 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4735e-05 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.600051+0000", 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.2245e-05 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.600061+0000", 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 9.7210000000000006e-06 2023-10-09T10:29:30.485 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.600120+0000", 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.8356999999999997e-05 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.600126+0000", 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 6.8260000000000004e-06 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.486 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:09.689855+0000", 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.089729108000000002 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "osd_repop(client.36870.0:192400 11.9 e543/516 11:94c025e2:::benchmark_data_smithi012_404402_object79617:head v 543'7068, mlcod=505'3217)", 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:27:30.609834+0000", 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 119.774187039, 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.07726152, 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:29:30.487 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.609834+0000", 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.609834+0000", 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.488 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.609832+0000", 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999986 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.609848+0000", 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.5693999999999999e-05 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.609866+0000", 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.753e-05 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.489 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.609876+0000", 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 1.0149000000000001e-05 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.615440+0000", 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.0055637710000000003 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.490 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.615464+0000", 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 2.4545e-05 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "commit_sent", 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.687055+0000", 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.071590547000000004 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:27:30.687095+0000", 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.0778999999999997e-05 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.491 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "description": "MOSDPGRecoveryDelete(11.5 e546,513 11:a0117700:::benchmark_data_smithi012_404402_object76538:head,543'6921)", 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "initiated_at": "2023-10-09T10:28:11.053951+0000", 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "age": 79.330069619, 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.098012659000000002, 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "type_data": { 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "flag_point": "started", 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "events": [ 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "initiated", 2023-10-09T10:29:30.492 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.053951+0000", 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "throttled", 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.053951+0000", 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "header_read", 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.053951+0000", 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4294967295.9999995 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.493 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "all_read", 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.053955+0000", 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.1459999999999998e-06 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "dispatched", 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.053963+0000", 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 8.1559999999999995e-06 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "queued_for_pg", 2023-10-09T10:29:30.494 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.053968+0000", 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.1699999999999996e-06 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "reached_pg", 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.054024+0000", 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 5.5538000000000003e-05 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "started", 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.054029+0000", 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 4.9910000000000002e-06 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2023-10-09T10:29:30.495 INFO:teuthology.orchestra.run.smithi012.stdout: { 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout: "event": "done", 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout: "time": "2023-10-09T10:28:11.151964+0000", 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout: "duration": 0.097935171000000001 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout: } 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2023-10-09T10:29:30.496 INFO:teuthology.orchestra.run.smithi012.stdout:} 2023-10-09T10:29:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[147926]: failed to encode map e589 with expected crc 2023-10-09T10:29:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3875153065' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:29:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[147926]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T10:29:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[147926]: failed to encode map e589 with expected crc 2023-10-09T10:29:30.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:30.678 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[152263]: failed to encode map e589 with expected crc 2023-10-09T10:29:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3875153065' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:29:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[152263]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T10:29:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[152263]: failed to encode map e589 with expected crc 2023-10-09T10:29:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:30.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:30 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:30.717 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T10:29:30.717 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T10:29:30.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:30 smithi110 ceph-mon[136032]: failed to encode map e589 with expected crc 2023-10-09T10:29:30.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:30 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3875153065' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:29:30.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:30 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:29:30.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:30 smithi110 ceph-mon[136032]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T10:29:30.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:30 smithi110 ceph-mon[136032]: failed to encode map e589 with expected crc 2023-10-09T10:29:30.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:30 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:30.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:30 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:31.352 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:31.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[147926]: pgmap v1642: 73 pgs: 3 active+undersized+remapped, 3 peering, 67 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 1.7 KiB/s wr, 0 op/s; 1.1 MiB/s, 14 keys/s, 17 objects/s recovering 2023-10-09T10:29:31.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[147926]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[147926]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[147926]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T10:29:31.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[147926]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[147926]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[147926]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[147926]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[152263]: pgmap v1642: 73 pgs: 3 active+undersized+remapped, 3 peering, 67 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 1.7 KiB/s wr, 0 op/s; 1.1 MiB/s, 14 keys/s, 17 objects/s recovering 2023-10-09T10:29:31.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[152263]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[152263]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[152263]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T10:29:31.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[152263]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[152263]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[152263]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:31 smithi012 ceph-mon[152263]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:31 smithi110 ceph-mon[136032]: pgmap v1642: 73 pgs: 3 active+undersized+remapped, 3 peering, 67 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 1.7 KiB/s wr, 0 op/s; 1.1 MiB/s, 14 keys/s, 17 objects/s recovering 2023-10-09T10:29:31.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:31 smithi110 ceph-mon[136032]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:31 smithi110 ceph-mon[136032]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:31 smithi110 ceph-mon[136032]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T10:29:31.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:31 smithi110 ceph-mon[136032]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:31 smithi110 ceph-mon[136032]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:31 smithi110 ceph-mon[136032]: failed to encode map e590 with expected crc 2023-10-09T10:29:31.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:31 smithi110 ceph-mon[136032]: failed to encode map e590 with expected crc 2023-10-09T10:29:32.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[147926]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[147926]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T10:29:32.608 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[147926]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.608 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[147926]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.608 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[147926]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.861 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a: {} 2023-10-09T10:29:32.871 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b: {} 2023-10-09T10:29:32.880 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c: {} 2023-10-09T10:29:32.882 INFO:teuthology.orchestra.run.smithi012.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T10:29:32.882 INFO:teuthology.orchestra.run.smithi012.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T10:29:32.882 INFO:teuthology.orchestra.run.smithi012.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T10:29:32.907 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[152263]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.907 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[152263]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T10:29:32.907 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[152263]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.908 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[152263]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.908 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:32 smithi012 ceph-mon[152263]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:32 smithi110 ceph-mon[136032]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:32 smithi110 ceph-mon[136032]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T10:29:32.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:32 smithi110 ceph-mon[136032]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:32 smithi110 ceph-mon[136032]: failed to encode map e591 with expected crc 2023-10-09T10:29:32.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:32 smithi110 ceph-mon[136032]: failed to encode map e591 with expected crc 2023-10-09T10:29:33.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T10:29:33.610 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: pgmap v1645: 73 pgs: 3 active+undersized+remapped, 2 peering, 68 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 1.7 KiB/s wr, 0 op/s; 1.0 MiB/s, 14 keys/s, 10 objects/s recovering 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/900722758' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:29:33.611 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: pgmap v1645: 73 pgs: 3 active+undersized+remapped, 2 peering, 68 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 1.7 KiB/s wr, 0 op/s; 1.0 MiB/s, 14 keys/s, 10 objects/s recovering 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/900722758' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:29:33.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:29:33.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:29:33.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:33 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:33.846 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: pgmap v1645: 73 pgs: 3 active+undersized+remapped, 2 peering, 68 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 1.7 KiB/s wr, 0 op/s; 1.0 MiB/s, 14 keys/s, 10 objects/s recovering 2023-10-09T10:29:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T10:29:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:33.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: failed to encode map e592 with expected crc 2023-10-09T10:29:33.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/900722758' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:29:33.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:29:33.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:29:33.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:33 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[147926]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T10:29:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[147926]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[147926]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[147926]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[147926]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[152263]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T10:29:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[152263]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[152263]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[152263]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:34 smithi012 ceph-mon[152263]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:34 smithi110 ceph-mon[136032]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T10:29:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:34 smithi110 ceph-mon[136032]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:34 smithi110 ceph-mon[136032]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:34 smithi110 ceph-mon[136032]: failed to encode map e593 with expected crc 2023-10-09T10:29:34.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:34 smithi110 ceph-mon[136032]: failed to encode map e593 with expected crc 2023-10-09T10:29:35.630 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T10:29:35.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:35 smithi012 ceph-mon[147926]: pgmap v1648: 73 pgs: 1 active+undersized+remapped, 2 peering, 70 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T10:29:35.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:35 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1667746222' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:35.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:35 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:35 smithi012 ceph-mon[152263]: pgmap v1648: 73 pgs: 1 active+undersized+remapped, 2 peering, 70 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T10:29:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:35 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1667746222' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:35.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:35 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:35.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:35 smithi110 ceph-mon[136032]: pgmap v1648: 73 pgs: 1 active+undersized+remapped, 2 peering, 70 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T10:29:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:35 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1667746222' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:35.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:35 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:36.081 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T10:29:36.690 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:36.904 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:36 smithi012 ceph-mon[147926]: failed to encode map e594 with expected crc 2023-10-09T10:29:36.904 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:36 smithi012 ceph-mon[147926]: failed to encode map e594 with expected crc 2023-10-09T10:29:36.904 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:36 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:29:36.904 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:36 smithi012 ceph-mon[147926]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T10:29:36.904 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:36 smithi012 ceph-mon[152263]: failed to encode map e594 with expected crc 2023-10-09T10:29:36.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:36 smithi012 ceph-mon[152263]: failed to encode map e594 with expected crc 2023-10-09T10:29:36.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:36 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:29:36.905 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:36 smithi012 ceph-mon[152263]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T10:29:36.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:36 smithi110 ceph-mon[136032]: failed to encode map e594 with expected crc 2023-10-09T10:29:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:36 smithi110 ceph-mon[136032]: failed to encode map e594 with expected crc 2023-10-09T10:29:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:36 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:29:36.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:36 smithi110 ceph-mon[136032]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T10:29:37.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-mon[147926]: pgmap v1650: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4 B/s, 8 objects/s recovering 2023-10-09T10:29:37.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:37.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:29:37.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-mon[147926]: failed to encode map e594 with expected crc 2023-10-09T10:29:37.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-mon[152263]: pgmap v1650: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4 B/s, 8 objects/s recovering 2023-10-09T10:29:37.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:37.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:29:37.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-mon[152263]: failed to encode map e594 with expected crc 2023-10-09T10:29:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:37 smithi110 ceph-mon[136032]: pgmap v1650: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4 B/s, 8 objects/s recovering 2023-10-09T10:29:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:37 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:37 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:29:37.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:37 smithi110 ceph-mon[136032]: failed to encode map e594 with expected crc 2023-10-09T10:29:38.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:29:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:29:37] "GET /metrics HTTP/1.1" 200 34267 "" "Prometheus/2.33.4" 2023-10-09T10:29:38.651 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T10:29:38.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:38 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2503396547' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:38.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:38 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2503396547' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:38.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:38 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2503396547' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:29:39.026 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T10:29:39.638 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:39.911 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:39 smithi012 ceph-mon[147926]: pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3 B/s, 5 objects/s recovering 2023-10-09T10:29:39.912 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:39 smithi012 ceph-mon[147926]: failed to encode map e595 with expected crc 2023-10-09T10:29:39.912 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:39 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2503396547' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:29:39.912 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:39 smithi012 ceph-mon[147926]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T10:29:39.912 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:39 smithi012 ceph-mon[152263]: pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3 B/s, 5 objects/s recovering 2023-10-09T10:29:39.912 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:39 smithi012 ceph-mon[152263]: failed to encode map e595 with expected crc 2023-10-09T10:29:39.912 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:39 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2503396547' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:29:39.912 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:39 smithi012 ceph-mon[152263]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T10:29:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:39 smithi110 ceph-mon[136032]: pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 3 B/s, 5 objects/s recovering 2023-10-09T10:29:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:39 smithi110 ceph-mon[136032]: failed to encode map e595 with expected crc 2023-10-09T10:29:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:39 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2503396547' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:29:39.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:39 smithi110 ceph-mon[136032]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T10:29:41.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:40 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:41.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:40 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:41.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:40 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:41.756 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T10:29:42.162 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:41 smithi012 ceph-mon[147926]: pgmap v1653: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 3 B/s, 3 objects/s recovering 2023-10-09T10:29:42.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2860091601' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:42.163 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:41 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:42.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:41 smithi012 ceph-mon[152263]: pgmap v1653: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 3 B/s, 3 objects/s recovering 2023-10-09T10:29:42.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2860091601' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:42.163 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:41 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:42.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T10:29:42.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:41 smithi110 ceph-mon[136032]: pgmap v1653: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 3 B/s, 3 objects/s recovering 2023-10-09T10:29:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2860091601' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:42.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:41 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:42.793 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:43.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:42 smithi012 ceph-mon[152263]: failed to encode map e596 with expected crc 2023-10-09T10:29:43.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:42 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:29:43.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:42 smithi012 ceph-mon[152263]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T10:29:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:42 smithi012 ceph-mon[147926]: failed to encode map e596 with expected crc 2023-10-09T10:29:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:42 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:29:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:42 smithi012 ceph-mon[147926]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T10:29:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:42 smithi110 ceph-mon[136032]: failed to encode map e596 with expected crc 2023-10-09T10:29:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:42 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:29:43.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:42 smithi110 ceph-mon[136032]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T10:29:44.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:43 smithi012 ceph-mon[147926]: pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:44.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:43 smithi012 ceph-mon[152263]: pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:44.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:43 smithi110 ceph-mon[136032]: pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:44.981 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T10:29:45.325 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:44 smithi012 ceph-mon[152263]: pgmap v1656: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:45.325 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:44 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:45.325 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:45.325 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3118046237' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:45.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:44 smithi012 ceph-mon[147926]: pgmap v1656: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:45.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:44 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:45.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:45.326 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3118046237' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:45.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T10:29:45.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:44 smithi110 ceph-mon[136032]: pgmap v1656: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:45.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:44 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:29:45.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:45.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3118046237' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:29:45.941 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:46.345 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:45 smithi012 ceph-mon[147926]: failed to encode map e597 with expected crc 2023-10-09T10:29:46.345 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:45 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3118046237' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:29:46.345 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:45 smithi012 ceph-mon[147926]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T10:29:46.345 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:45 smithi012 ceph-mon[152263]: failed to encode map e597 with expected crc 2023-10-09T10:29:46.345 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:45 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3118046237' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:29:46.346 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:45 smithi012 ceph-mon[152263]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T10:29:46.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:45 smithi110 ceph-mon[136032]: failed to encode map e597 with expected crc 2023-10-09T10:29:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:45 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3118046237' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:29:46.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:45 smithi110 ceph-mon[136032]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T10:29:47.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:46 smithi012 ceph-mon[147926]: pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:47.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:46 smithi012 ceph-mon[152263]: pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:47.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:46 smithi110 ceph-mon[136032]: pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:48.003 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T10:29:48.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:47 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3691450702' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:29:48.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:29:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:29:47] "GET /metrics HTTP/1.1" 200 34248 "" "Prometheus/2.33.4" 2023-10-09T10:29:48.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:47 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3691450702' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:29:48.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T10:29:48.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:47 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3691450702' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:29:48.991 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:49.250 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: pgmap v1659: 73 pgs: 73 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:49.250 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3691450702' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:29:49.250 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T10:29:49.250 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.250 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.250 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[147926]: failed to encode map e599 with expected crc 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: pgmap v1659: 73 pgs: 73 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3691450702' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.251 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T10:29:49.252 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:48 smithi012 ceph-mon[152263]: failed to encode map e599 with expected crc 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: pgmap v1659: 73 pgs: 73 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3691450702' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: failed to encode map e598 with expected crc 2023-10-09T10:29:49.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T10:29:49.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:48 smithi110 ceph-mon[136032]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[147926]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[147926]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[147926]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[147926]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[147926]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[147926]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[147926]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[147926]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T10:29:50.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[152263]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[152263]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[152263]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[152263]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[152263]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[152263]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[152263]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:49 smithi012 ceph-mon[152263]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T10:29:50.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:49 smithi110 ceph-mon[136032]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:49 smithi110 ceph-mon[136032]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:49 smithi110 ceph-mon[136032]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:49 smithi110 ceph-mon[136032]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:49 smithi110 ceph-mon[136032]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:49 smithi110 ceph-mon[136032]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:49 smithi110 ceph-mon[136032]: failed to encode map e599 with expected crc 2023-10-09T10:29:50.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:49 smithi110 ceph-mon[136032]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T10:29:50.557 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T10:29:51.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T10:29:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: pgmap v1663: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 14 B/s, 0 objects/s recovering 2023-10-09T10:29:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2702261570' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:29:51.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2702261570' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[152263]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: pgmap v1663: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 14 B/s, 0 objects/s recovering 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2702261570' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:51.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T10:29:51.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2702261570' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:29:51.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:50 smithi012 ceph-mon[147926]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: pgmap v1663: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 14 B/s, 0 objects/s recovering 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: failed to encode map e600 with expected crc 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2702261570' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2702261570' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:29:51.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:50 smithi110 ceph-mon[136032]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T10:29:51.624 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:52.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:51 smithi012 ceph-mon[152263]: failed to encode map e601 with expected crc 2023-10-09T10:29:52.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:51 smithi012 ceph-mon[152263]: failed to encode map e601 with expected crc 2023-10-09T10:29:52.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:51 smithi012 ceph-mon[152263]: failed to encode map e602 with expected crc 2023-10-09T10:29:52.360 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:51 smithi012 ceph-mon[152263]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T10:29:52.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:51 smithi012 ceph-mon[147926]: failed to encode map e601 with expected crc 2023-10-09T10:29:52.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:51 smithi012 ceph-mon[147926]: failed to encode map e601 with expected crc 2023-10-09T10:29:52.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:51 smithi012 ceph-mon[147926]: failed to encode map e602 with expected crc 2023-10-09T10:29:52.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:51 smithi012 ceph-mon[147926]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T10:29:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:51 smithi110 ceph-mon[136032]: failed to encode map e601 with expected crc 2023-10-09T10:29:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:51 smithi110 ceph-mon[136032]: failed to encode map e601 with expected crc 2023-10-09T10:29:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:51 smithi110 ceph-mon[136032]: failed to encode map e602 with expected crc 2023-10-09T10:29:52.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:51 smithi110 ceph-mon[136032]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T10:29:53.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:52 smithi012 ceph-mon[147926]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:52 smithi012 ceph-mon[147926]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:52 smithi012 ceph-mon[147926]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:52 smithi012 ceph-mon[147926]: pgmap v1666: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 17 B/s, 0 objects/s recovering 2023-10-09T10:29:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:52 smithi012 ceph-mon[152263]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:52 smithi012 ceph-mon[152263]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:52 smithi012 ceph-mon[152263]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:52 smithi012 ceph-mon[152263]: pgmap v1666: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 17 B/s, 0 objects/s recovering 2023-10-09T10:29:53.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:52 smithi110 ceph-mon[136032]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:52 smithi110 ceph-mon[136032]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:52 smithi110 ceph-mon[136032]: failed to encode map e602 with expected crc 2023-10-09T10:29:53.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:52 smithi110 ceph-mon[136032]: pgmap v1666: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 17 B/s, 0 objects/s recovering 2023-10-09T10:29:54.011 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T10:29:54.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:53 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2762284387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:54.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:53 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:54.401 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:53 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2762284387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:54.401 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:53 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:54.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:53 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2762284387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:54.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:53 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:54.463 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T10:29:55.083 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:55.333 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[152263]: pgmap v1667: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 116 B/s, 1 objects/s recovering 2023-10-09T10:29:55.333 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[152263]: failed to encode map e603 with expected crc 2023-10-09T10:29:55.333 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:29:55.333 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[152263]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T10:29:55.333 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[152263]: failed to encode map e603 with expected crc 2023-10-09T10:29:55.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[147926]: pgmap v1667: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 116 B/s, 1 objects/s recovering 2023-10-09T10:29:55.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[147926]: failed to encode map e603 with expected crc 2023-10-09T10:29:55.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:29:55.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[147926]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T10:29:55.334 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:55 smithi012 ceph-mon[147926]: failed to encode map e603 with expected crc 2023-10-09T10:29:55.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:55 smithi110 ceph-mon[136032]: pgmap v1667: 73 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 71 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2/84 objects misplaced (2.381%); 116 B/s, 1 objects/s recovering 2023-10-09T10:29:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:55 smithi110 ceph-mon[136032]: failed to encode map e603 with expected crc 2023-10-09T10:29:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:55 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:29:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:55 smithi110 ceph-mon[136032]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T10:29:55.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:55 smithi110 ceph-mon[136032]: failed to encode map e603 with expected crc 2023-10-09T10:29:56.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:56 smithi012 ceph-mon[147926]: failed to encode map e603 with expected crc 2023-10-09T10:29:56.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:56 smithi012 ceph-mon[147926]: failed to encode map e603 with expected crc 2023-10-09T10:29:56.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:56 smithi012 ceph-mon[152263]: failed to encode map e603 with expected crc 2023-10-09T10:29:56.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:56 smithi012 ceph-mon[152263]: failed to encode map e603 with expected crc 2023-10-09T10:29:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:56 smithi110 ceph-mon[136032]: failed to encode map e603 with expected crc 2023-10-09T10:29:56.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:56 smithi110 ceph-mon[136032]: failed to encode map e603 with expected crc 2023-10-09T10:29:57.021 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T10:29:57.418 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[147926]: pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 683 KiB/s, 2 objects/s recovering 2023-10-09T10:29:57.419 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:57.419 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[147926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T10:29:57.419 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:29:57.419 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2401355165' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:57.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[152263]: pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 683 KiB/s, 2 objects/s recovering 2023-10-09T10:29:57.419 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:57.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[152263]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T10:29:57.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:29:57.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2401355165' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:57.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T10:29:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:57 smithi110 ceph-mon[136032]: pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 683 KiB/s, 2 objects/s recovering 2023-10-09T10:29:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:57 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:29:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:57 smithi110 ceph-mon[136032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T10:29:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:57 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:29:57.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:57 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2401355165' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:29:58.036 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:29:58.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:29:57 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:29:57] "GET /metrics HTTP/1.1" 200 34248 "" "Prometheus/2.33.4" 2023-10-09T10:29:58.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2401355165' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:29:58.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[147926]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[147926]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[147926]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[147926]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[147926]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2401355165' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[152263]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[152263]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[152263]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[152263]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:58 smithi012 ceph-mon[152263]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:58 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2401355165' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:29:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:58 smithi110 ceph-mon[136032]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T10:29:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:58 smithi110 ceph-mon[136032]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:58 smithi110 ceph-mon[136032]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:58 smithi110 ceph-mon[136032]: failed to encode map e604 with expected crc 2023-10-09T10:29:58.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:58 smithi110 ceph-mon[136032]: failed to encode map e604 with expected crc 2023-10-09T10:29:59.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[147926]: pgmap v1671: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 645 KiB/s, 2 objects/s recovering 2023-10-09T10:29:59.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[147926]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[147926]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[147926]: osdmap e605: 8 total, 8 up, 8 in 2023-10-09T10:29:59.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[147926]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[147926]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.429 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:59.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[152263]: pgmap v1671: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 645 KiB/s, 2 objects/s recovering 2023-10-09T10:29:59.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[152263]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[152263]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[152263]: osdmap e605: 8 total, 8 up, 8 in 2023-10-09T10:29:59.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[152263]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[152263]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.430 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:29:59 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:29:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:59 smithi110 ceph-mon[136032]: pgmap v1671: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 645 KiB/s, 2 objects/s recovering 2023-10-09T10:29:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:59 smithi110 ceph-mon[136032]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:59 smithi110 ceph-mon[136032]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:59 smithi110 ceph-mon[136032]: osdmap e605: 8 total, 8 up, 8 in 2023-10-09T10:29:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:59 smithi110 ceph-mon[136032]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:59 smithi110 ceph-mon[136032]: failed to encode map e605 with expected crc 2023-10-09T10:29:59.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:29:59 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:00.038 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T10:30:00.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:00 smithi012 ceph-mon[147926]: failed to encode map e605 with expected crc 2023-10-09T10:30:00.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:00 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1199379178' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:00.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:00 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:00.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:00 smithi012 ceph-mon[147926]: overall HEALTH_OK 2023-10-09T10:30:00.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:00 smithi012 ceph-mon[152263]: failed to encode map e605 with expected crc 2023-10-09T10:30:00.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:00 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1199379178' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:00.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:00 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:00.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:00 smithi012 ceph-mon[152263]: overall HEALTH_OK 2023-10-09T10:30:00.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:00 smithi110 ceph-mon[136032]: failed to encode map e605 with expected crc 2023-10-09T10:30:00.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:00 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1199379178' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:00.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:00 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:00.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:00 smithi110 ceph-mon[136032]: overall HEALTH_OK 2023-10-09T10:30:00.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T10:30:01.074 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:01.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:01 smithi012 ceph-mon[152263]: pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 682 KiB/s, 1 objects/s recovering 2023-10-09T10:30:01.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:01 smithi012 ceph-mon[152263]: failed to encode map e606 with expected crc 2023-10-09T10:30:01.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:01 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1199379178' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:30:01.326 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:01 smithi012 ceph-mon[152263]: osdmap e606: 8 total, 8 up, 8 in 2023-10-09T10:30:01.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:01 smithi012 ceph-mon[147926]: pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 682 KiB/s, 1 objects/s recovering 2023-10-09T10:30:01.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:01 smithi012 ceph-mon[147926]: failed to encode map e606 with expected crc 2023-10-09T10:30:01.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:01 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1199379178' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:30:01.327 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:01 smithi012 ceph-mon[147926]: osdmap e606: 8 total, 8 up, 8 in 2023-10-09T10:30:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:01 smithi110 ceph-mon[136032]: pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 682 KiB/s, 1 objects/s recovering 2023-10-09T10:30:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:01 smithi110 ceph-mon[136032]: failed to encode map e606 with expected crc 2023-10-09T10:30:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:01 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1199379178' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:30:01.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:01 smithi110 ceph-mon[136032]: osdmap e606: 8 total, 8 up, 8 in 2023-10-09T10:30:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[147926]: failed to encode map e606 with expected crc 2023-10-09T10:30:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[147926]: failed to encode map e606 with expected crc 2023-10-09T10:30:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T10:30:02.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[152263]: failed to encode map e606 with expected crc 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[152263]: failed to encode map e606 with expected crc 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:02 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T10:30:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:02 smithi110 ceph-mon[136032]: failed to encode map e606 with expected crc 2023-10-09T10:30:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:02 smithi110 ceph-mon[136032]: failed to encode map e606 with expected crc 2023-10-09T10:30:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:02 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T10:30:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:30:02.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:02 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T10:30:03.058 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T10:30:03.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:03.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T10:30:03.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: osdmap e607: 8 total, 8 up, 8 in 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2792307099' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T10:30:03.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:30:03.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T10:30:03.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: osdmap e607: 8 total, 8 up, 8 in 2023-10-09T10:30:03.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2792307099' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:03.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:03 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: osdmap e607: 8 total, 8 up, 8 in 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: failed to encode map e607 with expected crc 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2792307099' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:03.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:03 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:30:03.507 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T10:30:04.123 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:04.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[147926]: failed to encode map e607 with expected crc 2023-10-09T10:30:04.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[147926]: failed to encode map e607 with expected crc 2023-10-09T10:30:04.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[147926]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.339 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:30:04.340 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[147926]: osdmap e608: 8 total, 8 up, 8 in 2023-10-09T10:30:04.340 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[147926]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.340 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[147926]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.340 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[147926]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[152263]: failed to encode map e607 with expected crc 2023-10-09T10:30:04.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[152263]: failed to encode map e607 with expected crc 2023-10-09T10:30:04.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[152263]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:30:04.341 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[152263]: osdmap e608: 8 total, 8 up, 8 in 2023-10-09T10:30:04.341 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[152263]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.341 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[152263]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.341 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:04 smithi012 ceph-mon[152263]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:04 smithi110 ceph-mon[136032]: failed to encode map e607 with expected crc 2023-10-09T10:30:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:04 smithi110 ceph-mon[136032]: failed to encode map e607 with expected crc 2023-10-09T10:30:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:04 smithi110 ceph-mon[136032]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:04 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:30:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:04 smithi110 ceph-mon[136032]: osdmap e608: 8 total, 8 up, 8 in 2023-10-09T10:30:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:04 smithi110 ceph-mon[136032]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:04 smithi110 ceph-mon[136032]: failed to encode map e608 with expected crc 2023-10-09T10:30:04.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:04 smithi110 ceph-mon[136032]: failed to encode map e608 with expected crc 2023-10-09T10:30:05.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:05 smithi012 ceph-mon[147926]: pgmap v1678: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:05 smithi012 ceph-mon[147926]: failed to encode map e609 with expected crc 2023-10-09T10:30:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:05 smithi012 ceph-mon[147926]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T10:30:05.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:05 smithi012 ceph-mon[147926]: failed to encode map e609 with expected crc 2023-10-09T10:30:05.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:05 smithi012 ceph-mon[152263]: pgmap v1678: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:05.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:05 smithi012 ceph-mon[152263]: failed to encode map e609 with expected crc 2023-10-09T10:30:05.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:05 smithi012 ceph-mon[152263]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T10:30:05.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:05 smithi012 ceph-mon[152263]: failed to encode map e609 with expected crc 2023-10-09T10:30:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:05 smithi110 ceph-mon[136032]: pgmap v1678: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:05 smithi110 ceph-mon[136032]: failed to encode map e609 with expected crc 2023-10-09T10:30:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:05 smithi110 ceph-mon[136032]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T10:30:05.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:05 smithi110 ceph-mon[136032]: failed to encode map e609 with expected crc 2023-10-09T10:30:06.068 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T10:30:06.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:06 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/769495542' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:06.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:06 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:06.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:06 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/769495542' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:06.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:06 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:06.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:06 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/769495542' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:06.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:06 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:06.456 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T10:30:07.068 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:07.328 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[147926]: pgmap v1680: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:07.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[147926]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:30:07.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[147926]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T10:30:07.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[147926]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[147926]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.329 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[147926]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[152263]: pgmap v1680: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:07.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[152263]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:30:07.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[152263]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T10:30:07.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[152263]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[152263]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-mon[152263]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:07 smithi110 ceph-mon[136032]: pgmap v1680: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:07 smithi110 ceph-mon[136032]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:07 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:30:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:07 smithi110 ceph-mon[136032]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T10:30:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:07 smithi110 ceph-mon[136032]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:07 smithi110 ceph-mon[136032]: failed to encode map e610 with expected crc 2023-10-09T10:30:07.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:07 smithi110 ceph-mon[136032]: failed to encode map e610 with expected crc 2023-10-09T10:30:08.093 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:07 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:30:07] "GET /metrics HTTP/1.1" 200 34251 "" "Prometheus/2.33.4" 2023-10-09T10:30:09.079 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T10:30:09.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:09 smithi012 ceph-mon[147926]: pgmap v1682: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:09.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:09 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1316801424' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:09.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:09 smithi012 ceph-mon[152263]: pgmap v1682: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:09.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:09 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1316801424' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:09.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:09 smithi110 ceph-mon[136032]: pgmap v1682: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:09.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:09 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1316801424' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:30:09.487 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T10:30:10.097 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:10.329 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:10 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/1316801424' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:30:10.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:10 smithi012 ceph-mon[152263]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T10:30:10.330 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:10 smithi012 ceph-mon[152263]: failed to encode map e611 with expected crc 2023-10-09T10:30:10.330 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:10 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/1316801424' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:30:10.330 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:10 smithi012 ceph-mon[147926]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T10:30:10.330 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:10 smithi012 ceph-mon[147926]: failed to encode map e611 with expected crc 2023-10-09T10:30:10.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:10 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/1316801424' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:30:10.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:10 smithi110 ceph-mon[136032]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T10:30:10.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:10 smithi110 ceph-mon[136032]: failed to encode map e611 with expected crc 2023-10-09T10:30:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[147926]: pgmap v1684: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[147926]: failed to encode map e612 with expected crc 2023-10-09T10:30:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[147926]: failed to encode map e611 with expected crc 2023-10-09T10:30:11.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[147926]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T10:30:11.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[152263]: pgmap v1684: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:11.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[152263]: failed to encode map e612 with expected crc 2023-10-09T10:30:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[152263]: failed to encode map e611 with expected crc 2023-10-09T10:30:11.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:11 smithi012 ceph-mon[152263]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T10:30:11.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:11 smithi110 ceph-mon[136032]: pgmap v1684: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:11.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:11 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:11.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:11 smithi110 ceph-mon[136032]: failed to encode map e612 with expected crc 2023-10-09T10:30:11.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:11 smithi110 ceph-mon[136032]: failed to encode map e611 with expected crc 2023-10-09T10:30:11.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:11 smithi110 ceph-mon[136032]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T10:30:12.090 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T10:30:12.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:12 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2053092013' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:30:12.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:12 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2053092013' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:30:12.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:12 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2053092013' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:30:12.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T10:30:13.139 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:13.356 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:13 smithi012 ceph-mon[147926]: pgmap v1686: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:13.356 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:13 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2053092013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:30:13.356 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:13 smithi012 ceph-mon[147926]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T10:30:13.356 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:13 smithi012 ceph-mon[152263]: pgmap v1686: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:13.356 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:13 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2053092013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:30:13.357 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:13 smithi012 ceph-mon[152263]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T10:30:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:13 smithi110 ceph-mon[136032]: pgmap v1686: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:13 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2053092013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:30:13.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:13 smithi110 ceph-mon[136032]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T10:30:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:14 smithi012 ceph-mon[147926]: failed to encode map e613 with expected crc 2023-10-09T10:30:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:14 smithi012 ceph-mon[147926]: failed to encode map e613 with expected crc 2023-10-09T10:30:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:14 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:14.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:14 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:14 smithi012 ceph-mon[152263]: failed to encode map e613 with expected crc 2023-10-09T10:30:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:14 smithi012 ceph-mon[152263]: failed to encode map e613 with expected crc 2023-10-09T10:30:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:14 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:14 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:14 smithi110 ceph-mon[136032]: failed to encode map e613 with expected crc 2023-10-09T10:30:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:14 smithi110 ceph-mon[136032]: failed to encode map e613 with expected crc 2023-10-09T10:30:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:14 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:14.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:14 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:15.103 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T10:30:15.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:15 smithi012 ceph-mon[147926]: pgmap v1688: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:15.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:15 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/248319451' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:30:15.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:15 smithi012 ceph-mon[152263]: pgmap v1688: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:15.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:15 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/248319451' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:30:15.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:15 smithi110 ceph-mon[136032]: pgmap v1688: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:15.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:15 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/248319451' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:30:15.572 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T10:30:16.188 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:16.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[147926]: failed to encode map e614 with expected crc 2023-10-09T10:30:16.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[147926]: failed to encode map e614 with expected crc 2023-10-09T10:30:16.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/248319451' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:30:16.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[147926]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T10:30:16.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[147926]: failed to encode map e614 with expected crc 2023-10-09T10:30:16.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[152263]: failed to encode map e614 with expected crc 2023-10-09T10:30:16.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[152263]: failed to encode map e614 with expected crc 2023-10-09T10:30:16.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/248319451' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:30:16.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[152263]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T10:30:16.413 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:16 smithi012 ceph-mon[152263]: failed to encode map e614 with expected crc 2023-10-09T10:30:16.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:16 smithi110 ceph-mon[136032]: failed to encode map e614 with expected crc 2023-10-09T10:30:16.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:16 smithi110 ceph-mon[136032]: failed to encode map e614 with expected crc 2023-10-09T10:30:16.446 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:16 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/248319451' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:30:16.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:16 smithi110 ceph-mon[136032]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T10:30:16.447 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:16 smithi110 ceph-mon[136032]: failed to encode map e614 with expected crc 2023-10-09T10:30:17.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[152263]: pgmap v1690: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:17.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T10:30:17.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:30:17.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[152263]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[152263]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.427 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[152263]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[152263]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T10:30:17.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[147926]: pgmap v1690: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:17.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T10:30:17.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:30:17.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[147926]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[147926]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[147926]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-mon[147926]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T10:30:17.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:17 smithi110 ceph-mon[136032]: pgmap v1690: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:17 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T10:30:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:17 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:30:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:17 smithi110 ceph-mon[136032]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:17 smithi110 ceph-mon[136032]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:17 smithi110 ceph-mon[136032]: failed to encode map e615 with expected crc 2023-10-09T10:30:17.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:17 smithi110 ceph-mon[136032]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T10:30:18.118 INFO:teuthology.orchestra.run.smithi012.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T10:30:18.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:18 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/658655353' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:18.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:17 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:30:17] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.33.4" 2023-10-09T10:30:18.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:18 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/658655353' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:18.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:18 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/658655353' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:18.526 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T10:30:19.142 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:19.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:19 smithi012 ceph-mon[147926]: pgmap v1692: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:19.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:19 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/658655353' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:30:19.361 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:19 smithi012 ceph-mon[147926]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T10:30:19.362 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:19 smithi012 ceph-mon[152263]: pgmap v1692: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:19.362 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:19 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/658655353' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:30:19.362 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:19 smithi012 ceph-mon[152263]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T10:30:19.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:19 smithi110 ceph-mon[136032]: pgmap v1692: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:19.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:19 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/658655353' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:30:19.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:19 smithi110 ceph-mon[136032]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T10:30:20.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:20 smithi012 ceph-mon[147926]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:20 smithi012 ceph-mon[147926]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:20 smithi012 ceph-mon[147926]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:20 smithi012 ceph-mon[147926]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:20 smithi012 ceph-mon[152263]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:20 smithi012 ceph-mon[152263]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:20 smithi012 ceph-mon[152263]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:20 smithi012 ceph-mon[152263]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:20 smithi110 ceph-mon[136032]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:20 smithi110 ceph-mon[136032]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:20 smithi110 ceph-mon[136032]: failed to encode map e616 with expected crc 2023-10-09T10:30:20.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:20 smithi110 ceph-mon[136032]: failed to encode map e616 with expected crc 2023-10-09T10:30:21.138 INFO:teuthology.orchestra.run.smithi012.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T10:30:21.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:21 smithi012 ceph-mon[147926]: pgmap v1694: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:21.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:21 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/980910887' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:21.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:21 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:21.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:21 smithi012 ceph-mon[152263]: pgmap v1694: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:21.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:21 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/980910887' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:21.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:21 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:21.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:21 smithi110 ceph-mon[136032]: pgmap v1694: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:21 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/980910887' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:21.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:21 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:30:21.580 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T10:30:21.580 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T10:30:21.581 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T10:30:21.581 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T10:30:21.581 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T10:30:21.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:30:22.196 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:22.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[147926]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[147926]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[147926]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[147926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:30:22.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[147926]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T10:30:22.412 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[147926]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[152263]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[152263]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[152263]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[152263]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:30:22.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[152263]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T10:30:22.412 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:22 smithi012 ceph-mon[152263]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:22 smithi110 ceph-mon[136032]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:22 smithi110 ceph-mon[136032]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:22 smithi110 ceph-mon[136032]: failed to encode map e617 with expected crc 2023-10-09T10:30:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:22 smithi110 ceph-mon[136032]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:30:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:22 smithi110 ceph-mon[136032]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T10:30:22.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:22 smithi110 ceph-mon[136032]: failed to encode map e617 with expected crc 2023-10-09T10:30:23.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: failed to encode map e617 with expected crc 2023-10-09T10:30:23.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: pgmap v1696: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:23.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:23.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[147926]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: failed to encode map e617 with expected crc 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: pgmap v1696: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T10:30:23.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.429 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:23 smithi012 ceph-mon[152263]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: failed to encode map e617 with expected crc 2023-10-09T10:30:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: pgmap v1696: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.443 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T10:30:23.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.444 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:23 smithi110 ceph-mon[136032]: failed to encode map e618 with expected crc 2023-10-09T10:30:23.988 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:30:23.988 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":618,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T10:30:22.122913+0000","last_up_change":"2023-10-09T10:15:41.261345+0000","last_in_change":"2023-10-09T10:29:29.575169+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:47:41.614981+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"580","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":580,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:48:24.361405+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:48:24.948128+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":75,"up_from":79,"up_thru":607,"down_at":76,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6803","nonce":3324439086}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6805","nonce":3324439086}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6809","nonce":3324439086}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6807","nonce":3324439086}]},"public_addr":"172.21.15.12:6803/3324439086","cluster_addr":"172.21.15.12:6805/3324439086","heartbeat_back_addr":"172.21.15.12:6809/3324439086","heartbeat_front_addr":"172.21.15.12:6807/3324439086","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":80,"up_from":84,"up_thru":598,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6811","nonce":715957322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6813","nonce":715957322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6817","nonce":715957322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6815","nonce":715957322}]},"public_addr":"172.21.15.12:6811/715957322","cluster_addr":"172.21.15.12:6813/715957322","heartbeat_back_addr":"172.21.15.12:6817/715957322","heartbeat_front_addr":"172.21.15.12:6815/715957322","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":89,"last_clean_end":212,"up_from":217,"up_thru":611,"down_at":213,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4176449052},{"type":"v1","addr":"172.21.15.12:6819","nonce":4176449052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4176449052},{"type":"v1","addr":"172.21.15.12:6821","nonce":4176449052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4176449052},{"type":"v1","addr":"172.21.15.12:6825","nonce":4176449052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4176449052},{"type":"v1","addr":"172.21.15.12:6823","nonce":4176449052}]},"public_addr":"172.21.15.12:6819/4176449052","cluster_addr":"172.21.15.12:6821/4176449052","heartbeat_back_addr":"172.21.15.12:6825/4176449052","heartbeat_front_addr":"172.21.15.12:6823/4176449052","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":250,"last_clean_end":344,"up_from":352,"up_thru":617,"down_at":345,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1499112407},{"type":"v1","addr":"172.21.15.12:6827","nonce":1499112407}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1499112407},{"type":"v1","addr":"172.21.15.12:6829","nonce":1499112407}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":1499112407},{"type":"v1","addr":"172.21.15.12:6833","nonce":1499112407}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1499112407},{"type":"v1","addr":"172.21.15.12:6831","nonce":1499112407}]},"public_addr":"172.21.15.12:6827/1499112407","cluster_addr":"172.21.15.12:6829/1499112407","heartbeat_back_addr":"172.21.15.12:6833/1499112407","heartbeat_front_addr":"172.21.15.12:6831/1499112407","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":238,"up_from":266,"up_thru":614,"down_at":239,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":881762686},{"type":"v1","addr":"172.21.15.110:6801","nonce":881762686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":881762686},{"type":"v1","addr":"172.21.15.110:6803","nonce":881762686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":881762686},{"type":"v1","addr":"172.21.15.110:6807","nonce":881762686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":881762686},{"type":"v1","addr":"172.21.15.110:6805","nonce":881762686}]},"public_addr":"172.21.15.110:6801/881762686","cluster_addr":"172.21.15.110:6803/881762686","heartbeat_back_addr":"172.21.15.110:6807/881762686","heartbeat_front_addr":"172.21.15.110:6805/881762686","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":607,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":607,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":608,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":75},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":80},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":212},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":344},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":238},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.b","mappings":[{"from":4,"to":0}]},{"pgid":"3.f","mappings":[{"from":4,"to":0}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/2293469660":"2023-10-10T09:50:12.280108+0000","172.21.15.12:0/2348702677":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/2990486696":"2023-10-10T09:51:58.776424+0000","172.21.15.110:0/1671269215":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/1146481933":"2023-10-10T09:51:58.776424+0000","172.21.15.110:0/3560901131":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/1925191424":"2023-10-10T09:50:12.280108+0000","172.21.15.12:6801/1925191424":"2023-10-10T09:50:12.280108+0000","172.21.15.110:0/4208451228":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6801/1146481933":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/1614086989":"2023-10-10T09:51:58.776424+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.110:0/1606345106":"2023-10-10T09:51:12.282860+0000","172.21.15.110:0/1952966694":"2023-10-10T09:51:12.282860+0000","172.21.15.12:0/3260703278":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/807677503":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/928078357":"2023-10-10T09:50:12.280108+0000","172.21.15.12:0/518580822":"2023-10-10T09:51:58.776424+0000","172.21.15.110:6835/4098121826":"2023-10-10T09:51:12.282860+0000","172.21.15.110:6834/4098121826":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.110:0/3728005719":"2023-10-10T09:51:12.282860+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:30:24.332 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T10:30:24.332 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:30:24.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:24 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/557174198' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:30:24.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:24 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/557174198' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:30:24.941 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:24.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:24 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/557174198' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:30:25.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:25 smithi012 ceph-mon[147926]: pgmap v1698: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:25.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:25 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:25.968 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:25 smithi012 ceph-mon[152263]: pgmap v1698: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:25.968 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:25 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:25 smithi110 ceph-mon[136032]: pgmap v1698: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:26.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:25 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:26.338 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:30:26.338 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":618,"fsid":"1881d6a6-6688-11ee-8db5-212e2dc638e7","created":"2023-10-09T09:42:53.020199+0000","modified":"2023-10-09T10:30:22.122913+0000","last_up_change":"2023-10-09T10:15:41.261345+0000","last_in_change":"2023-10-09T10:29:29.575169+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T09:45:36.218461+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T09:47:41.614981+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"580","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":580,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T09:48:24.361405+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T09:48:24.948128+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"23895d43-58c0-4692-9ca5-d05fc064a71e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":75,"up_from":79,"up_thru":607,"down_at":76,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6803","nonce":3324439086}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6805","nonce":3324439086}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6809","nonce":3324439086}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3324439086},{"type":"v1","addr":"172.21.15.12:6807","nonce":3324439086}]},"public_addr":"172.21.15.12:6803/3324439086","cluster_addr":"172.21.15.12:6805/3324439086","heartbeat_back_addr":"172.21.15.12:6809/3324439086","heartbeat_front_addr":"172.21.15.12:6807/3324439086","state":["exists","up"]},{"osd":1,"uuid":"8657aa57-580d-4318-a093-2cc78c0f72a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":80,"up_from":84,"up_thru":598,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6811","nonce":715957322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6813","nonce":715957322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6817","nonce":715957322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":715957322},{"type":"v1","addr":"172.21.15.12:6815","nonce":715957322}]},"public_addr":"172.21.15.12:6811/715957322","cluster_addr":"172.21.15.12:6813/715957322","heartbeat_back_addr":"172.21.15.12:6817/715957322","heartbeat_front_addr":"172.21.15.12:6815/715957322","state":["exists","up"]},{"osd":2,"uuid":"4bab5bc5-77e6-4b60-9cd1-511c5d18ff03","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":89,"last_clean_end":212,"up_from":217,"up_thru":611,"down_at":213,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4176449052},{"type":"v1","addr":"172.21.15.12:6819","nonce":4176449052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4176449052},{"type":"v1","addr":"172.21.15.12:6821","nonce":4176449052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4176449052},{"type":"v1","addr":"172.21.15.12:6825","nonce":4176449052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4176449052},{"type":"v1","addr":"172.21.15.12:6823","nonce":4176449052}]},"public_addr":"172.21.15.12:6819/4176449052","cluster_addr":"172.21.15.12:6821/4176449052","heartbeat_back_addr":"172.21.15.12:6825/4176449052","heartbeat_front_addr":"172.21.15.12:6823/4176449052","state":["exists","up"]},{"osd":3,"uuid":"bc089815-f2a8-41ef-9b69-505b7b51963e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":250,"last_clean_end":344,"up_from":352,"up_thru":617,"down_at":345,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":1499112407},{"type":"v1","addr":"172.21.15.12:6827","nonce":1499112407}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":1499112407},{"type":"v1","addr":"172.21.15.12:6829","nonce":1499112407}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":1499112407},{"type":"v1","addr":"172.21.15.12:6833","nonce":1499112407}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":1499112407},{"type":"v1","addr":"172.21.15.12:6831","nonce":1499112407}]},"public_addr":"172.21.15.12:6827/1499112407","cluster_addr":"172.21.15.12:6829/1499112407","heartbeat_back_addr":"172.21.15.12:6833/1499112407","heartbeat_front_addr":"172.21.15.12:6831/1499112407","state":["exists","up"]},{"osd":4,"uuid":"ad0b6f60-bf01-465b-acee-565d4d4a3df5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":238,"up_from":266,"up_thru":614,"down_at":239,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":881762686},{"type":"v1","addr":"172.21.15.110:6801","nonce":881762686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":881762686},{"type":"v1","addr":"172.21.15.110:6803","nonce":881762686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":881762686},{"type":"v1","addr":"172.21.15.110:6807","nonce":881762686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":881762686},{"type":"v1","addr":"172.21.15.110:6805","nonce":881762686}]},"public_addr":"172.21.15.110:6801/881762686","cluster_addr":"172.21.15.110:6803/881762686","heartbeat_back_addr":"172.21.15.110:6807/881762686","heartbeat_front_addr":"172.21.15.110:6805/881762686","state":["exists","up"]},{"osd":5,"uuid":"63ba8578-42c2-4852-acac-e616de4ccd47","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":607,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6809","nonce":3210978369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6811","nonce":3210978369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6815","nonce":3210978369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":3210978369},{"type":"v1","addr":"172.21.15.110:6813","nonce":3210978369}]},"public_addr":"172.21.15.110:6809/3210978369","cluster_addr":"172.21.15.110:6811/3210978369","heartbeat_back_addr":"172.21.15.110:6815/3210978369","heartbeat_front_addr":"172.21.15.110:6813/3210978369","state":["exists","up"]},{"osd":6,"uuid":"00113d3e-cc98-49fe-ba99-7172c2c8c80b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":607,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6817","nonce":3541801135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6819","nonce":3541801135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6823","nonce":3541801135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":3541801135},{"type":"v1","addr":"172.21.15.110:6821","nonce":3541801135}]},"public_addr":"172.21.15.110:6817/3541801135","cluster_addr":"172.21.15.110:6819/3541801135","heartbeat_back_addr":"172.21.15.110:6823/3541801135","heartbeat_front_addr":"172.21.15.110:6821/3541801135","state":["exists","up"]},{"osd":7,"uuid":"a34ee707-c52a-48c8-be23-cce0e1cc0801","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":608,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6825","nonce":4270614104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6827","nonce":4270614104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6831","nonce":4270614104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":4270614104},{"type":"v1","addr":"172.21.15.110:6829","nonce":4270614104}]},"public_addr":"172.21.15.110:6825/4270614104","cluster_addr":"172.21.15.110:6827/4270614104","heartbeat_back_addr":"172.21.15.110:6831/4270614104","heartbeat_front_addr":"172.21.15.110:6829/4270614104","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:44:55.332547+0000","dead_epoch":75},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:14.911918+0000","dead_epoch":80},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:34.267414+0000","dead_epoch":212},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:45:53.249590+0000","dead_epoch":344},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:11.930004+0000","dead_epoch":238},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:31.364016+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:46:50.227781+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T09:47:09.341651+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.b","mappings":[{"from":4,"to":0}]},{"pgid":"3.f","mappings":[{"from":4,"to":0}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/2348702677":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/1614086989":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/807677503":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/1937850108":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/847521526":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/497354987":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/2293469660":"2023-10-10T09:50:12.280108+0000","172.21.15.12:0/3530794450":"2023-10-10T09:47:27.376359+0000","172.21.15.12:0/826788601":"2023-10-10T09:43:46.740224+0000","172.21.15.12:0/1874684344":"2023-10-10T09:43:25.068166+0000","172.21.15.110:0/3560901131":"2023-10-10T09:51:12.282860+0000","172.21.15.12:0/518580822":"2023-10-10T09:51:58.776424+0000","172.21.15.12:0/928078357":"2023-10-10T09:50:12.280108+0000","172.21.15.12:6801/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.12:6800/1925191424":"2023-10-10T09:50:12.280108+0000","172.21.15.110:0/4208451228":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6800/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6801/1925191424":"2023-10-10T09:50:12.280108+0000","172.21.15.12:0/2990486696":"2023-10-10T09:51:58.776424+0000","172.21.15.12:6800/1146481933":"2023-10-10T09:51:58.776424+0000","172.21.15.110:0/1671269215":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6801/3616444435":"2023-10-10T09:43:46.740224+0000","172.21.15.12:6800/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:6800/106778378":"2023-10-10T09:47:27.376359+0000","172.21.15.110:0/3728005719":"2023-10-10T09:51:12.282860+0000","172.21.15.110:6834/4098121826":"2023-10-10T09:51:12.282860+0000","172.21.15.110:6835/4098121826":"2023-10-10T09:51:12.282860+0000","172.21.15.12:6801/1146481933":"2023-10-10T09:51:58.776424+0000","172.21.15.12:6801/819667578":"2023-10-10T09:43:25.068166+0000","172.21.15.12:0/3260703278":"2023-10-10T09:51:58.776424+0000","172.21.15.110:0/1952966694":"2023-10-10T09:51:12.282860+0000","172.21.15.110:0/1606345106":"2023-10-10T09:51:12.282860+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:30:26.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T10:30:26.661 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T10:30:26.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T10:30:26.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T10:30:26.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T10:30:26.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T10:30:26.663 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T10:30:26.664 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T10:30:26.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:26 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3497921306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:30:26.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:26 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3497921306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:30:27.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:26 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3497921306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:30:27.385 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:27.518 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:27.527 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:27.528 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:27.620 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:27.848 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:27.849 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:27.849 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:27.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:27 smithi012 ceph-mon[147926]: pgmap v1699: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:27.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:27 smithi012 ceph-mon[152263]: pgmap v1699: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:28.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:27 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:30:27] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.33.4" 2023-10-09T10:30:28.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:27 smithi110 ceph-mon[136032]: pgmap v1699: 73 pgs: 8 peering, 65 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:28.919 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:28 smithi012 ceph-mon[152263]: pgmap v1700: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:28.920 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:28 smithi012 ceph-mon[152263]: Health check update: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:28.920 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:28 smithi012 ceph-mon[147926]: pgmap v1700: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:28.920 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:28 smithi012 ceph-mon[147926]: Health check update: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:28 smithi110 ceph-mon[136032]: pgmap v1700: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:29.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:28 smithi110 ceph-mon[136032]: Health check update: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T10:30:29.887 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:29 smithi012 ceph-mon[147926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2023-10-09T10:30:29.888 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:29 smithi012 ceph-mon[147926]: Cluster is now healthy 2023-10-09T10:30:29.888 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:29 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:29.888 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:29 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:29.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:29 smithi012 ceph-mon[152263]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2023-10-09T10:30:29.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:29 smithi012 ceph-mon[152263]: Cluster is now healthy 2023-10-09T10:30:29.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:29 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:29.888 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:29 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:30.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:29 smithi110 ceph-mon[136032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2023-10-09T10:30:30.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:29 smithi110 ceph-mon[136032]: Cluster is now healthy 2023-10-09T10:30:30.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:29 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:30.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:29 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:30.831 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:30 smithi012 ceph-mon[147926]: pgmap v1701: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:30.832 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:30 smithi012 ceph-mon[152263]: pgmap v1701: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:31.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:30 smithi110 ceph-mon[136032]: pgmap v1701: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:32.607 INFO:teuthology.orchestra.run.smithi012.stdout:1511828488371 2023-10-09T10:30:32.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T10:30:33.197 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:32 smithi012 ceph-mon[147926]: pgmap v1702: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:33.197 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:32 smithi012 ceph-mon[152263]: pgmap v1702: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:33.435 INFO:teuthology.orchestra.run.smithi012.stdout:180388626959 2023-10-09T10:30:33.435 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T10:30:33.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:32 smithi110 ceph-mon[136032]: pgmap v1702: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:33.960 INFO:teuthology.orchestra.run.smithi012.stdout:1142461301001 2023-10-09T10:30:33.960 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T10:30:33.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:33 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:30:33.989 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:33 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:30:34.025 INFO:teuthology.orchestra.run.smithi012.stdout:339302416797 2023-10-09T10:30:34.025 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T10:30:34.442 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:33 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:30:35.186 INFO:teuthology.orchestra.run.smithi012.stdout:932007903532 2023-10-09T10:30:35.186 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T10:30:35.191 INFO:teuthology.orchestra.run.smithi012.stdout:206158430731 2023-10-09T10:30:35.191 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T10:30:35.306 INFO:teuthology.orchestra.run.smithi012.stdout:154618823186 2023-10-09T10:30:35.307 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T10:30:35.510 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:35.857 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:35.874 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:35.885 INFO:teuthology.orchestra.run.smithi012.stdout:360777253272 2023-10-09T10:30:35.886 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T10:30:36.044 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:36.149 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:35 smithi012 ceph-mon[147926]: pgmap v1703: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:36.149 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:35 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:36.149 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:35 smithi012 ceph-mon[152263]: pgmap v1703: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:36.149 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:35 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:36.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:35 smithi110 ceph-mon[136032]: pgmap v1703: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:36.193 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:35 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:36.427 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:37.939 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:38.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:37 smithi012 ceph-mon[147926]: pgmap v1704: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:38.178 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:37 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:30:37] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.33.4" 2023-10-09T10:30:38.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:37 smithi012 ceph-mon[152263]: pgmap v1704: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:38.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:37 smithi110 ceph-mon[136032]: pgmap v1704: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:39.328 INFO:teuthology.orchestra.run.smithi012.stdout:339302416798 2023-10-09T10:30:39.463 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:39 smithi012 ceph-mon[147926]: pgmap v1705: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:39 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:39.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:39 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:39.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:39 smithi012 ceph-mon[152263]: pgmap v1705: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:39.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:39 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:39.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:39 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:39.687 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:39.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:39 smithi110 ceph-mon[136032]: pgmap v1705: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:39 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:39.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:39 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:39.740 INFO:teuthology.orchestra.run.smithi012.stdout:1511828488373 2023-10-09T10:30:40.247 INFO:teuthology.orchestra.run.smithi012.stdout:1142461301002 2023-10-09T10:30:40.331 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3211836217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:30:40.331 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3142933820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:30:40.331 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:40 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3785590836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:30:40.590 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3211836217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:30:40.591 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3142933820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:30:40.591 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:40 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3785590836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:30:40.592 INFO:tasks.cephadm.ceph_manager.ceph:need seq 339302416797 got 339302416798 for osd.0 2023-10-09T10:30:40.592 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:40.633 INFO:teuthology.orchestra.run.smithi012.stdout:932007903534 2023-10-09T10:30:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3211836217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:30:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3142933820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:30:40.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:40 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3785590836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:30:40.925 INFO:teuthology.orchestra.run.smithi012.stdout:180388626960 2023-10-09T10:30:41.066 INFO:teuthology.orchestra.run.smithi012.stdout:206158430733 2023-10-09T10:30:41.183 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1511828488371 got 1511828488373 for osd.3 2023-10-09T10:30:41.183 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:41 smithi012 ceph-mon[147926]: pgmap v1706: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/462964231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:30:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2915203943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:30:41.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:41 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/411286643' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:30:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:41 smithi012 ceph-mon[152263]: pgmap v1706: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/462964231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:30:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2915203943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:30:41.678 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:41 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/411286643' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:30:41.692 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:41 smithi110 ceph-mon[136032]: pgmap v1706: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/462964231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:30:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2915203943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:30:41.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:41 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/411286643' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:30:41.916 INFO:tasks.cephadm.ceph_manager.ceph:need seq 932007903532 got 932007903534 for osd.2 2023-10-09T10:30:41.916 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:42.113 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1142461301001 got 1142461301002 for osd.4 2023-10-09T10:30:42.113 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:42.218 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430731 got 206158430733 for osd.7 2023-10-09T10:30:42.218 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:42.833 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626959 got 180388626960 for osd.6 2023-10-09T10:30:42.833 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:43.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:43 smithi012 ceph-mon[147926]: pgmap v1707: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:43.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:43 smithi012 ceph-mon[152263]: pgmap v1707: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:43.693 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:43 smithi110 ceph-mon[136032]: pgmap v1707: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:44.002 INFO:teuthology.orchestra.run.smithi012.stdout:154618823188 2023-10-09T10:30:44.542 INFO:teuthology.orchestra.run.smithi012.stdout:360777253274 2023-10-09T10:30:44.639 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:44 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:44.639 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/3504589766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:30:44.639 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:44 smithi012 ceph-mon[147926]: from='client.? 172.21.15.12:0/2250984231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:30:44.639 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:44 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:44.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/3504589766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:30:44.640 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:44 smithi012 ceph-mon[152263]: from='client.? 172.21.15.12:0/2250984231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:30:44.640 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823186 got 154618823188 for osd.5 2023-10-09T10:30:44.641 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:44.943 INFO:tasks.cephadm.ceph_manager.ceph:need seq 360777253272 got 360777253274 for osd.1 2023-10-09T10:30:44.943 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:44.943 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T10:30:44.943 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:30:44.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:44 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:30:44.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/3504589766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:30:44.944 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:44 smithi110 ceph-mon[136032]: from='client.? 172.21.15.12:0/2250984231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:30:45.572 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:45.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:45 smithi012 ceph-mon[147926]: pgmap v1708: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:45.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:45 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:45.814 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:45 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:45.814 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:45 smithi012 ceph-mon[152263]: pgmap v1708: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:45.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:45 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:45.815 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:45 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:45.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:45 smithi110 ceph-mon[136032]: pgmap v1708: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:45 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:45.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:45 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:46.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:46 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:30:46.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:46 smithi012 ceph-mon[147926]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:30:46.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:46 smithi012 ceph-mon[147926]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:46.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:46 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:30:46.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:46 smithi012 ceph-mon[152263]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:30:46.929 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:46 smithi012 ceph-mon[152263]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:46 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:30:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:46 smithi110 ceph-mon[136032]: from='mgr.25217 172.21.15.12:0/1559743134' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:30:46.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:46 smithi110 ceph-mon[136032]: from='mgr.25217 ' entity='mgr.y' 2023-10-09T10:30:46.972 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:30:46.973 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:30:47.391 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1709,"stamp":"2023-10-09T10:30:45.910993+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6263,"num_read_kb":13678,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":226,"num_bytes_recovered":11378044,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8281436,"kb_used_data":64860,"kb_used_omap":172,"kb_used_meta":8216339,"kb_avail":741679780,"statfs":{"total":767960285184,"available":759480094720,"internally_reserved":0,"allocated":66416640,"data_stored":52694030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":176164,"internal_metadata":8413532124},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002789"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":218,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130916+0000","last_change":"2023-10-09T10:29:30.611925+0000","last_active":"2023-10-09T10:30:21.130916+0000","last_peered":"2023-10-09T10:30:21.130916+0000","last_clean":"2023-10-09T10:30:21.130916+0000","last_became_active":"2023-10-09T10:29:30.611596+0000","last_became_peered":"2023-10-09T10:29:30.611596+0000","last_unstale":"2023-10-09T10:30:21.130916+0000","last_undegraded":"2023-10-09T10:30:21.130916+0000","last_fullsized":"2023-10-09T10:30:21.130916+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":464,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.411310+0000","last_change":"2023-10-09T10:30:22.411310+0000","last_active":"2023-10-09T10:30:22.411310+0000","last_peered":"2023-10-09T10:30:22.411310+0000","last_clean":"2023-10-09T10:30:22.411310+0000","last_became_active":"2023-10-09T10:30:22.410788+0000","last_became_peered":"2023-10-09T10:30:22.410788+0000","last_unstale":"2023-10-09T10:30:22.411310+0000","last_undegraded":"2023-10-09T10:30:22.411310+0000","last_fullsized":"2023-10-09T10:30:22.411310+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:24.212390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":39,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131956+0000","last_change":"2023-10-09T10:29:30.607630+0000","last_active":"2023-10-09T10:30:21.131956+0000","last_peered":"2023-10-09T10:30:21.131956+0000","last_clean":"2023-10-09T10:30:21.131956+0000","last_became_active":"2023-10-09T10:29:30.605951+0000","last_became_peered":"2023-10-09T10:29:30.605951+0000","last_unstale":"2023-10-09T10:30:21.131956+0000","last_undegraded":"2023-10-09T10:30:21.131956+0000","last_fullsized":"2023-10-09T10:30:21.131956+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:37.492242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":563,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.137924+0000","last_change":"2023-10-09T10:29:25.567076+0000","last_active":"2023-10-09T10:30:22.137924+0000","last_peered":"2023-10-09T10:30:22.137924+0000","last_clean":"2023-10-09T10:30:22.137924+0000","last_became_active":"2023-10-09T10:29:25.566709+0000","last_became_peered":"2023-10-09T10:29:25.566709+0000","last_unstale":"2023-10-09T10:30:22.137924+0000","last_undegraded":"2023-10-09T10:30:22.137924+0000","last_fullsized":"2023-10-09T10:30:22.137924+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:42.706058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":410,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.411227+0000","last_change":"2023-10-09T10:30:22.411227+0000","last_active":"2023-10-09T10:30:22.411227+0000","last_peered":"2023-10-09T10:30:22.411227+0000","last_clean":"2023-10-09T10:30:22.411227+0000","last_became_active":"2023-10-09T10:30:22.410870+0000","last_became_peered":"2023-10-09T10:30:22.410870+0000","last_unstale":"2023-10-09T10:30:22.411227+0000","last_undegraded":"2023-10-09T10:30:22.411227+0000","last_fullsized":"2023-10-09T10:30:22.411227+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:43.589572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":192,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132802+0000","last_change":"2023-10-09T10:30:03.061847+0000","last_active":"2023-10-09T10:30:21.132802+0000","last_peered":"2023-10-09T10:30:21.132802+0000","last_clean":"2023-10-09T10:30:21.132802+0000","last_became_active":"2023-10-09T10:30:03.061252+0000","last_became_peered":"2023-10-09T10:30:03.061252+0000","last_unstale":"2023-10-09T10:30:21.132802+0000","last_undegraded":"2023-10-09T10:30:21.132802+0000","last_fullsized":"2023-10-09T10:30:21.132802+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":596,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129171+0000","last_change":"2023-10-09T10:29:48.546235+0000","last_active":"2023-10-09T10:30:21.129171+0000","last_peered":"2023-10-09T10:30:21.129171+0000","last_clean":"2023-10-09T10:30:21.129171+0000","last_became_active":"2023-10-09T10:29:48.545757+0000","last_became_peered":"2023-10-09T10:29:48.545757+0000","last_unstale":"2023-10-09T10:30:21.129171+0000","last_undegraded":"2023-10-09T10:30:21.129171+0000","last_fullsized":"2023-10-09T10:30:21.129171+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:16.484807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":507,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.410402+0000","last_change":"2023-10-09T10:30:22.410402+0000","last_active":"2023-10-09T10:30:22.410402+0000","last_peered":"2023-10-09T10:30:22.410402+0000","last_clean":"2023-10-09T10:30:22.410402+0000","last_became_active":"2023-10-09T10:30:22.410019+0000","last_became_peered":"2023-10-09T10:30:22.410019+0000","last_unstale":"2023-10-09T10:30:22.410402+0000","last_undegraded":"2023-10-09T10:30:22.410402+0000","last_fullsized":"2023-10-09T10:30:22.410402+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:07.263314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":47,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131496+0000","last_change":"2023-10-09T10:29:51.562903+0000","last_active":"2023-10-09T10:30:21.131496+0000","last_peered":"2023-10-09T10:30:21.131496+0000","last_clean":"2023-10-09T10:30:21.131496+0000","last_became_active":"2023-10-09T10:29:51.562500+0000","last_became_peered":"2023-10-09T10:29:51.562500+0000","last_unstale":"2023-10-09T10:30:21.131496+0000","last_undegraded":"2023-10-09T10:30:21.131496+0000","last_fullsized":"2023-10-09T10:30:21.131496+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":531,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.146145+0000","last_change":"2023-10-09T10:30:22.146145+0000","last_active":"2023-10-09T10:30:22.146145+0000","last_peered":"2023-10-09T10:30:22.146145+0000","last_clean":"2023-10-09T10:30:22.146145+0000","last_became_active":"2023-10-09T10:30:22.145757+0000","last_became_peered":"2023-10-09T10:30:22.145757+0000","last_unstale":"2023-10-09T10:30:22.146145+0000","last_undegraded":"2023-10-09T10:30:22.146145+0000","last_fullsized":"2023-10-09T10:30:22.146145+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:10:21.864424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":209,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134523+0000","last_change":"2023-10-09T10:30:04.067294+0000","last_active":"2023-10-09T10:30:21.134523+0000","last_peered":"2023-10-09T10:30:21.134523+0000","last_clean":"2023-10-09T10:30:21.134523+0000","last_became_active":"2023-10-09T10:30:04.066841+0000","last_became_peered":"2023-10-09T10:30:04.066841+0000","last_unstale":"2023-10-09T10:30:21.134523+0000","last_undegraded":"2023-10-09T10:30:21.134523+0000","last_fullsized":"2023-10-09T10:30:21.134523+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":39,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132224+0000","last_change":"2023-10-09T10:29:30.609289+0000","last_active":"2023-10-09T10:30:21.132224+0000","last_peered":"2023-10-09T10:30:21.132224+0000","last_clean":"2023-10-09T10:30:21.132224+0000","last_became_active":"2023-10-09T10:29:30.609010+0000","last_became_peered":"2023-10-09T10:29:30.609010+0000","last_unstale":"2023-10-09T10:30:21.132224+0000","last_undegraded":"2023-10-09T10:30:21.132224+0000","last_fullsized":"2023-10-09T10:30:21.132224+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:47.993837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":286,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131823+0000","last_change":"2023-10-09T10:29:25.712940+0000","last_active":"2023-10-09T10:30:21.131823+0000","last_peered":"2023-10-09T10:30:21.131823+0000","last_clean":"2023-10-09T10:30:21.131823+0000","last_became_active":"2023-10-09T10:29:25.575382+0000","last_became_peered":"2023-10-09T10:29:25.575382+0000","last_unstale":"2023-10-09T10:30:21.131823+0000","last_undegraded":"2023-10-09T10:30:21.131823+0000","last_fullsized":"2023-10-09T10:30:21.131823+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":236,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131620+0000","last_change":"2023-10-09T10:29:30.608709+0000","last_active":"2023-10-09T10:30:21.131620+0000","last_peered":"2023-10-09T10:30:21.131620+0000","last_clean":"2023-10-09T10:30:21.131620+0000","last_became_active":"2023-10-09T10:29:30.608390+0000","last_became_peered":"2023-10-09T10:29:30.608390+0000","last_unstale":"2023-10-09T10:30:21.131620+0000","last_undegraded":"2023-10-09T10:30:21.131620+0000","last_fullsized":"2023-10-09T10:30:21.131620+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":48,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134109+0000","last_change":"2023-10-09T10:29:25.661460+0000","last_active":"2023-10-09T10:30:21.134109+0000","last_peered":"2023-10-09T10:30:21.134109+0000","last_clean":"2023-10-09T10:30:21.134109+0000","last_became_active":"2023-10-09T10:29:25.573543+0000","last_became_peered":"2023-10-09T10:29:25.573543+0000","last_unstale":"2023-10-09T10:30:21.134109+0000","last_undegraded":"2023-10-09T10:30:21.134109+0000","last_fullsized":"2023-10-09T10:30:21.134109+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":435,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.410384+0000","last_change":"2023-10-09T10:30:22.410384+0000","last_active":"2023-10-09T10:30:22.410384+0000","last_peered":"2023-10-09T10:30:22.410384+0000","last_clean":"2023-10-09T10:30:22.410384+0000","last_became_active":"2023-10-09T10:30:22.410048+0000","last_became_peered":"2023-10-09T10:30:22.410048+0000","last_unstale":"2023-10-09T10:30:22.410384+0000","last_undegraded":"2023-10-09T10:30:22.410384+0000","last_fullsized":"2023-10-09T10:30:22.410384+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:05.134050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":552,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.137298+0000","last_change":"2023-10-09T10:29:25.562515+0000","last_active":"2023-10-09T10:30:22.137298+0000","last_peered":"2023-10-09T10:30:22.137298+0000","last_clean":"2023-10-09T10:30:22.137298+0000","last_became_active":"2023-10-09T10:29:25.562150+0000","last_became_peered":"2023-10-09T10:29:25.562150+0000","last_unstale":"2023-10-09T10:30:22.137298+0000","last_undegraded":"2023-10-09T10:30:22.137298+0000","last_fullsized":"2023-10-09T10:30:22.137298+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:27.262966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131005+0000","last_change":"2023-10-09T10:29:48.553767+0000","last_active":"2023-10-09T10:30:21.131005+0000","last_peered":"2023-10-09T10:30:21.131005+0000","last_clean":"2023-10-09T10:30:21.131005+0000","last_became_active":"2023-10-09T10:29:48.553323+0000","last_became_peered":"2023-10-09T10:29:48.553323+0000","last_unstale":"2023-10-09T10:30:21.131005+0000","last_undegraded":"2023-10-09T10:30:21.131005+0000","last_fullsized":"2023-10-09T10:30:21.131005+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":215,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134382+0000","last_change":"2023-10-09T10:30:04.068976+0000","last_active":"2023-10-09T10:30:21.134382+0000","last_peered":"2023-10-09T10:30:21.134382+0000","last_clean":"2023-10-09T10:30:21.134382+0000","last_became_active":"2023-10-09T10:30:04.068637+0000","last_became_peered":"2023-10-09T10:30:04.068637+0000","last_unstale":"2023-10-09T10:30:21.134382+0000","last_undegraded":"2023-10-09T10:30:21.134382+0000","last_fullsized":"2023-10-09T10:30:21.134382+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":124,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.128766+0000","last_change":"2023-10-09T10:29:25.566025+0000","last_active":"2023-10-09T10:30:21.128766+0000","last_peered":"2023-10-09T10:30:21.128766+0000","last_clean":"2023-10-09T10:30:21.128766+0000","last_became_active":"2023-10-09T10:29:25.565569+0000","last_became_peered":"2023-10-09T10:29:25.565569+0000","last_unstale":"2023-10-09T10:30:21.128766+0000","last_undegraded":"2023-10-09T10:30:21.128766+0000","last_fullsized":"2023-10-09T10:30:21.128766+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:24:42.150298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":484,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132951+0000","last_change":"2023-10-09T10:29:58.037928+0000","last_active":"2023-10-09T10:30:21.132951+0000","last_peered":"2023-10-09T10:30:21.132951+0000","last_clean":"2023-10-09T10:30:21.132951+0000","last_became_active":"2023-10-09T10:29:58.037560+0000","last_became_peered":"2023-10-09T10:29:58.037560+0000","last_unstale":"2023-10-09T10:30:21.132951+0000","last_undegraded":"2023-10-09T10:30:21.132951+0000","last_fullsized":"2023-10-09T10:30:21.132951+0000","mapping_epoch":604,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":107,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.411522+0000","last_change":"2023-10-09T10:30:22.411522+0000","last_active":"2023-10-09T10:30:22.411522+0000","last_peered":"2023-10-09T10:30:22.411522+0000","last_clean":"2023-10-09T10:30:22.411522+0000","last_became_active":"2023-10-09T10:30:22.410935+0000","last_became_peered":"2023-10-09T10:30:22.410935+0000","last_unstale":"2023-10-09T10:30:22.411522+0000","last_undegraded":"2023-10-09T10:30:22.411522+0000","last_fullsized":"2023-10-09T10:30:22.411522+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:30.052943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":118,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134928+0000","last_change":"2023-10-09T10:30:04.065970+0000","last_active":"2023-10-09T10:30:21.134928+0000","last_peered":"2023-10-09T10:30:21.134928+0000","last_clean":"2023-10-09T10:30:21.134928+0000","last_became_active":"2023-10-09T10:30:04.065363+0000","last_became_peered":"2023-10-09T10:30:04.065363+0000","last_unstale":"2023-10-09T10:30:21.134928+0000","last_undegraded":"2023-10-09T10:30:21.134928+0000","last_fullsized":"2023-10-09T10:30:21.134928+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":198,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131440+0000","last_change":"2023-10-09T10:29:48.553090+0000","last_active":"2023-10-09T10:30:21.131440+0000","last_peered":"2023-10-09T10:30:21.131440+0000","last_clean":"2023-10-09T10:30:21.131440+0000","last_became_active":"2023-10-09T10:29:48.552702+0000","last_became_peered":"2023-10-09T10:29:48.552702+0000","last_unstale":"2023-10-09T10:30:21.131440+0000","last_undegraded":"2023-10-09T10:30:21.131440+0000","last_fullsized":"2023-10-09T10:30:21.131440+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":244,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134360+0000","last_change":"2023-10-09T10:30:04.065752+0000","last_active":"2023-10-09T10:30:21.134360+0000","last_peered":"2023-10-09T10:30:21.134360+0000","last_clean":"2023-10-09T10:30:21.134360+0000","last_became_active":"2023-10-09T10:30:04.065364+0000","last_became_peered":"2023-10-09T10:30:04.065364+0000","last_unstale":"2023-10-09T10:30:21.134360+0000","last_undegraded":"2023-10-09T10:30:21.134360+0000","last_fullsized":"2023-10-09T10:30:21.134360+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":215,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131022+0000","last_change":"2023-10-09T10:30:16.107607+0000","last_active":"2023-10-09T10:30:21.131022+0000","last_peered":"2023-10-09T10:30:21.131022+0000","last_clean":"2023-10-09T10:30:21.131022+0000","last_became_active":"2023-10-09T10:30:16.107315+0000","last_became_peered":"2023-10-09T10:30:16.107315+0000","last_unstale":"2023-10-09T10:30:21.131022+0000","last_undegraded":"2023-10-09T10:30:21.131022+0000","last_fullsized":"2023-10-09T10:30:21.131022+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":61,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134590+0000","last_change":"2023-10-09T10:30:04.069124+0000","last_active":"2023-10-09T10:30:21.134590+0000","last_peered":"2023-10-09T10:30:21.134590+0000","last_clean":"2023-10-09T10:30:21.134590+0000","last_became_active":"2023-10-09T10:30:04.068712+0000","last_became_peered":"2023-10-09T10:30:04.068712+0000","last_unstale":"2023-10-09T10:30:21.134590+0000","last_undegraded":"2023-10-09T10:30:21.134590+0000","last_fullsized":"2023-10-09T10:30:21.134590+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":139,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133006+0000","last_change":"2023-10-09T10:29:58.327402+0000","last_active":"2023-10-09T10:30:21.133006+0000","last_peered":"2023-10-09T10:30:21.133006+0000","last_clean":"2023-10-09T10:30:21.133006+0000","last_became_active":"2023-10-09T10:29:58.327044+0000","last_became_peered":"2023-10-09T10:29:58.327044+0000","last_unstale":"2023-10-09T10:30:21.133006+0000","last_undegraded":"2023-10-09T10:30:21.133006+0000","last_fullsized":"2023-10-09T10:30:21.133006+0000","mapping_epoch":604,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":292,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131740+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:30:21.131740+0000","last_peered":"2023-10-09T10:30:21.131740+0000","last_clean":"2023-10-09T10:30:21.131740+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:30:21.131740+0000","last_undegraded":"2023-10-09T10:30:21.131740+0000","last_fullsized":"2023-10-09T10:30:21.131740+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":538,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129076+0000","last_change":"2023-10-09T10:29:30.610242+0000","last_active":"2023-10-09T10:30:21.129076+0000","last_peered":"2023-10-09T10:30:21.129076+0000","last_clean":"2023-10-09T10:30:21.129076+0000","last_became_active":"2023-10-09T10:29:30.609891+0000","last_became_peered":"2023-10-09T10:29:30.609891+0000","last_unstale":"2023-10-09T10:30:21.129076+0000","last_undegraded":"2023-10-09T10:30:21.129076+0000","last_fullsized":"2023-10-09T10:30:21.129076+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:33.315269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":37,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.144565+0000","last_change":"2023-10-09T10:30:22.144565+0000","last_active":"2023-10-09T10:30:22.144565+0000","last_peered":"2023-10-09T10:30:22.144565+0000","last_clean":"2023-10-09T10:30:22.144565+0000","last_became_active":"2023-10-09T10:30:22.144253+0000","last_became_peered":"2023-10-09T10:30:22.144253+0000","last_unstale":"2023-10-09T10:30:22.144565+0000","last_undegraded":"2023-10-09T10:30:22.144565+0000","last_fullsized":"2023-10-09T10:30:22.144565+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:43.346142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":227,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131846+0000","last_change":"2023-10-09T10:30:16.509164+0000","last_active":"2023-10-09T10:30:21.131846+0000","last_peered":"2023-10-09T10:30:21.131846+0000","last_clean":"2023-10-09T10:30:21.131846+0000","last_became_active":"2023-10-09T10:30:16.508778+0000","last_became_peered":"2023-10-09T10:30:16.508778+0000","last_unstale":"2023-10-09T10:30:21.131846+0000","last_undegraded":"2023-10-09T10:30:21.131846+0000","last_fullsized":"2023-10-09T10:30:21.131846+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":224,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133112+0000","last_change":"2023-10-09T10:29:58.423808+0000","last_active":"2023-10-09T10:30:21.133112+0000","last_peered":"2023-10-09T10:30:21.133112+0000","last_clean":"2023-10-09T10:30:21.133112+0000","last_became_active":"2023-10-09T10:29:58.423466+0000","last_became_peered":"2023-10-09T10:29:58.423466+0000","last_unstale":"2023-10-09T10:30:21.133112+0000","last_undegraded":"2023-10-09T10:30:21.133112+0000","last_fullsized":"2023-10-09T10:30:21.133112+0000","mapping_epoch":604,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":338,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131810+0000","last_change":"2023-10-09T10:30:16.240089+0000","last_active":"2023-10-09T10:30:21.131810+0000","last_peered":"2023-10-09T10:30:21.131810+0000","last_clean":"2023-10-09T10:30:21.131810+0000","last_became_active":"2023-10-09T10:30:16.239747+0000","last_became_peered":"2023-10-09T10:30:16.239747+0000","last_unstale":"2023-10-09T10:30:21.131810+0000","last_undegraded":"2023-10-09T10:30:21.131810+0000","last_fullsized":"2023-10-09T10:30:21.131810+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":227,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131099+0000","last_change":"2023-10-09T10:29:25.567139+0000","last_active":"2023-10-09T10:30:21.131099+0000","last_peered":"2023-10-09T10:30:21.131099+0000","last_clean":"2023-10-09T10:30:21.131099+0000","last_became_active":"2023-10-09T10:29:25.566752+0000","last_became_peered":"2023-10-09T10:29:25.566752+0000","last_unstale":"2023-10-09T10:30:21.131099+0000","last_undegraded":"2023-10-09T10:30:21.131099+0000","last_fullsized":"2023-10-09T10:30:21.131099+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":245,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132330+0000","last_change":"2023-10-09T10:29:30.608624+0000","last_active":"2023-10-09T10:30:21.132330+0000","last_peered":"2023-10-09T10:30:21.132330+0000","last_clean":"2023-10-09T10:30:21.132330+0000","last_became_active":"2023-10-09T10:29:30.608310+0000","last_became_peered":"2023-10-09T10:29:30.608310+0000","last_unstale":"2023-10-09T10:30:21.132330+0000","last_undegraded":"2023-10-09T10:30:21.132330+0000","last_fullsized":"2023-10-09T10:30:21.132330+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":500,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.411701+0000","last_change":"2023-10-09T10:30:22.411701+0000","last_active":"2023-10-09T10:30:22.411701+0000","last_peered":"2023-10-09T10:30:22.411701+0000","last_clean":"2023-10-09T10:30:22.411701+0000","last_became_active":"2023-10-09T10:30:22.410855+0000","last_became_peered":"2023-10-09T10:30:22.410855+0000","last_unstale":"2023-10-09T10:30:22.411701+0000","last_undegraded":"2023-10-09T10:30:22.411701+0000","last_fullsized":"2023-10-09T10:30:22.411701+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:16.380309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":39,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133536+0000","last_change":"2023-10-09T10:29:30.608788+0000","last_active":"2023-10-09T10:30:21.133536+0000","last_peered":"2023-10-09T10:30:21.133536+0000","last_clean":"2023-10-09T10:30:21.133536+0000","last_became_active":"2023-10-09T10:29:30.607797+0000","last_became_peered":"2023-10-09T10:29:30.607797+0000","last_unstale":"2023-10-09T10:30:21.133536+0000","last_undegraded":"2023-10-09T10:30:21.133536+0000","last_fullsized":"2023-10-09T10:30:21.133536+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:45.991193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"580'175","reported_seq":1138,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.135090+0000","last_change":"2023-10-09T10:29:33.626271+0000","last_active":"2023-10-09T10:30:21.135090+0000","last_peered":"2023-10-09T10:30:21.135090+0000","last_clean":"2023-10-09T10:30:21.135090+0000","last_became_active":"2023-10-09T10:29:33.625801+0000","last_became_peered":"2023-10-09T10:29:33.625801+0000","last_unstale":"2023-10-09T10:30:21.135090+0000","last_undegraded":"2023-10-09T10:30:21.135090+0000","last_fullsized":"2023-10-09T10:30:21.135090+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:22.698773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072463300000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1289,"num_read_kb":2426,"num_write":156,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":28,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131097+0000","last_change":"2023-10-09T10:29:48.543406+0000","last_active":"2023-10-09T10:30:21.131097+0000","last_peered":"2023-10-09T10:30:21.131097+0000","last_clean":"2023-10-09T10:30:21.131097+0000","last_became_active":"2023-10-09T10:29:48.542916+0000","last_became_peered":"2023-10-09T10:29:48.542916+0000","last_unstale":"2023-10-09T10:30:21.131097+0000","last_undegraded":"2023-10-09T10:30:21.131097+0000","last_fullsized":"2023-10-09T10:30:21.131097+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132085+0000","last_change":"2023-10-09T10:29:48.551978+0000","last_active":"2023-10-09T10:30:21.132085+0000","last_peered":"2023-10-09T10:30:21.132085+0000","last_clean":"2023-10-09T10:30:21.132085+0000","last_became_active":"2023-10-09T10:29:48.551771+0000","last_became_peered":"2023-10-09T10:29:48.551771+0000","last_unstale":"2023-10-09T10:30:21.132085+0000","last_undegraded":"2023-10-09T10:30:21.132085+0000","last_fullsized":"2023-10-09T10:30:21.132085+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:02.207672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"580'79","reported_seq":782,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132920+0000","last_change":"2023-10-09T10:29:51.559548+0000","last_active":"2023-10-09T10:30:21.132920+0000","last_peered":"2023-10-09T10:30:21.132920+0000","last_clean":"2023-10-09T10:30:21.132920+0000","last_became_active":"2023-10-09T10:29:51.559055+0000","last_became_peered":"2023-10-09T10:29:51.559055+0000","last_unstale":"2023-10-09T10:30:21.132920+0000","last_undegraded":"2023-10-09T10:30:21.132920+0000","last_fullsized":"2023-10-09T10:30:21.132920+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":655,"num_read_kb":2665,"num_write":63,"num_write_kb":2097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":218,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131526+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:30:21.131526+0000","last_peered":"2023-10-09T10:30:21.131526+0000","last_clean":"2023-10-09T10:30:21.131526+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:30:21.131526+0000","last_undegraded":"2023-10-09T10:30:21.131526+0000","last_fullsized":"2023-10-09T10:30:21.131526+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131591+0000","last_change":"2023-10-09T10:30:03.067276+0000","last_active":"2023-10-09T10:30:21.131591+0000","last_peered":"2023-10-09T10:30:21.131591+0000","last_clean":"2023-10-09T10:30:21.131591+0000","last_became_active":"2023-10-09T10:30:03.066921+0000","last_became_peered":"2023-10-09T10:30:03.066921+0000","last_unstale":"2023-10-09T10:30:21.131591+0000","last_undegraded":"2023-10-09T10:30:21.131591+0000","last_fullsized":"2023-10-09T10:30:21.131591+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"580'96","reported_seq":886,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131809+0000","last_change":"2023-10-09T10:30:10.082719+0000","last_active":"2023-10-09T10:30:21.131809+0000","last_peered":"2023-10-09T10:30:21.131809+0000","last_clean":"2023-10-09T10:30:21.131809+0000","last_became_active":"2023-10-09T10:30:10.082270+0000","last_became_peered":"2023-10-09T10:30:10.082270+0000","last_unstale":"2023-10-09T10:30:21.131809+0000","last_undegraded":"2023-10-09T10:30:21.131809+0000","last_fullsized":"2023-10-09T10:30:21.131809+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:01.498272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016205779999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":527,"num_read_kb":2199,"num_write":93,"num_write_kb":1467,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":39,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133051+0000","last_change":"2023-10-09T10:29:30.606561+0000","last_active":"2023-10-09T10:30:21.133051+0000","last_peered":"2023-10-09T10:30:21.133051+0000","last_clean":"2023-10-09T10:30:21.133051+0000","last_became_active":"2023-10-09T10:29:30.605545+0000","last_became_peered":"2023-10-09T10:29:30.605545+0000","last_unstale":"2023-10-09T10:30:21.133051+0000","last_undegraded":"2023-10-09T10:30:21.133051+0000","last_fullsized":"2023-10-09T10:30:21.133051+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:05.844422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133564+0000","last_change":"2023-10-09T10:29:48.545839+0000","last_active":"2023-10-09T10:30:21.133564+0000","last_peered":"2023-10-09T10:30:21.133564+0000","last_clean":"2023-10-09T10:30:21.133564+0000","last_became_active":"2023-10-09T10:29:48.545458+0000","last_became_peered":"2023-10-09T10:29:48.545458+0000","last_unstale":"2023-10-09T10:30:21.133564+0000","last_undegraded":"2023-10-09T10:30:21.133564+0000","last_fullsized":"2023-10-09T10:30:21.133564+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:48.354699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"580'175","reported_seq":780,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131326+0000","last_change":"2023-10-09T10:29:51.563279+0000","last_active":"2023-10-09T10:30:21.131326+0000","last_peered":"2023-10-09T10:30:21.131326+0000","last_clean":"2023-10-09T10:30:21.131326+0000","last_became_active":"2023-10-09T10:29:51.562644+0000","last_became_peered":"2023-10-09T10:29:51.562644+0000","last_unstale":"2023-10-09T10:30:21.131326+0000","last_undegraded":"2023-10-09T10:30:21.131326+0000","last_fullsized":"2023-10-09T10:30:21.131326+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":573,"num_read_kb":1842,"num_write":165,"num_write_kb":6420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":801,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134241+0000","last_change":"2023-10-09T10:30:04.068948+0000","last_active":"2023-10-09T10:30:21.134241+0000","last_peered":"2023-10-09T10:30:21.134241+0000","last_clean":"2023-10-09T10:30:21.134241+0000","last_became_active":"2023-10-09T10:30:04.068426+0000","last_became_peered":"2023-10-09T10:30:04.068426+0000","last_unstale":"2023-10-09T10:30:21.134241+0000","last_undegraded":"2023-10-09T10:30:21.134241+0000","last_fullsized":"2023-10-09T10:30:21.134241+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":229,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.159966+0000","last_change":"2023-10-09T10:30:22.159966+0000","last_active":"2023-10-09T10:30:22.159966+0000","last_peered":"2023-10-09T10:30:22.159966+0000","last_clean":"2023-10-09T10:30:22.159966+0000","last_became_active":"2023-10-09T10:30:22.159743+0000","last_became_peered":"2023-10-09T10:30:22.159743+0000","last_unstale":"2023-10-09T10:30:22.159966+0000","last_undegraded":"2023-10-09T10:30:22.159966+0000","last_fullsized":"2023-10-09T10:30:22.159966+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:32.139439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":219,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133329+0000","last_change":"2023-10-09T10:29:58.424418+0000","last_active":"2023-10-09T10:30:21.133329+0000","last_peered":"2023-10-09T10:30:21.133329+0000","last_clean":"2023-10-09T10:30:21.133329+0000","last_became_active":"2023-10-09T10:29:58.423947+0000","last_became_peered":"2023-10-09T10:29:58.423947+0000","last_unstale":"2023-10-09T10:30:21.133329+0000","last_undegraded":"2023-10-09T10:30:21.133329+0000","last_fullsized":"2023-10-09T10:30:21.133329+0000","mapping_epoch":604,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"580'62","reported_seq":490,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134844+0000","last_change":"2023-10-09T10:30:04.065679+0000","last_active":"2023-10-09T10:30:21.134844+0000","last_peered":"2023-10-09T10:30:21.134844+0000","last_clean":"2023-10-09T10:30:21.134844+0000","last_became_active":"2023-10-09T10:30:04.065068+0000","last_became_peered":"2023-10-09T10:30:04.065068+0000","last_unstale":"2023-10-09T10:30:21.134844+0000","last_undegraded":"2023-10-09T10:30:21.134844+0000","last_fullsized":"2023-10-09T10:30:21.134844+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":272,"num_read_kb":1630,"num_write":58,"num_write_kb":1238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":564,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130632+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:30:21.130632+0000","last_peered":"2023-10-09T10:30:21.130632+0000","last_clean":"2023-10-09T10:30:21.130632+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:30:21.130632+0000","last_undegraded":"2023-10-09T10:30:21.130632+0000","last_fullsized":"2023-10-09T10:30:21.130632+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":151,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.128662+0000","last_change":"2023-10-09T10:30:03.059595+0000","last_active":"2023-10-09T10:30:21.128662+0000","last_peered":"2023-10-09T10:30:21.128662+0000","last_clean":"2023-10-09T10:30:21.128662+0000","last_became_active":"2023-10-09T10:30:03.059212+0000","last_became_peered":"2023-10-09T10:30:03.059212+0000","last_unstale":"2023-10-09T10:30:21.128662+0000","last_undegraded":"2023-10-09T10:30:21.128662+0000","last_fullsized":"2023-10-09T10:30:21.128662+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:02.823940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"580'221","reported_seq":989,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134453+0000","last_change":"2023-10-09T10:30:04.067345+0000","last_active":"2023-10-09T10:30:21.134453+0000","last_peered":"2023-10-09T10:30:21.134453+0000","last_clean":"2023-10-09T10:30:21.134453+0000","last_became_active":"2023-10-09T10:30:04.066626+0000","last_became_peered":"2023-10-09T10:30:04.066626+0000","last_unstale":"2023-10-09T10:30:21.134453+0000","last_undegraded":"2023-10-09T10:30:21.134453+0000","last_fullsized":"2023-10-09T10:30:21.134453+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":221,"log_dups_size":0,"ondisk_log_size":221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1677,"num_read_kb":1616,"num_write":189,"num_write_kb":178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.135200+0000","last_change":"2023-10-09T10:29:48.543560+0000","last_active":"2023-10-09T10:30:21.135200+0000","last_peered":"2023-10-09T10:30:21.135200+0000","last_clean":"2023-10-09T10:30:21.135200+0000","last_became_active":"2023-10-09T10:29:48.543228+0000","last_became_peered":"2023-10-09T10:29:48.543228+0000","last_unstale":"2023-10-09T10:30:21.135200+0000","last_undegraded":"2023-10-09T10:30:21.135200+0000","last_fullsized":"2023-10-09T10:30:21.135200+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:16.331966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":568,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129413+0000","last_change":"2023-10-09T10:29:48.548903+0000","last_active":"2023-10-09T10:30:21.129413+0000","last_peered":"2023-10-09T10:30:21.129413+0000","last_clean":"2023-10-09T10:30:21.129413+0000","last_became_active":"2023-10-09T10:29:48.548487+0000","last_became_peered":"2023-10-09T10:29:48.548487+0000","last_unstale":"2023-10-09T10:30:21.129413+0000","last_undegraded":"2023-10-09T10:30:21.129413+0000","last_fullsized":"2023-10-09T10:30:21.129413+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.121686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":378,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.146194+0000","last_change":"2023-10-09T10:30:22.146194+0000","last_active":"2023-10-09T10:30:22.146194+0000","last_peered":"2023-10-09T10:30:22.146194+0000","last_clean":"2023-10-09T10:30:22.146194+0000","last_became_active":"2023-10-09T10:30:22.145862+0000","last_became_peered":"2023-10-09T10:30:22.145862+0000","last_unstale":"2023-10-09T10:30:22.146194+0000","last_undegraded":"2023-10-09T10:30:22.146194+0000","last_fullsized":"2023-10-09T10:30:22.146194+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:53.763454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"580'95","reported_seq":56,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132278+0000","last_change":"2023-10-09T10:29:30.705557+0000","last_active":"2023-10-09T10:30:21.132278+0000","last_peered":"2023-10-09T10:30:21.132278+0000","last_clean":"2023-10-09T10:30:21.132278+0000","last_became_active":"2023-10-09T10:29:30.605870+0000","last_became_peered":"2023-10-09T10:29:30.605870+0000","last_unstale":"2023-10-09T10:30:21.132278+0000","last_undegraded":"2023-10-09T10:30:21.132278+0000","last_fullsized":"2023-10-09T10:30:21.132278+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":95,"log_dups_size":0,"ondisk_log_size":95,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:52:08.194203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":494,"num_read_kb":388,"num_write":57,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":401,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.146005+0000","last_change":"2023-10-09T10:30:22.146005+0000","last_active":"2023-10-09T10:30:22.146005+0000","last_peered":"2023-10-09T10:30:22.146005+0000","last_clean":"2023-10-09T10:30:22.146005+0000","last_became_active":"2023-10-09T10:30:22.145607+0000","last_became_peered":"2023-10-09T10:30:22.145607+0000","last_unstale":"2023-10-09T10:30:22.146005+0000","last_undegraded":"2023-10-09T10:30:22.146005+0000","last_fullsized":"2023-10-09T10:30:22.146005+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:45.404318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"580'73","reported_seq":393,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132910+0000","last_change":"2023-10-09T10:29:25.626159+0000","last_active":"2023-10-09T10:30:21.132910+0000","last_peered":"2023-10-09T10:30:21.132910+0000","last_clean":"2023-10-09T10:30:21.132910+0000","last_became_active":"2023-10-09T10:29:25.569871+0000","last_became_peered":"2023-10-09T10:29:25.569871+0000","last_unstale":"2023-10-09T10:30:21.132910+0000","last_undegraded":"2023-10-09T10:30:21.132910+0000","last_fullsized":"2023-10-09T10:30:21.132910+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":468,"num_read_kb":400,"num_write":65,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":158,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.128887+0000","last_change":"2023-10-09T10:29:30.629496+0000","last_active":"2023-10-09T10:30:21.128887+0000","last_peered":"2023-10-09T10:30:21.128887+0000","last_clean":"2023-10-09T10:30:21.128887+0000","last_became_active":"2023-10-09T10:29:30.610003+0000","last_became_peered":"2023-10-09T10:29:30.610003+0000","last_unstale":"2023-10-09T10:30:21.128887+0000","last_undegraded":"2023-10-09T10:30:21.128887+0000","last_fullsized":"2023-10-09T10:30:21.128887+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:10.345132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":205,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130768+0000","last_change":"2023-10-09T10:30:16.509241+0000","last_active":"2023-10-09T10:30:21.130768+0000","last_peered":"2023-10-09T10:30:21.130768+0000","last_clean":"2023-10-09T10:30:21.130768+0000","last_became_active":"2023-10-09T10:30:16.508882+0000","last_became_peered":"2023-10-09T10:30:16.508882+0000","last_unstale":"2023-10-09T10:30:21.130768+0000","last_undegraded":"2023-10-09T10:30:21.130768+0000","last_fullsized":"2023-10-09T10:30:21.130768+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":124,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129258+0000","last_change":"2023-10-09T10:29:25.562616+0000","last_active":"2023-10-09T10:30:21.129258+0000","last_peered":"2023-10-09T10:30:21.129258+0000","last_clean":"2023-10-09T10:30:21.129258+0000","last_became_active":"2023-10-09T10:29:25.562196+0000","last_became_peered":"2023-10-09T10:29:25.562196+0000","last_unstale":"2023-10-09T10:30:21.129258+0000","last_undegraded":"2023-10-09T10:30:21.129258+0000","last_fullsized":"2023-10-09T10:30:21.129258+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:29.218982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":32,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130712+0000","last_change":"2023-10-09T10:30:10.108309+0000","last_active":"2023-10-09T10:30:21.130712+0000","last_peered":"2023-10-09T10:30:21.130712+0000","last_clean":"2023-10-09T10:30:21.130712+0000","last_became_active":"2023-10-09T10:30:10.107974+0000","last_became_peered":"2023-10-09T10:30:10.107974+0000","last_unstale":"2023-10-09T10:30:21.130712+0000","last_undegraded":"2023-10-09T10:30:21.130712+0000","last_fullsized":"2023-10-09T10:30:21.130712+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:33.100341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":329,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131829+0000","last_change":"2023-10-09T10:29:51.563375+0000","last_active":"2023-10-09T10:30:21.131829+0000","last_peered":"2023-10-09T10:30:21.131829+0000","last_clean":"2023-10-09T10:30:21.131829+0000","last_became_active":"2023-10-09T10:29:51.562894+0000","last_became_peered":"2023-10-09T10:29:51.562894+0000","last_unstale":"2023-10-09T10:30:21.131829+0000","last_undegraded":"2023-10-09T10:30:21.131829+0000","last_fullsized":"2023-10-09T10:30:21.131829+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":218,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131718+0000","last_change":"2023-10-09T10:29:30.607297+0000","last_active":"2023-10-09T10:30:21.131718+0000","last_peered":"2023-10-09T10:30:21.131718+0000","last_clean":"2023-10-09T10:30:21.131718+0000","last_became_active":"2023-10-09T10:29:30.606901+0000","last_became_peered":"2023-10-09T10:29:30.606901+0000","last_unstale":"2023-10-09T10:30:21.131718+0000","last_undegraded":"2023-10-09T10:30:21.131718+0000","last_fullsized":"2023-10-09T10:30:21.131718+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":330,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131451+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:30:21.131451+0000","last_peered":"2023-10-09T10:30:21.131451+0000","last_clean":"2023-10-09T10:30:21.131451+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:30:21.131451+0000","last_undegraded":"2023-10-09T10:30:21.131451+0000","last_fullsized":"2023-10-09T10:30:21.131451+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":251,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130805+0000","last_change":"2023-10-09T10:30:16.509403+0000","last_active":"2023-10-09T10:30:21.130805+0000","last_peered":"2023-10-09T10:30:21.130805+0000","last_clean":"2023-10-09T10:30:21.130805+0000","last_became_active":"2023-10-09T10:30:16.509004+0000","last_became_peered":"2023-10-09T10:30:16.509004+0000","last_unstale":"2023-10-09T10:30:21.130805+0000","last_undegraded":"2023-10-09T10:30:21.130805+0000","last_fullsized":"2023-10-09T10:30:21.130805+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":113,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.145022+0000","last_change":"2023-10-09T10:30:22.145022+0000","last_active":"2023-10-09T10:30:22.145022+0000","last_peered":"2023-10-09T10:30:22.145022+0000","last_clean":"2023-10-09T10:30:22.145022+0000","last_became_active":"2023-10-09T10:30:22.144678+0000","last_became_peered":"2023-10-09T10:30:22.144678+0000","last_unstale":"2023-10-09T10:30:22.145022+0000","last_undegraded":"2023-10-09T10:30:22.145022+0000","last_fullsized":"2023-10-09T10:30:22.145022+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:18.217178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":571,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.141235+0000","last_change":"2023-10-09T10:30:22.141235+0000","last_active":"2023-10-09T10:30:22.141235+0000","last_peered":"2023-10-09T10:30:22.141235+0000","last_clean":"2023-10-09T10:30:22.141235+0000","last_became_active":"2023-10-09T10:30:22.140931+0000","last_became_peered":"2023-10-09T10:30:22.140931+0000","last_unstale":"2023-10-09T10:30:22.141235+0000","last_undegraded":"2023-10-09T10:30:22.141235+0000","last_fullsized":"2023-10-09T10:30:22.141235+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:52:37.696238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":166,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129000+0000","last_change":"2023-10-09T10:29:48.556175+0000","last_active":"2023-10-09T10:30:21.129000+0000","last_peered":"2023-10-09T10:30:21.129000+0000","last_clean":"2023-10-09T10:30:21.129000+0000","last_became_active":"2023-10-09T10:29:48.545116+0000","last_became_peered":"2023-10-09T10:29:48.545116+0000","last_unstale":"2023-10-09T10:30:21.129000+0000","last_undegraded":"2023-10-09T10:30:21.129000+0000","last_fullsized":"2023-10-09T10:30:21.129000+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:36:43.096055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8310,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":245,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130589+0000","last_change":"2023-10-09T10:29:30.608959+0000","last_active":"2023-10-09T10:30:21.130589+0000","last_peered":"2023-10-09T10:30:21.130589+0000","last_clean":"2023-10-09T10:30:21.130589+0000","last_became_active":"2023-10-09T10:29:30.608703+0000","last_became_peered":"2023-10-09T10:29:30.608703+0000","last_unstale":"2023-10-09T10:30:21.130589+0000","last_undegraded":"2023-10-09T10:30:21.130589+0000","last_fullsized":"2023-10-09T10:30:21.130589+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":37596,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5955,"num_read_kb":13166,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430734,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":922760,"kb_used_data":10304,"kb_used_omap":0,"kb_used_meta":912448,"kb_avail":92822392,"statfs":{"total":95995035648,"available":95050129408,"internally_reserved":0,"allocated":10551296,"data_stored":9500114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":934346752},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.5649999999999999,"15min":1.454},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.216},"max":{"1min":0.91400000000000003,"5min":34.241,"15min":34.241},"last":0.439},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.883,"15min":1.4970000000000001},"min":{"1min":0.315,"5min":0.247,"15min":0.22},"max":{"1min":0.86199999999999999,"5min":34.030000000000001,"15min":34.030000000000001},"last":0.66400000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:30:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":1.131,"15min":1.794},"min":{"1min":0.33500000000000002,"5min":0.27500000000000002,"15min":0.23799999999999999},"max":{"1min":0.83899999999999997,"5min":5.8460000000000001,"15min":129.53700000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.0129999999999999,"15min":1.849},"min":{"1min":0.32800000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":0.88800000000000001,"5min":6.9249999999999998,"15min":128.85599999999999},"last":0.68999999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.474,"15min":1.6919999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":0.86699999999999999,"5min":6.7270000000000003,"15min":42.725999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.391,"15min":1.6299999999999999},"min":{"1min":0.42399999999999999,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":0.998,"5min":6.9000000000000004,"15min":42.752000000000002},"last":0.60999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.7689999999999999,"15min":1.7030000000000001},"min":{"1min":0.41499999999999998,"5min":0.33900000000000002,"15min":0.307},"max":{"1min":0.91700000000000004,"5min":8.5299999999999994,"15min":12.436},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.6870000000000001,"15min":1.708},"min":{"1min":0.374,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":0.76800000000000002,"5min":10.691000000000001,"15min":12.161},"last":0.58099999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:30:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.67900000000000005,"15min":1.222},"min":{"1min":0.34499999999999997,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":0.72899999999999998,"5min":4.2039999999999997,"15min":17.995000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.98499999999999999,"15min":1.3879999999999999},"min":{"1min":0.39600000000000002,"5min":0.27700000000000002,"15min":0.25700000000000001},"max":{"1min":0.94099999999999995,"5min":7.0609999999999999,"15min":18.198},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.86299999999999999,"15min":1.3899999999999999},"min":{"1min":0.42299999999999999,"5min":0.308,"15min":0.23999999999999999},"max":{"1min":0.84799999999999998,"5min":5.0720000000000001,"15min":21.712},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.69299999999999995,"5min":1.0860000000000001,"15min":1.603},"min":{"1min":0.47699999999999998,"5min":0.307,"15min":0.26800000000000002},"max":{"1min":0.84199999999999997,"5min":8.266,"15min":22.824999999999999},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":1.1579999999999999,"15min":1.4399999999999999},"min":{"1min":0.44600000000000001,"5min":0.33300000000000002,"15min":0.28399999999999997},"max":{"1min":0.98099999999999998,"5min":13.496,"15min":14.407},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.93899999999999995,"15min":1.117},"min":{"1min":0.42999999999999999,"5min":0.311,"15min":0.307},"max":{"1min":0.85399999999999998,"5min":5.7510000000000003,"15min":7.7670000000000003},"last":0.53000000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626962,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515328,"kb_used_data":6200,"kb_used_omap":0,"kb_used_meta":1509120,"kb_avail":92229824,"statfs":{"total":95995035648,"available":94443339776,"internally_reserved":0,"allocated":6348800,"data_stored":5306783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1545338880},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:23 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":2.0219999999999998,"15min":1.627},"min":{"1min":0.318,"5min":0.219,"15min":0.216},"max":{"1min":0.83499999999999996,"5min":47.401000000000003,"15min":47.401000000000003},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.55100000000000005,"5min":2.0230000000000001,"15min":1.9610000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":0.82399999999999995,"5min":47.311999999999998,"15min":47.311999999999998},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.258,"15min":1.8839999999999999},"min":{"1min":0.27800000000000002,"5min":0.26000000000000001,"15min":0.23799999999999999},"max":{"1min":0.91800000000000004,"5min":7.415,"15min":155.01400000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.298,"15min":2.0800000000000001},"min":{"1min":0.35199999999999998,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":0.80200000000000005,"5min":6.3929999999999998,"15min":154.71100000000001},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:30:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.587,"15min":1.7250000000000001},"min":{"1min":0.372,"5min":0.309,"15min":0.24099999999999999},"max":{"1min":0.90300000000000002,"5min":10.907,"15min":29.367999999999999},"last":0.433},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.298,"15min":1.407},"min":{"1min":0.42899999999999999,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":0.86799999999999999,"5min":7.4480000000000004,"15min":8.4480000000000004},"last":0.60099999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.798,"15min":1.556},"min":{"1min":0.44700000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.94899999999999995,"5min":43.393000000000001,"15min":43.393000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.8899999999999999,"15min":1.7529999999999999},"min":{"1min":0.40300000000000002,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":0.88900000000000001,"5min":43.466000000000001,"15min":43.466000000000001},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":2.722,"15min":2.391},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.85699999999999998,"5min":177.929,"15min":177.929},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":2.4199999999999999,"15min":2.2120000000000002},"min":{"1min":0.34300000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.77900000000000003,"5min":178.16499999999999,"15min":178.16499999999999},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.4279999999999999,"15min":2.036},"min":{"1min":0.35499999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.70099999999999996,"5min":11.788,"15min":121.98999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.079,"15min":1.5269999999999999},"min":{"1min":0.314,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.873,"5min":7.0060000000000002,"15min":122.346},"last":0.56000000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.1279999999999999,"15min":1.8939999999999999},"min":{"1min":0.34899999999999998,"5min":0.32600000000000001,"15min":0.29199999999999998},"max":{"1min":0.97099999999999997,"5min":12.612,"15min":130.66999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.212,"15min":1.9510000000000001},"min":{"1min":0.40600000000000003,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":1.097,"5min":12.659000000000001,"15min":130.702},"last":0.624}]}]},{"osd":1,"up_from":84,"seq":360777253275,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836576,"kb_used_data":6872,"kb_used_omap":9,"kb_used_meta":829686,"kb_avail":92908576,"statfs":{"total":95995035648,"available":95138381824,"internally_reserved":0,"allocated":7036928,"data_stored":4847378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":849599465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":2.6520000000000001,"15min":3.0539999999999998},"min":{"1min":0.247,"5min":0.218,"15min":0.188},"max":{"1min":0.87,"5min":64.558000000000007,"15min":270.48000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":2.5169999999999999,"15min":3.1579999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.216},"max":{"1min":1.046,"5min":61.692,"15min":272.69299999999998},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:30:21 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.2290000000000001,"15min":1.631},"min":{"1min":0.27500000000000002,"5min":0.219,"15min":0.19600000000000001},"max":{"1min":0.88500000000000001,"5min":7.5780000000000003,"15min":20.306000000000001},"last":0.373},{"interface":"front","average":{"1min":0.44900000000000001,"5min":1.23,"15min":1.238},"min":{"1min":0.309,"5min":0.29099999999999998,"15min":0.23100000000000001},"max":{"1min":0.90400000000000003,"5min":7.8860000000000001,"15min":9.4830000000000005},"last":0.309}]},{"osd":3,"last update":"Mon Oct 9 10:30:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":1.554,"15min":1.327},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":0.95399999999999996,"5min":13.223000000000001,"15min":13.223000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":1.673,"15min":1.5429999999999999},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0149999999999999,"5min":20.091000000000001,"15min":20.091000000000001},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.05,"15min":1.196},"min":{"1min":0.42399999999999999,"5min":0.33100000000000002,"15min":0.30199999999999999},"max":{"1min":0.85199999999999998,"5min":7.8200000000000003,"15min":10.797000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.603,"15min":1.8029999999999999},"min":{"1min":0.48699999999999999,"5min":0.34000000000000002,"15min":0.314},"max":{"1min":0.92500000000000004,"5min":20.015999999999998,"15min":20.015999999999998},"last":0.61799999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.718,"15min":2.2679999999999998},"min":{"1min":0.40600000000000003,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":0.96799999999999997,"5min":20.120999999999999,"15min":92.177999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.629,"5min":1.4510000000000001,"15min":2.1309999999999998},"min":{"1min":0.41299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.874,"5min":10.675000000000001,"15min":91.869},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 10:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":2.0089999999999999,"15min":1.704},"min":{"1min":0.48999999999999999,"5min":0.371,"15min":0.33700000000000002},"max":{"1min":3.9670000000000001,"5min":81.216999999999999,"15min":81.216999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.82199999999999995,"5min":2.3879999999999999,"15min":2.0289999999999999},"min":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":4.0179999999999998,"5min":81.231999999999999,"15min":81.231999999999999},"last":0.69599999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.4830000000000001,"15min":1.8029999999999999},"min":{"1min":0.503,"5min":0.41799999999999998,"15min":0.32300000000000001},"max":{"1min":0.89800000000000002,"5min":10.621,"15min":11.186},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.258,"15min":1.4850000000000001},"min":{"1min":0.46300000000000002,"5min":0.40000000000000002,"15min":0.371},"max":{"1min":0.98699999999999999,"5min":7.9130000000000003,"15min":10.627000000000001},"last":0.57999999999999996}]}]},{"osd":0,"up_from":79,"seq":339302416799,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1050644,"kb_used_data":7692,"kb_used_omap":28,"kb_used_meta":1042915,"kb_avail":92694508,"statfs":{"total":95995035648,"available":94919176192,"internally_reserved":0,"allocated":7876608,"data_stored":5460138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1067945808},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:29:53 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.538,"15min":1.544},"min":{"1min":0.247,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":0.48899999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":1.766,"15min":1.6539999999999999},"min":{"1min":0.247,"5min":0.218,"15min":0.192},"max":{"1min":0.64900000000000002,"5min":16.818000000000001,"15min":56.950000000000003},"last":0.33300000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":1.7729999999999999,"15min":1.736},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.22800000000000001},"max":{"1min":0.57599999999999996,"5min":17.093,"15min":53.988999999999997},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.443,"5min":1.8,"15min":1.7969999999999999},"min":{"1min":0.317,"5min":0.29999999999999999,"15min":0.22600000000000001},"max":{"1min":0.61899999999999999,"5min":17.053999999999998,"15min":54.119},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:30:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":2.1789999999999998,"15min":1.746},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":0.78400000000000003,"5min":29.835999999999999,"15min":56.984999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.41699999999999998,"5min":2.3109999999999999,"15min":1.857},"min":{"1min":0.311,"5min":0.249,"15min":0.249},"max":{"1min":0.61099999999999999,"5min":29.776,"15min":56.475999999999999},"last":0.318}]},{"osd":4,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":1.6080000000000001,"15min":1.7709999999999999},"min":{"1min":0.46600000000000003,"5min":0.29199999999999998,"15min":0.251},"max":{"1min":0.90000000000000002,"5min":11.653,"15min":57.719000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.63,"5min":1.8799999999999999,"15min":1.958},"min":{"1min":0.45300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.90900000000000003,"5min":11.896000000000001,"15min":56.378},"last":0.63200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":3.1560000000000001,"15min":2.3769999999999998},"min":{"1min":0.379,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":0.85599999999999998,"5min":90.412000000000006,"15min":90.412000000000006},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":2.4700000000000002,"15min":2.0049999999999999},"min":{"1min":0.432,"5min":0.34399999999999997,"15min":0.30199999999999999},"max":{"1min":0.82799999999999996,"5min":90.393000000000001,"15min":90.393000000000001},"last":0.76900000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.5189999999999999,"15min":1.778},"min":{"1min":0.41299999999999998,"5min":0.38400000000000001,"15min":0.34000000000000002},"max":{"1min":0.92700000000000005,"5min":9.7840000000000007,"15min":54.482999999999997},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.8049999999999999,"15min":1.9910000000000001},"min":{"1min":0.46800000000000003,"5min":0.37,"15min":0.34799999999999998},"max":{"1min":0.95399999999999996,"5min":11.702999999999999,"15min":57.753},"last":0.69199999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.9410000000000001,"15min":2.0110000000000001},"min":{"1min":0.46700000000000003,"5min":0.38500000000000001,"15min":0.34999999999999998},"max":{"1min":0.94199999999999995,"5min":12.029999999999999,"15min":56.411999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.66900000000000004,"5min":1.7929999999999999,"15min":1.917},"min":{"1min":0.46600000000000003,"5min":0.44800000000000001,"15min":0.38300000000000001},"max":{"1min":0.91900000000000004,"5min":17.129999999999999,"15min":54.451999999999998},"last":0.73699999999999999}]}]},{"osd":2,"up_from":217,"seq":932007903535,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":895732,"kb_used_data":10988,"kb_used_omap":83,"kb_used_meta":884652,"kb_avail":92849420,"statfs":{"total":95995035648,"available":95077806080,"internally_reserved":0,"allocated":11251712,"data_stored":9045627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":905884143},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":1.9850000000000001,"15min":1.659},"min":{"1min":0.215,"5min":0.192,"15min":0.19},"max":{"1min":0.44400000000000001,"5min":31.641999999999999,"15min":48.990000000000002},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":1.5940000000000001,"15min":1.728},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.20899999999999999},"max":{"1min":0.65900000000000003,"5min":13.773,"15min":49.412999999999997},"last":0.35999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":2.1560000000000001,"15min":1.5900000000000001},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.20300000000000001},"max":{"1min":0.51400000000000001,"5min":27.097999999999999,"15min":34.893999999999998},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":1.246,"15min":1.4390000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25800000000000001},"max":{"1min":0.505,"5min":13.356999999999999,"15min":34.962000000000003},"last":0.39000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:30:01 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7090000000000001,"15min":1.4339999999999999},"min":{"1min":0.26300000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.47299999999999998,"5min":15.115,"15min":35.043999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.38100000000000001,"5min":1.673,"15min":1.4019999999999999},"min":{"1min":0.29299999999999998,"5min":0.28899999999999998,"15min":0.23699999999999999},"max":{"1min":0.63400000000000001,"5min":11.206,"15min":50.406999999999996},"last":0.38200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.167,"15min":1.3420000000000001},"min":{"1min":0.498,"5min":0.31900000000000001,"15min":0.29799999999999999},"max":{"1min":0.67500000000000004,"5min":8.1859999999999999,"15min":47.503999999999998},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.7090000000000001,"15min":1.653},"min":{"1min":0.46400000000000002,"5min":0.34000000000000002,"15min":0.29299999999999998},"max":{"1min":0.77800000000000002,"5min":11.298,"15min":47.704000000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.494,"15min":1.484},"min":{"1min":0.443,"5min":0.34300000000000003,"15min":0.308},"max":{"1min":0.747,"5min":11.266999999999999,"15min":24.23},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":2.1190000000000002,"15min":1.669},"min":{"1min":0.38600000000000001,"5min":0.34899999999999998,"15min":0.31},"max":{"1min":0.86599999999999999,"5min":26.997,"15min":26.997},"last":0.78700000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.206,"15min":1.54},"min":{"1min":0.46999999999999997,"5min":0.35399999999999998,"15min":0.35299999999999998},"max":{"1min":0.76100000000000001,"5min":7.9390000000000001,"15min":12.411},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.71,"15min":1.6180000000000001},"min":{"1min":0.41999999999999998,"5min":0.39300000000000002,"15min":0.38300000000000001},"max":{"1min":0.78700000000000003,"5min":13.975,"15min":24.379000000000001},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.363,"15min":1.7929999999999999},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":0.80200000000000005,"5min":9.5199999999999996,"15min":35.008000000000003},"last":0.752},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.6899999999999999,"15min":1.605},"min":{"1min":0.49299999999999999,"5min":0.40799999999999997,"15min":0.35899999999999999},"max":{"1min":0.83699999999999997,"5min":15.548,"15min":24.323},"last":0.70299999999999996}]}]},{"osd":3,"up_from":352,"seq":1511828488374,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":607032,"kb_used_data":7472,"kb_used_omap":15,"kb_used_meta":599536,"kb_avail":93138120,"statfs":{"total":95995035648,"available":95373434880,"internally_reserved":0,"allocated":7651328,"data_stored":4913179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":613925318},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":3.2589999999999999,"15min":2.0270000000000001},"min":{"1min":0.25,"5min":0.23899999999999999,"15min":0.19900000000000001},"max":{"1min":0.65400000000000003,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":3.3639999999999999,"15min":2.1309999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":0.62,"5min":66.802000000000007,"15min":66.802000000000007},"last":0.26500000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":1.2150000000000001,"15min":1.8420000000000001},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":0.63700000000000001,"5min":11.785,"15min":105.401},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.673,"15min":1.9930000000000001},"min":{"1min":0.28000000000000003,"5min":0.23400000000000001,"15min":0.23300000000000001},"max":{"1min":0.81999999999999995,"5min":10.422000000000001,"15min":104.914},"last":0.41299999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":2.036,"15min":1.8320000000000001},"min":{"1min":0.27600000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.59899999999999998,"5min":13.497999999999999,"15min":13.497999999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.502,"5min":2.0830000000000002,"15min":1.587},"min":{"1min":0.34000000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.78600000000000003,"5min":16.475000000000001,"15min":25.338999999999999},"last":0.46500000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":3.6080000000000001,"15min":2.306},"min":{"1min":0.42399999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.94399999999999995,"5min":178.72200000000001,"15min":178.72200000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":4.2270000000000003,"15min":2.7999999999999998},"min":{"1min":0.45300000000000001,"5min":0.314,"15min":0.29599999999999999},"max":{"1min":0.85299999999999998,"5min":179.08500000000001,"15min":179.08500000000001},"last":1.04}]},{"osd":5,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.4339999999999999,"15min":1.4379999999999999},"min":{"1min":0.441,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.754,"5min":11.363,"15min":25.251999999999999},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.67300000000000004,"5min":1.7549999999999999,"15min":1.776},"min":{"1min":0.45300000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.042,"5min":10.388,"15min":13.394},"last":0.94699999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.891,"15min":1.819},"min":{"1min":0.47899999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.74099999999999999,"5min":13.545999999999999,"15min":19.114000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.496,"15min":1.573},"min":{"1min":0.499,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":0.91500000000000004,"5min":11.32,"15min":25.306999999999999},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.6950000000000001,"15min":1.7390000000000001},"min":{"1min":0.54400000000000004,"5min":0.41299999999999998,"15min":0.38400000000000001},"max":{"1min":0.999,"5min":10.345000000000001,"15min":10.459},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.8819999999999999,"15min":1.875},"min":{"1min":0.46500000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.96899999999999997,"5min":12.4,"15min":19.280000000000001},"last":0.47699999999999998}]}]},{"osd":4,"up_from":266,"seq":1142461301004,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":934656,"kb_used_data":5624,"kb_used_omap":35,"kb_used_meta":928988,"kb_avail":92810496,"statfs":{"total":95995035648,"available":95037947904,"internally_reserved":0,"allocated":5758976,"data_stored":4715372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":951283950},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":2.7000000000000002,"15min":2.54},"min":{"1min":0.25900000000000001,"5min":0.22700000000000001,"15min":0.20599999999999999},"max":{"1min":1.9670000000000001,"5min":109.193,"15min":109.193},"last":0.44},{"interface":"front","average":{"1min":0.66200000000000003,"5min":3.0779999999999998,"15min":3.0779999999999998},"min":{"1min":0.318,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":1.982,"5min":110.438,"15min":110.438},"last":0.502}]},{"osd":1,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.458,"15min":1.95},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.24299999999999999},"max":{"1min":3.0310000000000001,"5min":12.209,"15min":71.457999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.679,"15min":2.008},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.248},"max":{"1min":1.9950000000000001,"5min":9.2040000000000006,"15min":71.658000000000001},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.4990000000000001,"15min":1.6419999999999999},"min":{"1min":0.40699999999999997,"5min":0.33200000000000002,"15min":0.25600000000000001},"max":{"1min":2.141,"5min":9.9039999999999999,"15min":15.944000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.3700000000000001,"15min":1.5089999999999999},"min":{"1min":0.45300000000000001,"5min":0.28000000000000003,"15min":0.255},"max":{"1min":2.2130000000000001,"5min":11.102,"15min":14.473000000000001},"last":0.35399999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:30:03 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.9359999999999999,"15min":1.9730000000000001},"min":{"1min":0.38,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":3.109,"5min":11.347,"15min":12.074},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.9950000000000001,"15min":1.9319999999999999},"min":{"1min":0.39000000000000001,"5min":0.32000000000000001,"15min":0.27100000000000002},"max":{"1min":3.214,"5min":11.369999999999999,"15min":16.067},"last":0.48299999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.99299999999999999,"15min":1.621},"min":{"1min":0.29199999999999998,"5min":0.26300000000000001,"15min":0.252},"max":{"1min":0.68899999999999995,"5min":10.007,"15min":16.094000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.79600000000000004,"15min":1.4199999999999999},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":0.66100000000000003,"5min":5.9509999999999996,"15min":49.915999999999997},"last":0.40300000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":1.139,"15min":1.3280000000000001},"min":{"1min":0.33700000000000002,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":0.69899999999999995,"5min":9.5389999999999997,"15min":14.499000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":1.29,"15min":1.7410000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28299999999999997},"max":{"1min":0.77200000000000002,"5min":12.242000000000001,"15min":18.952999999999999},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.3360000000000001,"15min":1.758},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.307},"max":{"1min":0.75900000000000001,"5min":12.391999999999999,"15min":17.369},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.1299999999999999,"15min":1.663},"min":{"1min":0.371,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.81399999999999995,"5min":9.5129999999999999,"15min":51.026000000000003},"last":0.51600000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823189,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518708,"kb_used_data":9708,"kb_used_omap":0,"kb_used_meta":1508992,"kb_avail":92226444,"statfs":{"total":95995035648,"available":94439878656,"internally_reserved":0,"allocated":9940992,"data_stored":8905439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1545207808},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:05 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":2.3500000000000001,"15min":1.992},"min":{"1min":0.29699999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.755,"5min":19.102,"15min":22.469999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.5349999999999999,"15min":1.6379999999999999},"min":{"1min":0.32500000000000001,"5min":0.24299999999999999,"15min":0.23200000000000001},"max":{"1min":0.83299999999999996,"5min":17.887,"15min":22.437000000000001},"last":0.50800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:29:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.9139999999999999,"15min":2},"min":{"1min":0.40000000000000002,"5min":0.27300000000000002,"15min":0.246},"max":{"1min":1.0209999999999999,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.37},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.236,"15min":1.607},"min":{"1min":0.34599999999999997,"5min":0.27300000000000002,"15min":0.23499999999999999},"max":{"1min":0.71999999999999997,"5min":8.9529999999999994,"15min":29.065000000000001},"last":0.56299999999999994}]},{"osd":2,"last update":"Mon Oct 9 10:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.579,"15min":1.8360000000000001},"min":{"1min":0.41199999999999998,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":0.82699999999999996,"5min":12.247999999999999,"15min":16.629999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.272,"15min":1.5960000000000001},"min":{"1min":0.376,"5min":0.32600000000000001,"15min":0.28399999999999997},"max":{"1min":0.90000000000000002,"5min":7.1479999999999997,"15min":13.095000000000001},"last":0.64000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:30:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.954,"15min":1.9019999999999999},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":0.84599999999999997,"5min":13.614000000000001,"15min":13.614000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.7190000000000001,"15min":1.7849999999999999},"min":{"1min":0.44900000000000001,"5min":0.38100000000000001,"15min":0.315},"max":{"1min":0.94899999999999995,"5min":11.124000000000001,"15min":13.624000000000001},"last":0.70399999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.4730000000000001,"15min":1.5720000000000001},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":0.71499999999999997,"5min":13.523999999999999,"15min":13.523999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.0489999999999999,"15min":1.446},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.23599999999999999},"max":{"1min":0.79800000000000004,"5min":8.8620000000000001,"15min":11.956},"last":0.67400000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:30:05 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.6519999999999999,"15min":1.776},"min":{"1min":0.315,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":0.67400000000000004,"5min":13.567,"15min":13.567},"last":0.502},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.4930000000000001,"15min":1.649},"min":{"1min":0.35399999999999998,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":0.91700000000000004,"5min":16.497,"15min":16.497},"last":0.53100000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:30:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.1259999999999999,"15min":1.6679999999999999},"min":{"1min":0.45700000000000002,"5min":0.36899999999999999,"15min":0.311},"max":{"1min":0.81699999999999995,"5min":7.3490000000000002,"15min":21.109000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.70899999999999996,"15min":1.129},"min":{"1min":0.38700000000000001,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":0.82699999999999996,"5min":5.5519999999999996,"15min":11.994},"last":0.73499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:30:47.397 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:30:47.677 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:47 smithi012 ceph-mon[147926]: pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:47.677 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:47 smithi012 ceph-mon[152263]: pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:47.943 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:47 smithi110 ceph-mon[136032]: pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:48.003 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:30:48.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:47 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: ::ffff:172.21.15.110 - - [09/Oct/2023:10:30:47] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.33.4" 2023-10-09T10:30:48.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:48 smithi012 ceph-mon[147926]: from='client.38280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:30:48.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:48 smithi012 ceph-mon[152263]: from='client.38280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:30:48.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:48 smithi110 ceph-mon[136032]: from='client.38280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:30:49.426 INFO:teuthology.orchestra.run.smithi012.stdout: 2023-10-09T10:30:49.428 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2023-10-09T10:30:49.787 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:49 smithi012 ceph-mon[147926]: pgmap v1710: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:49.787 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:49 smithi012 ceph-mon[152263]: pgmap v1710: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:49.823 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":1710,"stamp":"2023-10-09T10:30:47.911457+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6263,"num_read_kb":13678,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":226,"num_bytes_recovered":11378044,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8281436,"kb_used_data":64860,"kb_used_omap":172,"kb_used_meta":8216339,"kb_avail":741679780,"statfs":{"total":767960285184,"available":759480094720,"internally_reserved":0,"allocated":66416640,"data_stored":52694030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":176164,"internal_metadata":8413532124},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002802"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":218,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130916+0000","last_change":"2023-10-09T10:29:30.611925+0000","last_active":"2023-10-09T10:30:21.130916+0000","last_peered":"2023-10-09T10:30:21.130916+0000","last_clean":"2023-10-09T10:30:21.130916+0000","last_became_active":"2023-10-09T10:29:30.611596+0000","last_became_peered":"2023-10-09T10:29:30.611596+0000","last_unstale":"2023-10-09T10:30:21.130916+0000","last_undegraded":"2023-10-09T10:30:21.130916+0000","last_fullsized":"2023-10-09T10:30:21.130916+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":464,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.411310+0000","last_change":"2023-10-09T10:30:22.411310+0000","last_active":"2023-10-09T10:30:22.411310+0000","last_peered":"2023-10-09T10:30:22.411310+0000","last_clean":"2023-10-09T10:30:22.411310+0000","last_became_active":"2023-10-09T10:30:22.410788+0000","last_became_peered":"2023-10-09T10:30:22.410788+0000","last_unstale":"2023-10-09T10:30:22.411310+0000","last_undegraded":"2023-10-09T10:30:22.411310+0000","last_fullsized":"2023-10-09T10:30:22.411310+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:24.212390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":39,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131956+0000","last_change":"2023-10-09T10:29:30.607630+0000","last_active":"2023-10-09T10:30:21.131956+0000","last_peered":"2023-10-09T10:30:21.131956+0000","last_clean":"2023-10-09T10:30:21.131956+0000","last_became_active":"2023-10-09T10:29:30.605951+0000","last_became_peered":"2023-10-09T10:29:30.605951+0000","last_unstale":"2023-10-09T10:30:21.131956+0000","last_undegraded":"2023-10-09T10:30:21.131956+0000","last_fullsized":"2023-10-09T10:30:21.131956+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:37.492242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":563,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.137924+0000","last_change":"2023-10-09T10:29:25.567076+0000","last_active":"2023-10-09T10:30:22.137924+0000","last_peered":"2023-10-09T10:30:22.137924+0000","last_clean":"2023-10-09T10:30:22.137924+0000","last_became_active":"2023-10-09T10:29:25.566709+0000","last_became_peered":"2023-10-09T10:29:25.566709+0000","last_unstale":"2023-10-09T10:30:22.137924+0000","last_undegraded":"2023-10-09T10:30:22.137924+0000","last_fullsized":"2023-10-09T10:30:22.137924+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:42.706058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":410,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.411227+0000","last_change":"2023-10-09T10:30:22.411227+0000","last_active":"2023-10-09T10:30:22.411227+0000","last_peered":"2023-10-09T10:30:22.411227+0000","last_clean":"2023-10-09T10:30:22.411227+0000","last_became_active":"2023-10-09T10:30:22.410870+0000","last_became_peered":"2023-10-09T10:30:22.410870+0000","last_unstale":"2023-10-09T10:30:22.411227+0000","last_undegraded":"2023-10-09T10:30:22.411227+0000","last_fullsized":"2023-10-09T10:30:22.411227+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:43.589572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":192,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132802+0000","last_change":"2023-10-09T10:30:03.061847+0000","last_active":"2023-10-09T10:30:21.132802+0000","last_peered":"2023-10-09T10:30:21.132802+0000","last_clean":"2023-10-09T10:30:21.132802+0000","last_became_active":"2023-10-09T10:30:03.061252+0000","last_became_peered":"2023-10-09T10:30:03.061252+0000","last_unstale":"2023-10-09T10:30:21.132802+0000","last_undegraded":"2023-10-09T10:30:21.132802+0000","last_fullsized":"2023-10-09T10:30:21.132802+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":596,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129171+0000","last_change":"2023-10-09T10:29:48.546235+0000","last_active":"2023-10-09T10:30:21.129171+0000","last_peered":"2023-10-09T10:30:21.129171+0000","last_clean":"2023-10-09T10:30:21.129171+0000","last_became_active":"2023-10-09T10:29:48.545757+0000","last_became_peered":"2023-10-09T10:29:48.545757+0000","last_unstale":"2023-10-09T10:30:21.129171+0000","last_undegraded":"2023-10-09T10:30:21.129171+0000","last_fullsized":"2023-10-09T10:30:21.129171+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:16.484807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":507,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.410402+0000","last_change":"2023-10-09T10:30:22.410402+0000","last_active":"2023-10-09T10:30:22.410402+0000","last_peered":"2023-10-09T10:30:22.410402+0000","last_clean":"2023-10-09T10:30:22.410402+0000","last_became_active":"2023-10-09T10:30:22.410019+0000","last_became_peered":"2023-10-09T10:30:22.410019+0000","last_unstale":"2023-10-09T10:30:22.410402+0000","last_undegraded":"2023-10-09T10:30:22.410402+0000","last_fullsized":"2023-10-09T10:30:22.410402+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:07.263314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":47,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131496+0000","last_change":"2023-10-09T10:29:51.562903+0000","last_active":"2023-10-09T10:30:21.131496+0000","last_peered":"2023-10-09T10:30:21.131496+0000","last_clean":"2023-10-09T10:30:21.131496+0000","last_became_active":"2023-10-09T10:29:51.562500+0000","last_became_peered":"2023-10-09T10:29:51.562500+0000","last_unstale":"2023-10-09T10:30:21.131496+0000","last_undegraded":"2023-10-09T10:30:21.131496+0000","last_fullsized":"2023-10-09T10:30:21.131496+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":531,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.146145+0000","last_change":"2023-10-09T10:30:22.146145+0000","last_active":"2023-10-09T10:30:22.146145+0000","last_peered":"2023-10-09T10:30:22.146145+0000","last_clean":"2023-10-09T10:30:22.146145+0000","last_became_active":"2023-10-09T10:30:22.145757+0000","last_became_peered":"2023-10-09T10:30:22.145757+0000","last_unstale":"2023-10-09T10:30:22.146145+0000","last_undegraded":"2023-10-09T10:30:22.146145+0000","last_fullsized":"2023-10-09T10:30:22.146145+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:10:21.864424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":209,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134523+0000","last_change":"2023-10-09T10:30:04.067294+0000","last_active":"2023-10-09T10:30:21.134523+0000","last_peered":"2023-10-09T10:30:21.134523+0000","last_clean":"2023-10-09T10:30:21.134523+0000","last_became_active":"2023-10-09T10:30:04.066841+0000","last_became_peered":"2023-10-09T10:30:04.066841+0000","last_unstale":"2023-10-09T10:30:21.134523+0000","last_undegraded":"2023-10-09T10:30:21.134523+0000","last_fullsized":"2023-10-09T10:30:21.134523+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":39,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132224+0000","last_change":"2023-10-09T10:29:30.609289+0000","last_active":"2023-10-09T10:30:21.132224+0000","last_peered":"2023-10-09T10:30:21.132224+0000","last_clean":"2023-10-09T10:30:21.132224+0000","last_became_active":"2023-10-09T10:29:30.609010+0000","last_became_peered":"2023-10-09T10:29:30.609010+0000","last_unstale":"2023-10-09T10:30:21.132224+0000","last_undegraded":"2023-10-09T10:30:21.132224+0000","last_fullsized":"2023-10-09T10:30:21.132224+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:47.993837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":286,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131823+0000","last_change":"2023-10-09T10:29:25.712940+0000","last_active":"2023-10-09T10:30:21.131823+0000","last_peered":"2023-10-09T10:30:21.131823+0000","last_clean":"2023-10-09T10:30:21.131823+0000","last_became_active":"2023-10-09T10:29:25.575382+0000","last_became_peered":"2023-10-09T10:29:25.575382+0000","last_unstale":"2023-10-09T10:30:21.131823+0000","last_undegraded":"2023-10-09T10:30:21.131823+0000","last_fullsized":"2023-10-09T10:30:21.131823+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":236,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131620+0000","last_change":"2023-10-09T10:29:30.608709+0000","last_active":"2023-10-09T10:30:21.131620+0000","last_peered":"2023-10-09T10:30:21.131620+0000","last_clean":"2023-10-09T10:30:21.131620+0000","last_became_active":"2023-10-09T10:29:30.608390+0000","last_became_peered":"2023-10-09T10:29:30.608390+0000","last_unstale":"2023-10-09T10:30:21.131620+0000","last_undegraded":"2023-10-09T10:30:21.131620+0000","last_fullsized":"2023-10-09T10:30:21.131620+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":48,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134109+0000","last_change":"2023-10-09T10:29:25.661460+0000","last_active":"2023-10-09T10:30:21.134109+0000","last_peered":"2023-10-09T10:30:21.134109+0000","last_clean":"2023-10-09T10:30:21.134109+0000","last_became_active":"2023-10-09T10:29:25.573543+0000","last_became_peered":"2023-10-09T10:29:25.573543+0000","last_unstale":"2023-10-09T10:30:21.134109+0000","last_undegraded":"2023-10-09T10:30:21.134109+0000","last_fullsized":"2023-10-09T10:30:21.134109+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":435,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.410384+0000","last_change":"2023-10-09T10:30:22.410384+0000","last_active":"2023-10-09T10:30:22.410384+0000","last_peered":"2023-10-09T10:30:22.410384+0000","last_clean":"2023-10-09T10:30:22.410384+0000","last_became_active":"2023-10-09T10:30:22.410048+0000","last_became_peered":"2023-10-09T10:30:22.410048+0000","last_unstale":"2023-10-09T10:30:22.410384+0000","last_undegraded":"2023-10-09T10:30:22.410384+0000","last_fullsized":"2023-10-09T10:30:22.410384+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:05.134050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":552,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.137298+0000","last_change":"2023-10-09T10:29:25.562515+0000","last_active":"2023-10-09T10:30:22.137298+0000","last_peered":"2023-10-09T10:30:22.137298+0000","last_clean":"2023-10-09T10:30:22.137298+0000","last_became_active":"2023-10-09T10:29:25.562150+0000","last_became_peered":"2023-10-09T10:29:25.562150+0000","last_unstale":"2023-10-09T10:30:22.137298+0000","last_undegraded":"2023-10-09T10:30:22.137298+0000","last_fullsized":"2023-10-09T10:30:22.137298+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:27.262966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131005+0000","last_change":"2023-10-09T10:29:48.553767+0000","last_active":"2023-10-09T10:30:21.131005+0000","last_peered":"2023-10-09T10:30:21.131005+0000","last_clean":"2023-10-09T10:30:21.131005+0000","last_became_active":"2023-10-09T10:29:48.553323+0000","last_became_peered":"2023-10-09T10:29:48.553323+0000","last_unstale":"2023-10-09T10:30:21.131005+0000","last_undegraded":"2023-10-09T10:30:21.131005+0000","last_fullsized":"2023-10-09T10:30:21.131005+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":215,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134382+0000","last_change":"2023-10-09T10:30:04.068976+0000","last_active":"2023-10-09T10:30:21.134382+0000","last_peered":"2023-10-09T10:30:21.134382+0000","last_clean":"2023-10-09T10:30:21.134382+0000","last_became_active":"2023-10-09T10:30:04.068637+0000","last_became_peered":"2023-10-09T10:30:04.068637+0000","last_unstale":"2023-10-09T10:30:21.134382+0000","last_undegraded":"2023-10-09T10:30:21.134382+0000","last_fullsized":"2023-10-09T10:30:21.134382+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":124,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.128766+0000","last_change":"2023-10-09T10:29:25.566025+0000","last_active":"2023-10-09T10:30:21.128766+0000","last_peered":"2023-10-09T10:30:21.128766+0000","last_clean":"2023-10-09T10:30:21.128766+0000","last_became_active":"2023-10-09T10:29:25.565569+0000","last_became_peered":"2023-10-09T10:29:25.565569+0000","last_unstale":"2023-10-09T10:30:21.128766+0000","last_undegraded":"2023-10-09T10:30:21.128766+0000","last_fullsized":"2023-10-09T10:30:21.128766+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:24:42.150298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":484,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132951+0000","last_change":"2023-10-09T10:29:58.037928+0000","last_active":"2023-10-09T10:30:21.132951+0000","last_peered":"2023-10-09T10:30:21.132951+0000","last_clean":"2023-10-09T10:30:21.132951+0000","last_became_active":"2023-10-09T10:29:58.037560+0000","last_became_peered":"2023-10-09T10:29:58.037560+0000","last_unstale":"2023-10-09T10:30:21.132951+0000","last_undegraded":"2023-10-09T10:30:21.132951+0000","last_fullsized":"2023-10-09T10:30:21.132951+0000","mapping_epoch":604,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":107,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.411522+0000","last_change":"2023-10-09T10:30:22.411522+0000","last_active":"2023-10-09T10:30:22.411522+0000","last_peered":"2023-10-09T10:30:22.411522+0000","last_clean":"2023-10-09T10:30:22.411522+0000","last_became_active":"2023-10-09T10:30:22.410935+0000","last_became_peered":"2023-10-09T10:30:22.410935+0000","last_unstale":"2023-10-09T10:30:22.411522+0000","last_undegraded":"2023-10-09T10:30:22.411522+0000","last_fullsized":"2023-10-09T10:30:22.411522+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:30.052943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":118,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134928+0000","last_change":"2023-10-09T10:30:04.065970+0000","last_active":"2023-10-09T10:30:21.134928+0000","last_peered":"2023-10-09T10:30:21.134928+0000","last_clean":"2023-10-09T10:30:21.134928+0000","last_became_active":"2023-10-09T10:30:04.065363+0000","last_became_peered":"2023-10-09T10:30:04.065363+0000","last_unstale":"2023-10-09T10:30:21.134928+0000","last_undegraded":"2023-10-09T10:30:21.134928+0000","last_fullsized":"2023-10-09T10:30:21.134928+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":198,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131440+0000","last_change":"2023-10-09T10:29:48.553090+0000","last_active":"2023-10-09T10:30:21.131440+0000","last_peered":"2023-10-09T10:30:21.131440+0000","last_clean":"2023-10-09T10:30:21.131440+0000","last_became_active":"2023-10-09T10:29:48.552702+0000","last_became_peered":"2023-10-09T10:29:48.552702+0000","last_unstale":"2023-10-09T10:30:21.131440+0000","last_undegraded":"2023-10-09T10:30:21.131440+0000","last_fullsized":"2023-10-09T10:30:21.131440+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":244,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134360+0000","last_change":"2023-10-09T10:30:04.065752+0000","last_active":"2023-10-09T10:30:21.134360+0000","last_peered":"2023-10-09T10:30:21.134360+0000","last_clean":"2023-10-09T10:30:21.134360+0000","last_became_active":"2023-10-09T10:30:04.065364+0000","last_became_peered":"2023-10-09T10:30:04.065364+0000","last_unstale":"2023-10-09T10:30:21.134360+0000","last_undegraded":"2023-10-09T10:30:21.134360+0000","last_fullsized":"2023-10-09T10:30:21.134360+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":215,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131022+0000","last_change":"2023-10-09T10:30:16.107607+0000","last_active":"2023-10-09T10:30:21.131022+0000","last_peered":"2023-10-09T10:30:21.131022+0000","last_clean":"2023-10-09T10:30:21.131022+0000","last_became_active":"2023-10-09T10:30:16.107315+0000","last_became_peered":"2023-10-09T10:30:16.107315+0000","last_unstale":"2023-10-09T10:30:21.131022+0000","last_undegraded":"2023-10-09T10:30:21.131022+0000","last_fullsized":"2023-10-09T10:30:21.131022+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":61,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134590+0000","last_change":"2023-10-09T10:30:04.069124+0000","last_active":"2023-10-09T10:30:21.134590+0000","last_peered":"2023-10-09T10:30:21.134590+0000","last_clean":"2023-10-09T10:30:21.134590+0000","last_became_active":"2023-10-09T10:30:04.068712+0000","last_became_peered":"2023-10-09T10:30:04.068712+0000","last_unstale":"2023-10-09T10:30:21.134590+0000","last_undegraded":"2023-10-09T10:30:21.134590+0000","last_fullsized":"2023-10-09T10:30:21.134590+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":139,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133006+0000","last_change":"2023-10-09T10:29:58.327402+0000","last_active":"2023-10-09T10:30:21.133006+0000","last_peered":"2023-10-09T10:30:21.133006+0000","last_clean":"2023-10-09T10:30:21.133006+0000","last_became_active":"2023-10-09T10:29:58.327044+0000","last_became_peered":"2023-10-09T10:29:58.327044+0000","last_unstale":"2023-10-09T10:30:21.133006+0000","last_undegraded":"2023-10-09T10:30:21.133006+0000","last_fullsized":"2023-10-09T10:30:21.133006+0000","mapping_epoch":604,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":292,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131740+0000","last_change":"2023-10-09T10:24:40.654724+0000","last_active":"2023-10-09T10:30:21.131740+0000","last_peered":"2023-10-09T10:30:21.131740+0000","last_clean":"2023-10-09T10:30:21.131740+0000","last_became_active":"2023-10-09T10:24:40.653936+0000","last_became_peered":"2023-10-09T10:24:40.653936+0000","last_unstale":"2023-10-09T10:30:21.131740+0000","last_undegraded":"2023-10-09T10:30:21.131740+0000","last_fullsized":"2023-10-09T10:30:21.131740+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":538,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129076+0000","last_change":"2023-10-09T10:29:30.610242+0000","last_active":"2023-10-09T10:30:21.129076+0000","last_peered":"2023-10-09T10:30:21.129076+0000","last_clean":"2023-10-09T10:30:21.129076+0000","last_became_active":"2023-10-09T10:29:30.609891+0000","last_became_peered":"2023-10-09T10:29:30.609891+0000","last_unstale":"2023-10-09T10:30:21.129076+0000","last_undegraded":"2023-10-09T10:30:21.129076+0000","last_fullsized":"2023-10-09T10:30:21.129076+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:33.315269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":37,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.144565+0000","last_change":"2023-10-09T10:30:22.144565+0000","last_active":"2023-10-09T10:30:22.144565+0000","last_peered":"2023-10-09T10:30:22.144565+0000","last_clean":"2023-10-09T10:30:22.144565+0000","last_became_active":"2023-10-09T10:30:22.144253+0000","last_became_peered":"2023-10-09T10:30:22.144253+0000","last_unstale":"2023-10-09T10:30:22.144565+0000","last_undegraded":"2023-10-09T10:30:22.144565+0000","last_fullsized":"2023-10-09T10:30:22.144565+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:43.346142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":227,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131846+0000","last_change":"2023-10-09T10:30:16.509164+0000","last_active":"2023-10-09T10:30:21.131846+0000","last_peered":"2023-10-09T10:30:21.131846+0000","last_clean":"2023-10-09T10:30:21.131846+0000","last_became_active":"2023-10-09T10:30:16.508778+0000","last_became_peered":"2023-10-09T10:30:16.508778+0000","last_unstale":"2023-10-09T10:30:21.131846+0000","last_undegraded":"2023-10-09T10:30:21.131846+0000","last_fullsized":"2023-10-09T10:30:21.131846+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":224,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133112+0000","last_change":"2023-10-09T10:29:58.423808+0000","last_active":"2023-10-09T10:30:21.133112+0000","last_peered":"2023-10-09T10:30:21.133112+0000","last_clean":"2023-10-09T10:30:21.133112+0000","last_became_active":"2023-10-09T10:29:58.423466+0000","last_became_peered":"2023-10-09T10:29:58.423466+0000","last_unstale":"2023-10-09T10:30:21.133112+0000","last_undegraded":"2023-10-09T10:30:21.133112+0000","last_fullsized":"2023-10-09T10:30:21.133112+0000","mapping_epoch":604,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":338,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131810+0000","last_change":"2023-10-09T10:30:16.240089+0000","last_active":"2023-10-09T10:30:21.131810+0000","last_peered":"2023-10-09T10:30:21.131810+0000","last_clean":"2023-10-09T10:30:21.131810+0000","last_became_active":"2023-10-09T10:30:16.239747+0000","last_became_peered":"2023-10-09T10:30:16.239747+0000","last_unstale":"2023-10-09T10:30:21.131810+0000","last_undegraded":"2023-10-09T10:30:21.131810+0000","last_fullsized":"2023-10-09T10:30:21.131810+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":227,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131099+0000","last_change":"2023-10-09T10:29:25.567139+0000","last_active":"2023-10-09T10:30:21.131099+0000","last_peered":"2023-10-09T10:30:21.131099+0000","last_clean":"2023-10-09T10:30:21.131099+0000","last_became_active":"2023-10-09T10:29:25.566752+0000","last_became_peered":"2023-10-09T10:29:25.566752+0000","last_unstale":"2023-10-09T10:30:21.131099+0000","last_undegraded":"2023-10-09T10:30:21.131099+0000","last_fullsized":"2023-10-09T10:30:21.131099+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":245,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132330+0000","last_change":"2023-10-09T10:29:30.608624+0000","last_active":"2023-10-09T10:30:21.132330+0000","last_peered":"2023-10-09T10:30:21.132330+0000","last_clean":"2023-10-09T10:30:21.132330+0000","last_became_active":"2023-10-09T10:29:30.608310+0000","last_became_peered":"2023-10-09T10:29:30.608310+0000","last_unstale":"2023-10-09T10:30:21.132330+0000","last_undegraded":"2023-10-09T10:30:21.132330+0000","last_fullsized":"2023-10-09T10:30:21.132330+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":500,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.411701+0000","last_change":"2023-10-09T10:30:22.411701+0000","last_active":"2023-10-09T10:30:22.411701+0000","last_peered":"2023-10-09T10:30:22.411701+0000","last_clean":"2023-10-09T10:30:22.411701+0000","last_became_active":"2023-10-09T10:30:22.410855+0000","last_became_peered":"2023-10-09T10:30:22.410855+0000","last_unstale":"2023-10-09T10:30:22.411701+0000","last_undegraded":"2023-10-09T10:30:22.411701+0000","last_fullsized":"2023-10-09T10:30:22.411701+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:16.380309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":39,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133536+0000","last_change":"2023-10-09T10:29:30.608788+0000","last_active":"2023-10-09T10:30:21.133536+0000","last_peered":"2023-10-09T10:30:21.133536+0000","last_clean":"2023-10-09T10:30:21.133536+0000","last_became_active":"2023-10-09T10:29:30.607797+0000","last_became_peered":"2023-10-09T10:29:30.607797+0000","last_unstale":"2023-10-09T10:30:21.133536+0000","last_undegraded":"2023-10-09T10:30:21.133536+0000","last_fullsized":"2023-10-09T10:30:21.133536+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:45.991193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"580'175","reported_seq":1138,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.135090+0000","last_change":"2023-10-09T10:29:33.626271+0000","last_active":"2023-10-09T10:30:21.135090+0000","last_peered":"2023-10-09T10:30:21.135090+0000","last_clean":"2023-10-09T10:30:21.135090+0000","last_became_active":"2023-10-09T10:29:33.625801+0000","last_became_peered":"2023-10-09T10:29:33.625801+0000","last_unstale":"2023-10-09T10:30:21.135090+0000","last_undegraded":"2023-10-09T10:30:21.135090+0000","last_fullsized":"2023-10-09T10:30:21.135090+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:22.698773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072463300000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1289,"num_read_kb":2426,"num_write":156,"num_write_kb":1221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":28,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131097+0000","last_change":"2023-10-09T10:29:48.543406+0000","last_active":"2023-10-09T10:30:21.131097+0000","last_peered":"2023-10-09T10:30:21.131097+0000","last_clean":"2023-10-09T10:30:21.131097+0000","last_became_active":"2023-10-09T10:29:48.542916+0000","last_became_peered":"2023-10-09T10:29:48.542916+0000","last_unstale":"2023-10-09T10:30:21.131097+0000","last_undegraded":"2023-10-09T10:30:21.131097+0000","last_fullsized":"2023-10-09T10:30:21.131097+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132085+0000","last_change":"2023-10-09T10:29:48.551978+0000","last_active":"2023-10-09T10:30:21.132085+0000","last_peered":"2023-10-09T10:30:21.132085+0000","last_clean":"2023-10-09T10:30:21.132085+0000","last_became_active":"2023-10-09T10:29:48.551771+0000","last_became_peered":"2023-10-09T10:29:48.551771+0000","last_unstale":"2023-10-09T10:30:21.132085+0000","last_undegraded":"2023-10-09T10:30:21.132085+0000","last_fullsized":"2023-10-09T10:30:21.132085+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:02.207672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"580'79","reported_seq":782,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132920+0000","last_change":"2023-10-09T10:29:51.559548+0000","last_active":"2023-10-09T10:30:21.132920+0000","last_peered":"2023-10-09T10:30:21.132920+0000","last_clean":"2023-10-09T10:30:21.132920+0000","last_became_active":"2023-10-09T10:29:51.559055+0000","last_became_peered":"2023-10-09T10:29:51.559055+0000","last_unstale":"2023-10-09T10:30:21.132920+0000","last_undegraded":"2023-10-09T10:30:21.132920+0000","last_fullsized":"2023-10-09T10:30:21.132920+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":655,"num_read_kb":2665,"num_write":63,"num_write_kb":2097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":218,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131526+0000","last_change":"2023-10-09T10:24:40.657160+0000","last_active":"2023-10-09T10:30:21.131526+0000","last_peered":"2023-10-09T10:30:21.131526+0000","last_clean":"2023-10-09T10:30:21.131526+0000","last_became_active":"2023-10-09T10:24:40.656335+0000","last_became_peered":"2023-10-09T10:24:40.656335+0000","last_unstale":"2023-10-09T10:30:21.131526+0000","last_undegraded":"2023-10-09T10:30:21.131526+0000","last_fullsized":"2023-10-09T10:30:21.131526+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131591+0000","last_change":"2023-10-09T10:30:03.067276+0000","last_active":"2023-10-09T10:30:21.131591+0000","last_peered":"2023-10-09T10:30:21.131591+0000","last_clean":"2023-10-09T10:30:21.131591+0000","last_became_active":"2023-10-09T10:30:03.066921+0000","last_became_peered":"2023-10-09T10:30:03.066921+0000","last_unstale":"2023-10-09T10:30:21.131591+0000","last_undegraded":"2023-10-09T10:30:21.131591+0000","last_fullsized":"2023-10-09T10:30:21.131591+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"580'96","reported_seq":886,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131809+0000","last_change":"2023-10-09T10:30:10.082719+0000","last_active":"2023-10-09T10:30:21.131809+0000","last_peered":"2023-10-09T10:30:21.131809+0000","last_clean":"2023-10-09T10:30:21.131809+0000","last_became_active":"2023-10-09T10:30:10.082270+0000","last_became_peered":"2023-10-09T10:30:10.082270+0000","last_unstale":"2023-10-09T10:30:21.131809+0000","last_undegraded":"2023-10-09T10:30:21.131809+0000","last_fullsized":"2023-10-09T10:30:21.131809+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:01.498272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016205779999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":527,"num_read_kb":2199,"num_write":93,"num_write_kb":1467,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":39,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133051+0000","last_change":"2023-10-09T10:29:30.606561+0000","last_active":"2023-10-09T10:30:21.133051+0000","last_peered":"2023-10-09T10:30:21.133051+0000","last_clean":"2023-10-09T10:30:21.133051+0000","last_became_active":"2023-10-09T10:29:30.605545+0000","last_became_peered":"2023-10-09T10:29:30.605545+0000","last_unstale":"2023-10-09T10:30:21.133051+0000","last_undegraded":"2023-10-09T10:30:21.133051+0000","last_fullsized":"2023-10-09T10:30:21.133051+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:05.844422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133564+0000","last_change":"2023-10-09T10:29:48.545839+0000","last_active":"2023-10-09T10:30:21.133564+0000","last_peered":"2023-10-09T10:30:21.133564+0000","last_clean":"2023-10-09T10:30:21.133564+0000","last_became_active":"2023-10-09T10:29:48.545458+0000","last_became_peered":"2023-10-09T10:29:48.545458+0000","last_unstale":"2023-10-09T10:30:21.133564+0000","last_undegraded":"2023-10-09T10:30:21.133564+0000","last_fullsized":"2023-10-09T10:30:21.133564+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:48.354699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"580'175","reported_seq":780,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131326+0000","last_change":"2023-10-09T10:29:51.563279+0000","last_active":"2023-10-09T10:30:21.131326+0000","last_peered":"2023-10-09T10:30:21.131326+0000","last_clean":"2023-10-09T10:30:21.131326+0000","last_became_active":"2023-10-09T10:29:51.562644+0000","last_became_peered":"2023-10-09T10:29:51.562644+0000","last_unstale":"2023-10-09T10:30:21.131326+0000","last_undegraded":"2023-10-09T10:30:21.131326+0000","last_fullsized":"2023-10-09T10:30:21.131326+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":573,"num_read_kb":1842,"num_write":165,"num_write_kb":6420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":801,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134241+0000","last_change":"2023-10-09T10:30:04.068948+0000","last_active":"2023-10-09T10:30:21.134241+0000","last_peered":"2023-10-09T10:30:21.134241+0000","last_clean":"2023-10-09T10:30:21.134241+0000","last_became_active":"2023-10-09T10:30:04.068426+0000","last_became_peered":"2023-10-09T10:30:04.068426+0000","last_unstale":"2023-10-09T10:30:21.134241+0000","last_undegraded":"2023-10-09T10:30:21.134241+0000","last_fullsized":"2023-10-09T10:30:21.134241+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:45:36.759190+0000","last_clean_scrub_stamp":"2023-10-09T09:45:36.759190+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":229,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.159966+0000","last_change":"2023-10-09T10:30:22.159966+0000","last_active":"2023-10-09T10:30:22.159966+0000","last_peered":"2023-10-09T10:30:22.159966+0000","last_clean":"2023-10-09T10:30:22.159966+0000","last_became_active":"2023-10-09T10:30:22.159743+0000","last_became_peered":"2023-10-09T10:30:22.159743+0000","last_unstale":"2023-10-09T10:30:22.159966+0000","last_undegraded":"2023-10-09T10:30:22.159966+0000","last_fullsized":"2023-10-09T10:30:22.159966+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:32.139439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":219,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.133329+0000","last_change":"2023-10-09T10:29:58.424418+0000","last_active":"2023-10-09T10:30:21.133329+0000","last_peered":"2023-10-09T10:30:21.133329+0000","last_clean":"2023-10-09T10:30:21.133329+0000","last_became_active":"2023-10-09T10:29:58.423947+0000","last_became_peered":"2023-10-09T10:29:58.423947+0000","last_unstale":"2023-10-09T10:30:21.133329+0000","last_undegraded":"2023-10-09T10:30:21.133329+0000","last_fullsized":"2023-10-09T10:30:21.133329+0000","mapping_epoch":604,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"580'62","reported_seq":490,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134844+0000","last_change":"2023-10-09T10:30:04.065679+0000","last_active":"2023-10-09T10:30:21.134844+0000","last_peered":"2023-10-09T10:30:21.134844+0000","last_clean":"2023-10-09T10:30:21.134844+0000","last_became_active":"2023-10-09T10:30:04.065068+0000","last_became_peered":"2023-10-09T10:30:04.065068+0000","last_unstale":"2023-10-09T10:30:21.134844+0000","last_undegraded":"2023-10-09T10:30:21.134844+0000","last_fullsized":"2023-10-09T10:30:21.134844+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":272,"num_read_kb":1630,"num_write":58,"num_write_kb":1238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":564,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130632+0000","last_change":"2023-10-09T10:25:08.438752+0000","last_active":"2023-10-09T10:30:21.130632+0000","last_peered":"2023-10-09T10:30:21.130632+0000","last_clean":"2023-10-09T10:30:21.130632+0000","last_became_active":"2023-10-09T10:25:08.438449+0000","last_became_peered":"2023-10-09T10:25:08.438449+0000","last_unstale":"2023-10-09T10:30:21.130632+0000","last_undegraded":"2023-10-09T10:30:21.130632+0000","last_fullsized":"2023-10-09T10:30:21.130632+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:20.383114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":151,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.128662+0000","last_change":"2023-10-09T10:30:03.059595+0000","last_active":"2023-10-09T10:30:21.128662+0000","last_peered":"2023-10-09T10:30:21.128662+0000","last_clean":"2023-10-09T10:30:21.128662+0000","last_became_active":"2023-10-09T10:30:03.059212+0000","last_became_peered":"2023-10-09T10:30:03.059212+0000","last_unstale":"2023-10-09T10:30:21.128662+0000","last_undegraded":"2023-10-09T10:30:21.128662+0000","last_fullsized":"2023-10-09T10:30:21.128662+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:02.823940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"580'221","reported_seq":989,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.134453+0000","last_change":"2023-10-09T10:30:04.067345+0000","last_active":"2023-10-09T10:30:21.134453+0000","last_peered":"2023-10-09T10:30:21.134453+0000","last_clean":"2023-10-09T10:30:21.134453+0000","last_became_active":"2023-10-09T10:30:04.066626+0000","last_became_peered":"2023-10-09T10:30:04.066626+0000","last_unstale":"2023-10-09T10:30:21.134453+0000","last_undegraded":"2023-10-09T10:30:21.134453+0000","last_fullsized":"2023-10-09T10:30:21.134453+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":221,"log_dups_size":0,"ondisk_log_size":221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1677,"num_read_kb":1616,"num_write":189,"num_write_kb":178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.135200+0000","last_change":"2023-10-09T10:29:48.543560+0000","last_active":"2023-10-09T10:30:21.135200+0000","last_peered":"2023-10-09T10:30:21.135200+0000","last_clean":"2023-10-09T10:30:21.135200+0000","last_became_active":"2023-10-09T10:29:48.543228+0000","last_became_peered":"2023-10-09T10:29:48.543228+0000","last_unstale":"2023-10-09T10:30:21.135200+0000","last_undegraded":"2023-10-09T10:30:21.135200+0000","last_fullsized":"2023-10-09T10:30:21.135200+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:16.331966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":568,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129413+0000","last_change":"2023-10-09T10:29:48.548903+0000","last_active":"2023-10-09T10:30:21.129413+0000","last_peered":"2023-10-09T10:30:21.129413+0000","last_clean":"2023-10-09T10:30:21.129413+0000","last_became_active":"2023-10-09T10:29:48.548487+0000","last_became_peered":"2023-10-09T10:29:48.548487+0000","last_unstale":"2023-10-09T10:30:21.129413+0000","last_undegraded":"2023-10-09T10:30:21.129413+0000","last_fullsized":"2023-10-09T10:30:21.129413+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.121686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":378,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.146194+0000","last_change":"2023-10-09T10:30:22.146194+0000","last_active":"2023-10-09T10:30:22.146194+0000","last_peered":"2023-10-09T10:30:22.146194+0000","last_clean":"2023-10-09T10:30:22.146194+0000","last_became_active":"2023-10-09T10:30:22.145862+0000","last_became_peered":"2023-10-09T10:30:22.145862+0000","last_unstale":"2023-10-09T10:30:22.146194+0000","last_undegraded":"2023-10-09T10:30:22.146194+0000","last_fullsized":"2023-10-09T10:30:22.146194+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:53.763454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"580'95","reported_seq":56,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132278+0000","last_change":"2023-10-09T10:29:30.705557+0000","last_active":"2023-10-09T10:30:21.132278+0000","last_peered":"2023-10-09T10:30:21.132278+0000","last_clean":"2023-10-09T10:30:21.132278+0000","last_became_active":"2023-10-09T10:29:30.605870+0000","last_became_peered":"2023-10-09T10:29:30.605870+0000","last_unstale":"2023-10-09T10:30:21.132278+0000","last_undegraded":"2023-10-09T10:30:21.132278+0000","last_fullsized":"2023-10-09T10:30:21.132278+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":95,"log_dups_size":0,"ondisk_log_size":95,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:52:08.194203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":494,"num_read_kb":388,"num_write":57,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":401,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.146005+0000","last_change":"2023-10-09T10:30:22.146005+0000","last_active":"2023-10-09T10:30:22.146005+0000","last_peered":"2023-10-09T10:30:22.146005+0000","last_clean":"2023-10-09T10:30:22.146005+0000","last_became_active":"2023-10-09T10:30:22.145607+0000","last_became_peered":"2023-10-09T10:30:22.145607+0000","last_unstale":"2023-10-09T10:30:22.146005+0000","last_undegraded":"2023-10-09T10:30:22.146005+0000","last_fullsized":"2023-10-09T10:30:22.146005+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:45.404318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"580'73","reported_seq":393,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.132910+0000","last_change":"2023-10-09T10:29:25.626159+0000","last_active":"2023-10-09T10:30:21.132910+0000","last_peered":"2023-10-09T10:30:21.132910+0000","last_clean":"2023-10-09T10:30:21.132910+0000","last_became_active":"2023-10-09T10:29:25.569871+0000","last_became_peered":"2023-10-09T10:29:25.569871+0000","last_unstale":"2023-10-09T10:30:21.132910+0000","last_undegraded":"2023-10-09T10:30:21.132910+0000","last_fullsized":"2023-10-09T10:30:21.132910+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:47:41.873906+0000","last_clean_scrub_stamp":"2023-10-09T09:47:41.873906+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":468,"num_read_kb":400,"num_write":65,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":158,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.128887+0000","last_change":"2023-10-09T10:29:30.629496+0000","last_active":"2023-10-09T10:30:21.128887+0000","last_peered":"2023-10-09T10:30:21.128887+0000","last_clean":"2023-10-09T10:30:21.128887+0000","last_became_active":"2023-10-09T10:29:30.610003+0000","last_became_peered":"2023-10-09T10:29:30.610003+0000","last_unstale":"2023-10-09T10:30:21.128887+0000","last_undegraded":"2023-10-09T10:30:21.128887+0000","last_fullsized":"2023-10-09T10:30:21.128887+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:10.345132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":205,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130768+0000","last_change":"2023-10-09T10:30:16.509241+0000","last_active":"2023-10-09T10:30:21.130768+0000","last_peered":"2023-10-09T10:30:21.130768+0000","last_clean":"2023-10-09T10:30:21.130768+0000","last_became_active":"2023-10-09T10:30:16.508882+0000","last_became_peered":"2023-10-09T10:30:16.508882+0000","last_unstale":"2023-10-09T10:30:21.130768+0000","last_undegraded":"2023-10-09T10:30:21.130768+0000","last_fullsized":"2023-10-09T10:30:21.130768+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":124,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129258+0000","last_change":"2023-10-09T10:29:25.562616+0000","last_active":"2023-10-09T10:30:21.129258+0000","last_peered":"2023-10-09T10:30:21.129258+0000","last_clean":"2023-10-09T10:30:21.129258+0000","last_became_active":"2023-10-09T10:29:25.562196+0000","last_became_peered":"2023-10-09T10:29:25.562196+0000","last_unstale":"2023-10-09T10:30:21.129258+0000","last_undegraded":"2023-10-09T10:30:21.129258+0000","last_fullsized":"2023-10-09T10:30:21.129258+0000","mapping_epoch":584,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:29.218982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":32,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130712+0000","last_change":"2023-10-09T10:30:10.108309+0000","last_active":"2023-10-09T10:30:21.130712+0000","last_peered":"2023-10-09T10:30:21.130712+0000","last_clean":"2023-10-09T10:30:21.130712+0000","last_became_active":"2023-10-09T10:30:10.107974+0000","last_became_peered":"2023-10-09T10:30:10.107974+0000","last_unstale":"2023-10-09T10:30:21.130712+0000","last_undegraded":"2023-10-09T10:30:21.130712+0000","last_fullsized":"2023-10-09T10:30:21.130712+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:33.100341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":329,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131829+0000","last_change":"2023-10-09T10:29:51.563375+0000","last_active":"2023-10-09T10:30:21.131829+0000","last_peered":"2023-10-09T10:30:21.131829+0000","last_clean":"2023-10-09T10:30:21.131829+0000","last_became_active":"2023-10-09T10:29:51.562894+0000","last_became_peered":"2023-10-09T10:29:51.562894+0000","last_unstale":"2023-10-09T10:30:21.131829+0000","last_undegraded":"2023-10-09T10:30:21.131829+0000","last_fullsized":"2023-10-09T10:30:21.131829+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":218,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131718+0000","last_change":"2023-10-09T10:29:30.607297+0000","last_active":"2023-10-09T10:30:21.131718+0000","last_peered":"2023-10-09T10:30:21.131718+0000","last_clean":"2023-10-09T10:30:21.131718+0000","last_became_active":"2023-10-09T10:29:30.606901+0000","last_became_peered":"2023-10-09T10:29:30.606901+0000","last_unstale":"2023-10-09T10:30:21.131718+0000","last_undegraded":"2023-10-09T10:30:21.131718+0000","last_fullsized":"2023-10-09T10:30:21.131718+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":330,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.131451+0000","last_change":"2023-10-09T10:24:40.693007+0000","last_active":"2023-10-09T10:30:21.131451+0000","last_peered":"2023-10-09T10:30:21.131451+0000","last_clean":"2023-10-09T10:30:21.131451+0000","last_became_active":"2023-10-09T10:24:40.669853+0000","last_became_peered":"2023-10-09T10:24:40.669853+0000","last_unstale":"2023-10-09T10:30:21.131451+0000","last_undegraded":"2023-10-09T10:30:21.131451+0000","last_fullsized":"2023-10-09T10:30:21.131451+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":251,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130805+0000","last_change":"2023-10-09T10:30:16.509403+0000","last_active":"2023-10-09T10:30:21.130805+0000","last_peered":"2023-10-09T10:30:21.130805+0000","last_clean":"2023-10-09T10:30:21.130805+0000","last_became_active":"2023-10-09T10:30:16.509004+0000","last_became_peered":"2023-10-09T10:30:16.509004+0000","last_unstale":"2023-10-09T10:30:21.130805+0000","last_undegraded":"2023-10-09T10:30:21.130805+0000","last_fullsized":"2023-10-09T10:30:21.130805+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":113,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.145022+0000","last_change":"2023-10-09T10:30:22.145022+0000","last_active":"2023-10-09T10:30:22.145022+0000","last_peered":"2023-10-09T10:30:22.145022+0000","last_clean":"2023-10-09T10:30:22.145022+0000","last_became_active":"2023-10-09T10:30:22.144678+0000","last_became_peered":"2023-10-09T10:30:22.144678+0000","last_unstale":"2023-10-09T10:30:22.145022+0000","last_undegraded":"2023-10-09T10:30:22.145022+0000","last_fullsized":"2023-10-09T10:30:22.145022+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:18.217178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":571,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T10:30:22.141235+0000","last_change":"2023-10-09T10:30:22.141235+0000","last_active":"2023-10-09T10:30:22.141235+0000","last_peered":"2023-10-09T10:30:22.141235+0000","last_clean":"2023-10-09T10:30:22.141235+0000","last_became_active":"2023-10-09T10:30:22.140931+0000","last_became_peered":"2023-10-09T10:30:22.140931+0000","last_unstale":"2023-10-09T10:30:22.141235+0000","last_undegraded":"2023-10-09T10:30:22.141235+0000","last_fullsized":"2023-10-09T10:30:22.141235+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:52:37.696238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":166,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.129000+0000","last_change":"2023-10-09T10:29:48.556175+0000","last_active":"2023-10-09T10:30:21.129000+0000","last_peered":"2023-10-09T10:30:21.129000+0000","last_clean":"2023-10-09T10:30:21.129000+0000","last_became_active":"2023-10-09T10:29:48.545116+0000","last_became_peered":"2023-10-09T10:29:48.545116+0000","last_unstale":"2023-10-09T10:30:21.129000+0000","last_undegraded":"2023-10-09T10:30:21.129000+0000","last_fullsized":"2023-10-09T10:30:21.129000+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:24.497421+0000","last_clean_scrub_stamp":"2023-10-09T09:48:24.497421+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:36:43.096055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8310,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":245,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-10-09T10:30:21.130589+0000","last_change":"2023-10-09T10:29:30.608959+0000","last_active":"2023-10-09T10:30:21.130589+0000","last_peered":"2023-10-09T10:30:21.130589+0000","last_clean":"2023-10-09T10:30:21.130589+0000","last_became_active":"2023-10-09T10:29:30.608703+0000","last_became_peered":"2023-10-09T10:29:30.608703+0000","last_unstale":"2023-10-09T10:30:21.130589+0000","last_undegraded":"2023-10-09T10:30:21.130589+0000","last_fullsized":"2023-10-09T10:30:21.130589+0000","mapping_epoch":589,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T09:48:25.498867+0000","last_clean_scrub_stamp":"2023-10-09T09:48:25.498867+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":37596,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18611,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5955,"num_read_kb":13166,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430734,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":922760,"kb_used_data":10304,"kb_used_omap":0,"kb_used_meta":912448,"kb_avail":92822392,"statfs":{"total":95995035648,"available":95050129408,"internally_reserved":0,"allocated":10551296,"data_stored":9500114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":934346752},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.5649999999999999,"15min":1.454},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.216},"max":{"1min":0.91400000000000003,"5min":34.241,"15min":34.241},"last":0.439},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.883,"15min":1.4970000000000001},"min":{"1min":0.315,"5min":0.247,"15min":0.22},"max":{"1min":0.86199999999999999,"5min":34.030000000000001,"15min":34.030000000000001},"last":0.66400000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:30:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":1.131,"15min":1.794},"min":{"1min":0.33500000000000002,"5min":0.27500000000000002,"15min":0.23799999999999999},"max":{"1min":0.83899999999999997,"5min":5.8460000000000001,"15min":129.53700000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.0129999999999999,"15min":1.849},"min":{"1min":0.32800000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":0.88800000000000001,"5min":6.9249999999999998,"15min":128.85599999999999},"last":0.68999999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.474,"15min":1.6919999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":0.86699999999999999,"5min":6.7270000000000003,"15min":42.725999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.391,"15min":1.6299999999999999},"min":{"1min":0.42399999999999999,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":0.998,"5min":6.9000000000000004,"15min":42.752000000000002},"last":0.60999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.7689999999999999,"15min":1.7030000000000001},"min":{"1min":0.41499999999999998,"5min":0.33900000000000002,"15min":0.307},"max":{"1min":0.91700000000000004,"5min":8.5299999999999994,"15min":12.436},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.6870000000000001,"15min":1.708},"min":{"1min":0.374,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":0.76800000000000002,"5min":10.691000000000001,"15min":12.161},"last":0.58099999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:30:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.67900000000000005,"15min":1.222},"min":{"1min":0.34499999999999997,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":0.72899999999999998,"5min":4.2039999999999997,"15min":17.995000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.98499999999999999,"15min":1.3879999999999999},"min":{"1min":0.39600000000000002,"5min":0.27700000000000002,"15min":0.25700000000000001},"max":{"1min":0.94099999999999995,"5min":7.0609999999999999,"15min":18.198},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.86299999999999999,"15min":1.3899999999999999},"min":{"1min":0.42299999999999999,"5min":0.308,"15min":0.23999999999999999},"max":{"1min":0.84799999999999998,"5min":5.0720000000000001,"15min":21.712},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.69299999999999995,"5min":1.0860000000000001,"15min":1.603},"min":{"1min":0.47699999999999998,"5min":0.307,"15min":0.26800000000000002},"max":{"1min":0.84199999999999997,"5min":8.266,"15min":22.824999999999999},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":1.1579999999999999,"15min":1.4399999999999999},"min":{"1min":0.44600000000000001,"5min":0.33300000000000002,"15min":0.28399999999999997},"max":{"1min":0.98099999999999998,"5min":13.496,"15min":14.407},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.93899999999999995,"15min":1.117},"min":{"1min":0.42999999999999999,"5min":0.311,"15min":0.307},"max":{"1min":0.85399999999999998,"5min":5.7510000000000003,"15min":7.7670000000000003},"last":0.53000000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626962,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515328,"kb_used_data":6200,"kb_used_omap":0,"kb_used_meta":1509120,"kb_avail":92229824,"statfs":{"total":95995035648,"available":94443339776,"internally_reserved":0,"allocated":6348800,"data_stored":5306783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1545338880},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:23 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":2.0219999999999998,"15min":1.627},"min":{"1min":0.318,"5min":0.219,"15min":0.216},"max":{"1min":0.83499999999999996,"5min":47.401000000000003,"15min":47.401000000000003},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.55100000000000005,"5min":2.0230000000000001,"15min":1.9610000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":0.82399999999999995,"5min":47.311999999999998,"15min":47.311999999999998},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.258,"15min":1.8839999999999999},"min":{"1min":0.27800000000000002,"5min":0.26000000000000001,"15min":0.23799999999999999},"max":{"1min":0.91800000000000004,"5min":7.415,"15min":155.01400000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.298,"15min":2.0800000000000001},"min":{"1min":0.35199999999999998,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":0.80200000000000005,"5min":6.3929999999999998,"15min":154.71100000000001},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:30:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.587,"15min":1.7250000000000001},"min":{"1min":0.372,"5min":0.309,"15min":0.24099999999999999},"max":{"1min":0.90300000000000002,"5min":10.907,"15min":29.367999999999999},"last":0.433},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.298,"15min":1.407},"min":{"1min":0.42899999999999999,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":0.86799999999999999,"5min":7.4480000000000004,"15min":8.4480000000000004},"last":0.60099999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.798,"15min":1.556},"min":{"1min":0.44700000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.94899999999999995,"5min":43.393000000000001,"15min":43.393000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.8899999999999999,"15min":1.7529999999999999},"min":{"1min":0.40300000000000002,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":0.88900000000000001,"5min":43.466000000000001,"15min":43.466000000000001},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":2.722,"15min":2.391},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.85699999999999998,"5min":177.929,"15min":177.929},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":2.4199999999999999,"15min":2.2120000000000002},"min":{"1min":0.34300000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.77900000000000003,"5min":178.16499999999999,"15min":178.16499999999999},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.4279999999999999,"15min":2.036},"min":{"1min":0.35499999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.70099999999999996,"5min":11.788,"15min":121.98999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.079,"15min":1.5269999999999999},"min":{"1min":0.314,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.873,"5min":7.0060000000000002,"15min":122.346},"last":0.56000000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.1279999999999999,"15min":1.8939999999999999},"min":{"1min":0.34899999999999998,"5min":0.32600000000000001,"15min":0.29199999999999998},"max":{"1min":0.97099999999999997,"5min":12.612,"15min":130.66999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.212,"15min":1.9510000000000001},"min":{"1min":0.40600000000000003,"5min":0.374,"15min":0.34100000000000003},"max":{"1min":1.097,"5min":12.659000000000001,"15min":130.702},"last":0.624}]}]},{"osd":1,"up_from":84,"seq":360777253275,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836576,"kb_used_data":6872,"kb_used_omap":9,"kb_used_meta":829686,"kb_avail":92908576,"statfs":{"total":95995035648,"available":95138381824,"internally_reserved":0,"allocated":7036928,"data_stored":4847378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":849599465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":2.6520000000000001,"15min":3.0539999999999998},"min":{"1min":0.247,"5min":0.218,"15min":0.188},"max":{"1min":0.87,"5min":64.558000000000007,"15min":270.48000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":2.5169999999999999,"15min":3.1579999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.216},"max":{"1min":1.046,"5min":61.692,"15min":272.69299999999998},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:30:21 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.2290000000000001,"15min":1.631},"min":{"1min":0.27500000000000002,"5min":0.219,"15min":0.19600000000000001},"max":{"1min":0.88500000000000001,"5min":7.5780000000000003,"15min":20.306000000000001},"last":0.373},{"interface":"front","average":{"1min":0.44900000000000001,"5min":1.23,"15min":1.238},"min":{"1min":0.309,"5min":0.29099999999999998,"15min":0.23100000000000001},"max":{"1min":0.90400000000000003,"5min":7.8860000000000001,"15min":9.4830000000000005},"last":0.309}]},{"osd":3,"last update":"Mon Oct 9 10:30:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":1.554,"15min":1.327},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":0.95399999999999996,"5min":13.223000000000001,"15min":13.223000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":1.673,"15min":1.5429999999999999},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0149999999999999,"5min":20.091000000000001,"15min":20.091000000000001},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.05,"15min":1.196},"min":{"1min":0.42399999999999999,"5min":0.33100000000000002,"15min":0.30199999999999999},"max":{"1min":0.85199999999999998,"5min":7.8200000000000003,"15min":10.797000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.603,"15min":1.8029999999999999},"min":{"1min":0.48699999999999999,"5min":0.34000000000000002,"15min":0.314},"max":{"1min":0.92500000000000004,"5min":20.015999999999998,"15min":20.015999999999998},"last":0.61799999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.718,"15min":2.2679999999999998},"min":{"1min":0.40600000000000003,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":0.96799999999999997,"5min":20.120999999999999,"15min":92.177999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.629,"5min":1.4510000000000001,"15min":2.1309999999999998},"min":{"1min":0.41299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.874,"5min":10.675000000000001,"15min":91.869},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 10:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":2.0089999999999999,"15min":1.704},"min":{"1min":0.48999999999999999,"5min":0.371,"15min":0.33700000000000002},"max":{"1min":3.9670000000000001,"5min":81.216999999999999,"15min":81.216999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.82199999999999995,"5min":2.3879999999999999,"15min":2.0289999999999999},"min":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":4.0179999999999998,"5min":81.231999999999999,"15min":81.231999999999999},"last":0.69599999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.4830000000000001,"15min":1.8029999999999999},"min":{"1min":0.503,"5min":0.41799999999999998,"15min":0.32300000000000001},"max":{"1min":0.89800000000000002,"5min":10.621,"15min":11.186},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.258,"15min":1.4850000000000001},"min":{"1min":0.46300000000000002,"5min":0.40000000000000002,"15min":0.371},"max":{"1min":0.98699999999999999,"5min":7.9130000000000003,"15min":10.627000000000001},"last":0.57999999999999996}]}]},{"osd":0,"up_from":79,"seq":339302416800,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1050644,"kb_used_data":7692,"kb_used_omap":28,"kb_used_meta":1042915,"kb_avail":92694508,"statfs":{"total":95995035648,"available":94919176192,"internally_reserved":0,"allocated":7876608,"data_stored":5460138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1067945808},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:29:53 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.538,"15min":1.544},"min":{"1min":0.247,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":0.48899999999999999,"5min":17.161999999999999,"15min":54.518000000000001},"last":0.443},{"interface":"front","average":{"1min":0.40300000000000002,"5min":1.766,"15min":1.6539999999999999},"min":{"1min":0.247,"5min":0.218,"15min":0.192},"max":{"1min":0.64900000000000002,"5min":16.818000000000001,"15min":56.950000000000003},"last":0.35499999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":1.7729999999999999,"15min":1.736},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.22800000000000001},"max":{"1min":0.57599999999999996,"5min":17.093,"15min":53.988999999999997},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.443,"5min":1.8,"15min":1.7969999999999999},"min":{"1min":0.317,"5min":0.29999999999999999,"15min":0.22600000000000001},"max":{"1min":0.61899999999999999,"5min":17.053999999999998,"15min":54.119},"last":0.46899999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:30:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":2.1789999999999998,"15min":1.746},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":0.78400000000000003,"5min":29.835999999999999,"15min":56.984999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":2.3109999999999999,"15min":1.857},"min":{"1min":0.311,"5min":0.249,"15min":0.249},"max":{"1min":0.61099999999999999,"5min":29.776,"15min":56.475999999999999},"last":0.49099999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":1.6080000000000001,"15min":1.7709999999999999},"min":{"1min":0.46600000000000003,"5min":0.29199999999999998,"15min":0.251},"max":{"1min":0.90000000000000002,"5min":11.653,"15min":57.719000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.63,"5min":1.8799999999999999,"15min":1.958},"min":{"1min":0.45300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.90900000000000003,"5min":11.896000000000001,"15min":56.378},"last":0.63300000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":3.1560000000000001,"15min":2.3769999999999998},"min":{"1min":0.379,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":0.85599999999999998,"5min":90.412000000000006,"15min":90.412000000000006},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":2.4700000000000002,"15min":2.0049999999999999},"min":{"1min":0.432,"5min":0.34399999999999997,"15min":0.30199999999999999},"max":{"1min":0.82799999999999996,"5min":90.393000000000001,"15min":90.393000000000001},"last":0.59599999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.5189999999999999,"15min":1.778},"min":{"1min":0.41299999999999998,"5min":0.38400000000000001,"15min":0.34000000000000002},"max":{"1min":0.92700000000000005,"5min":9.7840000000000007,"15min":54.482999999999997},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.8049999999999999,"15min":1.9910000000000001},"min":{"1min":0.46800000000000003,"5min":0.37,"15min":0.34799999999999998},"max":{"1min":0.95399999999999996,"5min":11.702999999999999,"15min":57.753},"last":0.69899999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.9410000000000001,"15min":2.0110000000000001},"min":{"1min":0.46700000000000003,"5min":0.38500000000000001,"15min":0.34999999999999998},"max":{"1min":0.94199999999999995,"5min":12.029999999999999,"15min":56.411999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.66900000000000004,"5min":1.7929999999999999,"15min":1.917},"min":{"1min":0.46600000000000003,"5min":0.44800000000000001,"15min":0.38300000000000001},"max":{"1min":0.91900000000000004,"5min":17.129999999999999,"15min":54.451999999999998},"last":0.68799999999999994}]}]},{"osd":2,"up_from":217,"seq":932007903535,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":895732,"kb_used_data":10988,"kb_used_omap":83,"kb_used_meta":884652,"kb_avail":92849420,"statfs":{"total":95995035648,"available":95077806080,"internally_reserved":0,"allocated":11251712,"data_stored":9045627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85521,"internal_metadata":905884143},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":1.9850000000000001,"15min":1.659},"min":{"1min":0.215,"5min":0.192,"15min":0.19},"max":{"1min":0.44400000000000001,"5min":31.641999999999999,"15min":48.990000000000002},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":1.5940000000000001,"15min":1.728},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.20899999999999999},"max":{"1min":0.65900000000000003,"5min":13.773,"15min":49.412999999999997},"last":0.35999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":2.1560000000000001,"15min":1.5900000000000001},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.20300000000000001},"max":{"1min":0.51400000000000001,"5min":27.097999999999999,"15min":34.893999999999998},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":1.246,"15min":1.4390000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25800000000000001},"max":{"1min":0.505,"5min":13.356999999999999,"15min":34.962000000000003},"last":0.39000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:30:01 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7090000000000001,"15min":1.4339999999999999},"min":{"1min":0.26300000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.47299999999999998,"5min":15.115,"15min":35.043999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.38100000000000001,"5min":1.673,"15min":1.4019999999999999},"min":{"1min":0.29299999999999998,"5min":0.28899999999999998,"15min":0.23699999999999999},"max":{"1min":0.63400000000000001,"5min":11.206,"15min":50.406999999999996},"last":0.38200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.167,"15min":1.3420000000000001},"min":{"1min":0.498,"5min":0.31900000000000001,"15min":0.29799999999999999},"max":{"1min":0.67500000000000004,"5min":8.1859999999999999,"15min":47.503999999999998},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.7090000000000001,"15min":1.653},"min":{"1min":0.46400000000000002,"5min":0.34000000000000002,"15min":0.29299999999999998},"max":{"1min":0.77800000000000002,"5min":11.298,"15min":47.704000000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.494,"15min":1.484},"min":{"1min":0.443,"5min":0.34300000000000003,"15min":0.308},"max":{"1min":0.747,"5min":11.266999999999999,"15min":24.23},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":2.1190000000000002,"15min":1.669},"min":{"1min":0.38600000000000001,"5min":0.34899999999999998,"15min":0.31},"max":{"1min":0.86599999999999999,"5min":26.997,"15min":26.997},"last":0.78700000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.206,"15min":1.54},"min":{"1min":0.46999999999999997,"5min":0.35399999999999998,"15min":0.35299999999999998},"max":{"1min":0.76100000000000001,"5min":7.9390000000000001,"15min":12.411},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.71,"15min":1.6180000000000001},"min":{"1min":0.41999999999999998,"5min":0.39300000000000002,"15min":0.38300000000000001},"max":{"1min":0.78700000000000003,"5min":13.975,"15min":24.379000000000001},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:30:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.363,"15min":1.7929999999999999},"min":{"1min":0.434,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":0.80200000000000005,"5min":9.5199999999999996,"15min":35.008000000000003},"last":0.752},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.6899999999999999,"15min":1.605},"min":{"1min":0.49299999999999999,"5min":0.40799999999999997,"15min":0.35899999999999999},"max":{"1min":0.83699999999999997,"5min":15.548,"15min":24.323},"last":0.70299999999999996}]}]},{"osd":3,"up_from":352,"seq":1511828488375,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":607032,"kb_used_data":7472,"kb_used_omap":15,"kb_used_meta":599536,"kb_avail":93138120,"statfs":{"total":95995035648,"available":95373434880,"internally_reserved":0,"allocated":7651328,"data_stored":4913179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15930,"internal_metadata":613925318},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":3.2589999999999999,"15min":2.0270000000000001},"min":{"1min":0.25,"5min":0.23899999999999999,"15min":0.19900000000000001},"max":{"1min":0.65400000000000003,"5min":70.004000000000005,"15min":70.004000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":3.3639999999999999,"15min":2.1309999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":0.62,"5min":66.802000000000007,"15min":66.802000000000007},"last":0.46300000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":1.2150000000000001,"15min":1.8420000000000001},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":0.63700000000000001,"5min":11.785,"15min":105.401},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.673,"15min":1.9930000000000001},"min":{"1min":0.28000000000000003,"5min":0.23400000000000001,"15min":0.23300000000000001},"max":{"1min":0.81999999999999995,"5min":10.422000000000001,"15min":104.914},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":2.036,"15min":1.8320000000000001},"min":{"1min":0.27600000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.59899999999999998,"5min":13.497999999999999,"15min":13.497999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.502,"5min":2.0830000000000002,"15min":1.587},"min":{"1min":0.34000000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.78600000000000003,"5min":16.475000000000001,"15min":25.338999999999999},"last":0.443}]},{"osd":4,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":3.6080000000000001,"15min":2.306},"min":{"1min":0.42399999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.94399999999999995,"5min":178.72200000000001,"15min":178.72200000000001},"last":0.433},{"interface":"front","average":{"1min":0.61199999999999999,"5min":4.2270000000000003,"15min":2.7999999999999998},"min":{"1min":0.45300000000000001,"5min":0.314,"15min":0.29599999999999999},"max":{"1min":0.85299999999999998,"5min":179.08500000000001,"15min":179.08500000000001},"last":0.49399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.4339999999999999,"15min":1.4379999999999999},"min":{"1min":0.441,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.754,"5min":11.363,"15min":25.251999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.67300000000000004,"5min":1.7549999999999999,"15min":1.776},"min":{"1min":0.45300000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.042,"5min":10.388,"15min":13.394},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.891,"15min":1.819},"min":{"1min":0.47899999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.74099999999999999,"5min":13.545999999999999,"15min":19.114000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.496,"15min":1.573},"min":{"1min":0.499,"5min":0.35199999999999998,"15min":0.33900000000000002},"max":{"1min":0.91500000000000004,"5min":11.32,"15min":25.306999999999999},"last":0.57899999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.6950000000000001,"15min":1.7390000000000001},"min":{"1min":0.54400000000000004,"5min":0.41299999999999998,"15min":0.38400000000000001},"max":{"1min":0.999,"5min":10.345000000000001,"15min":10.459},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.8819999999999999,"15min":1.875},"min":{"1min":0.46500000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.96899999999999997,"5min":12.4,"15min":19.280000000000001},"last":0.68100000000000005}]}]},{"osd":4,"up_from":266,"seq":1142461301004,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":934656,"kb_used_data":5624,"kb_used_omap":35,"kb_used_meta":928988,"kb_avail":92810496,"statfs":{"total":95995035648,"available":95037947904,"internally_reserved":0,"allocated":5758976,"data_stored":4715372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36626,"internal_metadata":951283950},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":2.7000000000000002,"15min":2.54},"min":{"1min":0.25900000000000001,"5min":0.22700000000000001,"15min":0.20599999999999999},"max":{"1min":1.9670000000000001,"5min":109.193,"15min":109.193},"last":0.44},{"interface":"front","average":{"1min":0.66200000000000003,"5min":3.0779999999999998,"15min":3.0779999999999998},"min":{"1min":0.318,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":1.982,"5min":110.438,"15min":110.438},"last":0.502}]},{"osd":1,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.458,"15min":1.95},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.24299999999999999},"max":{"1min":3.0310000000000001,"5min":12.209,"15min":71.457999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.679,"15min":2.008},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.248},"max":{"1min":1.9950000000000001,"5min":9.2040000000000006,"15min":71.658000000000001},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.4990000000000001,"15min":1.6419999999999999},"min":{"1min":0.40699999999999997,"5min":0.33200000000000002,"15min":0.25600000000000001},"max":{"1min":2.141,"5min":9.9039999999999999,"15min":15.944000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":1.3700000000000001,"15min":1.5089999999999999},"min":{"1min":0.45300000000000001,"5min":0.28000000000000003,"15min":0.255},"max":{"1min":2.2130000000000001,"5min":11.102,"15min":14.473000000000001},"last":0.35399999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:30:03 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.9359999999999999,"15min":1.9730000000000001},"min":{"1min":0.38,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":3.109,"5min":11.347,"15min":12.074},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.9950000000000001,"15min":1.9319999999999999},"min":{"1min":0.39000000000000001,"5min":0.32000000000000001,"15min":0.27100000000000002},"max":{"1min":3.214,"5min":11.369999999999999,"15min":16.067},"last":0.48299999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.99299999999999999,"15min":1.621},"min":{"1min":0.29199999999999998,"5min":0.26300000000000001,"15min":0.252},"max":{"1min":0.68899999999999995,"5min":10.007,"15min":16.094000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.79600000000000004,"15min":1.4199999999999999},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":0.66100000000000003,"5min":5.9509999999999996,"15min":49.915999999999997},"last":0.40300000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":1.139,"15min":1.3280000000000001},"min":{"1min":0.33700000000000002,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":0.69899999999999995,"5min":9.5389999999999997,"15min":14.499000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":1.29,"15min":1.7410000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28299999999999997},"max":{"1min":0.77200000000000002,"5min":12.242000000000001,"15min":18.952999999999999},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.3360000000000001,"15min":1.758},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.307},"max":{"1min":0.75900000000000001,"5min":12.391999999999999,"15min":17.369},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.1299999999999999,"15min":1.663},"min":{"1min":0.371,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.81399999999999995,"5min":9.5129999999999999,"15min":51.026000000000003},"last":0.51600000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823189,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518708,"kb_used_data":9708,"kb_used_omap":0,"kb_used_meta":1508992,"kb_avail":92226444,"statfs":{"total":95995035648,"available":94439878656,"internally_reserved":0,"allocated":9940992,"data_stored":8905439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1545207808},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:30:05 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":2.3500000000000001,"15min":1.992},"min":{"1min":0.29699999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.755,"5min":19.102,"15min":22.469999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.5349999999999999,"15min":1.6379999999999999},"min":{"1min":0.32500000000000001,"5min":0.24299999999999999,"15min":0.23200000000000001},"max":{"1min":0.83299999999999996,"5min":17.887,"15min":22.437000000000001},"last":0.50800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:29:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.9139999999999999,"15min":2},"min":{"1min":0.40000000000000002,"5min":0.27300000000000002,"15min":0.246},"max":{"1min":1.0209999999999999,"5min":16.562999999999999,"15min":29.370000000000001},"last":0.37},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.236,"15min":1.607},"min":{"1min":0.34599999999999997,"5min":0.27300000000000002,"15min":0.23499999999999999},"max":{"1min":0.71999999999999997,"5min":8.9529999999999994,"15min":29.065000000000001},"last":0.56299999999999994}]},{"osd":2,"last update":"Mon Oct 9 10:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.579,"15min":1.8360000000000001},"min":{"1min":0.41199999999999998,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":0.82699999999999996,"5min":12.247999999999999,"15min":16.629999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.272,"15min":1.5960000000000001},"min":{"1min":0.376,"5min":0.32600000000000001,"15min":0.28399999999999997},"max":{"1min":0.90000000000000002,"5min":7.1479999999999997,"15min":13.095000000000001},"last":0.64000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:30:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.954,"15min":1.9019999999999999},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":0.84599999999999997,"5min":13.614000000000001,"15min":13.614000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.7190000000000001,"15min":1.7849999999999999},"min":{"1min":0.44900000000000001,"5min":0.38100000000000001,"15min":0.315},"max":{"1min":0.94899999999999995,"5min":11.124000000000001,"15min":13.624000000000001},"last":0.70399999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.4730000000000001,"15min":1.5720000000000001},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25800000000000001},"max":{"1min":0.71499999999999997,"5min":13.523999999999999,"15min":13.523999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.0489999999999999,"15min":1.446},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.23599999999999999},"max":{"1min":0.79800000000000004,"5min":8.8620000000000001,"15min":11.956},"last":0.67400000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:30:05 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.6519999999999999,"15min":1.776},"min":{"1min":0.315,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":0.67400000000000004,"5min":13.567,"15min":13.567},"last":0.502},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.4930000000000001,"15min":1.649},"min":{"1min":0.35399999999999998,"5min":0.29599999999999999,"15min":0.26700000000000002},"max":{"1min":0.91700000000000004,"5min":16.497,"15min":16.497},"last":0.53100000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:30:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.1259999999999999,"15min":1.6679999999999999},"min":{"1min":0.45700000000000002,"5min":0.36899999999999999,"15min":0.311},"max":{"1min":0.81699999999999995,"5min":7.3490000000000002,"15min":21.109000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.70899999999999996,"15min":1.129},"min":{"1min":0.38700000000000001,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":0.82699999999999996,"5min":5.5519999999999996,"15min":11.994},"last":0.73499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6346,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8354,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3911,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:30:49.829 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T10:30:49.829 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T10:30:49.903 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=e6fe39d8aab8493fa58e2b684bd53720 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T10:30:49.906 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T10:30:49.920 INFO:tasks.cephadm:Teardown begin 2023-10-09T10:30:49.921 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:30:49.942 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:49 smithi110 ceph-mon[136032]: pgmap v1710: 73 pgs: 73 active+clean; 4.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T10:30:49.972 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:30:50.066 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T10:30:50.066 DEBUG:teuthology.orchestra.run.smithi012:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T10:30:50.084 DEBUG:teuthology.orchestra.run.smithi110:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T10:30:50.102 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T10:30:50.105 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T10:30:50.105 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T10:30:50.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:50 smithi012 systemd[1]: Stopping Ceph mon.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:30:50.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:30:50.496+0000 7f85af1de700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:30:50.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Oct 09 10:30:50 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-a[147902]: 2023-10-09T10:30:50.496+0000 7f85af1de700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T10:30:51.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T10:30:51.179 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:30:51.179 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T10:30:51.179 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T10:30:51.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T10:30:51.573 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:51 smithi012 systemd[1]: Stopping Ceph mon.c for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:30:51.874 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:10:30:51] ENGINE Bus STOPPING 2023-10-09T10:30:51.874 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:10:30:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:30:51.875 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:10:30:51] ENGINE Bus STOPPED 2023-10-09T10:30:51.875 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:10:30:51] ENGINE Bus STARTING 2023-10-09T10:30:51.875 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:10:30:51] ENGINE Serving on http://:::9283 2023-10-09T10:30:51.875 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: [09/Oct/2023:10:30:51] ENGINE Bus STARTED 2023-10-09T10:30:51.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[152239]: 2023-10-09T10:30:51.571+0000 7f549f339700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:30:51.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:51 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c[152239]: 2023-10-09T10:30:51.571+0000 7f549f339700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T10:30:52.177 INFO:journalctl@ceph.mon.c.smithi012.stdout:Oct 09 10:30:51 smithi012 bash[462399]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T10:30:52.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T10:30:52.283 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:30:52.283 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T10:30:52.283 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T10:30:52.284 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T10:30:52.646 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:52 smithi110 systemd[1]: Stopping Ceph mon.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:30:52.940 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[136008]: 2023-10-09T10:30:52.644+0000 7ff1160ce700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:30:52.940 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b[136008]: 2023-10-09T10:30:52.644+0000 7ff1160ce700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T10:30:53.192 INFO:journalctl@ceph.mon.b.smithi110.stdout:Oct 09 10:30:53 smithi110 bash[284197]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T10:30:53.383 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T10:30:53.423 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:30:53.423 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T10:30:53.423 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T10:30:53.424 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T10:30:53.778 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:53 smithi012 systemd[1]: Stopping Ceph mgr.y for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:30:54.177 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:53 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y[142621]: 2023-10-09T10:30:53.776+0000 7f2cf0160700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:30:54.728 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Oct 09 10:30:54 smithi012 bash[462557]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T10:30:54.734 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T10:30:54.773 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:30:54.773 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T10:30:54.774 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T10:30:54.774 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T10:30:55.193 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 10:30:54 smithi110 systemd[1]: Stopping Ceph mgr.x for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:30:55.481 INFO:journalctl@ceph.mgr.x.smithi110.stdout:Oct 09 10:30:55 smithi110 bash[284339]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T10:30:55.671 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T10:30:55.708 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:30:55.708 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T10:30:55.708 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T10:30:55.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T10:30:56.072 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 10:30:55 smithi012 systemd[1]: Stopping Ceph osd.0 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:30:56.429 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 10:30:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[160095]: 2023-10-09T10:30:56.070+0000 7fe454281700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:30:56.430 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 10:30:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[160095]: 2023-10-09T10:30:56.070+0000 7fe454281700 -1 osd.0 618 *** Got signal Terminated *** 2023-10-09T10:30:56.430 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 10:30:56 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0[160095]: 2023-10-09T10:30:56.070+0000 7fe454281700 -1 osd.0 618 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:31:02.053 INFO:journalctl@ceph.osd.0.smithi012.stdout:Oct 09 10:31:01 smithi012 bash[462699]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T10:31:03.732 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T10:31:03.774 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:03.775 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T10:31:03.775 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T10:31:03.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T10:31:04.177 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 10:31:03 smithi012 systemd[1]: Stopping Ceph osd.1 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:04.677 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 10:31:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[163414]: 2023-10-09T10:31:04.176+0000 7fa377907700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:31:04.678 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 10:31:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[163414]: 2023-10-09T10:31:04.176+0000 7fa377907700 -1 osd.1 618 *** Got signal Terminated *** 2023-10-09T10:31:04.678 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 10:31:04 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1[163414]: 2023-10-09T10:31:04.176+0000 7fa377907700 -1 osd.1 618 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:31:09.928 INFO:journalctl@ceph.osd.1.smithi012.stdout:Oct 09 10:31:09 smithi012 bash[462962]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T10:31:11.334 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T10:31:11.377 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:11.377 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T10:31:11.377 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T10:31:11.377 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T10:31:11.677 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:31:11 smithi012 systemd[1]: Stopping Ceph osd.2 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:12.177 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:31:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[238878]: 2023-10-09T10:31:11.766+0000 7f8f29c27700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:31:12.177 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:31:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[238878]: 2023-10-09T10:31:11.766+0000 7f8f29c27700 -1 osd.2 618 *** Got signal Terminated *** 2023-10-09T10:31:12.177 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:31:11 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2[238878]: 2023-10-09T10:31:11.766+0000 7f8f29c27700 -1 osd.2 618 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:31:18.177 INFO:journalctl@ceph.osd.2.smithi012.stdout:Oct 09 10:31:17 smithi012 bash[463223]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T10:31:19.021 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T10:31:19.064 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:19.064 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T10:31:19.064 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T10:31:19.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T10:31:19.428 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:31:19 smithi012 systemd[1]: Stopping Ceph osd.3 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:19.927 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:31:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[330575]: 2023-10-09T10:31:19.465+0000 7f0efef11700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:31:19.927 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:31:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[330575]: 2023-10-09T10:31:19.465+0000 7f0efef11700 -1 osd.3 618 *** Got signal Terminated *** 2023-10-09T10:31:19.927 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:31:19 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3[330575]: 2023-10-09T10:31:19.465+0000 7f0efef11700 -1 osd.3 618 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:31:22.194 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:21 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:21.870+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:22.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:22.317+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:23.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:22 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:22.884+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:23.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:23.313+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:23.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:23.251+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:24.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:23 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:23.849+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:24.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:24.270+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:24.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:24.321+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:25.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:24 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:24.848+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:25.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:25.312+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:25.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:25.371+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:26.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:25.887+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:26.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:25.919Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.12:8765: connect: connection refused" 2023-10-09T10:31:26.193 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:25.919Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.12:8765: connect: connection refused" 2023-10-09T10:31:26.194 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:25.922Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.12:8765: connect: connection refused" 2023-10-09T10:31:26.194 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:25 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:25.922Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.12:8765: connect: connection refused" 2023-10-09T10:31:26.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Oct 09 10:31:26 smithi012 bash[463486]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T10:31:26.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:26.344+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:26.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:26.415+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:27.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:26.848+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:27.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:26 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:26.902+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:27.591 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T10:31:27.633 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:27.633 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T10:31:27.633 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T10:31:27.634 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T10:31:27.665 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:27.360+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:27.666 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:27.396+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:27.927 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:27 smithi110 systemd[1]: Stopping Ceph osd.4 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:27.928 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:27.894+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:28.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:27.926+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:28.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:27.992+0000 7feb8fc34700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:31:28.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:27.992+0000 7feb8fc34700 -1 osd.4 618 *** Got signal Terminated *** 2023-10-09T10:31:28.194 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:27 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:27.992+0000 7feb8fc34700 -1 osd.4 618 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:31:28.692 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:28.338+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:28.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:28.376+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:29.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:28.929+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:29.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:28 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:28.950+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:29.696 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:29.342+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:29.697 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:29.382+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:30.318 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:29.951+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:30.318 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:29.962+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:30.318 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:29 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:29.962+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:30.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:30.317+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:30.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:30.417+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:31.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:30 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:30.918+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:31.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:31.006+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:31.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:31.006+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:31.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:31.301+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:31.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:31.431+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:32.193 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:31.883+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:32.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:31.959+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:32.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:31 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:31.959+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:32.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:32.275+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:32.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:32.406+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:32.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:32.406+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:33.192 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4[188025]: debug 2023-10-09T10:31:32.889+0000 7feb8abad700 -1 osd.4 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.914047+0000 front 2023-10-09T10:30:58.913836+0000 (oldest deadline 2023-10-09T10:31:21.813940+0000) 2023-10-09T10:31:33.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:32.914+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:33.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:32 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:32.914+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:33.693 INFO:journalctl@ceph.osd.4.smithi110.stdout:Oct 09 10:31:33 smithi110 bash[284494]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T10:31:33.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:33.276+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:33.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:33.276+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.462144+0000 front 2023-10-09T10:31:07.462241+0000 (oldest deadline 2023-10-09T10:31:32.762041+0000) 2023-10-09T10:31:33.694 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:33.449+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:33.694 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:33.449+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:34.179 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:33.923+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:34.180 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:33 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:33.924+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:34.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:34.274+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:34.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:34.274+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.462144+0000 front 2023-10-09T10:31:07.462241+0000 (oldest deadline 2023-10-09T10:31:32.762041+0000) 2023-10-09T10:31:34.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:34.415+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:34.694 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:34.415+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:35.062 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:34.940+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:35.062 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:34 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:34.940+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:35.070 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T10:31:35.152 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:35.153 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T10:31:35.153 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T10:31:35.153 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T10:31:35.438 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:35 smithi110 systemd[1]: Stopping Ceph osd.5 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:35.439 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:35.274+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:35.439 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:35.274+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.462144+0000 front 2023-10-09T10:31:07.462241+0000 (oldest deadline 2023-10-09T10:31:32.762041+0000) 2023-10-09T10:31:35.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:35.438+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:35.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:35.438+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:35.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:35.524+0000 7f2240fdd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:31:35.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:35.524+0000 7f2240fdd700 -1 osd.5 618 *** Got signal Terminated *** 2023-10-09T10:31:35.694 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:35.524+0000 7f2240fdd700 -1 osd.5 618 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:31:36.192 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:35.923+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:36.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:35 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:35.923+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:36.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:36.392+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:36.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:36.392+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:36.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:36.261+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:36.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:36.261+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.462144+0000 front 2023-10-09T10:31:07.462241+0000 (oldest deadline 2023-10-09T10:31:32.762041+0000) 2023-10-09T10:31:37.192 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:36.920+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:37.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:36 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:36.920+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:37.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:37.397+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:37.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:37.397+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:37.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:37.218+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:37.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:37.218+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.462144+0000 front 2023-10-09T10:31:07.462241+0000 (oldest deadline 2023-10-09T10:31:32.762041+0000) 2023-10-09T10:31:38.209 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:37.946+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:38.209 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:37 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:37.946+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:38.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:38.425+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:38.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:38.425+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:38.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:38.209+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:38.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:38.209+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.462144+0000 front 2023-10-09T10:31:07.462241+0000 (oldest deadline 2023-10-09T10:31:32.762041+0000) 2023-10-09T10:31:39.253 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:38.945+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:39.253 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:38 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:38.945+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:39.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:39.252+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:39.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:39.252+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.462144+0000 front 2023-10-09T10:31:07.462241+0000 (oldest deadline 2023-10-09T10:31:32.762041+0000) 2023-10-09T10:31:39.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:39.471+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:39.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:39.471+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:39.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:39.471+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:40.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:39.941+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:40.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:39.941+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:40.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:39 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:39.941+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:15.617671+0000 front 2023-10-09T10:31:15.617796+0000 (oldest deadline 2023-10-09T10:31:39.117581+0000) 2023-10-09T10:31:40.692 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:40.292+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:59.459958+0000 front 2023-10-09T10:30:59.460138+0000 (oldest deadline 2023-10-09T10:31:22.359976+0000) 2023-10-09T10:31:40.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:40.292+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.462144+0000 front 2023-10-09T10:31:07.462241+0000 (oldest deadline 2023-10-09T10:31:32.762041+0000) 2023-10-09T10:31:40.693 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5[105384]: debug 2023-10-09T10:31:40.292+0000 7f223d759700 -1 osd.5 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:16.163969+0000 front 2023-10-09T10:31:16.163892+0000 (oldest deadline 2023-10-09T10:31:40.263852+0000) 2023-10-09T10:31:40.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:40.512+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:40.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:40.512+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:40.694 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:40.512+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:41.192 INFO:journalctl@ceph.osd.5.smithi110.stdout:Oct 09 10:31:41 smithi110 bash[284759]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T10:31:41.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:40.941+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:41.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:40.941+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:41.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:40 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:40.941+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:15.617671+0000 front 2023-10-09T10:31:15.617796+0000 (oldest deadline 2023-10-09T10:31:39.117581+0000) 2023-10-09T10:31:41.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:41.551+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:41.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:41.551+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:41.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:41.551+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:42.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:41.980+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:42.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:41.980+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:42.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:41 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:41.980+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:15.617671+0000 front 2023-10-09T10:31:15.617796+0000 (oldest deadline 2023-10-09T10:31:39.117581+0000) 2023-10-09T10:31:42.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T10:31:42.495 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:42.495 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T10:31:42.495 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T10:31:42.495 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T10:31:42.884 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:42 smithi110 systemd[1]: Stopping Ceph osd.6 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:42.884 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:42.519+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:42.884 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:42.519+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:42.884 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:42.519+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:43.192 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:42.883+0000 7f7cb72bc700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:31:43.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:42.883+0000 7f7cb72bc700 -1 osd.6 618 *** Got signal Terminated *** 2023-10-09T10:31:43.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:42.883+0000 7f7cb72bc700 -1 osd.6 618 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:31:43.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:42.976+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:43.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:42.976+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:43.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:42 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:42.976+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:15.617671+0000 front 2023-10-09T10:31:15.617796+0000 (oldest deadline 2023-10-09T10:31:39.117581+0000) 2023-10-09T10:31:43.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:43.482+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:43.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:43.482+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:43.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:43.482+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:44.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:43.974+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:44.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:43.974+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:44.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:43 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:43.974+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:15.617671+0000 front 2023-10-09T10:31:15.617796+0000 (oldest deadline 2023-10-09T10:31:39.117581+0000) 2023-10-09T10:31:44.942 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:44.511+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:44.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:44.511+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:44.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:44.511+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:45.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:44.979+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:45.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:44.979+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:45.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:44 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:44.979+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:15.617671+0000 front 2023-10-09T10:31:15.617796+0000 (oldest deadline 2023-10-09T10:31:39.117581+0000) 2023-10-09T10:31:45.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:45.476+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:45.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:45.476+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:45.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:45.476+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:46.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:45.951+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:46.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:45.951+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:46.443 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:45 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:45.951+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:15.617671+0000 front 2023-10-09T10:31:15.617796+0000 (oldest deadline 2023-10-09T10:31:39.117581+0000) 2023-10-09T10:31:46.921 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:46.473+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:46.922 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:46.473+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:46.922 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:46.473+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:47.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:46.921+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:56.514566+0000 front 2023-10-09T10:30:56.514442+0000 (oldest deadline 2023-10-09T10:31:25.914728+0000) 2023-10-09T10:31:47.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:46.921+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.615741+0000 front 2023-10-09T10:31:07.615807+0000 (oldest deadline 2023-10-09T10:31:29.915805+0000) 2023-10-09T10:31:47.193 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:46 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6[109535]: debug 2023-10-09T10:31:46.921+0000 7f7cb4239700 -1 osd.6 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:15.617671+0000 front 2023-10-09T10:31:15.617796+0000 (oldest deadline 2023-10-09T10:31:39.117581+0000) 2023-10-09T10:31:47.942 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:47 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:47.484+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:47.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:47 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:47.484+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:47.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:47 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:47.484+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:48.693 INFO:journalctl@ceph.osd.6.smithi110.stdout:Oct 09 10:31:48 smithi110 bash[285028]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T10:31:48.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:48.519+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:48.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:48.519+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:48.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:48.519+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:48.694 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:48 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:48.519+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2023-10-09T10:31:24.036683+0000 front 2023-10-09T10:31:24.036621+0000 (oldest deadline 2023-10-09T10:31:47.536577+0000) 2023-10-09T10:31:49.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:49.556+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:49.693 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:49.556+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:49.694 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:49.556+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:49.694 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:49 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:49.556+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2023-10-09T10:31:24.036683+0000 front 2023-10-09T10:31:24.036621+0000 (oldest deadline 2023-10-09T10:31:47.536577+0000) 2023-10-09T10:31:49.824 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T10:31:49.864 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:49.864 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T10:31:49.864 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T10:31:49.864 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T10:31:50.193 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:49 smithi110 systemd[1]: Stopping Ceph osd.7 for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:50.569 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:50.240+0000 7f1935a8c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T10:31:50.570 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:50.240+0000 7f1935a8c700 -1 osd.7 618 *** Got signal Terminated *** 2023-10-09T10:31:50.570 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:50.240+0000 7f1935a8c700 -1 osd.7 618 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:31:50.942 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:50.569+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:50.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:50.569+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:50.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:50.569+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:50.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:50 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:50.569+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2023-10-09T10:31:24.036683+0000 front 2023-10-09T10:31:24.036621+0000 (oldest deadline 2023-10-09T10:31:47.536577+0000) 2023-10-09T10:31:51.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:51.522+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:51.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:51.522+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:51.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:51.522+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:51.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:51 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:51.522+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2023-10-09T10:31:24.036683+0000 front 2023-10-09T10:31:24.036621+0000 (oldest deadline 2023-10-09T10:31:47.536577+0000) 2023-10-09T10:31:52.942 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:52.495+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:52.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:52.495+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:52.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:52.495+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:52.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:52 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:52.495+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2023-10-09T10:31:24.036683+0000 front 2023-10-09T10:31:24.036621+0000 (oldest deadline 2023-10-09T10:31:47.536577+0000) 2023-10-09T10:31:53.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:53.479+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:53.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:53.479+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:53.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:53.479+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:53.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:53.479+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2023-10-09T10:31:24.036683+0000 front 2023-10-09T10:31:24.036621+0000 (oldest deadline 2023-10-09T10:31:47.536577+0000) 2023-10-09T10:31:53.944 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:53 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:53.479+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.110:6804 osd.4 since back 2023-10-09T10:31:27.537001+0000 front 2023-10-09T10:31:27.536969+0000 (oldest deadline 2023-10-09T10:31:53.436888+0000) 2023-10-09T10:31:54.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:54.460+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2023-10-09T10:30:58.833597+0000 front 2023-10-09T10:30:58.833665+0000 (oldest deadline 2023-10-09T10:31:21.733570+0000) 2023-10-09T10:31:54.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:54.460+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2023-10-09T10:31:07.634351+0000 front 2023-10-09T10:31:07.634506+0000 (oldest deadline 2023-10-09T10:31:31.734377+0000) 2023-10-09T10:31:54.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:54.460+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2023-10-09T10:31:13.435367+0000 front 2023-10-09T10:31:13.435464+0000 (oldest deadline 2023-10-09T10:31:38.735359+0000) 2023-10-09T10:31:54.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:54.460+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2023-10-09T10:31:24.036683+0000 front 2023-10-09T10:31:24.036621+0000 (oldest deadline 2023-10-09T10:31:47.536577+0000) 2023-10-09T10:31:54.943 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:54 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7[113583]: debug 2023-10-09T10:31:54.460+0000 7f193320a700 -1 osd.7 618 heartbeat_check: no reply from 172.21.15.110:6804 osd.4 since back 2023-10-09T10:31:27.537001+0000 front 2023-10-09T10:31:27.536969+0000 (oldest deadline 2023-10-09T10:31:53.436888+0000) 2023-10-09T10:31:55.728 INFO:journalctl@ceph.osd.7.smithi110.stdout:Oct 09 10:31:55 smithi110 bash[285285]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T10:31:56.996 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T10:31:57.035 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:57.035 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T10:31:57.036 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T10:31:57.036 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T10:31:57.407 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 systemd[1]: Stopping Ceph prometheus.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:57.693 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:31:57.693 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T10:31:57.693 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T10:31:57.693 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T10:31:57.693 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T10:31:57.694 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T10:31:57.694 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T10:31:57.694 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T10:31:57.694 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.406Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T10:31:57.694 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.408Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T10:31:57.694 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.408Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T10:31:57.694 INFO:journalctl@ceph.prometheus.a.smithi110.stdout:Oct 09 10:31:57 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-prometheus-a[129085]: ts=2023-10-09T10:31:57.408Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T10:31:57.951 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T10:31:57.987 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:31:57.987 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T10:31:57.987 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T10:31:59.677 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 10:31:59 smithi012 systemd[1]: Stopping Ceph node-exporter.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:32:00.139 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 10:32:00 smithi012 bash[463987]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T10:32:00.427 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 10:32:00 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:32:00.427 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 10:32:00 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T10:32:00.427 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Oct 09 10:32:00 smithi012 systemd[1]: Stopped Ceph node-exporter.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T10:32:00.766 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 10:32:00 smithi012 systemd[1]: Stopping Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:32:01.030 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 10:32:00 smithi012 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a[141717]: level=info ts=2023-10-09T10:32:00.765Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:32:01.408 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 10:32:01 smithi012 bash[464111]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T10:32:01.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 10:32:01 smithi012 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T10:32:01.677 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Oct 09 10:32:01 smithi012 systemd[1]: Stopped Ceph alertmanager.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T10:32:07.168 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T10:32:08.693 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 10:32:08 smithi110 systemd[1]: Stopping Ceph node-exporter.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:32:09.226 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 10:32:09 smithi110 bash[285909]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T10:32:09.512 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 10:32:09 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:32:09.512 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 10:32:09 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T10:32:09.512 INFO:journalctl@ceph.node-exporter.b.smithi110.stdout:Oct 09 10:32:09 smithi110 systemd[1]: Stopped Ceph node-exporter.b for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T10:32:09.822 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 10:32:09 smithi110 systemd[1]: Stopping Ceph grafana.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7... 2023-10-09T10:32:10.075 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 10:32:09 smithi110 ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a[126241]: t=2023-10-09T10:32:09+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T10:32:10.076 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 10:32:10 smithi110 bash[286035]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T10:32:10.652 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 10:32:10 smithi110 systemd[1]: ceph-1881d6a6-6688-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T10:32:10.653 INFO:journalctl@ceph.grafana.a.smithi110.stdout:Oct 09 10:32:10 smithi110 systemd[1]: Stopped Ceph grafana.a for 1881d6a6-6688-11ee-8db5-212e2dc638e7. 2023-10-09T10:32:14.056 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:14.094 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:14.123 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T10:32:14.125 DEBUG:teuthology.misc:Transferring archived files from smithi012:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635/remote/smithi012/crash 2023-10-09T10:32:14.126 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T10:32:14.198 INFO:teuthology.orchestra.run.smithi012.stderr:tar: /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T10:32:14.198 INFO:teuthology.orchestra.run.smithi012.stderr:tar: Error is not recoverable: exiting now 2023-10-09T10:32:14.200 DEBUG:teuthology.misc:Transferring archived files from smithi110:/var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635/remote/smithi110/crash 2023-10-09T10:32:14.201 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T10:32:14.286 INFO:teuthology.orchestra.run.smithi110.stderr:tar: /var/lib/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T10:32:14.287 INFO:teuthology.orchestra.run.smithi110.stderr:tar: Error is not recoverable: exiting now 2023-10-09T10:32:14.287 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T10:32:14.288 DEBUG:teuthology.orchestra.run.smithi012:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T10:32:14.322 INFO:teuthology.orchestra.run.smithi012.stderr:grep: /var/log/ceph/1881d6a6-6688-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T10:32:14.325 INFO:tasks.cephadm:Compressing logs... 2023-10-09T10:32:14.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T10:32:14.367 DEBUG:teuthology.orchestra.run.smithi110:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T10:32:14.419 INFO:teuthology.orchestra.run.smithi110.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T10:32:14.465 INFO:teuthology.orchestra.run.smithi012.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T10:42:28.787 INFO:tasks.cephadm:Archiving logs... 2023-10-09T10:42:28.788 DEBUG:teuthology.misc:Transferring archived files from smithi012:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635/remote/smithi012/log 2023-10-09T10:42:28.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T10:43:40.710 DEBUG:teuthology.misc:Transferring archived files from smithi110:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635/remote/smithi110/log 2023-10-09T10:43:40.712 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T10:44:39.957 INFO:tasks.cephadm:Removing cluster... 2023-10-09T10:44:40.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 --force 2023-10-09T10:44:41.796 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1881d6a6-6688-11ee-8db5-212e2dc638e7 --force 2023-10-09T10:44:43.337 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T10:44:43.338 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T10:44:43.360 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T10:44:43.384 INFO:tasks.cephadm:Teardown complete 2023-10-09T10:44:43.384 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T10:44:43.472 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T10:44:43.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T10:44:43.475 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T10:44:43.579 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T10:44:43.589 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T10:44:43.590 DEBUG:teuthology.orchestra.run.smithi012:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T10:44:43.592 DEBUG:teuthology.orchestra.run.smithi110:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T10:44:43.609 INFO:teuthology.orchestra.run.smithi110.stderr:bash: ntpq: command not found 2023-10-09T10:44:43.610 INFO:teuthology.orchestra.run.smithi012.stderr:bash: ntpq: command not found 2023-10-09T10:44:43.631 INFO:teuthology.orchestra.run.smithi110.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T10:44:43.631 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================== 2023-10-09T10:44:43.632 INFO:teuthology.orchestra.run.smithi110.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 46 -112us[ -118us] +/- 89ms 2023-10-09T10:44:43.632 INFO:teuthology.orchestra.run.smithi110.stdout:^* hv02.front.sepia.ceph.com 3 6 377 41 +544us[ +539us] +/- 60ms 2023-10-09T10:44:43.632 INFO:teuthology.orchestra.run.smithi110.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 40 -461us[ -461us] +/- 78ms 2023-10-09T10:44:43.632 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:44:43.634 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T10:44:43.634 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2023-10-09T10:44:43.634 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 35 -153us[ -153us] +/- 89ms 2023-10-09T10:44:43.634 INFO:teuthology.orchestra.run.smithi012.stdout:^* hv02.front.sepia.ceph.com 3 6 377 36 +478us[ +481us] +/- 60ms 2023-10-09T10:44:43.634 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 35 -505us[ -505us] +/- 78ms 2023-10-09T10:44:43.634 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:44:43.635 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T10:44:43.644 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T10:44:43.645 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T10:44:43.670 DEBUG:teuthology.orchestra.run.smithi012:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T10:44:43.822 DEBUG:teuthology.orchestra.run.smithi110:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T10:44:43.967 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T10:44:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:44:44.003 DEBUG:teuthology.orchestra.run.smithi110:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T10:44:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:44:44.034 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T10:44:44.045 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T10:44:44.063 INFO:teuthology.task.internal:Duration was 4593.352484 seconds 2023-10-09T10:44:44.064 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T10:44:44.095 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T10:44:44.095 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T10:44:44.097 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T10:44:44.155 INFO:teuthology.orchestra.run.smithi110.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T10:44:44.161 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T10:44:44.466 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T10:44:44.466 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi012.front.sepia.ceph.com 2023-10-09T10:44:44.467 DEBUG:teuthology.orchestra.run.smithi012:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T10:44:44.496 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi110.front.sepia.ceph.com 2023-10-09T10:44:44.497 DEBUG:teuthology.orchestra.run.smithi110:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T10:44:44.521 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T10:44:44.521 DEBUG:teuthology.orchestra.run.smithi012:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T10:44:44.539 DEBUG:teuthology.orchestra.run.smithi110:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T10:44:44.722 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T10:44:44.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T10:44:44.724 DEBUG:teuthology.orchestra.run.smithi110:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T10:44:45.127 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T10:44:45.138 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T10:44:45.139 DEBUG:teuthology.orchestra.run.smithi012:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T10:44:45.178 DEBUG:teuthology.orchestra.run.smithi110:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T10:44:45.249 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T10:44:45.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T10:44:45.262 DEBUG:teuthology.orchestra.run.smithi110:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T10:44:45.294 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern = core 2023-10-09T10:44:45.300 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = core 2023-10-09T10:44:45.325 DEBUG:teuthology.orchestra.run.smithi012:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T10:44:45.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:44:45.372 DEBUG:teuthology.orchestra.run.smithi110:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T10:44:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:44:45.390 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T10:44:45.400 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T10:44:45.402 DEBUG:teuthology.misc:Transferring archived files from smithi012:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635/remote/smithi012 2023-10-09T10:44:45.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T10:44:45.477 DEBUG:teuthology.misc:Transferring archived files from smithi110:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635/remote/smithi110 2023-10-09T10:44:45.477 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T10:44:45.526 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T10:44:45.527 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T10:44:45.529 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T10:44:45.573 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T10:44:45.585 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T10:44:45.585 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T10:44:45.606 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T10:44:45.606 DEBUG:teuthology.orchestra.run.smithi012:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T10:44:45.609 DEBUG:teuthology.orchestra.run.smithi110:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T10:44:45.623 INFO:teuthology.orchestra.run.smithi012.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 10:44 /home/ubuntu/cephtest 2023-10-09T10:44:45.623 INFO:teuthology.orchestra.run.smithi012.stdout: 4325377 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 10:29 /home/ubuntu/cephtest/mnt.0 2023-10-09T10:44:45.623 INFO:teuthology.orchestra.run.smithi012.stdout: 4325378 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 10:29 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:44:45.629 INFO:teuthology.orchestra.run.smithi110.stdout: 262320 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 10:44 /home/ubuntu/cephtest 2023-10-09T10:44:45.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:44:45.639 INFO:teuthology.orchestra.run.smithi012.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T10:44:45.639 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi012 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T10:44:45.646 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T10:44:45.657 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T10:44:45.682 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi012 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T10:44:45.683 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T10:44:45.708 DEBUG:teuthology.nuke:shortname: smithi012 2023-10-09T10:44:45.709 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T10:44:45.732 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi012.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 09:22:55.238270', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T10:44:45.736 DEBUG:teuthology.nuke:shortname: smithi110 2023-10-09T10:44:45.736 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T10:44:45.758 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi110.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417635', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 09:22:55.240068', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T10:44:45.791 INFO:teuthology.orchestra.console.smithi012:Power off 2023-10-09T10:44:45.791 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T10:44:45.810 INFO:teuthology.orchestra.console.smithi110:Power off 2023-10-09T10:44:45.811 DEBUG:teuthology.orchestra.console.smithi110:pexpect command: ipmitool -H smithi110.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T10:44:45.820 DEBUG:teuthology.orchestra.console.smithi012:power off output: Chassis Power Control: Down/Off 2023-10-09T10:44:45.820 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:44:45.833 DEBUG:teuthology.orchestra.console.smithi110:power off output: Chassis Power Control: Down/Off 2023-10-09T10:44:45.833 DEBUG:teuthology.orchestra.console.smithi110:pexpect command: ipmitool -H smithi110.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:44:45.844 DEBUG:teuthology.orchestra.console.smithi012:check power output: Chassis Power is on 2023-10-09T10:44:45.857 DEBUG:teuthology.orchestra.console.smithi110:check power output: Chassis Power is on 2023-10-09T10:44:49.845 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:44:49.858 DEBUG:teuthology.orchestra.console.smithi110:pexpect command: ipmitool -H smithi110.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:44:49.956 DEBUG:teuthology.orchestra.console.smithi012:check power output: Chassis Power is on 2023-10-09T10:44:49.968 DEBUG:teuthology.orchestra.console.smithi110:check power output: Chassis Power is on 2023-10-09T10:44:53.961 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:44:53.970 DEBUG:teuthology.orchestra.console.smithi110:pexpect command: ipmitool -H smithi110.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:44:54.072 DEBUG:teuthology.orchestra.console.smithi012:check power output: Chassis Power is on 2023-10-09T10:44:54.079 DEBUG:teuthology.orchestra.console.smithi110:check power output: Chassis Power is off 2023-10-09T10:44:54.180 INFO:teuthology.orchestra.console.smithi110:Power off completed 2023-10-09T10:44:58.077 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T10:44:58.187 DEBUG:teuthology.orchestra.console.smithi012:check power output: Chassis Power is off 2023-10-09T10:44:58.288 INFO:teuthology.orchestra.console.smithi012:Power off completed 2023-10-09T10:44:58.392 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 4593.352484226227 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=e6fe39d8aab8493fa58e2b684bd53720 status: fail success: false 2023-10-09T10:44:58.392 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T10:44:58.469 INFO:teuthology.run:FAIL